Disable Stealth
When scraping the web in headless mode, WebScrapingAPI implements evasion techniques by default, to prevent detection. Should you need, you can disable this feature at any time.
Websites take all sorts of measures to prevent scraping. By default, WebScrapingAPI tries to overcome those measures by implementing evasion techniques, making it harder for the targeted website to detect us.
However, certain websites may not require such technology. In this case, you can disable stealth by passing the
disable_stealth=1 parameter to your request. Your full GET request should then be sent to the following address:
https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=<TARGETED_URL>&disable_stealth=1
cURL
NodeJS
Python
PHP
Go
Java
.NET
Ruby
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://httpbin.org/get&disable_stealth=1&render_js=1"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&disable_stealth=1&render_js=1",
"headers": {}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/'
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"disable_stealth":1
}
response = requests.get(SCRAPER_URL, params=PARAMS)
print(response.text)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&disable_stealth=1&render_js=1",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&disable_stealth=1&render_js=1"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&disable_stealth=1&render_js=1")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&disable_stealth=1&render_js=1");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&disable_stealth=1&render_js=1")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
Important! The
url parameter has to be encoded. ( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F )<html>
<head></head>
<body>
<pre style="word-wrap: break-word; white-space: pre-wrap;">{
"args": {},
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Cookie": "MUIDB=09BEED778B5E6BBF0F38FCE58A816AF1",
"Host": "httpbin.org",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.0 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-6267e32d-2cb7e3a46d827e293918711c"
},
"origin": "196.243.24.17",
"url": "https://httpbin.org/get"
}
</pre>
</body>
</html>
Last modified 7mo ago