Comment on page
Custom Headers
WebScrapingAPI allows you to add your own headers to a request and get customised results.
Custom HTTP headers are useful when you want to provide additional information to your request. If your request requires passing custom HTTP headers, all you have to do is to append
WSA- to your header's name and add them to your request. For example, the
Sec-Fetch-Dest: document header will be passed as WSA-Sec-Fetch-Dest: document.Only use this feature to get customized results, do not use this feature to avoid blocks. WebScrapingAPI has a powerful internal engine that takes care of everything for you.
cURL
NodeJS
Python
PHP
Go
Java
.NET
Ruby
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fheaders" --header "WSA-My-header: test" --header "WSA-User-Agent: potato"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=http%3A%2F%2Fhttpbin.org%2Fheaders",
"headers": {
"Wsa-My-header": "test",
"Wsa-User-Agent": "potato"
}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/headers'
HEADERS = {
'WSA-My-header': "test",
'WSA-User-Agent': "potato"
}
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"render_js":1
}
response = requests.get(SCRAPER_URL, params=PARAMS, headers=HEADERS)
print(response.text)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=http%3A%2F%2Fhttpbin.org%2Fheaders",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Wsa-My-header: test",
"Wsa-User-Agent: potato"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=http%3A%2F%2Fhttpbin.org%2Fheaders"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Wsa-My-header", "test")
req.Header.Add("Wsa-User-Agent", "potato")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=http%3A%2F%2Fhttpbin.org%2Fheaders")
.header("Wsa-My-header", "test")
.header("Wsa-User-Agent", "potato")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=http%3A%2F%2Fhttpbin.org%2Fheaders");
var request = new RestRequest(Method.GET);
request.AddHeader("Wsa-My-header", "test");
request.AddHeader("Wsa-User-Agent", "potato");
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=http%3A%2F%2Fhttpbin.org%2Fheaders")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
request["Wsa-My-header"] = 'test'
request["Wsa-User-Agent"] = 'potato'
response = http.request(request)
puts response.read_body
Important! The
url parameter has to be encoded. ( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F ){
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Host": "httpbin.org",
"My-Header": "test",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "potato",
"X-Amzn-Trace-Id": "Root=1-6267c064-44fa9993017cfcc32e41dfc1"
}
}
By default, when you add custom headers to your request, WebScrapingAPI will pass the original headers alongside the ones you specified. If you do not wish to pass the original headers to your request, you will have to disable this feature by passing the
keep_headers=0 parameter. We've kept the example above and just added the
keep_headers=0 parameter to the request and here is how the response differs from the original one:keep_headers=0
keep_headers=1 (default)
{
"headers": {
"Accept-Encoding": "gzip, deflate, br",
"Host": "httpbin.org",
"My-Header": "test",
"User-Agent": "potato",
"X-Amzn-Trace-Id": "Root=1-6320123d-7d995c9d1c8f6cd66937f114"
}
}
{
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Host": "httpbin.org",
"My-Header": "test",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "potato",
"X-Amzn-Trace-Id": "Root=1-63201227-389a26f76a53f31b6ed255c4"
}
}
Last modified 3mo ago