Custom Cookies
Add your own cookies to BrowserAPI and receive customised responses.
Custom cookies are an extremely powerful feature of BrowserAPI. For example, they can be successfully used to scrape personalized content on a e-commerce site.
BrowserAPI does not allow the use of custom cookies for authentication purposes.
Because BrowserAPI will no longer have complete control over the scraping performance, using this feature means charging 100% of the requests (both successful and failed).
Custom Cookies without JavaScript Rendering
When you are not using JavaScript rendering, custom cookies should be added in the WSA-Cookie
custom header. Simply pass the name and value of the cookie(s) to this header when you send your request.
Integration Examples
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies" --header "Wsa-Accept: application/json" --header "WSA-Cookie: name1=value1; name2=value2"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies",
"headers": {
"Wsa-Accept": "application/json",
"Wsa-Cookie": "name1=value1; name2=value2"
}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/headers'
HEADERS = {
"WSA-Accept": "application/json",
"WSA-Cookie": "name1=value1; name2=value2"
}
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"render_js":1
}
response = requests.get(SCRAPER_URL, params=PARAMS, headers=HEADERS)
print(response.text)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Wsa-Accept: application/json",
"Wsa-Cookie: name1=value1; name2=value2"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Wsa-Accept", "application/json")
req.Header.Add("Wsa-Cookie", "name1=value1; name2=value2")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies")
.header("Wsa-Accept", "application/json")
.header("Wsa-Cookie", "name1=value1; name2=value2")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies");
var request = new RestRequest(Method.GET);
request.AddHeader("Wsa-Accept", "application/json");
request.AddHeader("Wsa-Cookie", "name1=value1; name2=value2");
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=http%3A%2F%2Fhttpbin.org%2Fcookies")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
request["Wsa-Accept"] = 'application/json'
request["Wsa-Cookie"] = 'name1=value1; name2=value2'
response = http.request(request)
puts response.read_body
Custom Cookies with JavaScript Rendering
When using JavaScript rendering, custom cookies should be passed in a list in the cookies
parameter. Each cookie object will have 3 properties: name, value and domain. Here is an example:
[{"name":"cookie_name1", "value":"cookie_value1", "domain":"cookie_domain1"},{"name":"cookie_name2", "value":"cookie_value2", "domain":"cookie_domain2"},{"name":"cookie_name3", "value":"cookie_value3", "domain":"cookie_domain3"}]
And this is the URL encoded value from the example above:
%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22cookie_domain1%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22cookie_domain2%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22cookie_domain3%22%7D%5D
Warning! Please make sure that you passed the correct domain value in the cookie objects. We used "domain":"httpbin.org"
in the examples below because we want those cookies to be set on the httpbin.org domain.
Integration Examples
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D"
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
COOKIES = '%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D'
TARGET_URL = 'https://httpbin.org/cookies'
PARAMS = {
"api_key": API_KEY,
"url": TARGET_URL,
"render_js": 1,
"cookies": COOKIES
}
response = requests.get(SCRAPER_URL, params=PARAMS, headers=HEADERS)
print(response.text)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET"
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&render_js=1&url=http%3A%2F%2Fhttpbin.org%2Fcookies&cookies=%5B%7B%22name%22%3A%22cookie_name1%22%2C%20%22value%22%3A%22cookie_value1%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name2%22%2C%20%22value%22%3A%22cookie_value2%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%2C%7B%22name%22%3A%22cookie_name3%22%2C%20%22value%22%3A%22cookie_value3%22%2C%20%22domain%22%3A%22httpbin.org%22%7D%5D")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
Response Example
<html>
<head>
<meta name="color-scheme" content="light dark">
</head>
<body>
<pre style="word-wrap: break-word; white-space: pre-wrap;">{
"cookies": {
"cookie_name1": "cookie_value1",
"cookie_name2": "cookie_value2",
"cookie_name3": "cookie_value3"
}
}
</pre>
</body>
</html>
Important! The url
parameter has to be encoded.
( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F )
Last updated