JavaScript Instructions
Interact with the website you want to scrape and get an optimal response.
This parameter only works with
render_js=1
Interacting with the website you are scraping is very important, especially if you want to retrieve custom information. WebScrapingAPI allows you to do so, by simply adding the
js_instructions
parameter to your request.This parameter's value is a
stringified object
, which accepts the following options:Parameter | Type | Description |
---|---|---|
action
Required | string | The action that you want to perform. Accepted values are: - click - click on an element specified by the selector option- scrollTo - scroll to an element specified by the selector option- scrollPx - scroll a specified distance in pixels specified by the pixels option- scrollInf - scroll a specified number of times specified by the count option- focus - sets the focus on a HTML element- value - sets the value of a HTML input- submit - works only on FORM elements. |
selector | string | The CSS selector on which you want to perform your action. |
block | string | The area of the HTML element identified by the specified CSS selector. Used with the scrollTo action. Accepted values are start , center and end . The default value is start . |
value | string | The value of a HTML input identified by the specified CSS selector. Used with the value action. |
pixels | string | The distance for vertically scrolling. Used with the scrollPx action. |
count | int | The number of times to perform a vertically scroll. Used with the scrollInf action. The scrolling distance is calculated by the window height. Useful for lazy-loaded lists. |
timeout | int | Perform a delay after each JavaScript instruction. For the scrollInf action, the delay is performed after each individual scroll. Measured in ms . |
A full example of how this parameter would look like in production is:
js_instructions='[{"action":"scrollTo","selector":"div.navFooterBackToTop","timeout": 5000, "block": "start"}]'
The js_instructions parameter will accept a URL encoded JSON string that represents an array. This is a mock example that uses multiple actions:
[{"action":"click","selector":"<selector>","timeout":2000},{"action":"click","selector":"<selector>","timeout":2000},{"action":"click","selector":"<selector>","timeout":2000},{"action":"scrollTo","selector":"<selector>","timeout": 2000, "block": "end"}]
And this is the URL encoded value from the example above:
%5B%7B%22action%22%3A%22click%22%2C%22selector%22%3A%22%3Cselector%3E%22%2C%22timeout%22%3A2000%7D%2C%7B%22action%22%3A%22click%22%2C%22selector%22%3A%22%3Cselector%3E%22%2C%22timeout%22%3A2000%7D%2C%7B%22action%22%3A%22click%22%2C%22selector%22%3A%22%3Cselector%3E%22%2C%22timeout%22%3A2000%7D%2C%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22%3Cselector%3E%22%2C%22timeout%22%3A%202000%2C%20%22block%22%3A%20%22end%22%7D%5D
get
https://api.webscrapingapi.com/
v1
JavaScript Instructions Passed to Request
The full GET request for the
js_instructions
should be:https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://webscrapingapi.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A
cURL
NodeJS
Python
PHP
Go
Java
.NET
Ruby
curl "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A",
"headers": {}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://webscrapingapi.com/'
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"render_js": 1,
"js_instructions":'[{"action":"scrollTo","selector":"div.navFooterBackToTop","timeout": 5000, "block": "start"}]'
}
response = requests.get(SCRAPER_URL, params=PARAMS)
print(response.text)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://webscrapingapi.com&render_js=1&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D%0A")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
Important! The
url
& js_instructions
parameters have to be encoded. ( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div... )<html>
<head>
<meta name="color-scheme" content="light dark">
</head>
<body>
<pre style="word-wrap: break-word; white-space: pre-wrap;">{
"args": {
"json": ""
},
"data": "",
"files": {},
"form": {},
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Host": "httpbin.org",
"Sec-Ch-Ua": "\"Google Chrome\";v=\"105\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"105\"",
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": "\"macOS\"",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-632aed04-47d111760275920f3389aaed"
},
"json": null,
"method": "GET",
"origin": "46.226.151.166",
"url": "https://httpbin.org/anything?json"
}
</pre>
</body>
</html>ht
Last modified 21d ago