Full JSON Response

Have your response organised in a JSON response for easier integration with third party apps.

The default response body received from the WebScrapingAPI contains the plain HTML scraped from the targeted URL. This approach is ideal for HTML parsing, however it can be harder to integrate with third party apps thatb only accept JSON format.

If you want WebScrapingAPI to organise the response in a JSON format, all you have to do is to add json_response=1 to your request. The format of the response will then be:

Element	Type	Description
headers	Object	key : value pair of haders received from the server.
cost	int	Credit usage of your request.
initial-status-code	int	Status code received from the server.
resolved-url	string	URL in case of redirects.
body	string	Scraped content.
type	string	Response type. Can be html , json or b64 (for files, images, PDFs etc.)
screenshot	string	base64 encrypted image.
cookies	Array	Array of cookies sent from the server.
xhr	Array	Array of XHR / Ajax requests sent by the browser durring scraping.
metadata	Object	Microdata and JSON-LD extracted from the HTML file.

Your full GET request should then be sent to the following address:

https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=<TARGETED_URL>&json_response=1

This parameter is available both with JavaScript Rendering turned on (render_js=1) or off (render_js=0)

Full JSON Response Integration Examples

cURL

curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D"

NodeJS

const http = require("https");

const options = {
  "method": "GET",
  "hostname": "api.webscrapingapi.com",
  "port": null,
  "path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D",
  "headers": {}
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on("data", function (chunk) {
    chunks.push(chunk);
  });

  res.on("end", function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.end();

Python

import requests

API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'

TARGET_URL = 'https://httpbin.org/'

PARAMS = {
    "api_key":API_KEY,
    "url": TARGET_URL,
    "screenshot":1,
    "render_js":1,
    "extract_rules":'{"paragraphs":{"selector": "p","output":"html"}}',
    "json_response":1
}

response = requests.get(SCRAPER_URL, params=PARAMS)

print(response)

PHP

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

Go

package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func main() {

	url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D"

	req, _ := http.NewRequest("GET", url, nil)

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

Java

HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D")
  .asString();

.NET

var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);

Ruby

require 'uri'
require 'net/http'
require 'openssl'

url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https://httpbin.org/&render_js=1&screenshot=1&json_response=1&extract_rules=%7B%22paragraphs%22%3A%7B%22selector%22%3A%20%22p%22%2C%22output%22%3A%22html%22%7D%7D")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

request = Net::HTTP::Get.new(url)

response = http.request(request)
puts response.read_body

Important! The url parameter has to be encoded. ( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F )

Response Example

{
    "headers": {
        "access-control-allow-credentials": "true",
        "access-control-allow-origin": "*",
        "content-length": "9593",
        "content-type": "text/html; charset=utf-8",
        "date": "Fri, 23 Sep 2022 09:05:27 GMT",
        "server": "gunicorn/19.9.0"
    },
    "cost": 1,
    "initial-status-code": 200,
    "resolved-url": "https://httpbin.org/",
    "body": "{\"screenshot\":\"/9j/4AAQSkZJRgABAQAAAQABAAD/...",
    "cookies": [],
    "evaluate_results": "{\"paragraphs\":[\"<p>A simple HTTP Request &amp; Response Service.\\n                                        <br>\\n                                        <br>\\n                                        <b>Run locally: </b>\\n                                        <code>$ docker run -p 80:80 kennethreitz/httpbin</code>\\n                                    </p>\"]}",
    "xhr": [],
    "metadata": {
        "microdata": {
            "items": []
        },
        "json-ld": []
    }
}