Proxies
Using the WebScrapingAPI, you can bypass security rules on hard to scrape websites by sending requests from residential IP addresses.
The WebScrapingAPI is making use of a pool of 100+ million IP addresses worldwide, making your requests impossible to get blocked.
When interacting with the targeted URL, you can specify if your request should be processed through a
residential or a datacenter proxy. By default, the proxy_type parameter is set to residential.Your full GET request should then be sent to the following address:
https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=<TARGETED_URL>&proxy_type=residential
Datacenter proxies are proxies that are not affiliated with an Internet Service Provider (ISP). They come from a secondary corporation and provide you with completely private IP authentication and anonymity.
Usually coming from cloud server providers, datacenter proxies are used by many at the same time. Since they are not listed as ISP providers, these IPs can already be flagged by some targets and certain precautionary measures might be taken.
However, this is not the case with WebScrapingAPI datacenter proxies. All WebScrapingAPI datacenter proxies are private proxies, and ensure little to no IP blacklisting.
A dedicated residential proxy is an IP address provided by an ISP (Internet Service Provider) to a homeowner. It is a real IP address attached to a physical location. So basically, whenever you move to a new location and set up your internet, your ISP will provide you with an IP address.
While these proxies are the most common proxies used on the Internet, they are also much more likely to get blocked on an IP-basis when attempting to scrape data.
The main difference between datacenter and residential proxies lies in the source of the IP. All other differences that arise rely on the differences between household machines and servers.
Enabling
proxy_type=residential counts as 10 API calls when render_js is off and 25 API calls when the render_js is on.The default value for
proxy_type will be decided based on your plan. If your plan supports residential proxies then the default value will be residential. Otherwise the default value will be datacenter.cURL
NodeJS
Python
php
Golang
Java
.NET
Ruby
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://httpbin.org/get&proxy_type=datacenter"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=datacenter",
"headers": {}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/'
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"proxy_type":'datacenter'
}
response = requests.get(SCRAPER_URL, params=PARAMS)
print(response)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=datacenter",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=datacenter"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=datacenter")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=datacenter");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=datacenter")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
Important! The
url parameter has to be encoded. ( i.e. &url=https%3A%2F%2Fwww.webscrapingapi.com%2F ){
"args": {},
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Host": "httpbin.org",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-6267d94c-6139009e7aa96c016119654f"
},
"origin": "5.153.235.233",
"url": "https://httpbin.org/get"
}
cURL
NodeJS
Python
php
Golang
Java
.NET
Ruby
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https://httpbin.org/get&proxy_type=residential
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=residential",
"headers": {}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/'
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
"proxy_type":'residential'
}
response = requests.get(SCRAPER_URL, params=PARAMS)
print(response)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=residential",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=residential"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=residential")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=residential");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fhttpbin.org%2Fget&proxy_type=residential")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
{
"args": {},
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Host": "httpbin.org",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-6267d9e9-7b7217ea171b487901c4bfab"
},
"origin": "71.167.129.216",
"url": "https://httpbin.org/get"
}
Last modified 7mo ago