Links

Proxy Mode

WebScrapingAPI offers a proxy front-end to the API, making the integration with third-party tools much easier.
Interacting with WebScrapingAPI via the proxy mode only changes the access method with our API; apart from that, full functionality is still available to the end user. Both costs, parameters and response format remain the same.
Note that our proxy mode supports GET, POST and PUT requests.

Accessing the API via Proxy Mode

The Proxy Mode is available via:
  • HTTPS at proxy.webscrapingapi.com:8000
  • HTTP at proxy.webscrapingapi.com:80
Authentication via proxy mode is done using the username:password@proxy.webscrapingapi.com:8000 method. However, the username part also holds the parameters you want to pass to your request. Hence, the configuration you need to pass to your request is:
  • Username: webscrapingapi, followed by the parameters you want to pass to your request (separated by dots)
  • Password: your personal API key
The most basic URL example for an authenticated request on our API via proxy mode is:
https://webscrapingapi.<parameter_1.parameter_2. ... .parameter_n>:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000

Proxy Mode Requirements and Recommendations

If you decide to use the proxy mode, please make sure you understand and follow these requirements:
  1. 1.
    Disable SSL certificates verification (-k switch in cURL, verify=false in Python etc.)
  2. 2.
    If you use proxy mode with a browser automation library (e.g. Selenium, Puppeteer etc.), it is highly recommended to disable JavaScript rendering (render_js=0) or to limit the amount of resources requested by your browser.

Proxy Mode Integration Examples

#1: GET Requests

cURL
NodeJS
Python
PHP
Go
Java
.NET
Ruby
curl --proxy https://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X GET http://httpbin.org/get -k
const axios = require('axios');
axios.get('http://httpbin.org/get', {
proxy: {
host: 'proxy.webscrapingapi.com',
port: 80,
auth: {
username: 'webscrapingapi.proxy_type=datacenter.device=desktop',
password: '<YOUR_API_KEY>'
}
}
}).then(function (response) {
console.log(response.data);
}, (err) => {
console.log(err)
})
import requests
def get_params(object):
params = ''
for key,value in object.items():
if list(object).index(key) < len(object) - 1:
params += f"{key}={value}."
else:
params += f"{key}={value}"
return params
API_KEY = '<YOUR_API_KEY>'
TARGET_URL = 'http://httpbin.org/get'
PARAMETERS = {
"proxy_type":"datacenter",
"device":"desktop"
}
PROXY = {
"http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
"https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}
response = requests.get(
url=TARGET_URL,
proxies=PROXY,
verify=False
)
print(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'http://httpbin.org/get');
curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
$response = curl_exec($ch);
if (!$response) {
die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}
echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;
curl_close($ch);
?>
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func send_proxy() {
proxyStr := "http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
proxyURL, err := url.Parse(proxyStr)
if err != nil {
fmt.Println(err)
}
transport := &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{
Transport: transport,
}
req, err := http.NewRequest("GET", "http://httpbin.org/get", nil)
parseFormErr := req.ParseForm()
if parseFormErr != nil {
fmt.Println(parseFormErr)
}
res, err := client.Do(req)
if err != nil {
fmt.Println("Failure : ", err)
}
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
func main() {
send_proxy()
}
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
public class Main {
public static void main(String [] args) throws IOException {
final String PROXY_HOST = "proxy.webscrapingapi.com";
final int PROXY_PORT = 80;
final String PROXY_USER = "webscrapingapi.proxy_type=datacenter.device=desktop";
final String PROXY_PASS = "<YOUR-API-KEY-HERE>";
final String TARGET_URL = "http://httpbin.org/get";
HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(myProxy)
, new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));
HttpClient client = HttpClients.custom()
.setConnectionManager(new BasicHttpClientConnectionManager())
.setProxy(myProxy)
.setDefaultCredentialsProvider(credentialsProvider)
.build();
HttpGet request = new HttpGet(TARGET_URL);
HttpResponse response = client.execute(request);
System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));
}
}
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
class Program
{
static async Task Main() {
var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
{
Credentials = new NetworkCredential("webscrapingapi.proxy_type=datacenter.device=desktop", "<YOUR_API_KEY>"),
};
var clientHandler = new HttpClientHandler
{
Proxy = proxy,
};
var client = new HttpClient(clientHandler);
var response = await client.GetAsync("http://httpbin.org/get");
Console.WriteLine(await response.Content.ReadAsStringAsync());
}
}net
require 'uri'
require 'net/http'
require 'openssl'
proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.proxy_type=datacenter.device=desktop"
proxy_pass = "<YOUR_API_KEY>"
url = URI("http://httpbin.org/get")
proxy = Net::HTTP::Proxy(proxy_host,
proxy_port,
proxy_user,
proxy_pass)
req = Net::HTTP::Get.new(url.path)
response = proxy.start(url.host,url.port) do |http|
http.request(req)
end
puts response.body
{
args: {},
headers: {
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
Host: 'httpbin.org',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.80 Safari/537.36',
'X-Amzn-Trace-Id': 'Root=1-632993e3-29380e237fe8c362301e8a57'
},
origin: '198.46.171.167',
url: 'http://httpbin.org/get'
}

#2: POST Requests

cURL
NodeJS
Python
PHP
Go
Java
.NET
Ruby
curl --proxy https://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X POST https://httpbin.org/post --data '{"foo":"bar"}' -k
const axios = require('axios');
let data = { 'foo':'bar' }
axios.post('http://httpbin.org/post', data, {
proxy: {
host: 'proxy.webscrapingapi.com',
port: 80,
auth: {
username: 'webscrapingapi.proxy_type=datacenter.device=desktop',
password: '<YOUR_API_KEY>'
}
}
}).then(function (response) {
console.log(response.data);
}, (err) => {
console.log(err)
})
import requests
def get_params(object):
params = ''
for key,value in object.items():
if list(object).index(key) < len(object) - 1:
params += f"{key}={value}."
else:
params += f"{key}={value}"
return params
API_KEY = '<YOUR_API_KEY>'
TARGET_URL = 'https://httpbin.org/post'
DATA = {"foo":"bar"}
PARAMETERS = {
"proxy_type":"datacenter",
"device":"desktop"
}
PROXY = {
"http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
"https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}
response = requests.post(
url=TARGET_URL,
data=DATA,
proxies=PROXY,
verify=False
)
print(response.text)
<?php
$data = json_encode(array("foo" => "bar"));
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://httpbin.org/post');
curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');
curl_setopt( $ch, CURLOPT_POSTFIELDS, $data);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
$response = curl_exec($ch);
if (!$response) {
die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}
echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;
curl_close($ch);
?>
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func send_proxy() {
proxyStr := "http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
proxyURL, err := url.Parse(proxyStr)
if err != nil {
fmt.Println(err)
}
transport := &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{
Transport: transport,
}
data := url.Values{}
data.Set("foo","bar")
req, err := http.NewRequest("POST", "http://httpbin.org/post", strings.NewReader(data.Encode()))
parseFormErr := req.ParseForm()
if parseFormErr != nil {
fmt.Println(parseFormErr)
}
res, err := client.Do(req)
if err != nil {
fmt.Println("Failure : ", err)
}
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
func main() {
send_proxy()
}
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
public class Main {
public static void main(String [] args) throws IOException {
final String PROXY_HOST = "proxy.webscrapingapi.com";
final int PROXY_PORT = 80;
final String PROXY_USER = "webscrapingapi.proxy_type=datacenter.device=desktop";
final String PROXY_PASS = "<YOUR_API_KEY>";
final String TARGET_URL = "http://httpbin.org/post";
final String BODY = "{\"foo\":\"bar\"}";
HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(myProxy)
, new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));
HttpClient client = HttpClients.custom()
.setConnectionManager(new BasicHttpClientConnectionManager())
.setProxy(myProxy)
.setDefaultCredentialsProvider(credentialsProvider)
.build();
HttpPost request = new HttpPost(TARGET_URL);
HttpEntity stringEntity = new StringEntity(BODY, ContentType.APPLICATION_JSON);
request.setEntity(stringEntity);
HttpResponse response = client.execute(request);
System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));
}
}
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
class Program
{
static async Task Main() {
var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
{
Credentials = new NetworkCredential("webscrapingapi.proxy_type=datacenter.device=desktop", "<YOUR_API_KEY>"),
};
var clientHandler = new HttpClientHandler
{
Proxy = proxy,
};
var data = new FormUrlEncodedContent(new[]
{
new KeyValuePair<string, string>("foo", "bar"),
});
var client = new HttpClient(clientHandler);
var response = await client.PostAsync("http://httpbin.org/post", data);
Console.WriteLine(await response.Content.ReadAsStringAsync());
}
}
require 'uri'
require 'net/http'
require 'openssl'
proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.proxy_type=datacenter.device=desktop"
proxy_pass = "<YOUR_API_KEY>"
url = URI("http://httpbin.org/post")
proxy = Net::HTTP::Proxy(proxy_host,
proxy_port,
proxy_user,
proxy_pass)
req = Net::HTTP::Post.new(url.path)
req.set_form_data({'foo' => 'bar'})
response = proxy.start(url.host,url.port) do |http|
http.request(req)
end
puts response.body
{
"args": {},
"data": "",
"files": {},
"form": {
"{\"foo\":\"bar\"}": ""
},
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Content-Length": "28",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "httpbin.org",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-631edd38-2bbe74166ca70a5b438fa87f"
},
"json": null,
"origin": "196.244.10.12",
"url": "https://httpbin.org/post"
}

#3: PUT Requests

cURL
NodeJS
Python
PHP
Go
Java
.NET
Ruby
curl --proxy https://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X PUT https://httpbin.org/put --data '{"foo":"bar"}' -k
const axios = require('axios');
let data = { 'foo':'bar' }
axios.put('http://httpbin.org/put', data, {
proxy: {
host: 'proxy.webscrapingapi.com',
port: 80,
auth: {
username: 'webscrapingapi.proxy_type=datacenter.device=desktop',
password: '<YOUR_API_KEY>'
}
}
}).then(function (response) {
console.log(response.data);
}, (err) => {
console.log(err)
})
import requests
def get_params(object):
params = ''
for key,value in object.items():
if list(object).index(key) < len(object) - 1:
params += f"{key}={value}."
else:
params += f"{key}={value}"
return params
API_KEY = '<YOUR_API_KEY>'
TARGET_URL = 'https://httpbin.org/put'
DATA = {"foo":"bar"}
PARAMETERS = {
"proxy_type":"datacenter",
"device":"desktop"
}
PROXY = {
"http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
"https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}
response = requests.put(
url=TARGET_URL,
data=DATA,
proxies=PROXY,
verify=False
)
print(response.text)
<?php
$data = json_encode(array("foo" => "bar"));
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://httpbin.org/put');
curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');
curl_setopt( $ch, CURLOPT_POSTFIELDS, $data);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
$response = curl_exec($ch);
if (!$response) {
die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}
echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;
curl_close($ch);
?>
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func send_proxy() {
proxyStr := "http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
proxyURL, err := url.Parse(proxyStr)
if err != nil {
fmt.Println(err)
}
transport := &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{
Transport: transport,
}
data := url.Values{}
data.Set("foo","bar")
req, err := http.NewRequest("PUT", "http://httpbin.org/put", strings.NewReader(data.Encode()))
parseFormErr := req.ParseForm()
if parseFormErr != nil {
fmt.Println(parseFormErr)
}
res, err := client.Do(req)
if err != nil {
fmt.Println("Failure : ", err)
}
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
func main() {
send_proxy()
}
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
public class Main {
public static void main(String [] args) throws IOException {
final String PROXY_HOST = "proxy.webscrapingapi.com";
final int PROXY_PORT = 80;
final String PROXY_USER = "webscrapingapi.proxy_type=datacenter.device=desktop";
final String PROXY_PASS = "<YOUR_API_KEY>";
final String TARGET_URL = "http://httpbin.org/put";
final String BODY = "{\"foo\":\"bar\"}";
HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(myProxy)
, new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));
HttpClient client = HttpClients.custom()
.setConnectionManager(new BasicHttpClientConnectionManager())
.setProxy(myProxy)
.setDefaultCredentialsProvider(credentialsProvider)
.build();
HttpPut request = new HttpPut(TARGET_URL);
HttpEntity stringEntity = new StringEntity(BODY, ContentType.APPLICATION_JSON);
request.setEntity(stringEntity);
HttpResponse response = client.execute(request);
System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));
}
}
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
class Program
{
static async Task Main() {
var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
{
Credentials = new NetworkCredential("webscrapingapi.proxy_type=datacenter.device=desktop", "<YOUR_API_KEY>"),
};