Proxy Mode
WebScrapingAPI offers a proxy front-end to the API, making the integration with third-party tools much easier.
Interacting with WebScrapingAPI via the proxy mode only changes the access method with our API; apart from that, full functionality is still available to the end user. Both costs, parameters and response format remain the same.
Note that our proxy mode supports GET, POST and PUT requests.

Accessing the API via Proxy Mode

The Proxy Mode is available via:
  • HTTPS at proxy.webscrapingapi.com:8000
  • HTTP at proxy.webscrapingapi.com:80
Authentication via proxy mode is done using the username:password@proxy.webscrapingapi.com:8000 method. However, the username part also holds the parameters you want to pass to your request. Hence, the configuration you need to pass to your request is:
  • Username: webscrapingapi, followed by the parameters you want to pass to your request (separated by dots)
  • Password: your personal API key
The most basic URL example for an authenticated request on our API via proxy mode is:
https://webscrapingapi.<parameter_1.parameter_2. ... .parameter_n>:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000

Proxy Mode Requirements and Recommendations

If you decide to use the proxy mode, please make sure you understand and follow these requirements:
  1. 1.
    Disable SSL certificates verification (-k switch in cURL, verify=false in Python etc.)
  2. 2.
    If you use proxy mode with a browser automation library (e.g. Selenium, Puppeteer etc.), it is highly recommended to disable JavaScript rendering (render_js=0) or to limit the amount of resources requested by your browser.

Proxy Mode Integration Examples

#1: GET Requests

cURL
NodeJS
Python
PHP
Go
Ruby
curl --proxy https://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X GET http://httpbin.org/get -k
const axios = require('axios');
axios.get('http://httpbin.org/get', {
proxy: {
host: 'proxy.webscrapingapi.com',
port: 80,
auth: {
username: 'webscrapingapi.proxy_type=datacenter.device=desktop',
password: '<YOUR_API_KEY>'
}
}
}).then(function (response) {
console.log(response.data);
}, (err) => {
console.log(err)
})
import requests
def get_params(object):
params = ''
for key,value in object.items():
if list(object).index(key) < len(object) - 1:
params += f"{key}={value}."
else:
params += f"{key}={value}"
return params
API_KEY = '<YOUR_API_KEY>'
TARGET_URL = 'http://httpbin.org/get'
PARAMETERS = {
"proxy_type":"datacenter",
"device":"desktop"
}
PROXY = {
"https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}
response = requests.get(
url=TARGET_URL,
proxies=PROXY,
verify=False
)
print(response.text)
<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'http://httpbin.org/get');
curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
$response = curl_exec($ch);
if (!$response) {
die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}
echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;
curl_close($ch);
?>
package main
import (
"fmt"
"crypto/tls"
"io/ioutil"
"net/http"
"net/url"
)
func send_proxy() {
proxyStr := "http://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
proxyURL, err := url.Parse(proxyStr)
if err != nil {
fmt.Println(err)
}
transport := &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{
Transport: transport,
}
req, err := http.NewRequest("GET", "http://httpbin.org/get", nil)
parseFormErr := req.ParseForm()
if parseFormErr != nil {
fmt.Println(parseFormErr)
}
resp, err := client.Do(req)
if err != nil {
fmt.Println("Failure : ", err)
}
respBody, _ := ioutil.ReadAll(resp.Body)
fmt.Println(res)
fmt.Println(string(body))
}
func main() {
send_proxy()
}
require 'httparty'
HTTParty::Basement.default_options.update(verify: false)
response = HTTParty.get('http://httpbin.org/get', {
http_proxyaddr: 'proxy.webscrapingapi.com',
http_proxyport: '80',
http_proxyuser: 'webscrapingapi.proxy_type=datacenter.device=desktop',
http_proxypass: '<YOUR_API_KEY>'
})
puts response.read_body
Response Example

#2: POST Requests

cURL
Python
curl --proxy https://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X POST https://httpbin.org/post --data '{"foo":"bar"}' -k
import requests
def get_params(object):
params = ''
for key,value in object.items():
if list(object).index(key) < len(object) - 1:
params += f"{key}={value}."
else:
params += f"{key}={value}"
return params
API_KEY = '<YOUR_API_KEY>'
TARGET_URL = 'https://httpbin.org/post'
DATA = {"foo":"bar"}
PARAMETERS = {
"proxy_type":"datacenter",
"device":"desktop"
}
PROXY = {
"https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}
response = requests.post(
url=TARGET_URL,
data=DATA,
proxies=PROXY,
verify=False
)
print(response.text)
Response Example

#3: PUT Requests

cURL
Python
curl --proxy https://webscrapingapi.proxy_type=datacenter.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X PUT https://httpbin.org/put --data '{"foo":"bar"}' -k
import requests
def get_params(object):
params = ''
for key,value in object.items():
if list(object).index(key) < len(object) - 1:
params += f"{key}={value}."
else:
params += f"{key}={value}"
return params
API_KEY = '<YOUR_API_KEY>'
TARGET_URL = 'https://httpbin.org/put'
DATA = {"foo":"bar"}
PARAMETERS = {
"proxy_type":"datacenter",
"device":"desktop"
}
PROXY = {
"https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}
response = requests.put(
url=TARGET_URL,
data=DATA,
proxies=PROXY,
verify=False
)
print(response.text)
Response Example
Copy link
Outline
Accessing the API via Proxy Mode
Proxy Mode Requirements and Recommendations
Proxy Mode Integration Examples