Proxy Mode

WebScrapingAPI offers a proxy front-end to the API, making the integration with third-party tools much easier.

Interacting with BrowserAPI via the proxy mode only changes the access method with our API; apart from that, full functionality is still available to the end user. Both costs, parameters and response format remain the same.

Note that our proxy mode supports GET, POST and PUT requests.

Accessing the API via Proxy Mode

The Proxy Mode is available via:

• HTTPS at proxy.webscrapingapi.com:8000

• HTTP at proxy.webscrapingapi.com:80

Authentication via proxy mode is done using the username:[e-mail]:8000 method. However, the username part also holds the parameters you want to pass to your request. Hence, the configuration you need to pass to your request is:

• Username: webscrapingapi, followed by the parameters you want to pass to your request (separated by dots)

• Password: your personal API key

The most basic URL example for an authenticated request on our API via proxy mode is:

https://webscrapingapi.<parameter_1.parameter_2. ... .parameter_n>:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000

Proxy Mode Requirements and Recommendations

If you decide to use the proxy mode, please make sure you disable SSL certificates verification (-k switch in cURL, verify=false in Python etc.);

Proxy Mode Integration Examples

#1: GET Requests

cURL

curl --proxy https://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X GET http://httpbin.org/get -k

NodeJS

const axios = require('axios');
    
axios.get('http://httpbin.org/get', {
      proxy: {
          host: 'proxy.webscrapingapi.com',
          port: 80,
          auth: {
            username: 'webscrapingapi.device=desktop', 
            password: '<YOUR_API_KEY>'
          }
      }
    }).then(function (response) {
        console.log(response.data);
    }, (err) => {
      console.log(err)
})

Python

import requests

def get_params(object):
    params = ''
    for key,value in object.items():
        if list(object).index(key) < len(object) - 1:
            params += f"{key}={value}."
        else:
            params += f"{key}={value}"
    return params

API_KEY = '<YOUR_API_KEY>'

TARGET_URL = 'http://httpbin.org/get'

PARAMETERS = {
    "device":"desktop"
}

PROXY = {
    "http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
    "https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}

response = requests.get(
    url=TARGET_URL,
    proxies=PROXY,
    verify=False
)

print(response.text)

PHP

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'http://httpbin.org/get');

curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');

curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

$response = curl_exec($ch);

if (!$response) {
  die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}

echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;

curl_close($ch);
?>

Go

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func send_proxy() {
	proxyStr := "http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
	proxyURL, err := url.Parse(proxyStr)

	if err != nil {
		fmt.Println(err)
	}

	transport := &http.Transport{
		Proxy:           http.ProxyURL(proxyURL),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client := &http.Client{
		Transport: transport,
	}

	req, err := http.NewRequest("GET", "http://httpbin.org/get", nil)

	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	res, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
}

func main() {
	send_proxy()
}

Java

import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class Main {
    public static void main(String [] args) throws IOException {

        final String PROXY_HOST = "proxy.webscrapingapi.com";
        final int PROXY_PORT = 80;
        final String PROXY_USER = "webscrapingapi.device=desktop";
        final String PROXY_PASS = "<YOUR-API-KEY-HERE>";

        final String TARGET_URL = "http://httpbin.org/get";

        HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(myProxy)
                , new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));

        HttpClient client = HttpClients.custom()
                .setConnectionManager(new BasicHttpClientConnectionManager())
                .setProxy(myProxy)
                .setDefaultCredentialsProvider(credentialsProvider)
                .build();


        HttpGet request = new HttpGet(TARGET_URL);
        HttpResponse response = client.execute(request);
        System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));

    }
}

.NET

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main() {

        var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
        {
            Credentials = new NetworkCredential("webscrapingapi.device=desktop", "<YOUR_API_KEY>"),
        };

        var clientHandler = new HttpClientHandler
        {
            Proxy = proxy,
        };

        var client = new HttpClient(clientHandler);
        var response = await client.GetAsync("http://httpbin.org/get");
        Console.WriteLine(await response.Content.ReadAsStringAsync());
    }
}net

Ruby

require 'uri'
require 'net/http'
require 'openssl'

proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.device=desktop"
proxy_pass = "<YOUR_API_KEY>"

url = URI("http://httpbin.org/get")

proxy = Net::HTTP::Proxy(proxy_host,
                         proxy_port,
                         proxy_user,
                         proxy_pass)

req = Net::HTTP::Get.new(url.path)

response = proxy.start(url.host,url.port) do |http|
  http.request(req)
end
    
puts response.body

Response Example

{
  args: {},
  headers: {
    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    Host: 'httpbin.org',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.80 Safari/537.36',
    'X-Amzn-Trace-Id': 'Root=1-632993e3-29380e237fe8c362301e8a57'
  },
  origin: '198.46.171.167',
  url: 'http://httpbin.org/get'
}

#2: POST Requests

cURL

curl --proxy https://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X POST https://httpbin.org/post --data '{"foo":"bar"}' -k      

NodeJS

const axios = require('axios');
    
let data = { 'foo':'bar' }
axios.post('http://httpbin.org/post', data, {
      proxy: {
          host: 'proxy.webscrapingapi.com',
          port: 80,
          auth: {
            username: 'webscrapingapi.device=desktop', 
            password: '<YOUR_API_KEY>'
          }
      }
    }).then(function (response) {
        console.log(response.data);
    }, (err) => {
      console.log(err)
})

Python

import requests

def get_params(object):
    params = ''
    for key,value in object.items():
        if list(object).index(key) < len(object) - 1:
            params += f"{key}={value}."
        else:
            params += f"{key}={value}"
    return params

API_KEY = '<YOUR_API_KEY>'

TARGET_URL = 'https://httpbin.org/post'

DATA = {"foo":"bar"}

PARAMETERS = {
    "device":"desktop"
}

PROXY = {
    "http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
    "https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}

response = requests.post(
    url=TARGET_URL,
    data=DATA,
    proxies=PROXY,
    verify=False
)

print(response.text)

PHP

<?php

$data = json_encode(array("foo" => "bar"));

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://httpbin.org/post');

curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');

curl_setopt( $ch, CURLOPT_POSTFIELDS, $data);

curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

$response = curl_exec($ch);

if (!$response) {
  die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}

echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;

curl_close($ch);
?>

Go

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func send_proxy() {
	proxyStr := "http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
	proxyURL, err := url.Parse(proxyStr)

	if err != nil {
		fmt.Println(err)
	}

	transport := &http.Transport{
		Proxy:           http.ProxyURL(proxyURL),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client := &http.Client{
		Transport: transport,
	}
	
	 data := url.Values{}
	 data.Set("foo","bar")

	req, err := http.NewRequest("POST", "http://httpbin.org/post", strings.NewReader(data.Encode()))

	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	res, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
}

func main() {
	send_proxy()
}

Java

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class Main {
    public static void main(String [] args) throws IOException {

        final String PROXY_HOST = "proxy.webscrapingapi.com";
        final int PROXY_PORT = 80;
        final String PROXY_USER = "webscrapingapi.device=desktop";
        final String PROXY_PASS = "<YOUR_API_KEY>";

        final String TARGET_URL = "http://httpbin.org/post";
        final String BODY = "{\"foo\":\"bar\"}";

        HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(myProxy)
                , new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));

        HttpClient client = HttpClients.custom()
                .setConnectionManager(new BasicHttpClientConnectionManager())
                .setProxy(myProxy)
                .setDefaultCredentialsProvider(credentialsProvider)
                .build();


        HttpPost request = new HttpPost(TARGET_URL);
        HttpEntity stringEntity = new StringEntity(BODY, ContentType.APPLICATION_JSON);
        request.setEntity(stringEntity);

        HttpResponse response = client.execute(request);
        System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));

    }
}

.NET

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main() {

        var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
        {
            Credentials = new NetworkCredential("webscrapingapi.device=desktop", "<YOUR_API_KEY>"),
        };

        var clientHandler = new HttpClientHandler
        {
            Proxy = proxy,
        };

        var data = new FormUrlEncodedContent(new[]
        {
            new KeyValuePair<string, string>("foo", "bar"),
        });

        var client = new HttpClient(clientHandler);
        var response = await client.PostAsync("http://httpbin.org/post", data);
        Console.WriteLine(await response.Content.ReadAsStringAsync());
    }
}

Ruby

require 'uri'
require 'net/http'
require 'openssl'

proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.device=desktop"
proxy_pass = "<YOUR_API_KEY>"

url = URI("http://httpbin.org/post")

proxy = Net::HTTP::Proxy(proxy_host,
                         proxy_port,
                         proxy_user,
                         proxy_pass)

req = Net::HTTP::Post.new(url.path)
req.set_form_data({'foo' => 'bar'})

response = proxy.start(url.host,url.port) do |http|
  http.request(req)
end
    
puts response.body

Response Example

{
  "args": {},
  "data": "",
  "files": {},
  "form": {
    "{\"foo\":\"bar\"}": ""
  },
  "headers": {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Content-Length": "28",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "httpbin.org",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
    "X-Amzn-Trace-Id": "Root=1-631edd38-2bbe74166ca70a5b438fa87f"
  },
  "json": null,
  "origin": "196.244.10.12",
  "url": "https://httpbin.org/post"
}

#3: PUT Requests

cURL

curl --proxy https://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X PUT https://httpbin.org/put --data '{"foo":"bar"}' -k      

NodeJS

const axios = require('axios');
    
let data = { 'foo':'bar' }
axios.put('http://httpbin.org/put', data, {
      proxy: {
          host: 'proxy.webscrapingapi.com',
          port: 80,
          auth: {
            username: 'webscrapingapi.device=desktop', 
            password: '<YOUR_API_KEY>'
          }
      }
    }).then(function (response) {
        console.log(response.data);
    }, (err) => {
      console.log(err)
})

Python

import requests

def get_params(object):
    params = ''
    for key,value in object.items():
        if list(object).index(key) < len(object) - 1:
            params += f"{key}={value}."
        else:
            params += f"{key}={value}"
    return params

API_KEY = '<YOUR_API_KEY>'

TARGET_URL = 'https://httpbin.org/put'

DATA = {"foo":"bar"}

PARAMETERS = {
    "device":"desktop"
}

PROXY = {
    "http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
    "https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}

response = requests.put(
    url=TARGET_URL,
    data=DATA,
    proxies=PROXY,
    verify=False
)

print(response.text)

PHP

<?php

$data = json_encode(array("foo" => "bar"));

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://httpbin.org/put');

curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');

curl_setopt( $ch, CURLOPT_POSTFIELDS, $data);

curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT');

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

$response = curl_exec($ch);

if (!$response) {
  die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}

echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;

curl_close($ch);
?>

Go

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func send_proxy() {
	proxyStr := "http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
	proxyURL, err := url.Parse(proxyStr)

	if err != nil {
		fmt.Println(err)
	}

	transport := &http.Transport{
		Proxy:           http.ProxyURL(proxyURL),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client := &http.Client{
		Transport: transport,
	}
	
	 data := url.Values{}
	 data.Set("foo","bar")

	req, err := http.NewRequest("PUT", "http://httpbin.org/put", strings.NewReader(data.Encode()))

	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	res, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
}

func main() {
	send_proxy()
}

Java

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class Main {
    public static void main(String [] args) throws IOException {

        final String PROXY_HOST = "proxy.webscrapingapi.com";
        final int PROXY_PORT = 80;
        final String PROXY_USER = "webscrapingapi.device=desktop";
        final String PROXY_PASS = "<YOUR_API_KEY>";

        final String TARGET_URL = "http://httpbin.org/put";
        final String BODY = "{\"foo\":\"bar\"}";

        HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(myProxy)
                , new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));

        HttpClient client = HttpClients.custom()
                .setConnectionManager(new BasicHttpClientConnectionManager())
                .setProxy(myProxy)
                .setDefaultCredentialsProvider(credentialsProvider)
                .build();


        HttpPut request = new HttpPut(TARGET_URL);
        HttpEntity stringEntity = new StringEntity(BODY, ContentType.APPLICATION_JSON);
        request.setEntity(stringEntity);

        HttpResponse response = client.execute(request);
        System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));

    }
}

.NET

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main() {

        var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
        {
            Credentials = new NetworkCredential("webscrapingapi.device=desktop", "<YOUR_API_KEY>"),
        };

        var clientHandler = new HttpClientHandler
        {
            Proxy = proxy,
        };

        var data = new FormUrlEncodedContent(new[]
        {
            new KeyValuePair<string, string>("foo", "bar"),
        });

        var client = new HttpClient(clientHandler);
        var response = await client.PutAsync("http://httpbin.org/put", data);
        Console.WriteLine(await response.Content.ReadAsStringAsync());
    }
}

Ruby

require 'uri'
require 'net/http'
require 'openssl'

proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.device=desktop"
proxy_pass = "<YOUR_API_KEY>"

url = URI("http://httpbin.org/put")

proxy = Net::HTTP::Proxy(proxy_host,
                         proxy_port,
                         proxy_user,
                         proxy_pass)

req = Net::HTTP::Put.new(url.path)
req.set_form_data({'foo' => 'bar'})

response = proxy.start(url.host,url.port) do |http|
  http.request(req)
end
    
puts response.body

Response Example

{
  "args": {},
  "data": "",
  "files": {},
  "form": {
    "{\"foo\":\"bar\"}": ""
  },
  "headers": {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Content-Length": "28",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "httpbin.org",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
    "X-Amzn-Trace-Id": "Root=1-631edd38-2bbe74166ca70a5b438fa87f"
  },
  "json": null,
  "origin": "196.244.10.12",
  "url": "https://httpbin.org/put"
}