Proxy Mode

WebScrapingAPI offers a proxy front-end to the API, making the integration with third-party tools much easier.

Interacting with BrowserAPI via the proxy mode only changes the access method with our API; apart from that, full functionality is still available to the end user. Both costs, parameters and response format remain the same.

Note that our proxy mode supports GET, POST and PUT requests.

Accessing the API via Proxy Mode

The Proxy Mode is available via:

HTTPS at proxy.webscrapingapi.com:8000
HTTP at proxy.webscrapingapi.com:80

Authentication via proxy mode is done using the username:password@proxy.webscrapingapi.com:8000 method. However, the username part also holds the parameters you want to pass to your request. Hence, the configuration you need to pass to your request is:

Username: webscrapingapi, followed by the parameters you want to pass to your request (separated by dots)
Password: your personal API key

The most basic URL example for an authenticated request on our API via proxy mode is:

https://webscrapingapi.<parameter_1.parameter_2. ... .parameter_n>:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000

Proxy Mode Requirements and Recommendations

If you decide to use the proxy mode, please make sure you disable SSL certificates verification (-k switch in cURL, verify=false in Python etc.);

Proxy Mode Integration Examples

#1: GET Requests

curl --proxy https://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X GET http://httpbin.org/get -k

const axios = require('axios');
    
axios.get('http://httpbin.org/get', {
      proxy: {
          host: 'proxy.webscrapingapi.com',
          port: 80,
          auth: {
            username: 'webscrapingapi.device=desktop', 
            password: '<YOUR_API_KEY>'
          }
      }
    }).then(function (response) {
        console.log(response.data);
    }, (err) => {
      console.log(err)
})

import requests

def get_params(object):
    params = ''
    for key,value in object.items():
        if list(object).index(key) < len(object) - 1:
            params += f"{key}={value}."
        else:
            params += f"{key}={value}"
    return params

API_KEY = '<YOUR_API_KEY>'

TARGET_URL = 'http://httpbin.org/get'

PARAMETERS = {
    "device":"desktop"
}

PROXY = {
    "http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
    "https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}

response = requests.get(
    url=TARGET_URL,
    proxies=PROXY,
    verify=False
)

print(response.text)

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'http://httpbin.org/get');

curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');

curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

$response = curl_exec($ch);

if (!$response) {
  die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}

echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;

curl_close($ch);
?>

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

func send_proxy() {
	proxyStr := "http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
	proxyURL, err := url.Parse(proxyStr)

	if err != nil {
		fmt.Println(err)
	}

	transport := &http.Transport{
		Proxy:           http.ProxyURL(proxyURL),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client := &http.Client{
		Transport: transport,
	}

	req, err := http.NewRequest("GET", "http://httpbin.org/get", nil)

	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	res, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
}

func main() {
	send_proxy()
}

import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class Main {
    public static void main(String [] args) throws IOException {

        final String PROXY_HOST = "proxy.webscrapingapi.com";
        final int PROXY_PORT = 80;
        final String PROXY_USER = "webscrapingapi.device=desktop";
        final String PROXY_PASS = "<YOUR-API-KEY-HERE>";

        final String TARGET_URL = "http://httpbin.org/get";

        HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(myProxy)
                , new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));

        HttpClient client = HttpClients.custom()
                .setConnectionManager(new BasicHttpClientConnectionManager())
                .setProxy(myProxy)
                .setDefaultCredentialsProvider(credentialsProvider)
                .build();


        HttpGet request = new HttpGet(TARGET_URL);
        HttpResponse response = client.execute(request);
        System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));

    }
}

using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main() {

        var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
        {
            Credentials = new NetworkCredential("webscrapingapi.device=desktop", "<YOUR_API_KEY>"),
        };

        var clientHandler = new HttpClientHandler
        {
            Proxy = proxy,
        };

        var client = new HttpClient(clientHandler);
        var response = await client.GetAsync("http://httpbin.org/get");
        Console.WriteLine(await response.Content.ReadAsStringAsync());
    }
}net

require 'uri'
require 'net/http'
require 'openssl'

proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.device=desktop"
proxy_pass = "<YOUR_API_KEY>"

url = URI("http://httpbin.org/get")

proxy = Net::HTTP::Proxy(proxy_host,
                         proxy_port,
                         proxy_user,
                         proxy_pass)

req = Net::HTTP::Get.new(url.path)

response = proxy.start(url.host,url.port) do |http|
  http.request(req)
end
    
puts response.body

Response Example

{
  args: {},
  headers: {
    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    Host: 'httpbin.org',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.80 Safari/537.36',
    'X-Amzn-Trace-Id': 'Root=1-632993e3-29380e237fe8c362301e8a57'
  },
  origin: '198.46.171.167',
  url: 'http://httpbin.org/get'
}

#2: POST Requests

curl --proxy https://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X POST https://httpbin.org/post --data '{"foo":"bar"}' -k

const axios = require('axios');
    
let data = { 'foo':'bar' }
axios.post('http://httpbin.org/post', data, {
      proxy: {
          host: 'proxy.webscrapingapi.com',
          port: 80,
          auth: {
            username: 'webscrapingapi.device=desktop', 
            password: '<YOUR_API_KEY>'
          }
      }
    }).then(function (response) {
        console.log(response.data);
    }, (err) => {
      console.log(err)
})

import requests

def get_params(object):
    params = ''
    for key,value in object.items():
        if list(object).index(key) < len(object) - 1:
            params += f"{key}={value}."
        else:
            params += f"{key}={value}"
    return params

API_KEY = '<YOUR_API_KEY>'

TARGET_URL = 'https://httpbin.org/post'

DATA = {"foo":"bar"}

PARAMETERS = {
    "device":"desktop"
}

PROXY = {
    "http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
    "https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}

response = requests.post(
    url=TARGET_URL,
    data=DATA,
    proxies=PROXY,
    verify=False
)

print(response.text)

<?php

$data = json_encode(array("foo" => "bar"));

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://httpbin.org/post');

curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');

curl_setopt( $ch, CURLOPT_POSTFIELDS, $data);

curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

$response = curl_exec($ch);

if (!$response) {
  die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}

echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;

curl_close($ch);
?>

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func send_proxy() {
	proxyStr := "http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
	proxyURL, err := url.Parse(proxyStr)

	if err != nil {
		fmt.Println(err)
	}

	transport := &http.Transport{
		Proxy:           http.ProxyURL(proxyURL),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client := &http.Client{
		Transport: transport,
	}
	
	 data := url.Values{}
	 data.Set("foo","bar")

	req, err := http.NewRequest("POST", "http://httpbin.org/post", strings.NewReader(data.Encode()))

	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	res, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
}

func main() {
	send_proxy()
}

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class Main {
    public static void main(String [] args) throws IOException {

        final String PROXY_HOST = "proxy.webscrapingapi.com";
        final int PROXY_PORT = 80;
        final String PROXY_USER = "webscrapingapi.device=desktop";
        final String PROXY_PASS = "<YOUR_API_KEY>";

        final String TARGET_URL = "http://httpbin.org/post";
        final String BODY = "{\"foo\":\"bar\"}";

        HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(myProxy)
                , new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));

        HttpClient client = HttpClients.custom()
                .setConnectionManager(new BasicHttpClientConnectionManager())
                .setProxy(myProxy)
                .setDefaultCredentialsProvider(credentialsProvider)
                .build();


        HttpPost request = new HttpPost(TARGET_URL);
        HttpEntity stringEntity = new StringEntity(BODY, ContentType.APPLICATION_JSON);
        request.setEntity(stringEntity);

        HttpResponse response = client.execute(request);
        System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));

    }
}

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main() {

        var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
        {
            Credentials = new NetworkCredential("webscrapingapi.device=desktop", "<YOUR_API_KEY>"),
        };

        var clientHandler = new HttpClientHandler
        {
            Proxy = proxy,
        };

        var data = new FormUrlEncodedContent(new[]
        {
            new KeyValuePair<string, string>("foo", "bar"),
        });

        var client = new HttpClient(clientHandler);
        var response = await client.PostAsync("http://httpbin.org/post", data);
        Console.WriteLine(await response.Content.ReadAsStringAsync());
    }
}

require 'uri'
require 'net/http'
require 'openssl'

proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.device=desktop"
proxy_pass = "<YOUR_API_KEY>"

url = URI("http://httpbin.org/post")

proxy = Net::HTTP::Proxy(proxy_host,
                         proxy_port,
                         proxy_user,
                         proxy_pass)

req = Net::HTTP::Post.new(url.path)
req.set_form_data({'foo' => 'bar'})

response = proxy.start(url.host,url.port) do |http|
  http.request(req)
end
    
puts response.body

Response Example

{
  "args": {},
  "data": "",
  "files": {},
  "form": {
    "{\"foo\":\"bar\"}": ""
  },
  "headers": {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Content-Length": "28",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "httpbin.org",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
    "X-Amzn-Trace-Id": "Root=1-631edd38-2bbe74166ca70a5b438fa87f"
  },
  "json": null,
  "origin": "196.244.10.12",
  "url": "https://httpbin.org/post"
}

#3: PUT Requests

curl --proxy https://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:8000 -X PUT https://httpbin.org/put --data '{"foo":"bar"}' -k

const axios = require('axios');
    
let data = { 'foo':'bar' }
axios.put('http://httpbin.org/put', data, {
      proxy: {
          host: 'proxy.webscrapingapi.com',
          port: 80,
          auth: {
            username: 'webscrapingapi.device=desktop', 
            password: '<YOUR_API_KEY>'
          }
      }
    }).then(function (response) {
        console.log(response.data);
    }, (err) => {
      console.log(err)
})

import requests

def get_params(object):
    params = ''
    for key,value in object.items():
        if list(object).index(key) < len(object) - 1:
            params += f"{key}={value}."
        else:
            params += f"{key}={value}"
    return params

API_KEY = '<YOUR_API_KEY>'

TARGET_URL = 'https://httpbin.org/put'

DATA = {"foo":"bar"}

PARAMETERS = {
    "device":"desktop"
}

PROXY = {
    "http": f"http://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:80",
    "https": f"https://webscrapingapi.{ get_params(PARAMETERS) }:{ API_KEY }@proxy.webscrapingapi.com:8000"
}

response = requests.put(
    url=TARGET_URL,
    data=DATA,
    proxies=PROXY,
    verify=False
)

print(response.text)

<?php

$data = json_encode(array("foo" => "bar"));

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, 'https://httpbin.org/put');

curl_setopt($ch, CURLOPT_PROXY, 'http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80');

curl_setopt( $ch, CURLOPT_POSTFIELDS, $data);

curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT');

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

$response = curl_exec($ch);

if (!$response) {
  die('Error: "'.curl_error($ch).'" - Code: '.curl_errno($ch));
}

echo 'HTTP Status Code: '.curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: '.$response . PHP_EOL;

curl_close($ch);
?>

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func send_proxy() {
	proxyStr := "http://webscrapingapi.device=desktop:<YOUR_API_KEY>@proxy.webscrapingapi.com:80"
	proxyURL, err := url.Parse(proxyStr)

	if err != nil {
		fmt.Println(err)
	}

	transport := &http.Transport{
		Proxy:           http.ProxyURL(proxyURL),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client := &http.Client{
		Transport: transport,
	}
	
	 data := url.Values{}
	 data.Set("foo","bar")

	req, err := http.NewRequest("PUT", "http://httpbin.org/put", strings.NewReader(data.Encode()))

	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	res, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
}

func main() {
	send_proxy()
}

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class Main {
    public static void main(String [] args) throws IOException {

        final String PROXY_HOST = "proxy.webscrapingapi.com";
        final int PROXY_PORT = 80;
        final String PROXY_USER = "webscrapingapi.device=desktop";
        final String PROXY_PASS = "<YOUR_API_KEY>";

        final String TARGET_URL = "http://httpbin.org/put";
        final String BODY = "{\"foo\":\"bar\"}";

        HttpHost myProxy = new HttpHost(PROXY_HOST, PROXY_PORT);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(myProxy)
                , new UsernamePasswordCredentials(PROXY_USER,PROXY_PASS));

        HttpClient client = HttpClients.custom()
                .setConnectionManager(new BasicHttpClientConnectionManager())
                .setProxy(myProxy)
                .setDefaultCredentialsProvider(credentialsProvider)
                .build();


        HttpPut request = new HttpPut(TARGET_URL);
        HttpEntity stringEntity = new StringEntity(BODY, ContentType.APPLICATION_JSON);
        request.setEntity(stringEntity);

        HttpResponse response = client.execute(request);
        System.out.println(EntityUtils.toString(response.getEntity(), "UTF-8"));

    }
}

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main() {

        var proxy = new WebProxy("http://proxy.webscrapingapi.com:80")
        {
            Credentials = new NetworkCredential("webscrapingapi.device=desktop", "<YOUR_API_KEY>"),
        };

        var clientHandler = new HttpClientHandler
        {
            Proxy = proxy,
        };

        var data = new FormUrlEncodedContent(new[]
        {
            new KeyValuePair<string, string>("foo", "bar"),
        });

        var client = new HttpClient(clientHandler);
        var response = await client.PutAsync("http://httpbin.org/put", data);
        Console.WriteLine(await response.Content.ReadAsStringAsync());
    }
}

require 'uri'
require 'net/http'
require 'openssl'

proxy_host = "proxy.webscrapingapi.com"
proxy_port = 80
proxy_user = "webscrapingapi.device=desktop"
proxy_pass = "<YOUR_API_KEY>"

url = URI("http://httpbin.org/put")

proxy = Net::HTTP::Proxy(proxy_host,
                         proxy_port,
                         proxy_user,
                         proxy_pass)

req = Net::HTTP::Put.new(url.path)
req.set_form_data({'foo' => 'bar'})

response = proxy.start(url.host,url.port) do |http|
  http.request(req)
end
    
puts response.body

Response Example

{
  "args": {},
  "data": "",
  "files": {},
  "form": {
    "{\"foo\":\"bar\"}": ""
  },
  "headers": {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Content-Length": "28",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "httpbin.org",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
    "X-Amzn-Trace-Id": "Root=1-631edd38-2bbe74166ca70a5b438fa87f"
  },
  "json": null,
  "origin": "196.244.10.12",
  "url": "https://httpbin.org/put"
}

PreviousPOST, PUT & PATCH Requests NextAdvanced API Features

Last updated 1 year ago