WebScrapingAPI Docs

Code Samples

cURL, Node.js, and Python examples for production WebScrapingAPI integrations.

The examples below avoid putting API keys in source code. Set WSA_API_KEY in your runtime environment.

Browser API with extraction rules

export WSA_API_KEY="YOUR_API_KEY"

RULES='{"heading":{"selector":"h1","output":"text","clean":1}}'

curl --get "https://api.webscrapingapi.com/v1" \
  --data-urlencode "api_key=$WSA_API_KEY" \
  --data-urlencode "url=https://example.com" \
  --data-urlencode "wait_until=domcontentloaded" \
  --data-urlencode "extract_rules=$RULES"
const apiKey = process.env.WSA_API_KEY;
const params = new URLSearchParams({
  api_key: apiKey,
  url: "https://example.com",
  wait_until: "domcontentloaded",
  extract_rules: JSON.stringify({
    heading: { selector: "h1", output: "text", clean: 1 },
  }),
});

const response = await fetch(`https://api.webscrapingapi.com/v1?${params}`);

if (!response.ok) {
  throw new Error(`Browser API failed: ${response.status} ${await response.text()}`);
}

const body = await response.text();
console.log(body);
import os
import requests

params = {
    "api_key": os.environ["WSA_API_KEY"],
    "url": "https://example.com",
    "wait_until": "domcontentloaded",
    "extract_rules": '{"heading":{"selector":"h1","output":"text","clean":1}}',
}

response = requests.get("https://api.webscrapingapi.com/v1", params=params, timeout=130)
response.raise_for_status()
print(response.text)

Web Unblocker JSON response

const apiKey = process.env.WSA_API_KEY;
const params = new URLSearchParams({
  api_key: apiKey,
  url: "https://httpbin.org/html",
  country: "us",
  json_response: "1",
});

const response = await fetch(`https://api.webscrapingapi.com/v2?${params}`);
const data = await response.json();

if (!response.ok) {
  throw new Error(data.error ?? data.error_message ?? `HTTP ${response.status}`);
}

console.log(data.body);

Browser API POST to target

curl --request POST \
  "https://api.webscrapingapi.com/v1?api_key=$WSA_API_KEY&url=https%3A%2F%2Fhttpbin.org%2Fpost&method=POST" \
  --header "Content-Type: application/json" \
  --header "WSA-Content-Type: application/json" \
  --data '{"hello":"world"}'

SERP request

const apiKey = process.env.WSA_API_KEY;
const params = new URLSearchParams({
  api_key: apiKey,
  engine: "google",
  q: "coffee",
  response: "json",
  device: "desktop",
});

const response = await fetch(`https://serpapi.webscrapingapi.com/v2?${params}`);
const data = await response.json();

if (!response.ok) {
  throw new Error(data.error ?? data.error_message ?? `HTTP ${response.status}`);
}

console.log(data.organic ?? data.organic_results ?? data);

Async polling helper

async function pollWebUnblockerSnapshot(apiKey, snapshotId, { attempts = 30, delayMs = 10000 } = {}) {
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    const params = new URLSearchParams({
      api_key: apiKey,
      snapshot_id: snapshotId,
      json_response: "1",
    });

    const response = await fetch(`https://api.webscrapingapi.com/v2?${params}`);
    const text = await response.text();

    if (response.status === 202 || /running/i.test(text)) {
      await new Promise((resolve) => setTimeout(resolve, delayMs));
      continue;
    }

    if (!response.ok) {
      throw new Error(`Snapshot failed: ${response.status} ${text}`);
    }

    try {
      return JSON.parse(text);
    } catch {
      return text;
    }
  }

  throw new Error(`Snapshot ${snapshotId} was not ready after ${attempts} attempts`);
}

Async commerce submit

import os
import time
import requests

api_key = os.environ["WSA_API_KEY"]

submit = requests.post(
    "https://ecom.webscrapingapi.com/v1",
    params={
        "api_key": api_key,
        "engine": "bestbuy_async",
        "type": "product",
    },
    json=[
        {
            "url": "https://www.bestbuy.com/site/singsation-freestyle-wireless-karaoke-system-blue/6523913.p?skuId=6523913"
        }
    ],
    timeout=60,
)
submit.raise_for_status()
snapshot_id = submit.json()["snapshot_id"]

for _ in range(30):
    result = requests.get(
        "https://ecom.webscrapingapi.com/v1",
        params={"api_key": api_key, "snapshot_id": snapshot_id},
        timeout=60,
    )
    if result.status_code == 202 or "running" in result.text.lower():
        time.sleep(10)
        continue
    result.raise_for_status()
    print(result.json())
    break
else:
    raise TimeoutError(f"Snapshot {snapshot_id} was not ready")

SearchData commerce examples

These examples cover supported /v1 commerce engines. Some engines do not have a dedicated guide yet, so use this section as the quick-start reference.

Crate & Barrel product by URL:

curl --get "https://ecom.webscrapingapi.com/v1" \
  --data-urlencode "api_key=$WSA_API_KEY" \
  --data-urlencode "engine=crateandbarrel" \
  --data-urlencode "type=product" \
  --data-urlencode "url=https://www.crateandbarrel.com/example-product/s123456"

Freedom product by id:

curl --get "https://ecom.webscrapingapi.com/v1" \
  --data-urlencode "api_key=$WSA_API_KEY" \
  --data-urlencode "engine=freedom" \
  --data-urlencode "type=product" \
  --data-urlencode "product_id=123456"

John Lewis reviews by URL:

curl --get "https://ecom.webscrapingapi.com/v1" \
  --data-urlencode "api_key=$WSA_API_KEY" \
  --data-urlencode "engine=johnlewis" \
  --data-urlencode "type=reviews" \
  --data-urlencode "url=https://www.johnlewis.com/example-product/p123456"

West Elm product by URL:

curl --get "https://ecom.webscrapingapi.com/v1" \
  --data-urlencode "api_key=$WSA_API_KEY" \
  --data-urlencode "engine=westelm" \
  --data-urlencode "type=product" \
  --data-urlencode "url=https://www.westelm.com/products/example-product/"

Target product with related products disabled:

curl --get "https://ecom.webscrapingapi.com/v1" \
  --data-urlencode "api_key=$WSA_API_KEY" \
  --data-urlencode "engine=target" \
  --data-urlencode "type=product" \
  --data-urlencode "product_id=A-89846564" \
  --data-urlencode "force_related_products=0"

On this page