Code Samples
cURL, Node.js, and Python examples for production WebScrapingAPI integrations.
The examples below avoid putting API keys in source code. Set WSA_API_KEY in your runtime environment.
Browser API with extraction rules
export WSA_API_KEY="YOUR_API_KEY"
RULES='{"heading":{"selector":"h1","output":"text","clean":1}}'
curl --get "https://api.webscrapingapi.com/v1" \
--data-urlencode "api_key=$WSA_API_KEY" \
--data-urlencode "url=https://example.com" \
--data-urlencode "wait_until=domcontentloaded" \
--data-urlencode "extract_rules=$RULES"const apiKey = process.env.WSA_API_KEY;
const params = new URLSearchParams({
api_key: apiKey,
url: "https://example.com",
wait_until: "domcontentloaded",
extract_rules: JSON.stringify({
heading: { selector: "h1", output: "text", clean: 1 },
}),
});
const response = await fetch(`https://api.webscrapingapi.com/v1?${params}`);
if (!response.ok) {
throw new Error(`Browser API failed: ${response.status} ${await response.text()}`);
}
const body = await response.text();
console.log(body);import os
import requests
params = {
"api_key": os.environ["WSA_API_KEY"],
"url": "https://example.com",
"wait_until": "domcontentloaded",
"extract_rules": '{"heading":{"selector":"h1","output":"text","clean":1}}',
}
response = requests.get("https://api.webscrapingapi.com/v1", params=params, timeout=130)
response.raise_for_status()
print(response.text)Web Unblocker JSON response
const apiKey = process.env.WSA_API_KEY;
const params = new URLSearchParams({
api_key: apiKey,
url: "https://httpbin.org/html",
country: "us",
json_response: "1",
});
const response = await fetch(`https://api.webscrapingapi.com/v2?${params}`);
const data = await response.json();
if (!response.ok) {
throw new Error(data.error ?? data.error_message ?? `HTTP ${response.status}`);
}
console.log(data.body);Browser API POST to target
curl --request POST \
"https://api.webscrapingapi.com/v1?api_key=$WSA_API_KEY&url=https%3A%2F%2Fhttpbin.org%2Fpost&method=POST" \
--header "Content-Type: application/json" \
--header "WSA-Content-Type: application/json" \
--data '{"hello":"world"}'SERP request
const apiKey = process.env.WSA_API_KEY;
const params = new URLSearchParams({
api_key: apiKey,
engine: "google",
q: "coffee",
response: "json",
device: "desktop",
});
const response = await fetch(`https://serpapi.webscrapingapi.com/v2?${params}`);
const data = await response.json();
if (!response.ok) {
throw new Error(data.error ?? data.error_message ?? `HTTP ${response.status}`);
}
console.log(data.organic ?? data.organic_results ?? data);Async polling helper
async function pollWebUnblockerSnapshot(apiKey, snapshotId, { attempts = 30, delayMs = 10000 } = {}) {
for (let attempt = 1; attempt <= attempts; attempt += 1) {
const params = new URLSearchParams({
api_key: apiKey,
snapshot_id: snapshotId,
json_response: "1",
});
const response = await fetch(`https://api.webscrapingapi.com/v2?${params}`);
const text = await response.text();
if (response.status === 202 || /running/i.test(text)) {
await new Promise((resolve) => setTimeout(resolve, delayMs));
continue;
}
if (!response.ok) {
throw new Error(`Snapshot failed: ${response.status} ${text}`);
}
try {
return JSON.parse(text);
} catch {
return text;
}
}
throw new Error(`Snapshot ${snapshotId} was not ready after ${attempts} attempts`);
}Async commerce submit
import os
import time
import requests
api_key = os.environ["WSA_API_KEY"]
submit = requests.post(
"https://ecom.webscrapingapi.com/v1",
params={
"api_key": api_key,
"engine": "bestbuy_async",
"type": "product",
},
json=[
{
"url": "https://www.bestbuy.com/site/singsation-freestyle-wireless-karaoke-system-blue/6523913.p?skuId=6523913"
}
],
timeout=60,
)
submit.raise_for_status()
snapshot_id = submit.json()["snapshot_id"]
for _ in range(30):
result = requests.get(
"https://ecom.webscrapingapi.com/v1",
params={"api_key": api_key, "snapshot_id": snapshot_id},
timeout=60,
)
if result.status_code == 202 or "running" in result.text.lower():
time.sleep(10)
continue
result.raise_for_status()
print(result.json())
break
else:
raise TimeoutError(f"Snapshot {snapshot_id} was not ready")SearchData commerce examples
These examples cover supported /v1 commerce engines. Some engines do not have a dedicated guide yet, so use this section as the quick-start reference.
Crate & Barrel product by URL:
curl --get "https://ecom.webscrapingapi.com/v1" \
--data-urlencode "api_key=$WSA_API_KEY" \
--data-urlencode "engine=crateandbarrel" \
--data-urlencode "type=product" \
--data-urlencode "url=https://www.crateandbarrel.com/example-product/s123456"Freedom product by id:
curl --get "https://ecom.webscrapingapi.com/v1" \
--data-urlencode "api_key=$WSA_API_KEY" \
--data-urlencode "engine=freedom" \
--data-urlencode "type=product" \
--data-urlencode "product_id=123456"John Lewis reviews by URL:
curl --get "https://ecom.webscrapingapi.com/v1" \
--data-urlencode "api_key=$WSA_API_KEY" \
--data-urlencode "engine=johnlewis" \
--data-urlencode "type=reviews" \
--data-urlencode "url=https://www.johnlewis.com/example-product/p123456"West Elm product by URL:
curl --get "https://ecom.webscrapingapi.com/v1" \
--data-urlencode "api_key=$WSA_API_KEY" \
--data-urlencode "engine=westelm" \
--data-urlencode "type=product" \
--data-urlencode "url=https://www.westelm.com/products/example-product/"Target product with related products disabled:
curl --get "https://ecom.webscrapingapi.com/v1" \
--data-urlencode "api_key=$WSA_API_KEY" \
--data-urlencode "engine=target" \
--data-urlencode "type=product" \
--data-urlencode "product_id=A-89846564" \
--data-urlencode "force_related_products=0"