Async Request

Queue long-running WebScrapingAPI scrapes and retrieve finished results from the Snapshot API.

Use an async request when a scrape may take longer to finish. WebScrapingAPI accepts the request immediately, queues the scrape, and returns a response id that you can use to retrieve the finished result.

The async flow has two steps:

Send an async GET request with api_key, url, and async=true.
Poll the Snapshot API with the returned response id once the result is ready.

Please make sure to escape the url parameter's value.

Params

Name	Required	Type	Description
`api_key`	✅	`string`	Your WebScrapingAPI key. Find out how to obtain an API key.
`url`	✅	`string`	The target page URL to scrape. URL-encode this value when building the query manually.
`async`	✅	`boolean`	Set to `true` to queue the scrape and return immediately with a response id.

Async GET Request

GET https://api.webscrapingapi.com/v2?api_key=<YOUR_API_KEY>&url=https://webscrapingapi.com/&async=true

curl --get "https://api.webscrapingapi.com/v2" \
  --data-urlencode "api_key=<YOUR_API_KEY>" \
  --data-urlencode "url=https://webscrapingapi.com/" \
  --data-urlencode "async=true"

const params = new URLSearchParams({
  api_key: "<YOUR_API_KEY>",
  url: "https://webscrapingapi.com/",
  async: "true",
});

const response = await fetch(`https://api.webscrapingapi.com/v2?${params}`);
const data = await response.json();

console.log(data);

import requests

params = {
    "api_key": "<YOUR_API_KEY>",
    "url": "https://webscrapingapi.com/",
    "async": "true",
}

response = requests.get("https://api.webscrapingapi.com/v2", params=params)

print(response.json())

<?php

$query = http_build_query([
    'api_key' => '<YOUR_API_KEY>',
    'url' => 'https://webscrapingapi.com/',
    'async' => 'true',
]);

$response = file_get_contents('https://api.webscrapingapi.com/v2?' . $query);

echo $response;

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
)

func main() {
	params := url.Values{}
	params.Add("api_key", "<YOUR_API_KEY>")
	params.Add("url", "https://webscrapingapi.com/")
	params.Add("async", "true")

	requestURL := "https://api.webscrapingapi.com/v2?" + params.Encode()

	response, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	defer response.Body.Close()

	body, err := io.ReadAll(response.Body)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(body))
}

import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) throws Exception {
        String apiKey = "<YOUR_API_KEY>";
        String targetUrl = "https://webscrapingapi.com/";

        String query = "api_key=" + URLEncoder.encode(apiKey, StandardCharsets.UTF_8)
            + "&url=" + URLEncoder.encode(targetUrl, StandardCharsets.UTF_8)
            + "&async=true";

        HttpRequest request = HttpRequest.newBuilder()
            .uri(URI.create("https://api.webscrapingapi.com/v2?" + query))
            .GET()
            .build();

        HttpResponse<String> response = HttpClient.newHttpClient()
            .send(request, HttpResponse.BodyHandlers.ofString());

        System.out.println(response.body());
    }
}

using System;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main()
    {
        var apiKey = "<YOUR_API_KEY>";
        var targetUrl = Uri.EscapeDataString("https://webscrapingapi.com/");
        var requestUrl = $"https://api.webscrapingapi.com/v2?api_key={apiKey}&url={targetUrl}&async=true";

        using var client = new HttpClient();
        var response = await client.GetStringAsync(requestUrl);

        Console.WriteLine(response);
    }
}

require 'net/http'
require 'uri'

uri = URI('https://api.webscrapingapi.com/v2')
uri.query = URI.encode_www_form(
  api_key: '<YOUR_API_KEY>',
  url: 'https://webscrapingapi.com/',
  async: 'true'
)

response = Net::HTTP.get(uri)

puts response

Response

The initial async request returns a response id. Use this value as the snapshot_id when you retrieve the finished scrape.

{
  "response_id": "s3w1t1751043071383r7bcfdskr5q8"
}

Retrieve results

Use the Snapshot API to retrieve the processed result. Replace the snapshot_id value with the response id returned by the async request.

Code Samples

curl --get "https://api.webscrapingapi.com/v2" \
  --data-urlencode "api_key=<YOUR_API_KEY>" \
  --data-urlencode "snapshot_id=s3w1t1751043071383r7bcfdskr5q8"

const params = new URLSearchParams({
  api_key: "<YOUR_API_KEY>",
  snapshot_id: "s3w1t1751043071383r7bcfdskr5q8",
});

const response = await fetch(`https://api.webscrapingapi.com/v2?${params}`);
const data = await response.text();

console.log(data);

import requests

params = {
    "api_key": "<YOUR_API_KEY>",
    "snapshot_id": "s3w1t1751043071383r7bcfdskr5q8",
}

response = requests.get("https://api.webscrapingapi.com/v2", params=params)

print(response.text)

<?php

$query = http_build_query([
    'api_key' => '<YOUR_API_KEY>',
    'snapshot_id' => 's3w1t1751043071383r7bcfdskr5q8',
]);

$response = file_get_contents('https://api.webscrapingapi.com/v2?' . $query);

echo $response;

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
)

func main() {
	params := url.Values{}
	params.Add("api_key", "<YOUR_API_KEY>")
	params.Add("snapshot_id", "s3w1t1751043071383r7bcfdskr5q8")

	requestURL := "https://api.webscrapingapi.com/v2?" + params.Encode()

	response, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	defer response.Body.Close()

	body, err := io.ReadAll(response.Body)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(string(body))
}

import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) throws Exception {
        String apiKey = "<YOUR_API_KEY>";
        String snapshotId = "s3w1t1751043071383r7bcfdskr5q8";

        String query = "api_key=" + URLEncoder.encode(apiKey, StandardCharsets.UTF_8)
            + "&snapshot_id=" + URLEncoder.encode(snapshotId, StandardCharsets.UTF_8);

        HttpRequest request = HttpRequest.newBuilder()
            .uri(URI.create("https://api.webscrapingapi.com/v2?" + query))
            .GET()
            .build();

        HttpResponse<String> response = HttpClient.newHttpClient()
            .send(request, HttpResponse.BodyHandlers.ofString());

        System.out.println(response.body());
    }
}

using System;
using System.Net.Http;
using System.Threading.Tasks;

class Program
{
    static async Task Main()
    {
        var apiKey = "<YOUR_API_KEY>";
        var snapshotId = Uri.EscapeDataString("s3w1t1751043071383r7bcfdskr5q8");
        var requestUrl = $"https://api.webscrapingapi.com/v2?api_key={apiKey}&snapshot_id={snapshotId}";

        using var client = new HttpClient();
        var response = await client.GetStringAsync(requestUrl);

        Console.WriteLine(response);
    }
}

require 'net/http'
require 'uri'

uri = URI('https://api.webscrapingapi.com/v2')
uri.query = URI.encode_www_form(
  api_key: '<YOUR_API_KEY>',
  snapshot_id: 's3w1t1751043071383r7bcfdskr5q8'
)

response = Net::HTTP.get(uri)

puts response

When the result is ready, the API returns the scraped page response.

<!doctype html>
<html lang="en" class="light">
<head>
  <title>WebScrapingAPI - Leading REST APIs & Services For Web Scraping</title>
  <meta name="description" content="Explore WebScrapingAPI - your trusted source for top-tier web scraping APIs and services." />
</head>
<body>
  ...
</body>
</html>