GET Request
Web scraping via GET requests requires users to pass two mandatory parameters to our API's endpoint:
api_key- used to authenticate the request on our serversurl- used to indicate the targeted URL that you want to scrape
Please make sure to escape the
url parameter's value!The most basic URL example for an authorised request on our API is:
https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=<TARGETED_URL>
get
https://api.webscrapingapi.com/v1
?api_key=<YOUR_API_KEY>&url=https://webscrapingapi.com/
Basic GET Request Example
Below are the integration examples of the URL we have discussed earlier:
cURL
NodeJS
Python
PHP
Golang
Java
.NET
Ruby
curl --request GET --url "https://api.webscrapingapi.com/v1?api_key=<YOUR_API_KEY>&url=https%3A%2F%2Fbing.com"
const http = require("https");
const options = {
"method": "GET",
"hostname": "api.webscrapingapi.com",
"port": null,
"path": "/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fbing.com",
"headers": {}
};
const req = http.request(options, function (res) {
const chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
const body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
import requests
API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'
TARGET_URL = 'https://httpbin.org/post'
PARAMS = {
"api_key":API_KEY,
"url": TARGET_URL,
}
response = requests.get(SCRAPER_URL, params=PARAMS)
print(response)
<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fbing.com",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fbing.com"
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
HttpResponse<String> response = Unirest.get("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fbing.com")
.asString();
var client = new RestClient("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fbing.com");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'
url = URI("https://api.webscrapingapi.com/v1?api_key=%7B%7Bapi_key%7D%7D&url=https%3A%2F%2Fbing.com")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
<!doctype html>
<html lang="es" dir="ltr">
<head>
<meta name="theme-color" content="#4F4F4F" />
<meta name="description"
content="Bing te ayuda a convertir la información en acción, ya que facilita y acelera la transición de la búsqueda a la actividad concreta." />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta property="fb:app_id" content="570810223073062" />
<meta property="og:type" content="website" />
<meta property="og:title" content="Info" />
<meta property="og:image" content="https://www.bing.com/th?id=OHR.Hunebourg_ROW9321634276_tmb.jpg&rf=" />
<meta property="og:image:width" content="1366" />
<meta property="og:image:height" content="768" />
<meta property="og:url" content="https://www.bing.com/?form=HPFBBK&ssd=20220426_0700&mkt=es-ES" />
<meta property="og:site_name" content="Bing" />
<meta property="og:description" content="We're enjoying the view of Château de Hunebourg, n" />
<title>Bing</title>
<link rel="shortcut icon" href="/sa/simg/favicon-2x.ico" />
<link rel="preload" href="/th?id=OHR.Hunebourg_ROW9321634276_1920x1080.jpg&rf=LaDigue_1920x1080.jpg&qlt=40"
as="image" id="preloadBg" />
<link rel="preload" href="https://r.bing.com/rp/lmu8EBCaPRMKtay8LSArGyY3mv4.br.js" as="script" />
<link rel="preconnect" href="https://r.bing.com" />
<link rel="preconnect" href="https://r.bing.com" crossorigin />
<link rel="dns-prefetch" href="https://r.bing.com" />
<link rel="dns-prefetch" href="https://r.bing.com" crossorigin />
<style type="text/css">
@media(max-width:1237px) {
#id_n {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
max-width: 100px;
display: inline-block
}
}
.rwds_svg {
vertical-align: top;
display: inline-block
}
.rwds_svg.mobile {
float: left
}
.rwds_svg.serp {
margin: 8px 0 0 8px
}
.rwds_svg.serp.mobile {
margin: 8px 4px 0 8px
}
.rwds_svg.hp {
margin: 5px 4px 0 8px
}
.rwds_svg.hp.mobile {
margin: 5px 4px 0 10px
}
.rhlined,
.rhfill {
vertical-align: top;
width: 32px;
height: 32px
}
#id_rh,
#id_rbh {
position: relative
}
.rhcoinflip {
vertical-align: top;
width: 60px;
height: 60px;
position: absolute;
right: -6px;
top: -6px
}
#idCont .rhcoinflip {
position: absolute;
right: -5px;
top: -9px
}
#rh_meter {
vertical-align: top;
width: 40px;
height: 40px;
margin-left: -36px;
margin-top: -4px
}
.rh_reedm .rhlined,
.rhfill,
.rh_reedm .meter {
display: none
}
.rhlined,
.rh_reedm .rhfill,
#rh_meter {
display: inline-block
}
.rd_hide {
visibility: hidden
}
.rhlined.hp .meter,
.rhfill.hp .meter {
stroke: rgba(255, 255, 255, .4)
}
.noBg .rhlined.hp .meter,
.noBg .rhfill.hp .meter,
.rhlined.serp .meter,
.rhfill.serp .meter {
stroke: rgba(177, 177, 177, .4)
}
.rhlined.hp .medal,
.rhfill.hp .medal {
fill: #fff
}
.noBg .rhlined.hp .medal,
.rhlined.serp .medal {
fill: #919191
}
.noBg .rh_reedm .rhfill.hp .medal,
.rh_reedm .rhfill.serp .medal {
fill: #00809d
}
#rh_animcrcl {
fill: none;
stroke: transparent;
stroke-width: 0
}
#rh_animcrcl.anim {
stroke-width: 2
}
#rh_animcrcl.anim.hp {
stroke: #fff
}
@media screen and (-ms-high-contrast:black-on-white) {
.rhlined.hp .medal,
.rhfill.hp .medal {
fill: #000
}
#rh_animcrcl.anim.hp {
stroke: #000
}
}
#rh_animcrcl.serp.anim {
stroke: #00809d
}
#rh_animpath.serp.anim {
fill: #00809d
}
.noBg #rh_animcrcl.anim.hp {
stroke: #00809d
}
.rh_scale .rhfill,
.rh_scale #rh_meter {
animation: scaling .4s cubic-bezier(.3, .55, .1, 1)
}
@-webkit-keyframes scaling {
0% {
transform: scale(1, 1)
}
50% {
transform: scale(1.3, 1.3)
}
100% {
transform: scale(1, 1)
}
}
@keyframes scaling {
0% {
transform: scale(1, 1)
}
50% {
transform: scale(1.3, 1.3)
}
100% {
transform: scale(1, 1)
}
}
#tallhead #idMobileCont {
position: absolute
}
#tallhead #idMobileCont>* {
top: 3px
}
#idMobileCont>* {
vertical-align: top;
display: inline-block;
top: 10px
}
#idMobileCont #id_rh,
#idMobileCont #id_rbh {
font-size: .8125rem;
position: relative;
margin-top: 2px
}
#idMobileCont #id_rh #id_rc.hp,
#idMobileCont #id_rbh #id_rc.hp,
#idMobileCont #id_rh #id_rbc.hp,
#idMobileCont #id_rbh #id_rbc.hp {
color: #fff;
line-height: 38px
}
#idMobileCont #id_rh #id_rc.serp,
#idMobileCont #id_rbh #id_rc.serp,
#idMobileCont #id_rh #id_rbc.serp,
#idMobileCont #id_rbh #id_rbc.serp {
color: #000;
line-height: 45px
}
#id_rbh.hp {
padding-right: 12px
}
#id_rbh.hp.reverse {
padding-left: 8px;
padding-right: 0
}
#id_rbh.serp {
padding-right: 0
}
#id_rbh.serp.reverse {
padding-left: 14px
}
#id_rbh.hp svg {
height: 30px;
fill: white
}
#id_rbh.serp svg {
height: 25px;
fill: #818285
}
@media(max-width:1356.9px) {
#id_h #id_rbh,
#idCont #id_h #id_rbh {
display: none
}
}
.bnc-hci {
position: relative;
top: 4px;
padding-left: 8px
}
html,
body {
font-family: "Segoe UI", Segoe, Tahoma, Arial, Verdana, sans-serif;
background-color: rgba(34, 34, 34, .9);
overflow: hidden;
width: 100%;
height: 100%
}
html,
body,
a,
div,
span,
table,
tr,
td,
strong,
ul,
ol,
li,
h1,
h2,
h3,
p,
input {
font-weight: inherit;
font-size: inherit;
list-style: none;
border-spacing: 0;
border: 0;
border-collapse: collapse;
text-decoration: none;
padding: 0;
margin: 0
}
a {
color: initial
}
a:hover {
text-decoration: none
}
object {
position: absolute
}
video::-webkit-media-controls {
display: none !important
}
input[type="search"]::-webkit-search-decoration,
input[type="search"]::-webkit-search-cancel-button,
input[type="search"]::-webkit-search-results-button,
input[type="search"]::-webkit-search-results-decoration {
display: none
}
body:not(.tabbing) *:focus {
outline: none
}
.hpapp {
height: 100%
}
.hp_body,
.hp_cont {
height: 100%
}
.hide,
.b_hide,
.bnc-hide,
.invtesthooks {
display: none !important
}
.show {
display: block !important
}
.hidden {
visibility: hidden
}
.hp_body {
min-width: 768px;
min-height: 360px;
background-color: #555
}
.hp_body.no_image {
background-color: #f5f5f5
}
.hp_body.no_image .img_cont,
.hp_body.no_image .img_uhd,
.hp_body.no_image .hp_top_cover {
background-color: #f5f5f5;
background-image: none !important
}
.hp_body.no_image .shaders,
.hp_body.no_image .dimmer,
.hp_body.no_image .birthday {
display: none
}
.hp_body.no_image #bLogo {
filter: none
}
.hp_body.no_image #bLogo .ms_text,
.hp_body.no_image #bLogo .b_text {
fill: #666
}
.hp_body.no_image .hp_top_cover_dim {
display: none
}
.hp_body.no_image .sbox {
left: 0;
right: 0;
margin: auto
}
.hp_body.no_image .sbox .below_sbox {
color: #666;
text-shadow: none
}
.hp_body.no_image .sbox .below_sbox .text,
.hp_body.no_image .sbox .below_sbox a {
color: #666
}
.hp_body.no_image .holiday_controls,
.hp_body.no_image .holiday_video,
.hp_body.no_image .holiday_effects {
display: none !important
}
.hp_media_container {
position: fixed;
left: 0;
right: 0;
top: 0;
bottom: 0;
overflow: hidden
}
.hp_top_cover {
top: 0;
left: 0;
width: 100%;
height: 100vh;
z-index: 2;
background-position: center;
background-size: cover
}
.hp_top_cover_container {
overflow: hidden;
position: fixed;
top: 0;
height: 194px;
z-index: 2;
width: 100%
}
.hp_top_cover_dim {
background: #555;
height: 100%;
width: 100%;
opacity: 0
}
.img_cont {
background-position: center;
background-size: cover;
bottom: 0;
right: 0;
left: 0;
top: 0;
position: absolute
}
.img_cont .img_uhd {
opacity: 0;
background-size: cover;
background-position: center;
bottom: 0;
right: 0;
left: 0;
top: 0;
position: absolute
}
.img_cont .bg_video {
object-fit: cover;
left: 50%;
top: 50%;
transform: translate3d(-50%, -50%, 0);
position: absolute;
height: 100%;
width: 100%
}
.img_cont .bg_video.fadeOut {
opacity: 0
}
.img_cont .bg_audio {
display: none
}
.hp_cont {
width: 80%;
margin: auto;
position: relative
}
@media screen and (min-width:2240px) {
.hp_cont {
width: 1792px
}
}
.dimmer {
position: fixed;
width: 100%;
height: 100%;
left: 0;
top: 0;
pointer-events: none;
transition: background-color .2s
}
.dimmer.dim {
z-index: 2;
pointer-events: all;
background-color: rgba(0, 0, 0, .6)
}
.shaders {
position: fixed;
height: 100vh;
width: 100%;
left: 0;
top: 0
}
.shaders .topLeft,
.shaders .topRight,
.shaders .bottom {
height: 28%;
position: absolute
}
.shaders .topLeft {
left: 0;
width: 100%;
background: linear-gradient(to bottom left, rgba(0, 0, 0, .5) -20%, transparent 30%)
}
.shaders .topRight {
right: 0;
width: 100%;
background: linear-gradient(to bottom right, rgba(0, 0, 0, .5) -20%, transparent 30%)
}
.shaders .bottom {
background: linear-gradient(to bottom, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, .6) 100%);
bottom: 0;
left: 0;
width: 100%
}
.hp_top_cover_container .shaders {
position: absolute
}
.hp_media_container .shaders {
z-index: 1
}
.sbox {
top: 20%;
height: 100%;
width: 100%;
width: 62.4%;
box-sizing: border-box;
min-height: 40px;
max-height: 52px;
min-width: 580px;
max-width: 596px;
white-space: nowrap;
position: fixed;
z-index: 2;
margin: 0 0 0 228px
}
.sbox * {
box-sizing: inherit
}
.sbox.fix {
top: 90px !important;
transform: none !important;
position: fixed
}
.sbox .logo,
.sbox .sb_form {
display: inline-block;
vertical-align: middle
}
.sbox .logo {
width: 20%;
height: 100%
}
.sbox .logo svg {
height: 100%;
width: 100%
}
.sbox .sb_form {
width: 100%;
background-color: transparent;
font: normal 1rem "Segoe UI", Arial, Helvetica, Sans-Serif;
padding-right: 10px;
box-shadow: 0 2px 4px rgba(0, 0, 0, .3);
align-items: center;
justify-content: flex-end;
display: inline-flex;
border-radius: 24px;
height: 100%;
position: relative;
background-color: #fff
}
.sbox .sb_form.as_show {
border-radius: 24px 24px 0 0
}
.sbox .sb_form.as_show.as_nextword {
border-radius: 24px
}
.sbox .sb_form svg {
height: 100%;
width: 100%
}
.sbox .sb_form input[type=search]::-ms-clear,
.sbox .sb_form input[type=search]::-ms-reveal {
display: none;
width: 0;
height: 0
}
.sbox .sb_form .sb_form_q {
background-color: transparent;
padding: 12px 10px 12px 16px;
flex: 1;
border: 0;
outline: none;
-webkit-appearance: none;
font-size: 1.125rem;
font: inherit
}
.sbox .sb_form .sb_form_q:focus~#sw_as,
.sbox .sb_form .sb_form_q:active~#sw_as {
display: block
}
.sbox .sb_form .sb_form_q:focus~#sw_as .sa_as,
.sbox .sb_form .sb_form_q:active~#sw_as .sa_as {
display: block
}
.sbox .sb_form .icon {
position: relative;
min-width: 28px;
margin: auto 4px;
height: 45%;
width: 5%
}
.sbox .sb_form #vkeyIcon {
width: 100%;
height: 100%
}
.sbox .sb_form .search,
.sbox .sb_form #vkeyIcon {
cursor: pointer
}
.sbox .sb_form .largeTargNoTT {
cursor: pointer;
margin: 0 0 0 5px;
padding: 13px 0 13px 0;
box-sizing: content-box
}
.sbox .sb_form #sb_form_go {
padding: 0;
height: 0;
width: 0;
outline: none;