Code examples - Zyte documentation

Running the scrollBottom action

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

C#

using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using HtmlAgilityPack;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var input = new Dictionary<string, object>(){
    {"url", "https://quotes.toscrape.com/scroll"},
    {"browserHtml", true},
    {
        "actions",
        new List<Dictionary<string, object>>()
        {
            new Dictionary<string, object>()
            {
                {"action", "scrollBottom"}
            }
        }
    }
};
var inputJson = JsonSerializer.Serialize(input);
var content = new StringContent(inputJson, Encoding.UTF8, "application/json");

HttpResponseMessage response = await client.PostAsync("https://api.zyte.com/v1/extract", content);
var body = await response.Content.ReadAsByteArrayAsync();

var data = JsonDocument.Parse(body);
var browserHtml = data.RootElement.GetProperty("browserHtml").ToString();
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(browserHtml);
var navigator = htmlDocument.CreateNavigator();
var quoteCount = (double)navigator.Evaluate("count(//*[@class=\"quote\"])");

CLI client

input.jsonl#

{"url": "https://quotes.toscrape.com/scroll", "browserHtml": true, "actions": [{"action": "scrollBottom"}]}

zyte-api input.jsonl \
    | jq --raw-output .browserHtml \
    | xmllint --html --xpath 'count(//*[@class="quote"])' - 2> /dev/null

curl

input.json#

{
    "url": "https://quotes.toscrape.com/scroll",
    "browserHtml": true,
    "actions": [
        {
            "action": "scrollBottom"
        }
    ]
}

curl \
    --user YOUR_API_KEY: \
    --header 'Content-Type: application/json' \
    --data @input.json \
    --compressed \
    https://api.zyte.com/v1/extract \
    | jq --raw-output .browserHtml \
    | xmllint --html --xpath 'count(//*[@class="quote"])' - 2> /dev/null

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collections;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

class Example {

  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> action = ImmutableMap.of("action", "scrollBottom");
    Map<String, Object> parameters =
        ImmutableMap.of(
            "url",
            "https://quotes.toscrape.com/scroll",
            "browserHtml",
            true,
            "actions",
            Collections.singletonList(action));
    String requestBody = new Gson().toJson(parameters);

    HttpPost request = new HttpPost("https://api.zyte.com/v1/extract");
    request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    request.setEntity(new StringEntity(requestBody));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        request,
        response -> {
          HttpEntity entity = response.getEntity();
          String apiResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
          JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
          String browserHtml = jsonObject.get("browserHtml").getAsString();
          Document document = Jsoup.parse(browserHtml);
          int quoteCount = document.select(".quote").size();
          System.out.println(quoteCount);
          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')
const cheerio = require('cheerio')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://quotes.toscrape.com/scroll',
    browserHtml: true,
    actions: [
      {
        action: 'scrollBottom'
      }
    ]
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((response) => {
  const browserHtml = response.data.browserHtml
  const $ = cheerio.load(browserHtml)
  const quoteCount = $('.quote').length
})

PHP

<?php

$client = new GuzzleHttp\Client();
$response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://quotes.toscrape.com/scroll',
        'browserHtml' => true,
        'actions' => [
            ['action' => 'scrollBottom'],
        ],
    ],
]);
$data = json_decode($response->getBody());
$doc = new DOMDocument();
$doc->loadHTML($data->browserHtml);
$xpath = new DOMXPath($doc);
$quote_count = $xpath->query("//*[@class='quote']")->count();

Python

import requests
from parsel import Selector

api_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://quotes.toscrape.com/scroll",
        "browserHtml": True,
        "actions": [
            {
                "action": "scrollBottom",
            },
        ],
    },
)
browser_html = api_response.json()["browserHtml"]
quote_count = len(Selector(browser_html).css(".quote"))

Python client

import asyncio

from parsel import Selector
from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    api_response = await client.get(
        {
            "url": "https://quotes.toscrape.com/scroll",
            "browserHtml": True,
            "actions": [
                {
                    "action": "scrollBottom",
                },
            ],
        },
    )
    browser_html = api_response["browserHtml"]
    quote_count = len(Selector(browser_html).css(".quote"))
    print(quote_count)


asyncio.run(main())

Scrapy

from scrapy import Request, Spider


class QuotesToScrapeComSpider(Spider):
    name = "quotes_toscrape_com"

    def start_requests(self):
        yield Request(
            "https://quotes.toscrape.com/scroll",
            meta={
                "zyte_api_automap": {
                    "browserHtml": True,
                    "actions": [
                        {
                            "action": "scrollBottom",
                        },
                    ],
                },
            },
        )

    def parse(self, response):
        quote_count = len(response.css(".quote"))

Output:

Getting an HTTP response body

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output (first 5 lines):

<!DOCTYPE html>
<html lang="en">
    <head>
        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
        <title>Scraping Sandbox</title>

Setting a Referer header in a browser request

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

C#

using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml.XPath;
using HtmlAgilityPack;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var input = new Dictionary<string, object>(){
    {"url", "https://httpbin.org/anything"},
    {"browserHtml", true},
    {
        "requestHeaders",
        new Dictionary<string, object>()
        {
            {"referer", "https://example.org/"}
        }
    }
};
var inputJson = JsonSerializer.Serialize(input);
var content = new StringContent(inputJson, Encoding.UTF8, "application/json");

HttpResponseMessage response = await client.PostAsync("https://api.zyte.com/v1/extract", content);
var body = await response.Content.ReadAsByteArrayAsync();

var data = JsonDocument.Parse(body);
var browserHtml = data.RootElement.GetProperty("browserHtml").ToString();
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(browserHtml);
var navigator = htmlDocument.CreateNavigator();
var nodeIterator = (XPathNodeIterator)navigator.Evaluate("//text()");
nodeIterator.MoveNext();
var responseJson = nodeIterator.Current.ToString();
var responseData = JsonDocument.Parse(responseJson);
var headerEnumerator = responseData.RootElement.GetProperty("headers").EnumerateObject();
var headers = new Dictionary<string, string>();
while (headerEnumerator.MoveNext())
{
    headers.Add(
        headerEnumerator.Current.Name.ToString(),
        headerEnumerator.Current.Value.ToString()
    );
}

CLI client

input.jsonl#

{"url": "https://httpbin.org/anything", "browserHtml": true, "requestHeaders": {"referer": "https://example.org/"}}

zyte-api input.jsonl \
    | jq --raw-output .browserHtml \
    | xmllint --html --xpath '//text()' - 2> /dev/null \
    | jq .headers

curl

input.json#

{
    "url": "https://httpbin.org/anything",
    "browserHtml": true,
    "requestHeaders": {
        "referer": "https://example.org/"
    }
}

curl \
    --user YOUR_API_KEY: \
    --header 'Content-Type: application/json' \
    --data @input.json \
    --compressed \
    https://api.zyte.com/v1/extract \
    | jq --raw-output .browserHtml \
    | xmllint --html --xpath '//text()' - 2> /dev/null \
    | jq .headers

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

class Example {
  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> requestHeaders = ImmutableMap.of("referer", "https://example.org/");
    Map<String, Object> parameters =
        ImmutableMap.of(
            "url",
            "https://httpbin.org/anything",
            "browserHtml",
            true,
            "requestHeaders",
            requestHeaders);
    String requestBody = new Gson().toJson(parameters);

    HttpPost request = new HttpPost("https://api.zyte.com/v1/extract");
    request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    request.setEntity(new StringEntity(requestBody));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        request,
        response -> {
          HttpEntity entity = response.getEntity();
          String apiResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
          JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
          String browserHtml = jsonObject.get("browserHtml").getAsString();
          Document document = Jsoup.parse(browserHtml);
          JsonObject data = JsonParser.parseString(document.text()).getAsJsonObject();
          JsonObject headers = data.get("headers").getAsJsonObject();
          Gson gson = new GsonBuilder().setPrettyPrinting().create();
          System.out.println(gson.toJson(headers));
          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')
const cheerio = require('cheerio')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://httpbin.org/anything',
    browserHtml: true,
    requestHeaders: {
      referer: 'https://example.org/'
    }
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((response) => {
  const $ = cheerio.load(response.data.browserHtml)
  const data = JSON.parse($.text())
  const headers = data.headers
})

PHP

<?php

$client = new GuzzleHttp\Client();
$response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://httpbin.org/anything',
        'browserHtml' => true,
        'requestHeaders' => [
            'referer' => 'https://example.org/',
        ],
    ],
]);
$api = json_decode($response->getBody());
$doc = new DOMDocument();
$doc->loadHTML($api->browserHtml);
$data = json_decode($doc->textContent);
$headers = $data->headers;

Python

import json

import requests
from parsel import Selector

api_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://httpbin.org/anything",
        "browserHtml": True,
        "requestHeaders": {
            "referer": "https://example.org/",
        },
    },
)
browser_html = api_response.json()["browserHtml"]
selector = Selector(browser_html)
response_json = selector.xpath("//text()").get()
response_data = json.loads(response_json)
headers = response_data["headers"]

Python client

import asyncio
import json

from parsel import Selector
from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    api_response = await client.get(
        {
            "url": "https://httpbin.org/anything",
            "browserHtml": True,
            "requestHeaders": {
                "referer": "https://example.org/",
            },
        }
    )
    browser_html = api_response["browserHtml"]
    selector = Selector(browser_html)
    response_json = selector.xpath("//text()").get()
    response_data = json.loads(response_json)
    print(json.dumps(response_data["headers"], indent=2))


asyncio.run(main())

Scrapy

import json

from scrapy import Request, Spider


class HTTPBinOrgSpider(Spider):
    name = "httpbin_org"

    def start_requests(self):
        yield Request(
            "https://httpbin.org/anything",
            headers={"Referer": "https://example.org/"},
            meta={
                "zyte_api_automap": {
                    "browserHtml": True,
                },
            },
        )

    def parse(self, response):
        response_json = response.xpath("//text()").get()
        response_data = json.loads(response_json)
        headers = response_data["headers"]

Output ("Referer" line):

  "Referer": "https://example.org/",

Getting browser HTML

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output (first 5 lines):

<!DOCTYPE html><html lang="en"><head>
        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
        <title>Scraping Sandbox</title>
        <link href="./css/bootstrap.min.css" rel="stylesheet">
        <link href="./css/main.css" rel="stylesheet">

Reusing browser cookies on HTTP requests

Send a browser request to the home page of a website, and use its response cookies as request cookies in an HTTP request to a different URL of that website.

C#

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var browserInput = new Dictionary<string, object>(){
    {"url", "https://toscrape.com/"},
    {"browserHtml", true},
    {"responseCookies", true}
};
var browserInputJson = JsonSerializer.Serialize(browserInput);
var browserContent = new StringContent(browserInputJson, Encoding.UTF8, "application/json");
HttpResponseMessage browserResponse = await client.PostAsync("https://api.zyte.com/v1/extract", browserContent);
var browserResponseBody = await browserResponse.Content.ReadAsByteArrayAsync();
var browserData = JsonDocument.Parse(browserResponseBody);

var httpInput = new Dictionary<string, object>(){
    {"url", "https://toscrape.com/"},
    {"httpResponseBody", true},
    {"requestCookies", browserData.RootElement.GetProperty("responseCookies")}
};
var httpInputJson = JsonSerializer.Serialize(httpInput);
var httpContent = new StringContent(httpInputJson, Encoding.UTF8, "application/json");
HttpResponseMessage httpResponse = await client.PostAsync("https://api.zyte.com/v1/extract", httpContent);
var httpResponseBody = await httpResponse.Content.ReadAsByteArrayAsync();
var httpData = JsonDocument.Parse(httpResponseBody);
var base64HttpResponseBodyField = httpData.RootElement.GetProperty("httpResponseBody").ToString();
var httpResponseBodyField = System.Convert.FromBase64String(base64HttpResponseBodyField);
var result = System.Text.Encoding.UTF8.GetString(httpResponseBodyField);

Console.WriteLine(result);

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;

class Example {
  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> browserParameters =
        ImmutableMap.of(
            "url", "https://toscrape.com/", "browserHtml", true, "responseCookies", true);
    String browserRequestBody = new Gson().toJson(browserParameters);

    HttpPost browserRequest = new HttpPost("https://api.zyte.com/v1/extract");
    browserRequest.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    browserRequest.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    browserRequest.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    browserRequest.setEntity(new StringEntity(browserRequestBody));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        browserRequest,
        browserResponse -> {
          HttpEntity browserEntity = browserResponse.getEntity();
          String browserApiResponse = EntityUtils.toString(browserEntity, StandardCharsets.UTF_8);
          JsonObject browserJsonObject =
              JsonParser.parseString(browserApiResponse).getAsJsonObject();

          Map<String, Object> httpParameters =
              ImmutableMap.of(
                  "url",
                  "https://books.toscrape.com/",
                  "httpResponseBody",
                  true,
                  "requestCookies",
                  browserJsonObject.get("responseCookies"));
          String httpRequestBody = new Gson().toJson(httpParameters);

          HttpPost httpRequest = new HttpPost("https://api.zyte.com/v1/extract");
          httpRequest.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
          httpRequest.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
          httpRequest.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
          httpRequest.setEntity(new StringEntity(httpRequestBody));

          client.execute(
              httpRequest,
              httpResponse -> {
                HttpEntity httpEntity = httpResponse.getEntity();
                String httpApiResponse = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8);
                JsonObject httpJsonObject =
                    JsonParser.parseString(httpApiResponse).getAsJsonObject();
                String base64HttpResponseBody =
                    httpJsonObject.get("httpResponseBody").getAsString();
                byte[] httpResponseBodyBytes = Base64.getDecoder().decode(base64HttpResponseBody);
                String httpResponseBody = new String(httpResponseBodyBytes, StandardCharsets.UTF_8);
                System.out.println(httpResponseBody);
                return null;
              });
          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://toscrape.com/',
    browserHtml: true,
    responseCookies: true
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((browserResponse) => {
  axios.post(
    'https://api.zyte.com/v1/extract',
    {
      url: 'https://books.toscrape.com/',
      httpResponseBody: true,
      requestCookies: browserResponse.data.responseCookies
    },
    {
      auth: { username: 'YOUR_API_KEY' }
    }
  ).then((httpResponse) => {
    const httpResponseBody = Buffer.from(
      httpResponse.data.httpResponseBody,
      'base64'
    )
    console.log(httpResponseBody.toString())
  })
})

PHP

<?php

$client = new GuzzleHttp\Client();
$browser_response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://toscrape.com/',
        'browserHtml' => true,
        'responseCookies' => true,
    ],
]);
$browser_data = json_decode($browser_response->getBody());
$http_response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://books.toscrape.com/',
        'httpResponseBody' => true,
        'requestCookies' => $browser_data->responseCookies,
    ],
]);
$http_data = json_decode($http_response->getBody());
$http_response_body = base64_decode($http_data->httpResponseBody);
echo $http_response_body;

Python

from base64 import b64decode

import requests

browser_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://toscrape.com/",
        "browserHtml": True,
        "responseCookies": True,
    },
)
http_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://books.toscrape.com/",
        "httpResponseBody": True,
        "requestCookies": browser_response.json()["responseCookies"],
    },
)
http_response_body = b64decode(http_response.json()["httpResponseBody"])
print(http_response_body.decode())

Python client

import asyncio
from base64 import b64decode

from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    browser_response = await client.get(
        {
            "url": "https://toscrape.com/",
            "browserHtml": True,
            "responseCookies": True,
        }
    )
    http_response = await client.get(
        {
            "url": "https://books.toscrape.com/",
            "httpResponseBody": True,
            "requestCookies": browser_response["responseCookies"],
        }
    )
    http_response_body = b64decode(http_response["httpResponseBody"]).decode()
    print(http_response_body)


asyncio.run(main())

Scrapy

from scrapy import Request, Spider


class ToScrapeComSpider(Spider):
    name = "toscrape_com"

    def start_requests(self):
        yield Request(
            "https://toscrape.com/",
            callback=self.parse_browser,
            meta={
                "zyte_api_automap": {
                    "browserHtml": True,
                    "responseCookies": True,
                },
            },
        )

    def parse_browser(self, response):
        yield response.follow(
            "https://books.toscrape.com/",
            callback=self.parse_http,
            meta={
                "zyte_api_automap": {
                    "requestCookies": response.raw_api_response["responseCookies"],
                },
            },
        )

    def parse_http(self, response):
        print(response.text)

Setting a geolocation

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output:

AU

Making an HTTP request seem like it comes from a mobile device

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Example output (may vary):

Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Mobile Safari/537.36

Getting structured data from a product details page of an e-commerce website

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

C#

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var input = new Dictionary<string, object>(){
    {"url", "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html"},
    {"product", true}
};
var inputJson = JsonSerializer.Serialize(input);
var content = new StringContent(inputJson, Encoding.UTF8, "application/json");

HttpResponseMessage response = await client.PostAsync("https://api.zyte.com/v1/extract", content);
var body = await response.Content.ReadAsByteArrayAsync();

var data = JsonDocument.Parse(body);
var product = data.RootElement.GetProperty("product").ToString();

Console.WriteLine(product);

CLI client

input.jsonl#

{"url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html", "product": true}

zyte-api input.jsonl \
    | jq --raw-output .product

curl

input.json#

{
    "url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",
    "product": true
}

curl \
    --user YOUR_API_KEY: \
    --header 'Content-Type: application/json' \
    --data @input.json \
    --compressed \
    https://api.zyte.com/v1/extract \
    | jq --raw-output .product

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;

class Example {
  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> parameters =
        ImmutableMap.of(
            "url",
            "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",
            "product",
            true);
    String requestBody = new Gson().toJson(parameters);

    HttpPost request = new HttpPost("https://api.zyte.com/v1/extract");
    request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    request.setEntity(new StringEntity(requestBody));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        request,
        response -> {
          HttpEntity entity = response.getEntity();
          String apiResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
          JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
          JsonObject product = jsonObject.get("product").getAsJsonObject();
          Gson gson = new GsonBuilder().setPrettyPrinting().create();
          System.out.println(gson.toJson(product));
          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    product: true
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((response) => {
  const product = response.data.product
  console.log(product)
})

PHP

<?php

$client = new GuzzleHttp\Client();
$response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
        'product' => true,
    ],
]);
$data = json_decode($response->getBody());
$product = json_encode($data->product);
echo $product.PHP_EOL;

Python

import requests

api_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": (
            "https://books.toscrape.com/catalogue"
            "/a-light-in-the-attic_1000/index.html"
        ),
        "product": True,
    },
)
product = api_response.json()["product"]
print(product)

Python client

import asyncio
import json

from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    api_response = await client.get(
        {
            "url": (
                "https://books.toscrape.com/catalogue"
                "/a-light-in-the-attic_1000/index.html"
            ),
            "product": True,
        }
    )
    product = api_response["product"]
    print(json.dumps(product, indent=2, ensure_ascii=False))


asyncio.run(main())

Scrapy

from scrapy import Request, Spider


class BooksToScrapeComSpider(Spider):
    name = "books_toscrape_com"

    def start_requests(self):
        yield Request(
            (
                "https://books.toscrape.com/catalogue"
                "/a-light-in-the-attic_1000/index.html"
            ),
            meta={
                "zyte_api_automap": {
                    "product": True,
                },
            },
        )

    def parse(self, response):
        product = response.raw_api_response["product"]
        print(product)

Output (first 5 lines):

{
  "name": "A Light in the Attic",
  "price": "51.77",
  "currency": "GBP",
  "currencyRaw": "£",

Submitting an HTML form with an HTTP request

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

In https://quotes.toscrape.com/search.aspx you get an HTML form that could be stripped down to:

<form action="/filter.aspx" method="post" >
    <select name="author">
        <option>----------</option>
        <option value="Albert Einstein">
            Albert Einstein
        </option>
        <!-- [more options] -->
    </select>
    <select name="tag">
        <option>----------</option>
    </select>
    <input type="hidden" name="__VIEWSTATE" value="ZTYzZDZ…">
</form>

When you select an Author (e.g. Albert Einstein), a form request is sent, and the Tag options fill up.

To reproduce that:

C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml.XPath;
using System.Web;
using HtmlAgilityPack;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var input1 = new Dictionary<string, object>(){
    {"url", "https://quotes.toscrape.com/search.aspx"},
    {"httpResponseBody", true}
};
var inputJson1 = JsonSerializer.Serialize(input1);
var content1 = new StringContent(inputJson1, Encoding.UTF8, "application/json");

HttpResponseMessage response1 = await client.PostAsync("https://api.zyte.com/v1/extract", content1);
var body1 = await response1.Content.ReadAsByteArrayAsync();
var data1 = JsonDocument.Parse(body1);
var base64HttpResponseBody1 = data1.RootElement.GetProperty("httpResponseBody").ToString();
var httpResponseBodyBytes1 = System.Convert.FromBase64String(base64HttpResponseBody1);
var httpResponseBody1 = System.Text.Encoding.UTF8.GetString(httpResponseBodyBytes1);
var htmlDocument1 = new HtmlDocument();
htmlDocument1.LoadHtml(httpResponseBody1);
var navigator1 = htmlDocument1.CreateNavigator();
var nodeIterator = (XPathNodeIterator)navigator1.Evaluate("//*[@name='__VIEWSTATE']/@value");
nodeIterator.MoveNext();
var viewState = nodeIterator.Current.ToString();

var httpRequestTextParameters = new Dictionary<string, string>
{
    { "author", "Albert Einstein" },
    { "tag", "----------" },
    { "__VIEWSTATE", viewState}
};
var httpRequestText = string.Join("&",
    httpRequestTextParameters.Select(kvp => $"{HttpUtility.UrlEncode(kvp.Key)}={HttpUtility.UrlEncode(kvp.Value)}"));

var input2 = new Dictionary<string, object>(){
    {"url", "https://quotes.toscrape.com/filter.aspx"},
    {"httpResponseBody", true},
    {"httpRequestMethod", "POST"},
    {
        "customHttpRequestHeaders",
        new List<Dictionary<string, object>>()
        {
            new Dictionary<string, object>()
            {
                {"name", "Content-Type"},
                {"value", "application/x-www-form-urlencoded"}
            }
        }
    },
    {"httpRequestText", httpRequestText}
};
var inputJson2 = JsonSerializer.Serialize(input2);
var content2 = new StringContent(inputJson2, Encoding.UTF8, "application/json");

HttpResponseMessage response2 = await client.PostAsync("https://api.zyte.com/v1/extract", content2);
var body2 = await response2.Content.ReadAsByteArrayAsync();
var data2 = JsonDocument.Parse(body2);
var base64HttpResponseBody2 = data2.RootElement.GetProperty("httpResponseBody").ToString();
var httpResponseBodyBytes2 = System.Convert.FromBase64String(base64HttpResponseBody2);
var httpResponseBody2 = System.Text.Encoding.UTF8.GetString(httpResponseBodyBytes2);
var htmlDocument2 = new HtmlDocument();
htmlDocument2.LoadHtml(httpResponseBody2);
var navigator2 = htmlDocument2.CreateNavigator();
var nodeIterator2 = (XPathNodeIterator)navigator2.Evaluate("//*[@name='tag']//option");
int tagCount = 0;
while (nodeIterator2.MoveNext())
{
    tagCount++;
}
Console.WriteLine($"{tagCount}");

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

class Example {
  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> parameters1 =
        ImmutableMap.of("url", "https://quotes.toscrape.com/search.aspx", "httpResponseBody", true);
    String requestBody1 = new Gson().toJson(parameters1);

    HttpPost request1 = new HttpPost("https://api.zyte.com/v1/extract");
    request1.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    request1.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    request1.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    request1.setEntity(new StringEntity(requestBody1));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        request1,
        (response1) -> {
          HttpEntity httpEntity1 = response1.getEntity();
          String httpApiResponse1 = EntityUtils.toString(httpEntity1, StandardCharsets.UTF_8);
          JsonObject httpJsonObject1 = JsonParser.parseString(httpApiResponse1).getAsJsonObject();
          String base64HttpResponseBody1 = httpJsonObject1.get("httpResponseBody").getAsString();
          byte[] httpResponseBodyBytes1 = Base64.getDecoder().decode(base64HttpResponseBody1);
          String httpResponseBody1 = new String(httpResponseBodyBytes1, StandardCharsets.UTF_8);
          Document document1 = Jsoup.parse(httpResponseBody1);
          String viewState = document1.select("[name='__VIEWSTATE']").attr("value");
          Map<String, String> params =
              ImmutableMap.of(
                  "author", "Albert Einstein",
                  "tag", "----------",
                  "__VIEWSTATE", viewState);
          List<NameValuePair> formParams = new ArrayList<>();
          for (Map.Entry<String, String> entry : params.entrySet()) {
            formParams.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
          }
          UrlEncodedFormEntity entity =
              new UrlEncodedFormEntity(formParams, StandardCharsets.UTF_8);
          String httpRequestText = EntityUtils.toString(entity);
          Map<String, Object> customHttpRequestHeader =
              ImmutableMap.of("name", "Content-Type", "value", "application/x-www-form-urlencoded");
          Map<String, Object> parameters2 =
              ImmutableMap.of(
                  "url",
                  "https://quotes.toscrape.com/filter.aspx",
                  "httpResponseBody",
                  true,
                  "httpRequestMethod",
                  "POST",
                  "customHttpRequestHeaders",
                  Collections.singletonList(customHttpRequestHeader),
                  "httpRequestText",
                  httpRequestText);
          String requestBody2 = new Gson().toJson(parameters2);

          HttpPost request2 = new HttpPost("https://api.zyte.com/v1/extract");
          request2.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
          request2.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
          request2.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
          request2.setEntity(new StringEntity(requestBody2));

          client.execute(
              request2,
              (response2) -> {
                HttpEntity httpEntity2 = response2.getEntity();
                String httpApiResponse2 = EntityUtils.toString(httpEntity2, StandardCharsets.UTF_8);
                JsonObject httpJsonObject2 =
                    JsonParser.parseString(httpApiResponse2).getAsJsonObject();
                String base64HttpResponseBody2 =
                    httpJsonObject2.get("httpResponseBody").getAsString();
                byte[] httpResponseBodyBytes2 = Base64.getDecoder().decode(base64HttpResponseBody2);
                String httpResponseBody2 =
                    new String(httpResponseBodyBytes2, StandardCharsets.UTF_8);
                Document document2 = Jsoup.parse(httpResponseBody2);
                Elements tags = document2.select("select[name='tag'] option");
                System.out.println(tags.size());
                return null;
              });

          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')
const cheerio = require('cheerio')
const querystring = require('querystring')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://quotes.toscrape.com/search.aspx',
    httpResponseBody: true
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((response) => {
  const httpResponseBody = Buffer.from(
    response.data.httpResponseBody,
    'base64'
  )
  const $ = cheerio.load(httpResponseBody)
  const viewState = $('[name="__VIEWSTATE"]').get(0).attribs.value
  const httpRequestText = querystring.stringify(
    {
      author: 'Albert Einstein',
      tag: '----------',
      __VIEWSTATE: viewState
    }
  )
  axios.post(
    'https://api.zyte.com/v1/extract',
    {
      url: 'https://quotes.toscrape.com/filter.aspx',
      httpResponseBody: true,
      httpRequestMethod: 'POST',
      customHttpRequestHeaders: [
        {
          name: 'Content-Type',
          value: 'application/x-www-form-urlencoded'
        }
      ],
      httpRequestText
    },
    {
      auth: { username: 'YOUR_API_KEY' }
    }
  ).then((response) => {
    const httpResponseBody = Buffer.from(
      response.data.httpResponseBody,
      'base64'
    )
    const $ = cheerio.load(httpResponseBody)
    console.log($('select[name="tag"] option').length)
  })
})

PHP

<?php

$client = new GuzzleHttp\Client();
$response_1 = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://quotes.toscrape.com/search.aspx',
        'httpResponseBody' => true,
    ],
]);
$data = json_decode($response_1->getBody());
$http_response_body = base64_decode($data->httpResponseBody);
$doc = new DOMDocument();
$doc->loadHTML($http_response_body);
$xpath_1 = new DOMXPath($doc);
$view_state = $xpath_1->query('//*[@name="__VIEWSTATE"]/@value')->item(0)->nodeValue;
$http_request_text = http_build_query(
    [
        'author' => 'Albert Einstein',
        'tag' => '----------',
        '__VIEWSTATE' => $view_state,
    ]
);
$response_2 = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://quotes.toscrape.com/filter.aspx',
        'httpResponseBody' => true,
        'httpRequestMethod' => 'POST',
        'customHttpRequestHeaders' => [
            [
                'name' => 'Content-Type',
                'value' => 'application/x-www-form-urlencoded',
            ],
        ],
        'httpRequestText' => $http_request_text,
    ],
]);
$data = json_decode($response_2->getBody());
$http_response_body = base64_decode($data->httpResponseBody);
$doc->loadHTML($http_response_body);
$xpath_2 = new DOMXPath($doc);
$tags = $xpath_2->query('//*[@name="tag"]/option');
echo count($tags).PHP_EOL;

Python

Install form2request, which makes it easier to handle HTML forms in Python.

Then:

from base64 import b64decode

from form2request import form2request
from parsel import Selector
import requests

api_response_1 = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://quotes.toscrape.com/search.aspx",
        "httpResponseBody": True,
    },
)
api_response_1_data = api_response_1.json()
http_response_body_1 = b64decode(api_response_1_data["httpResponseBody"])
selector_1 = Selector(body=http_response_body_1, base_url=api_response_1_data["url"])
form = selector_1.css("form")
request = form2request(form, {"author": "Albert Einstein"}, click=False)
api_response_2 = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": request.url,
        "httpRequestMethod": request.method,
        "customHttpRequestHeaders": [
            {"name": k, "value": v} for k, v in request.headers
        ],
        "httpRequestText": request.body.decode(),
        "httpResponseBody": True,
    },
)
http_response_body_2 = b64decode(api_response_2.json()["httpResponseBody"])
selector_2 = Selector(body=http_response_body_2)
print(len(selector_2.css("select[name='tag'] option")))

Python client

Install form2request, which makes it easier to handle HTML forms in Python.

Then:

import asyncio
from base64 import b64decode

from form2request import form2request
from parsel import Selector
from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    api_response_1 = await client.get(
        {
            "url": "https://quotes.toscrape.com/search.aspx",
            "httpResponseBody": True,
        }
    )
    http_response_body_1 = b64decode(api_response_1["httpResponseBody"])
    selector_1 = Selector(body=http_response_body_1, base_url=api_response_1["url"])
    form = selector_1.css("form")
    request = form2request(form, {"author": "Albert Einstein"}, click=False)
    api_response_2 = await client.get(
        {
            "url": request.url,
            "httpRequestMethod": request.method,
            "customHttpRequestHeaders": [
                {"name": k, "value": v} for k, v in request.headers
            ],
            "httpRequestText": request.body.decode(),
            "httpResponseBody": True,
        }
    )
    http_response_body_2 = b64decode(api_response_2["httpResponseBody"])
    selector_2 = Selector(body=http_response_body_2)
    print(len(selector_2.css("select[name='tag'] option")))


asyncio.run(main())

Scrapy

Install form2request, which makes it easier to handle HTML forms in Scrapy.

Then, use it and let transparent mode take care of the rest:

from form2request import form2request
from scrapy import Spider


class QuotesToScrapeComSpider(Spider):
    name = "quotes_toscrape_com"
    start_urls = ["https://quotes.toscrape.com/search.aspx"]

    def parse(self, response):
        form = response.css("form")
        request = form2request(form, {"author": "Albert Einstein"}, click=False)
        yield request.to_scrapy(callback=self.parse_tags)

    def parse_tags(self, response):
        print(len(response.css("select[name='tag'] option")))

Output (number of Tag options):

Decoding HTML from an HTTP response body (i.e. from bytes to text)

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

curl

Use file to find the media type of a previously-downloaded response based solely on its body (i.e. not following the HTML encoding sniffing algorithm).

file --mime-encoding output.html

JS

Use content-type-parser, html-encoding-sniffer and whatwg-encoding:

const contentTypeParser = require('content-type-parser')
const htmlEncodingSniffer = require('html-encoding-sniffer')
const whatwgEncoding = require('whatwg-encoding')

// …

const httpResponseHeaders = response.data.httpResponseHeaders
let contentTypeCharset
httpResponseHeaders.forEach(function (item) {
  if (item.name.toLowerCase() === 'content-type') {
    contentTypeCharset = contentTypeParser(item.value).get('charset')
  }
})
const httpResponseBody = Buffer.from(response.data.httpResponseBody, 'base64')
const encoding = htmlEncodingSniffer(httpResponseBody, {
  transportLayerEncodingLabel: contentTypeCharset
})
const html = whatwgEncoding.decode(httpResponseBody, encoding)

Python

web-poet provides a response wrapper that automatically decodes the response body following an encoding sniffing algorithm similar to the one defined in the HTML standard.

Provided that you have extracted a response with both body and headers, and you have Base64-decoded the response body, you can decode the HTML bytes as follows:

from web_poet import HttpResponse

# …

headers = tuple(
    (item['name'], item['value'])
    for item in http_response_headers
)
response = HttpResponse(
    url='https://example.com',
    body=http_response_body,
    status=200,
    headers=headers,
)
html = response.text

Scrapy

In transparent mode, regular Scrapy requests targeting HTML resources decode them by default. See Zyte API HTTP requests.

Setting arbitrary headers in HTTP requests

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output (first 5 lines):

{
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
  "Accept-Encoding": "gzip, deflate, br",
  "Accept-Language": "fa",
  "Host": "httpbin.org",

Forcing data center or residential traffic

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output:

[A web hosting company]
[An Internet service provider]

Disabling JavaScript in a browser request

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

C#

using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml.XPath;
using HtmlAgilityPack;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var input = new Dictionary<string, object>(){
    {"url", "https://www.whatismybrowser.com/detect/is-javascript-enabled"},
    {"browserHtml", true},
    {"javascript", false}
};
var inputJson = JsonSerializer.Serialize(input);
var content = new StringContent(inputJson, Encoding.UTF8, "application/json");

HttpResponseMessage response = await client.PostAsync("https://api.zyte.com/v1/extract", content);
var body = await response.Content.ReadAsByteArrayAsync();

var data = JsonDocument.Parse(body);
var browserHtml = data.RootElement.GetProperty("browserHtml").ToString();
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(browserHtml);
var navigator = htmlDocument.CreateNavigator();
var nodeIterator = (XPathNodeIterator)navigator.Evaluate("//*[@id=\"detected_value\"]/text()");
nodeIterator.MoveNext();
var isJavaScriptEnabled = nodeIterator.Current.ToString();

CLI client

input.jsonl#

{"url": "https://www.whatismybrowser.com/detect/is-javascript-enabled", "browserHtml": true, "javascript": false}

zyte-api input.jsonl \
    | jq --raw-output .browserHtml \
    | xmllint --html --xpath '//*[@id="detected_value"]/text()' - 2> /dev/null

curl

input.json#

{
    "url": "https://www.whatismybrowser.com/detect/is-javascript-enabled",
    "browserHtml": true,
    "javascript": false
}

curl \
    --user YOUR_API_KEY: \
    --header 'Content-Type: application/json' \
    --data @input.json \
    --compressed \
    https://api.zyte.com/v1/extract \
    | jq --raw-output .browserHtml \
    | xmllint --html --xpath '//*[@id="detected_value"]/text()' - 2> /dev/null

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

class Example {
  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> parameters =
        ImmutableMap.of(
            "url",
            "https://www.whatismybrowser.com/detect/is-javascript-enabled",
            "browserHtml",
            true,
            "javascript",
            false);
    String requestBody = new Gson().toJson(parameters);

    HttpPost request = new HttpPost("https://api.zyte.com/v1/extract");
    request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    request.setEntity(new StringEntity(requestBody));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        request,
        response -> {
          HttpEntity entity = response.getEntity();
          String apiResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
          JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
          String browserHtml = jsonObject.get("browserHtml").getAsString();
          Document document = Jsoup.parse(browserHtml);
          String isJavaScriptEnabled = document.select("#detected_value").text();
          System.out.println(isJavaScriptEnabled);
          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')
const cheerio = require('cheerio')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://www.whatismybrowser.com/detect/is-javascript-enabled',
    browserHtml: true,
    javascript: false
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((response) => {
  const $ = cheerio.load(response.data.browserHtml)
  const isJavaScriptEnabled = $('#detected_value').text()
})

PHP

<?php

$client = new GuzzleHttp\Client();
$response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://www.whatismybrowser.com/detect/is-javascript-enabled',
        'browserHtml' => true,
        'javascript' => false,
    ],
]);
$api = json_decode($response->getBody());
$doc = new DOMDocument();
$doc->loadHTML($api->browserHtml);
$xpath = new DOMXPath($doc);
$is_javascript_enabled = $xpath->query("//*[@id='detected_value']")->item(0)->textContent;

Python

import requests
from parsel import Selector

api_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://www.whatismybrowser.com/detect/is-javascript-enabled",
        "browserHtml": True,
        "javascript": False,
    },
)
browser_html = api_response.json()["browserHtml"]
selector = Selector(browser_html)
is_javascript_enabled: str = selector.css("#detected_value::text").get()

Python client

import asyncio

from parsel import Selector
from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    api_response = await client.get(
        {
            "url": "https://www.whatismybrowser.com/detect/is-javascript-enabled",
            "browserHtml": True,
            "javascript": False,
        }
    )
    browser_html = api_response["browserHtml"]
    selector = Selector(browser_html)
    is_javascript_enabled = selector.css("#detected_value::text").get()
    print(is_javascript_enabled)


asyncio.run(main())

Scrapy

from scrapy import Request, Spider


class WhatIsMyBrowserComSpider(Spider):
    name = "whatismybrowser_com"

    def start_requests(self):
        yield Request(
            "https://www.whatismybrowser.com/detect/is-javascript-enabled",
            meta={
                "zyte_api_automap": {
                    "browserHtml": True,
                    "javascript": False,
                },
            },
        )

    def parse(self, response):
        is_javascript_enabled: str = response.css("#detected_value::text").get()

Output:

No

Appending arbitrary metadata to a request

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output:

{"url": "https://quotes.toscrape.com/", "statusCode": 200, "browserHtml": "<!DOCTYPE html><html lang=\"en\"><head>\n\t<meta charset=\"UTF-8\">\n\t<title>Quotes to Scrape</title>\n    <link rel=\"stylesheet\" href=\"/static/bootstrap.min.css\">\n    <link rel=\"stylesheet\" href=\"/static/main.css\">\n</head>\n<body>\n    <div class=\"container\">\n        <div class=\"row header-box\">\n            <div class=\"col-md-8\">\n                <h1>\n                    <a href=\"/\" style=\"text-decoration: none\">Quotes to Scrape</a>\n                </h1>\n            </div>\n            <div class=\"col-md-4\">\n                <p>\n                \n                    <a href=\"/login\">Login</a>\n                \n                </p>\n            </div>\n        </div>\n    \n\n<div class=\"row\">\n    <div class=\"col-md-8\">\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Albert Einstein</small>\n        <a href=\"/author/Albert-Einstein\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"change,deep-thoughts,thinking,world\"> \n            \n            <a class=\"tag\" href=\"/tag/change/page/1/\">change</a>\n            \n            <a class=\"tag\" href=\"/tag/deep-thoughts/page/1/\">deep-thoughts</a>\n            \n            <a class=\"tag\" href=\"/tag/thinking/page/1/\">thinking</a>\n            \n            <a class=\"tag\" href=\"/tag/world/page/1/\">world</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">J.K. Rowling</small>\n        <a href=\"/author/J-K-Rowling\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"abilities,choices\"> \n            \n            <a class=\"tag\" href=\"/tag/abilities/page/1/\">abilities</a>\n            \n            <a class=\"tag\" href=\"/tag/choices/page/1/\">choices</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Albert Einstein</small>\n        <a href=\"/author/Albert-Einstein\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"inspirational,life,live,miracle,miracles\"> \n            \n            <a class=\"tag\" href=\"/tag/inspirational/page/1/\">inspirational</a>\n            \n            <a class=\"tag\" href=\"/tag/life/page/1/\">life</a>\n            \n            <a class=\"tag\" href=\"/tag/live/page/1/\">live</a>\n            \n            <a class=\"tag\" href=\"/tag/miracle/page/1/\">miracle</a>\n            \n            <a class=\"tag\" href=\"/tag/miracles/page/1/\">miracles</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Jane Austen</small>\n        <a href=\"/author/Jane-Austen\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"aliteracy,books,classic,humor\"> \n            \n            <a class=\"tag\" href=\"/tag/aliteracy/page/1/\">aliteracy</a>\n            \n            <a class=\"tag\" href=\"/tag/books/page/1/\">books</a>\n            \n            <a class=\"tag\" href=\"/tag/classic/page/1/\">classic</a>\n            \n            <a class=\"tag\" href=\"/tag/humor/page/1/\">humor</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Marilyn Monroe</small>\n        <a href=\"/author/Marilyn-Monroe\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"be-yourself,inspirational\"> \n            \n            <a class=\"tag\" href=\"/tag/be-yourself/page/1/\">be-yourself</a>\n            \n            <a class=\"tag\" href=\"/tag/inspirational/page/1/\">inspirational</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“Try not to become a man of success. Rather become a man of value.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Albert Einstein</small>\n        <a href=\"/author/Albert-Einstein\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"adulthood,success,value\"> \n            \n            <a class=\"tag\" href=\"/tag/adulthood/page/1/\">adulthood</a>\n            \n            <a class=\"tag\" href=\"/tag/success/page/1/\">success</a>\n            \n            <a class=\"tag\" href=\"/tag/value/page/1/\">value</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“It is better to be hated for what you are than to be loved for what you are not.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">André Gide</small>\n        <a href=\"/author/Andre-Gide\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"life,love\"> \n            \n            <a class=\"tag\" href=\"/tag/life/page/1/\">life</a>\n            \n            <a class=\"tag\" href=\"/tag/love/page/1/\">love</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“I have not failed. I've just found 10,000 ways that won't work.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Thomas A. Edison</small>\n        <a href=\"/author/Thomas-A-Edison\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"edison,failure,inspirational,paraphrased\"> \n            \n            <a class=\"tag\" href=\"/tag/edison/page/1/\">edison</a>\n            \n            <a class=\"tag\" href=\"/tag/failure/page/1/\">failure</a>\n            \n            <a class=\"tag\" href=\"/tag/inspirational/page/1/\">inspirational</a>\n            \n            <a class=\"tag\" href=\"/tag/paraphrased/page/1/\">paraphrased</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Eleanor Roosevelt</small>\n        <a href=\"/author/Eleanor-Roosevelt\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"misattributed-eleanor-roosevelt\"> \n            \n            <a class=\"tag\" href=\"/tag/misattributed-eleanor-roosevelt/page/1/\">misattributed-eleanor-roosevelt</a>\n            \n        </div>\n    </div>\n\n    <div class=\"quote\" itemscope=\"\" itemtype=\"http://schema.org/CreativeWork\">\n        <span class=\"text\" itemprop=\"text\">“A day without sunshine is like, you know, night.”</span>\n        <span>by <small class=\"author\" itemprop=\"author\">Steve Martin</small>\n        <a href=\"/author/Steve-Martin\">(about)</a>\n        </span>\n        <div class=\"tags\">\n            Tags:\n            <meta class=\"keywords\" itemprop=\"keywords\" content=\"humor,obvious,simile\"> \n            \n            <a class=\"tag\" href=\"/tag/humor/page/1/\">humor</a>\n            \n            <a class=\"tag\" href=\"/tag/obvious/page/1/\">obvious</a>\n            \n            <a class=\"tag\" href=\"/tag/simile/page/1/\">simile</a>\n            \n        </div>\n    </div>\n\n    <nav>\n        <ul class=\"pager\">\n            \n            \n            <li class=\"next\">\n                <a href=\"/page/2/\">Next <span aria-hidden=\"true\">→</span></a>\n            </li>\n            \n        </ul>\n    </nav>\n    </div>\n    <div class=\"col-md-4 tags-box\">\n        \n            <h2>Top Ten tags</h2>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 28px\" href=\"/tag/love/\">love</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 26px\" href=\"/tag/inspirational/\">inspirational</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 26px\" href=\"/tag/life/\">life</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 24px\" href=\"/tag/humor/\">humor</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 22px\" href=\"/tag/books/\">books</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 14px\" href=\"/tag/reading/\">reading</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 10px\" href=\"/tag/friendship/\">friendship</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 8px\" href=\"/tag/friends/\">friends</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 8px\" href=\"/tag/truth/\">truth</a>\n            </span>\n            \n            <span class=\"tag-item\">\n            <a class=\"tag\" style=\"font-size: 6px\" href=\"/tag/simile/\">simile</a>\n            </span>\n            \n        \n    </div>\n</div>\n\n    </div>\n    <footer class=\"footer\">\n        <div class=\"container\">\n            <p class=\"text-muted\">\n                Quotes by: <a href=\"https://www.goodreads.com/quotes\">GoodReads.com</a>\n            </p>\n            <p class=\"copyright\">\n                Made with <span class=\"zyte\">❤</span> by <a class=\"zyte\" href=\"https://www.zyte.com\">Zyte</a>\n            </p>\n        </div>\n    </footer>\n\n</body></html>", "echoData": 3}
{"url": "https://books.toscrape.com/", "statusCode": 200, "browserHtml": "<!DOCTYPE html><!--[if lt IE 7]>      <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8 lt-ie7\"> <![endif]--><!--[if IE 7]>         <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8\"> <![endif]--><!--[if IE 8]>         <html lang=\"en-us\" class=\"no-js lt-ie9\"> <![endif]--><!--[if gt IE 8]><!--><html lang=\"en-us\" class=\"no-js\"><!--<![endif]--><head>\n        <title>\n    All products | Books to Scrape - Sandbox\n</title>\n\n        <meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">\n        <meta name=\"created\" content=\"24th Jun 2016 09:29\">\n        <meta name=\"description\" content=\"\">\n        <meta name=\"viewport\" content=\"width=device-width\">\n        <meta name=\"robots\" content=\"NOARCHIVE,NOCACHE\">\n\n        <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->\n        <!--[if lt IE 9]>\n        <script src=\"//html5shim.googlecode.com/svn/trunk/html5.js\"></script>\n        <![endif]-->\n\n        \n            <link rel=\"shortcut icon\" href=\"static/oscar/favicon.ico\">\n        \n\n        \n        \n    \n    \n        <link rel=\"stylesheet\" type=\"text/css\" href=\"static/oscar/css/styles.css\">\n    \n    <link rel=\"stylesheet\" href=\"static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.css\">\n    <link rel=\"stylesheet\" type=\"text/css\" href=\"static/oscar/css/datetimepicker.css\">\n\n\n        \n        \n\n        \n\n        \n            \n            \n\n        \n    </head>\n\n    <body id=\"default\" class=\"default\">\n        \n        \n    \n    \n    <header class=\"header container-fluid\">\n        <div class=\"page_inner\">\n            <div class=\"row\">\n                <div class=\"col-sm-8 h1\"><a href=\"index.html\">Books to Scrape</a><small> We love being scraped!</small>\n</div>\n\n                \n            </div>\n        </div>\n    </header>\n\n    \n    \n<div class=\"container-fluid page\">\n    <div class=\"page_inner\">\n        \n    <ul class=\"breadcrumb\">\n        <li>\n            <a href=\"index.html\">Home</a>\n        </li>\n        <li class=\"active\">All products</li>\n    </ul>\n\n        <div class=\"row\">\n\n            <aside class=\"sidebar col-sm-4 col-md-3\">\n                \n                <div id=\"promotions_left\">\n                    \n                </div>\n                \n    \n    \n        \n        <div class=\"side_categories\">\n            <ul class=\"nav nav-list\">\n                \n                    <li>\n                        <a href=\"catalogue/category/books_1/index.html\">\n                            \n                                Books\n                            \n                        </a>\n\n                        <ul>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/travel_2/index.html\">\n                            \n                                Travel\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/mystery_3/index.html\">\n                            \n                                Mystery\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/historical-fiction_4/index.html\">\n                            \n                                Historical Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/sequential-art_5/index.html\">\n                            \n                                Sequential Art\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/classics_6/index.html\">\n                            \n                                Classics\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/philosophy_7/index.html\">\n                            \n                                Philosophy\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/romance_8/index.html\">\n                            \n                                Romance\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/womens-fiction_9/index.html\">\n                            \n                                Womens Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/fiction_10/index.html\">\n                            \n                                Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/childrens_11/index.html\">\n                            \n                                Childrens\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/religion_12/index.html\">\n                            \n                                Religion\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/nonfiction_13/index.html\">\n                            \n                                Nonfiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/music_14/index.html\">\n                            \n                                Music\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/default_15/index.html\">\n                            \n                                Default\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/science-fiction_16/index.html\">\n                            \n                                Science Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/sports-and-games_17/index.html\">\n                            \n                                Sports and Games\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/add-a-comment_18/index.html\">\n                            \n                                Add a comment\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/fantasy_19/index.html\">\n                            \n                                Fantasy\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/new-adult_20/index.html\">\n                            \n                                New Adult\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/young-adult_21/index.html\">\n                            \n                                Young Adult\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/science_22/index.html\">\n                            \n                                Science\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/poetry_23/index.html\">\n                            \n                                Poetry\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/paranormal_24/index.html\">\n                            \n                                Paranormal\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/art_25/index.html\">\n                            \n                                Art\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/psychology_26/index.html\">\n                            \n                                Psychology\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/autobiography_27/index.html\">\n                            \n                                Autobiography\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/parenting_28/index.html\">\n                            \n                                Parenting\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/adult-fiction_29/index.html\">\n                            \n                                Adult Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/humor_30/index.html\">\n                            \n                                Humor\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/horror_31/index.html\">\n                            \n                                Horror\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/history_32/index.html\">\n                            \n                                History\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/food-and-drink_33/index.html\">\n                            \n                                Food and Drink\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/christian-fiction_34/index.html\">\n                            \n                                Christian Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/business_35/index.html\">\n                            \n                                Business\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/biography_36/index.html\">\n                            \n                                Biography\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/thriller_37/index.html\">\n                            \n                                Thriller\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/contemporary_38/index.html\">\n                            \n                                Contemporary\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/spirituality_39/index.html\">\n                            \n                                Spirituality\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/academic_40/index.html\">\n                            \n                                Academic\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/self-help_41/index.html\">\n                            \n                                Self Help\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/historical_42/index.html\">\n                            \n                                Historical\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/christian_43/index.html\">\n                            \n                                Christian\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/suspense_44/index.html\">\n                            \n                                Suspense\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/short-stories_45/index.html\">\n                            \n                                Short Stories\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/novels_46/index.html\">\n                            \n                                Novels\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/health_47/index.html\">\n                            \n                                Health\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/politics_48/index.html\">\n                            \n                                Politics\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/cultural_49/index.html\">\n                            \n                                Cultural\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/erotica_50/index.html\">\n                            \n                                Erotica\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"catalogue/category/books/crime_51/index.html\">\n                            \n                                Crime\n                            \n                        </a>\n\n                        </li>\n                        \n                            </ul></li>\n                        \n                \n            </ul>\n        </div>\n    \n    \n\n            </aside>\n\n            <div class=\"col-sm-8 col-md-9\">\n                \n                <div class=\"page-header action\">\n                    <h1>All products</h1>\n                </div>\n                \n\n                \n\n\n\n<div id=\"messages\">\n\n</div>\n\n\n                <div id=\"promotions\">\n                    \n                </div>\n\n                \n    <form method=\"get\" class=\"form-horizontal\">\n        \n        <div style=\"display:none\">\n            \n            \n        </div>\n\n        \n            \n                \n                    <strong>1000</strong> results - showing <strong>1</strong> to <strong>20</strong>.\n                \n            \n            \n        \n    </form>\n    \n        <section>\n            <div class=\"alert alert-warning\" role=\"alert\"><strong>Warning!</strong> This is a demo website for web scraping purposes. Prices and ratings here were randomly assigned and have no real meaning.</div>\n\n            <div>\n                <ol class=\"row\">\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/a-light-in-the-attic_1000/index.html\"><img src=\"media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg\" alt=\"A Light in the Attic\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/a-light-in-the-attic_1000/index.html\" title=\"A Light in the Attic\">A Light in the ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£51.77</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/tipping-the-velvet_999/index.html\"><img src=\"media/cache/26/0c/260c6ae16bce31c8f8c95daddd9f4a1c.jpg\" alt=\"Tipping the Velvet\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/tipping-the-velvet_999/index.html\" title=\"Tipping the Velvet\">Tipping the Velvet</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£53.74</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/soumission_998/index.html\"><img src=\"media/cache/3e/ef/3eef99c9d9adef34639f510662022830.jpg\" alt=\"Soumission\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/soumission_998/index.html\" title=\"Soumission\">Soumission</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£50.10</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/sharp-objects_997/index.html\"><img src=\"media/cache/32/51/3251cf3a3412f53f339e42cac2134093.jpg\" alt=\"Sharp Objects\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/sharp-objects_997/index.html\" title=\"Sharp Objects\">Sharp Objects</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£47.82</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/sapiens-a-brief-history-of-humankind_996/index.html\"><img src=\"media/cache/be/a5/bea5697f2534a2f86a3ef27b5a8c12a6.jpg\" alt=\"Sapiens: A Brief History of Humankind\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/sapiens-a-brief-history-of-humankind_996/index.html\" title=\"Sapiens: A Brief History of Humankind\">Sapiens: A Brief History ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£54.23</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/the-requiem-red_995/index.html\"><img src=\"media/cache/68/33/68339b4c9bc034267e1da611ab3b34f8.jpg\" alt=\"The Requiem Red\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/the-requiem-red_995/index.html\" title=\"The Requiem Red\">The Requiem Red</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£22.65</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/the-dirty-little-secrets-of-getting-your-dream-job_994/index.html\"><img src=\"media/cache/92/27/92274a95b7c251fea59a2b8a78275ab4.jpg\" alt=\"The Dirty Little Secrets of Getting Your Dream Job\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/the-dirty-little-secrets-of-getting-your-dream-job_994/index.html\" title=\"The Dirty Little Secrets of Getting Your Dream Job\">The Dirty Little Secrets ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£33.34</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/the-coming-woman-a-novel-based-on-the-life-of-the-infamous-feminist-victoria-woodhull_993/index.html\"><img src=\"media/cache/3d/54/3d54940e57e662c4dd1f3ff00c78cc64.jpg\" alt=\"The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/the-coming-woman-a-novel-based-on-the-life-of-the-infamous-feminist-victoria-woodhull_993/index.html\" title=\"The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull\">The Coming Woman: A ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£17.93</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/the-boys-in-the-boat-nine-americans-and-their-epic-quest-for-gold-at-the-1936-berlin-olympics_992/index.html\"><img src=\"media/cache/66/88/66883b91f6804b2323c8369331cb7dd1.jpg\" alt=\"The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/the-boys-in-the-boat-nine-americans-and-their-epic-quest-for-gold-at-the-1936-berlin-olympics_992/index.html\" title=\"The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics\">The Boys in the ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£22.60</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/the-black-maria_991/index.html\"><img src=\"media/cache/58/46/5846057e28022268153beff6d352b06c.jpg\" alt=\"The Black Maria\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/the-black-maria_991/index.html\" title=\"The Black Maria\">The Black Maria</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£52.15</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/starving-hearts-triangular-trade-trilogy-1_990/index.html\"><img src=\"media/cache/be/f4/bef44da28c98f905a3ebec0b87be8530.jpg\" alt=\"Starving Hearts (Triangular Trade Trilogy, #1)\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/starving-hearts-triangular-trade-trilogy-1_990/index.html\" title=\"Starving Hearts (Triangular Trade Trilogy, #1)\">Starving Hearts (Triangular Trade ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£13.99</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/shakespeares-sonnets_989/index.html\"><img src=\"media/cache/10/48/1048f63d3b5061cd2f424d20b3f9b666.jpg\" alt=\"Shakespeare's Sonnets\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/shakespeares-sonnets_989/index.html\" title=\"Shakespeare's Sonnets\">Shakespeare's Sonnets</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£20.66</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/set-me-free_988/index.html\"><img src=\"media/cache/5b/88/5b88c52633f53cacf162c15f4f823153.jpg\" alt=\"Set Me Free\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/set-me-free_988/index.html\" title=\"Set Me Free\">Set Me Free</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£17.46</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/scott-pilgrims-precious-little-life-scott-pilgrim-1_987/index.html\"><img src=\"media/cache/94/b1/94b1b8b244bce9677c2f29ccc890d4d2.jpg\" alt=\"Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/scott-pilgrims-precious-little-life-scott-pilgrim-1_987/index.html\" title=\"Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)\">Scott Pilgrim's Precious Little ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£52.29</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/rip-it-up-and-start-again_986/index.html\"><img src=\"media/cache/81/c4/81c4a973364e17d01f217e1188253d5e.jpg\" alt=\"Rip it Up and Start Again\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/rip-it-up-and-start-again_986/index.html\" title=\"Rip it Up and Start Again\">Rip it Up and ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£35.02</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/our-band-could-be-your-life-scenes-from-the-american-indie-underground-1981-1991_985/index.html\"><img src=\"media/cache/54/60/54607fe8945897cdcced0044103b10b6.jpg\" alt=\"Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/our-band-could-be-your-life-scenes-from-the-american-indie-underground-1981-1991_985/index.html\" title=\"Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991\">Our Band Could Be ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£57.25</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/olio_984/index.html\"><img src=\"media/cache/55/33/553310a7162dfbc2c6d19a84da0df9e1.jpg\" alt=\"Olio\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/olio_984/index.html\" title=\"Olio\">Olio</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£23.88</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/mesaerion-the-best-science-fiction-stories-1800-1849_983/index.html\"><img src=\"media/cache/09/a3/09a3aef48557576e1a85ba7efea8ecb7.jpg\" alt=\"Mesaerion: The Best Science Fiction Stories 1800-1849\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/mesaerion-the-best-science-fiction-stories-1800-1849_983/index.html\" title=\"Mesaerion: The Best Science Fiction Stories 1800-1849\">Mesaerion: The Best Science ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£37.59</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/libertarianism-for-beginners_982/index.html\"><img src=\"media/cache/0b/bc/0bbcd0a6f4bcd81ccb1049a52736406e.jpg\" alt=\"Libertarianism for Beginners\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/libertarianism-for-beginners_982/index.html\" title=\"Libertarianism for Beginners\">Libertarianism for Beginners</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£51.33</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"catalogue/its-only-the-himalayas_981/index.html\"><img src=\"media/cache/27/a5/27a53d0bb95bdd88288eaf66c9230d7e.jpg\" alt=\"It's Only the Himalayas\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"catalogue/its-only-the-himalayas_981/index.html\" title=\"It's Only the Himalayas\">It's Only the Himalayas</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£45.17</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                </ol>\n                \n\n\n\n    <div>\n        <ul class=\"pager\">\n            \n            <li class=\"current\">\n            \n                Page 1 of 50\n            \n            </li>\n            \n                <li class=\"next\"><a href=\"catalogue/page-2.html\">next</a></li>\n            \n        </ul>\n    </div>\n\n\n            </div>\n        </section>\n    \n\n\n            </div>\n\n        </div><!-- /row -->\n    </div><!-- /page_inner -->\n</div><!-- /container-fluid -->\n\n\n    \n<footer class=\"footer container-fluid\">\n    \n        \n    \n</footer>\n\n\n        \n        \n  \n            <!-- jQuery -->\n            <script src=\"http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js\"></script>\n            <script>window.jQuery || document.write('<script src=\"static/oscar/js/jquery/jquery-1.9.1.min.js\"><\\/script>')</script><script src=\"static/oscar/js/jquery/jquery-1.9.1.min.js\"></script>\n        \n  \n\n\n        \n        \n    \n        \n    <!-- Twitter Bootstrap -->\n    <script type=\"text/javascript\" src=\"static/oscar/js/bootstrap3/bootstrap.min.js\"></script>\n    <!-- Oscar -->\n    <script src=\"static/oscar/js/oscar/ui.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n\n    <script src=\"static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n    <script src=\"static/oscar/js/bootstrap-datetimepicker/locales/bootstrap-datetimepicker.all.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n\n\n        \n        \n    \n\n    \n\n\n        \n        <script type=\"text/javascript\">\n            $(function() {\n                \n    \n    \n    oscar.init();\n\n    oscar.search.init();\n\n            });\n        </script>\n\n        \n        <!-- Version: N/A -->\n        \n    \n\n</body></html>", "echoData": 2}
{"url": "https://toscrape.com/", "statusCode": 200, "browserHtml": "<!DOCTYPE html><html lang=\"en\"><head>\n        <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n        <title>Scraping Sandbox</title>\n        <link href=\"./css/bootstrap.min.css\" rel=\"stylesheet\">\n        <link href=\"./css/main.css\" rel=\"stylesheet\">\n    </head>\n    <body>\n        <div class=\"container\">\n            <div class=\"row\">\n                <div class=\"col-md-1\"></div>\n                <div class=\"col-md-10 well\">\n                    <img class=\"logo\" src=\"img/zyte.png\" width=\"200px\">\n                    <h1 class=\"text-right\">Web Scraping Sandbox</h1>\n                </div>\n            </div>\n\n            <div class=\"row\">\n                <div class=\"col-md-1\"></div>\n                <div class=\"col-md-10\">\n                    <h2>Books</h2>\n                    <p>A <a href=\"http://books.toscrape.com\">fictional bookstore</a> that desperately wants to be scraped. It's a safe place for beginners learning web scraping and for developers validating their scraping technologies as well. Available at: <a href=\"http://books.toscrape.com\">books.toscrape.com</a></p>\n                    <div class=\"col-md-6\">\n                        <a href=\"http://books.toscrape.com\"><img src=\"./img/books.png\" class=\"img-thumbnail\"></a>\n                    </div>\n                    <div class=\"col-md-6\">\n                        <table class=\"table table-hover\">\n                            <tbody><tr><th colspan=\"2\">Details</th></tr>\n                            <tr><td>Amount of items </td><td>1000</td></tr>\n                            <tr><td>Pagination </td><td>✔</td></tr>\n                            <tr><td>Items per page </td><td>max 20</td></tr>\n                            <tr><td>Requires JavaScript </td><td>✘</td></tr>\n                        </tbody></table>\n                    </div>\n                </div>\n            </div>\n\n            <div class=\"row\">\n                <div class=\"col-md-1\"></div>\n                <div class=\"col-md-10\">\n                    <h2>Quotes</h2>\n                    <p><a href=\"http://quotes.toscrape.com/\">A website</a> that lists quotes from famous people. It has many endpoints showing the quotes in many different ways, each of them including new scraping challenges for you, as described below.</p>\n                    <div class=\"col-md-6\">\n                        <a href=\"http://quotes.toscrape.com\"><img src=\"./img/quotes.png\" class=\"img-thumbnail\"></a>\n                    </div>\n                    <div class=\"col-md-6\">\n                        <table class=\"table table-hover\">\n                            <tbody><tr><th colspan=\"2\">Endpoints</th></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/\">Default</a></td><td>Microdata and pagination</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/scroll\">Scroll</a> </td><td>infinite scrolling pagination</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/js\">JavaScript</a> </td><td>JavaScript generated content</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/js-delayed\">Delayed</a> </td><td>Same as JavaScript but with a delay (?delay=10000)</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/tableful\">Tableful</a> </td><td>a table based messed-up layout</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/login\">Login</a> </td><td>login with CSRF token (any user/passwd works)</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/search.aspx\">ViewState</a> </td><td>an AJAX based filter form with ViewStates</td></tr>\n                            <tr><td><a href=\"http://quotes.toscrape.com/random\">Random</a> </td><td>a single random quote</td></tr>\n                        </tbody></table>\n                    </div>\n                </div>\n            </div>\n        </div>\n    \n\n</body></html>", "echoData": 1}

Sending a POST request

Tip

For a more complete example featuring a request body and headers, see the HTML form example.

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

Output:

"POST"

Using network capture to intercept background requests sent during browser rendering

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

C#

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

HttpClientHandler handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var input = new Dictionary<string, object>(){
    {"url", "https://quotes.toscrape.com/scroll"},
    {"browserHtml", true},
    {
        "networkCapture",
        new List<Dictionary<string, object>>()
        {
            new Dictionary<string, object>()
            {
                {"filterType", "url"},
                {"httpResponseBody", true},
                {"value", "/api/"},
                {"matchType", "contains"}
            }
        }
    }
};
var inputJson = JsonSerializer.Serialize(input);
var content = new StringContent(inputJson, Encoding.UTF8, "application/json");

HttpResponseMessage response = await client.PostAsync("https://api.zyte.com/v1/extract", content);
var apiBody = await response.Content.ReadAsByteArrayAsync();

var data = JsonDocument.Parse(apiBody);
var captureEnumerator = data.RootElement.GetProperty("networkCapture").EnumerateArray();
captureEnumerator.MoveNext();
var capture = captureEnumerator.Current;
var base64Body = capture.GetProperty("httpResponseBody").ToString();
var body = System.Convert.FromBase64String(base64Body);

var captureData = JsonDocument.Parse(body);
var quoteEnumerator = captureData.RootElement.GetProperty("quotes").EnumerateArray();
quoteEnumerator.MoveNext();
var quote = quoteEnumerator.Current;
var authorEnumerator = quote.GetProperty("author").EnumerateObject();
while (authorEnumerator.MoveNext())
{
    if (authorEnumerator.Current.Name.ToString() == "name")
    {
        Console.WriteLine(authorEnumerator.Current.Value.ToString());
        break;
    }
}

CLI client

input.jsonl#

{"url": "https://quotes.toscrape.com/scroll", "browserHtml": true, "networkCapture": [{"filterType": "url", "httpResponseBody": true, "value": "/api/", "matchType": "contains"}]}

zyte-api input.jsonl \
    | jq --raw-output ".networkCapture[0].httpResponseBody" \
    | base64 --decode \
    | jq --raw-output ".quotes[0].author.name"

curl

input.json#

{
    "url": "https://quotes.toscrape.com/scroll",
    "browserHtml": true,
    "networkCapture": [
        {
            "filterType": "url",
            "httpResponseBody": true,
            "value": "/api/",
            "matchType": "contains"
        }
    ]
}

curl \
    --user YOUR_API_KEY: \
    --header 'Content-Type: application/json' \
    --data @input.json \
    --compressed \
    https://api.zyte.com/v1/extract \
    | jq --raw-output ".networkCapture[0].httpResponseBody" \
    | base64 --decode \
    | jq --raw-output ".quotes[0].author.name"

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collections;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;

class Example {

  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws InterruptedException, IOException, ParseException {
    Map<String, Object> filter =
        ImmutableMap.of(
            "filterType",
            "url",
            "httpResponseBody",
            true,
            "value",
            "/api/",
            "matchType",
            "contains");
    Map<String, Object> parameters =
        ImmutableMap.of(
            "url",
            "https://quotes.toscrape.com/scroll",
            "browserHtml",
            true,
            "networkCapture",
            Collections.singletonList(filter));
    String requestBody = new Gson().toJson(parameters);

    HttpPost request = new HttpPost("https://api.zyte.com/v1/extract");
    request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
    request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
    request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
    request.setEntity(new StringEntity(requestBody));

    CloseableHttpClient client = HttpClients.createDefault();
    client.execute(
        request,
        response -> {
          HttpEntity entity = response.getEntity();
          String apiResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
          JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
          JsonArray captures = jsonObject.get("networkCapture").getAsJsonArray();
          JsonObject capture = captures.get(0).getAsJsonObject();
          byte[] bodyBytes =
              Base64.getDecoder().decode(capture.get("httpResponseBody").getAsString());
          String body = new String(bodyBytes, StandardCharsets.UTF_8);
          JsonObject data = JsonParser.parseString(body).getAsJsonObject();
          JsonObject quote = data.get("quotes").getAsJsonArray().get(0).getAsJsonObject();
          String authorName = quote.get("author").getAsJsonObject().get("name").getAsString();
          System.out.println(authorName);
          return null;
        });
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const axios = require('axios')

axios.post(
  'https://api.zyte.com/v1/extract',
  {
    url: 'https://quotes.toscrape.com/scroll',
    browserHtml: true,
    networkCapture: [
      {
        filterType: 'url',
        httpResponseBody: true,
        value: '/api/',
        matchType: 'contains'
      }
    ]
  },
  {
    auth: { username: 'YOUR_API_KEY' }
  }
).then((response) => {
  const capture = response.data.networkCapture[0]
  const data = JSON.parse(Buffer.from(capture.httpResponseBody, 'base64'))
  console.log(data.quotes[0].author.name)
})

PHP

<?php

$client = new GuzzleHttp\Client();
$response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
    'auth' => ['YOUR_API_KEY', ''],
    'headers' => ['Accept-Encoding' => 'gzip'],
    'json' => [
        'url' => 'https://quotes.toscrape.com/scroll',
        'browserHtml' => true,
        'networkCapture' => [
            [
                'filterType' => 'url',
                'httpResponseBody' => true,
                'value' => '/api/',
                'matchType' => 'contains',
            ],
        ],
    ],
]);
$api_response = json_decode($response->getBody());
$capture = $api_response->networkCapture[0];
$data = json_decode(base64_decode($capture->httpResponseBody));
echo $data->quotes[0]->author->name.PHP_EOL;

Python

import json
from base64 import b64decode

import requests

api_response = requests.post(
    "https://api.zyte.com/v1/extract",
    auth=("YOUR_API_KEY", ""),
    json={
        "url": "https://quotes.toscrape.com/scroll",
        "browserHtml": True,
        "networkCapture": [
            {
                "filterType": "url",
                "httpResponseBody": True,
                "value": "/api/",
                "matchType": "contains",
            },
        ],
    },
)
capture = api_response.json()["networkCapture"][0]
data = json.loads(b64decode(capture["httpResponseBody"]).decode())
print(data["quotes"][0]["author"]["name"])

Python client

import asyncio
import json
from base64 import b64decode

from zyte_api import AsyncZyteAPI


async def main():
    client = AsyncZyteAPI()
    api_response = await client.get(
        {
            "url": "https://quotes.toscrape.com/scroll",
            "browserHtml": True,
            "networkCapture": [
                {
                    "filterType": "url",
                    "httpResponseBody": True,
                    "value": "/api/",
                    "matchType": "contains",
                },
            ],
        },
    )
    capture = api_response["networkCapture"][0]
    data = json.loads(b64decode(capture["httpResponseBody"]).decode())
    print(data["quotes"][0]["author"]["name"])


asyncio.run(main())

Scrapy

import json
from base64 import b64decode

from scrapy import Request, Spider


class QuotesToScrapeComSpider(Spider):
    name = "quotes_toscrape_com"

    def start_requests(self):
        yield Request(
            "https://quotes.toscrape.com/scroll",
            meta={
                "zyte_api_automap": {
                    "browserHtml": True,
                    "networkCapture": [
                        {
                            "filterType": "url",
                            "httpResponseBody": True,
                            "value": "/api/",
                            "matchType": "contains",
                        },
                    ],
                },
            },
        )

    def parse(self, response):
        capture = response.raw_api_response["networkCapture"][0]
        data = json.loads(b64decode(capture["httpResponseBody"]).decode())
        print(data["quotes"][0]["author"]["name"])

Output:

Albert Einstein

Sending multiple requests in parallel

Note

Install and configure code example requirements and the Zyte CA certificate to run the example below.

C#

using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

var urls = new string[2];
urls[0] = "https://books.toscrape.com/catalogue/page-1.html";
urls[1] = "https://books.toscrape.com/catalogue/page-2.html";
var output = new List<HttpResponseMessage>();

var handler = new HttpClientHandler()
{
    AutomaticDecompression = DecompressionMethods.All,
    MaxConnectionsPerServer = 15
};
var client = new HttpClient(handler);

var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);

client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");

var responseTasks = new List<Task<HttpResponseMessage>>();
foreach (var url in urls)
{
    var input = new Dictionary<string, object>(){
        {"url", url},
        {"browserHtml", true}
    };
    var inputJson = JsonSerializer.Serialize(input);
    var content = new StringContent(inputJson, Encoding.UTF8, "application/json");
    var responseTask = client.PostAsync("https://api.zyte.com/v1/extract", content);
    responseTasks.Add(responseTask);
}

while (responseTasks.Any())
{
    var responseTask = await Task.WhenAny(responseTasks);
    responseTasks.Remove(responseTask);
    var response = await responseTask;
    output.Add(response);
}

CLI client

input.jsonl#

{"url": "https://books.toscrape.com/catalogue/page-1.html", "browserHtml": true}
{"url": "https://books.toscrape.com/catalogue/page-2.html", "browserHtml": true}

zyte-api --n-conn 15 input.jsonl -o output.jsonl

curl

input.jsonl#

{"url": "https://books.toscrape.com/catalogue/page-1.html", "browserHtml": true}
{"url": "https://books.toscrape.com/catalogue/page-2.html", "browserHtml": true}

cat input.jsonl \
    | xargs -P 15 -d\\n -n 1 \
    bash -c "
        curl \
            --user YOUR_API_KEY: \
            --header 'Content-Type: application/json' \
            --data \"\$0\" \
            --compressed \
            https://api.zyte.com/v1/extract \
        | awk '{print \$1}' \
        >> output.jsonl
"

Java

import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.hc.client5.http.async.methods.SimpleHttpRequest;
import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
import org.apache.hc.client5.http.impl.async.HttpAsyncClients;
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
import org.apache.hc.core5.concurrent.FutureCallback;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;

class Example {
  private static final String API_KEY = "YOUR_API_KEY";

  public static void main(final String[] args)
      throws ExecutionException, InterruptedException, IOException, ParseException {

    String[] urls = {
      "https://books.toscrape.com/catalogue/page-1.html",
      "https://books.toscrape.com/catalogue/page-2.html"
    };
    List<Future> futures = new ArrayList<Future>();
    List<String> output = new ArrayList<String>();

    int concurrency = 15;

    // https://issues.apache.org/jira/browse/HTTPCLIENT-2219
    final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create().useSystemProperties().build();

    PoolingAsyncClientConnectionManager connectionManager =
        PoolingAsyncClientConnectionManagerBuilder.create().setTlsStrategy(tlsStrategy).build();
    connectionManager.setMaxTotal(concurrency);
    connectionManager.setDefaultMaxPerRoute(concurrency);

    CloseableHttpAsyncClient client =
        HttpAsyncClients.custom().setConnectionManager(connectionManager).build();
    try {
      client.start();
      for (int i = 0; i < urls.length; i++) {
        Map<String, Object> parameters = ImmutableMap.of("url", urls[i], "browserHtml", true);
        String requestBody = new Gson().toJson(parameters);

        SimpleHttpRequest request =
            new SimpleHttpRequest("POST", "https://api.zyte.com/v1/extract");
        request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
        request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
        request.setBody(requestBody, ContentType.APPLICATION_JSON);

        final Future<SimpleHttpResponse> future =
            client.execute(
                request,
                new FutureCallback<SimpleHttpResponse>() {
                  public void completed(final SimpleHttpResponse response) {
                    String apiResponse = response.getBodyText();
                    JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
                    String browserHtml = jsonObject.get("browserHtml").getAsString();
                    output.add(browserHtml);
                  }

                  public void failed(final Exception ex) {}

                  public void cancelled() {}
                });
        futures.add(future);
      }
      for (int i = 0; i < futures.size(); i++) {
        futures.get(i).get();
      }
    } finally {
      client.close();
    }
  }

  private static String buildAuthHeader() {
    String auth = API_KEY + ":";
    String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
    return "Basic " + encodedAuth;
  }
}

JS

const { ConcurrencyManager } = require('axios-concurrency')
const axios = require('axios')

const urls = [
  'https://books.toscrape.com/catalogue/page-1.html',
  'https://books.toscrape.com/catalogue/page-2.html'
]
const output = []

const client = axios.create()
ConcurrencyManager(client, 15)

Promise.all(
  urls.map((url) =>
    client.post(
      'https://api.zyte.com/v1/extract',
      { url, browserHtml: true },
      {
        auth: { username: 'YOUR_API_KEY' }
      }
    ).then((response) => output.push(response.data))
  )
)

PHP

<?php

$urls = [
  'https://books.toscrape.com/catalogue/page-1.html',
  'https://books.toscrape.com/catalogue/page-2.html',
];
$output = [];
$promises = [];

$client = new GuzzleHttp\Client();

foreach ($urls as $url) {
    $options = [
        'auth' => ['YOUR_API_KEY', ''],
        'headers' => ['Accept-Encoding' => 'gzip'],
        'json' => [
            'url' => $url,
            'browserHtml' => true,
        ],
    ];
    $request = new \GuzzleHttp\Psr7\Request('POST', 'https://api.zyte.com/v1/extract');
    global $promises;
    $promises[] = $client->sendAsync($request, $options)->then(function ($response) {
        global $output;
        $output[] = json_decode($response->getBody());
    });
}

foreach ($promises as $promise) {
    $promise->wait();
}

Python

import asyncio

import aiohttp

urls = [
    "https://books.toscrape.com/catalogue/page-1.html",
    "https://books.toscrape.com/catalogue/page-2.html",
]
output = []


async def extract(client, url):
    response = await client.post(
        "https://api.zyte.com/v1/extract",
        json={"url": url, "browserHtml": True},
        auth=aiohttp.BasicAuth("YOUR_API_KEY"),
    )
    output.append(await response.json())


async def main():
    connector = aiohttp.TCPConnector(limit_per_host=15)
    async with aiohttp.ClientSession(connector=connector) as client:
        await asyncio.gather(*[extract(client, url) for url in urls])


asyncio.run(main())

Python client

import asyncio

from zyte_api import AsyncZyteAPI

urls = [
    "https://books.toscrape.com/catalogue/page-1.html",
    "https://books.toscrape.com/catalogue/page-2.html",
]


async def main():
    client = AsyncZyteAPI(n_conn=15)
    queries = [{"url": url, "browserHtml": True} for url in urls]
    async with client.session() as session:
        for future in session.iter(queries):
            response = await future
            print(response)


asyncio.run(main())

Scrapy

from scrapy import Request, Spider

urls = [
    "https://books.toscrape.com/catalogue/page-1.html",
    "https://books.toscrape.com/catalogue/page-2.html",
]


class ToScrapeSpider(Spider):
    name = "toscrape_com"

    custom_settings = {
        "CONCURRENT_REQUESTS": 15,
        "CONCURRENT_REQUESTS_PER_DOMAIN": 15,
    }

    def start_requests(self):
        for url in urls:
            yield Request(
                url,
                meta={
                    "zyte_api_automap": {
                        "browserHtml": True,
                    },
                },
            )

    def parse(self, response):
        yield {
            "url": response.url,
            "browserHtml": response.text,
        }

Output:

{"url": "https://books.toscrape.com/catalogue/page-1.html", "statusCode": 200, "browserHtml": "<!DOCTYPE html><!--[if lt IE 7]>      <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8 lt-ie7\"> <![endif]--><!--[if IE 7]>         <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8\"> <![endif]--><!--[if IE 8]>         <html lang=\"en-us\" class=\"no-js lt-ie9\"> <![endif]--><!--[if gt IE 8]><!--><html lang=\"en-us\" class=\"no-js\"><!--<![endif]--><head>\n        <title>\n    All products | Books to Scrape - Sandbox\n</title>\n\n        <meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">\n        <meta name=\"created\" content=\"24th Jun 2016 09:30\">\n        <meta name=\"description\" content=\"\">\n        <meta name=\"viewport\" content=\"width=device-width\">\n        <meta name=\"robots\" content=\"NOARCHIVE,NOCACHE\">\n\n        <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->\n        <!--[if lt IE 9]>\n        <script src=\"//html5shim.googlecode.com/svn/trunk/html5.js\"></script>\n        <![endif]-->\n\n        \n            <link rel=\"shortcut icon\" href=\"../static/oscar/favicon.ico\">\n        \n\n        \n        \n    \n    \n        <link rel=\"stylesheet\" type=\"text/css\" href=\"../static/oscar/css/styles.css\">\n    \n    <link rel=\"stylesheet\" href=\"../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.css\">\n    <link rel=\"stylesheet\" type=\"text/css\" href=\"../static/oscar/css/datetimepicker.css\">\n\n\n        \n        \n\n        \n\n        \n            \n            \n\n        \n    </head>\n\n    <body id=\"default\" class=\"default\">\n        \n        \n    \n    \n    <header class=\"header container-fluid\">\n        <div class=\"page_inner\">\n            <div class=\"row\">\n                <div class=\"col-sm-8 h1\"><a href=\"../index.html\">Books to Scrape</a><small> We love being scraped!</small>\n</div>\n\n                \n            </div>\n        </div>\n    </header>\n\n    \n    \n<div class=\"container-fluid page\">\n    <div class=\"page_inner\">\n        \n    <ul class=\"breadcrumb\">\n        <li>\n            <a href=\"../index.html\">Home</a>\n        </li>\n        <li class=\"active\">All products</li>\n    </ul>\n\n        <div class=\"row\">\n\n            <aside class=\"sidebar col-sm-4 col-md-3\">\n                \n                <div id=\"promotions_left\">\n                    \n                </div>\n                \n    \n    \n        \n        <div class=\"side_categories\">\n            <ul class=\"nav nav-list\">\n                \n                    <li>\n                        <a href=\"category/books_1/index.html\">\n                            \n                                Books\n                            \n                        </a>\n\n                        <ul>\n                        \n                \n                    <li>\n                        <a href=\"category/books/travel_2/index.html\">\n                            \n                                Travel\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/mystery_3/index.html\">\n                            \n                                Mystery\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/historical-fiction_4/index.html\">\n                            \n                                Historical Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/sequential-art_5/index.html\">\n                            \n                                Sequential Art\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/classics_6/index.html\">\n                            \n                                Classics\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/philosophy_7/index.html\">\n                            \n                                Philosophy\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/romance_8/index.html\">\n                            \n                                Romance\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/womens-fiction_9/index.html\">\n                            \n                                Womens Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/fiction_10/index.html\">\n                            \n                                Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/childrens_11/index.html\">\n                            \n                                Childrens\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/religion_12/index.html\">\n                            \n                                Religion\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/nonfiction_13/index.html\">\n                            \n                                Nonfiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/music_14/index.html\">\n                            \n                                Music\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/default_15/index.html\">\n                            \n                                Default\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/science-fiction_16/index.html\">\n                            \n                                Science Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/sports-and-games_17/index.html\">\n                            \n                                Sports and Games\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/add-a-comment_18/index.html\">\n                            \n                                Add a comment\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/fantasy_19/index.html\">\n                            \n                                Fantasy\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/new-adult_20/index.html\">\n                            \n                                New Adult\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/young-adult_21/index.html\">\n                            \n                                Young Adult\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/science_22/index.html\">\n                            \n                                Science\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/poetry_23/index.html\">\n                            \n                                Poetry\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/paranormal_24/index.html\">\n                            \n                                Paranormal\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/art_25/index.html\">\n                            \n                                Art\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/psychology_26/index.html\">\n                            \n                                Psychology\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/autobiography_27/index.html\">\n                            \n                                Autobiography\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/parenting_28/index.html\">\n                            \n                                Parenting\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/adult-fiction_29/index.html\">\n                            \n                                Adult Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/humor_30/index.html\">\n                            \n                                Humor\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/horror_31/index.html\">\n                            \n                                Horror\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/history_32/index.html\">\n                            \n                                History\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/food-and-drink_33/index.html\">\n                            \n                                Food and Drink\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/christian-fiction_34/index.html\">\n                            \n                                Christian Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/business_35/index.html\">\n                            \n                                Business\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/biography_36/index.html\">\n                            \n                                Biography\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/thriller_37/index.html\">\n                            \n                                Thriller\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/contemporary_38/index.html\">\n                            \n                                Contemporary\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/spirituality_39/index.html\">\n                            \n                                Spirituality\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/academic_40/index.html\">\n                            \n                                Academic\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/self-help_41/index.html\">\n                            \n                                Self Help\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/historical_42/index.html\">\n                            \n                                Historical\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/christian_43/index.html\">\n                            \n                                Christian\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/suspense_44/index.html\">\n                            \n                                Suspense\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/short-stories_45/index.html\">\n                            \n                                Short Stories\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/novels_46/index.html\">\n                            \n                                Novels\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/health_47/index.html\">\n                            \n                                Health\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/politics_48/index.html\">\n                            \n                                Politics\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/cultural_49/index.html\">\n                            \n                                Cultural\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/erotica_50/index.html\">\n                            \n                                Erotica\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/crime_51/index.html\">\n                            \n                                Crime\n                            \n                        </a>\n\n                        </li>\n                        \n                            </ul></li>\n                        \n                \n            </ul>\n        </div>\n    \n    \n\n            </aside>\n\n            <div class=\"col-sm-8 col-md-9\">\n                \n                <div class=\"page-header action\">\n                    <h1>All products</h1>\n                </div>\n                \n\n                \n\n\n\n<div id=\"messages\">\n\n</div>\n\n\n                <div id=\"promotions\">\n                    \n                </div>\n\n                \n    <form method=\"get\" class=\"form-horizontal\">\n        \n        <div style=\"display:none\">\n            \n            \n        </div>\n\n        \n            \n                \n                    <strong>1000</strong> results - showing <strong>1</strong> to <strong>20</strong>.\n                \n            \n            \n        \n    </form>\n    \n        <section>\n            <div class=\"alert alert-warning\" role=\"alert\"><strong>Warning!</strong> This is a demo website for web scraping purposes. Prices and ratings here were randomly assigned and have no real meaning.</div>\n\n            <div>\n                <ol class=\"row\">\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"a-light-in-the-attic_1000/index.html\"><img src=\"../media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg\" alt=\"A Light in the Attic\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"a-light-in-the-attic_1000/index.html\" title=\"A Light in the Attic\">A Light in the ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£51.77</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"tipping-the-velvet_999/index.html\"><img src=\"../media/cache/26/0c/260c6ae16bce31c8f8c95daddd9f4a1c.jpg\" alt=\"Tipping the Velvet\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"tipping-the-velvet_999/index.html\" title=\"Tipping the Velvet\">Tipping the Velvet</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£53.74</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"soumission_998/index.html\"><img src=\"../media/cache/3e/ef/3eef99c9d9adef34639f510662022830.jpg\" alt=\"Soumission\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"soumission_998/index.html\" title=\"Soumission\">Soumission</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£50.10</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"sharp-objects_997/index.html\"><img src=\"../media/cache/32/51/3251cf3a3412f53f339e42cac2134093.jpg\" alt=\"Sharp Objects\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"sharp-objects_997/index.html\" title=\"Sharp Objects\">Sharp Objects</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£47.82</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"sapiens-a-brief-history-of-humankind_996/index.html\"><img src=\"../media/cache/be/a5/bea5697f2534a2f86a3ef27b5a8c12a6.jpg\" alt=\"Sapiens: A Brief History of Humankind\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"sapiens-a-brief-history-of-humankind_996/index.html\" title=\"Sapiens: A Brief History of Humankind\">Sapiens: A Brief History ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£54.23</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"the-requiem-red_995/index.html\"><img src=\"../media/cache/68/33/68339b4c9bc034267e1da611ab3b34f8.jpg\" alt=\"The Requiem Red\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"the-requiem-red_995/index.html\" title=\"The Requiem Red\">The Requiem Red</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£22.65</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"the-dirty-little-secrets-of-getting-your-dream-job_994/index.html\"><img src=\"../media/cache/92/27/92274a95b7c251fea59a2b8a78275ab4.jpg\" alt=\"The Dirty Little Secrets of Getting Your Dream Job\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"the-dirty-little-secrets-of-getting-your-dream-job_994/index.html\" title=\"The Dirty Little Secrets of Getting Your Dream Job\">The Dirty Little Secrets ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£33.34</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"the-coming-woman-a-novel-based-on-the-life-of-the-infamous-feminist-victoria-woodhull_993/index.html\"><img src=\"../media/cache/3d/54/3d54940e57e662c4dd1f3ff00c78cc64.jpg\" alt=\"The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"the-coming-woman-a-novel-based-on-the-life-of-the-infamous-feminist-victoria-woodhull_993/index.html\" title=\"The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull\">The Coming Woman: A ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£17.93</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"the-boys-in-the-boat-nine-americans-and-their-epic-quest-for-gold-at-the-1936-berlin-olympics_992/index.html\"><img src=\"../media/cache/66/88/66883b91f6804b2323c8369331cb7dd1.jpg\" alt=\"The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"the-boys-in-the-boat-nine-americans-and-their-epic-quest-for-gold-at-the-1936-berlin-olympics_992/index.html\" title=\"The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics\">The Boys in the ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£22.60</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"the-black-maria_991/index.html\"><img src=\"../media/cache/58/46/5846057e28022268153beff6d352b06c.jpg\" alt=\"The Black Maria\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"the-black-maria_991/index.html\" title=\"The Black Maria\">The Black Maria</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£52.15</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"starving-hearts-triangular-trade-trilogy-1_990/index.html\"><img src=\"../media/cache/be/f4/bef44da28c98f905a3ebec0b87be8530.jpg\" alt=\"Starving Hearts (Triangular Trade Trilogy, #1)\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"starving-hearts-triangular-trade-trilogy-1_990/index.html\" title=\"Starving Hearts (Triangular Trade Trilogy, #1)\">Starving Hearts (Triangular Trade ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£13.99</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"shakespeares-sonnets_989/index.html\"><img src=\"../media/cache/10/48/1048f63d3b5061cd2f424d20b3f9b666.jpg\" alt=\"Shakespeare's Sonnets\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"shakespeares-sonnets_989/index.html\" title=\"Shakespeare's Sonnets\">Shakespeare's Sonnets</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£20.66</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"set-me-free_988/index.html\"><img src=\"../media/cache/5b/88/5b88c52633f53cacf162c15f4f823153.jpg\" alt=\"Set Me Free\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"set-me-free_988/index.html\" title=\"Set Me Free\">Set Me Free</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£17.46</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"scott-pilgrims-precious-little-life-scott-pilgrim-1_987/index.html\"><img src=\"../media/cache/94/b1/94b1b8b244bce9677c2f29ccc890d4d2.jpg\" alt=\"Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"scott-pilgrims-precious-little-life-scott-pilgrim-1_987/index.html\" title=\"Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)\">Scott Pilgrim's Precious Little ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£52.29</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"rip-it-up-and-start-again_986/index.html\"><img src=\"../media/cache/81/c4/81c4a973364e17d01f217e1188253d5e.jpg\" alt=\"Rip it Up and Start Again\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"rip-it-up-and-start-again_986/index.html\" title=\"Rip it Up and Start Again\">Rip it Up and ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£35.02</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"our-band-could-be-your-life-scenes-from-the-american-indie-underground-1981-1991_985/index.html\"><img src=\"../media/cache/54/60/54607fe8945897cdcced0044103b10b6.jpg\" alt=\"Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"our-band-could-be-your-life-scenes-from-the-american-indie-underground-1981-1991_985/index.html\" title=\"Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991\">Our Band Could Be ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£57.25</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"olio_984/index.html\"><img src=\"../media/cache/55/33/553310a7162dfbc2c6d19a84da0df9e1.jpg\" alt=\"Olio\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"olio_984/index.html\" title=\"Olio\">Olio</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£23.88</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"mesaerion-the-best-science-fiction-stories-1800-1849_983/index.html\"><img src=\"../media/cache/09/a3/09a3aef48557576e1a85ba7efea8ecb7.jpg\" alt=\"Mesaerion: The Best Science Fiction Stories 1800-1849\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"mesaerion-the-best-science-fiction-stories-1800-1849_983/index.html\" title=\"Mesaerion: The Best Science Fiction Stories 1800-1849\">Mesaerion: The Best Science ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£37.59</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"libertarianism-for-beginners_982/index.html\"><img src=\"../media/cache/0b/bc/0bbcd0a6f4bcd81ccb1049a52736406e.jpg\" alt=\"Libertarianism for Beginners\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"libertarianism-for-beginners_982/index.html\" title=\"Libertarianism for Beginners\">Libertarianism for Beginners</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£51.33</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"its-only-the-himalayas_981/index.html\"><img src=\"../media/cache/27/a5/27a53d0bb95bdd88288eaf66c9230d7e.jpg\" alt=\"It's Only the Himalayas\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"its-only-the-himalayas_981/index.html\" title=\"It's Only the Himalayas\">It's Only the Himalayas</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£45.17</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                </ol>\n                \n\n\n\n    <div>\n        <ul class=\"pager\">\n            \n            <li class=\"current\">\n            \n                Page 1 of 50\n            \n            </li>\n            \n                <li class=\"next\"><a href=\"page-2.html\">next</a></li>\n            \n        </ul>\n    </div>\n\n\n            </div>\n        </section>\n    \n\n\n            </div>\n\n        </div><!-- /row -->\n    </div><!-- /page_inner -->\n</div><!-- /container-fluid -->\n\n\n    \n<footer class=\"footer container-fluid\">\n    \n        \n    \n</footer>\n\n\n        \n        \n  \n            <!-- jQuery -->\n            <script src=\"http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js\"></script>\n            <script>window.jQuery || document.write('<script src=\"../static/oscar/js/jquery/jquery-1.9.1.min.js\"><\\/script>')</script><script src=\"../static/oscar/js/jquery/jquery-1.9.1.min.js\"></script>\n        \n  \n\n\n        \n        \n    \n        \n    <!-- Twitter Bootstrap -->\n    <script type=\"text/javascript\" src=\"../static/oscar/js/bootstrap3/bootstrap.min.js\"></script>\n    <!-- Oscar -->\n    <script src=\"../static/oscar/js/oscar/ui.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n\n    <script src=\"../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n    <script src=\"../static/oscar/js/bootstrap-datetimepicker/locales/bootstrap-datetimepicker.all.js\" type=\"text/javascript\" charset=\"utf-8\"></script>\n\n\n        \n        \n    \n\n    \n\n\n        \n        <script type=\"text/javascript\">\n            $(function() {\n                \n    \n    \n    oscar.init();\n\n    oscar.search.init();\n\n            });\n        </script>\n\n        \n        <!-- Version: N/A -->\n        \n    \n\n</body></html>", "echoData": "https://books.toscrape.com/catalogue/page-1.html"}
{"url": "https://books.toscrape.com/catalogue/page-2.html", "statusCode": 200, "browserHtml": "<!DOCTYPE html><!--[if lt IE 7]>      <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8 lt-ie7\"> <![endif]--><!--[if IE 7]>         <html lang=\"en-us\" class=\"no-js lt-ie9 lt-ie8\"> <![endif]--><!--[if IE 8]>         <html lang=\"en-us\" class=\"no-js lt-ie9\"> <![endif]--><!--[if gt IE 8]><!--><html lang=\"en-us\" class=\"no-js\"><!--<![endif]--><head>\n        <title>\n    All products | Books to Scrape - Sandbox\n</title>\n\n        <meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">\n        <meta name=\"created\" content=\"24th Jun 2016 09:29\">\n        <meta name=\"description\" content=\"\">\n        <meta name=\"viewport\" content=\"width=device-width\">\n        <meta name=\"robots\" content=\"NOARCHIVE,NOCACHE\">\n\n        <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->\n        <!--[if lt IE 9]>\n        <script src=\"//html5shim.googlecode.com/svn/trunk/html5.js\"></script>\n        <![endif]-->\n\n        \n            <link rel=\"shortcut icon\" href=\"../static/oscar/favicon.ico\">\n        \n\n        \n        \n    \n    \n        <link rel=\"stylesheet\" type=\"text/css\" href=\"../static/oscar/css/styles.css\">\n    \n    <link rel=\"stylesheet\" href=\"../static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.css\">\n    <link rel=\"stylesheet\" type=\"text/css\" href=\"../static/oscar/css/datetimepicker.css\">\n\n\n        \n        \n\n        \n\n        \n            \n            \n\n        \n    </head>\n\n    <body id=\"default\" class=\"default\">\n        \n        \n    \n    \n    <header class=\"header container-fluid\">\n        <div class=\"page_inner\">\n            <div class=\"row\">\n                <div class=\"col-sm-8 h1\"><a href=\"../index.html\">Books to Scrape</a><small> We love being scraped!</small>\n</div>\n\n                \n            </div>\n        </div>\n    </header>\n\n    \n    \n<div class=\"container-fluid page\">\n    <div class=\"page_inner\">\n        \n    <ul class=\"breadcrumb\">\n        <li>\n            <a href=\"../index.html\">Home</a>\n        </li>\n        <li class=\"active\">All products</li>\n    </ul>\n\n        <div class=\"row\">\n\n            <aside class=\"sidebar col-sm-4 col-md-3\">\n                \n                <div id=\"promotions_left\">\n                    \n                </div>\n                \n    \n    \n        \n        <div class=\"side_categories\">\n            <ul class=\"nav nav-list\">\n                \n                    <li>\n                        <a href=\"category/books_1/index.html\">\n                            \n                                Books\n                            \n                        </a>\n\n                        <ul>\n                        \n                \n                    <li>\n                        <a href=\"category/books/travel_2/index.html\">\n                            \n                                Travel\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/mystery_3/index.html\">\n                            \n                                Mystery\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/historical-fiction_4/index.html\">\n                            \n                                Historical Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/sequential-art_5/index.html\">\n                            \n                                Sequential Art\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/classics_6/index.html\">\n                            \n                                Classics\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/philosophy_7/index.html\">\n                            \n                                Philosophy\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/romance_8/index.html\">\n                            \n                                Romance\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/womens-fiction_9/index.html\">\n                            \n                                Womens Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/fiction_10/index.html\">\n                            \n                                Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/childrens_11/index.html\">\n                            \n                                Childrens\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/religion_12/index.html\">\n                            \n                                Religion\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/nonfiction_13/index.html\">\n                            \n                                Nonfiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/music_14/index.html\">\n                            \n                                Music\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/default_15/index.html\">\n                            \n                                Default\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/science-fiction_16/index.html\">\n                            \n                                Science Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/sports-and-games_17/index.html\">\n                            \n                                Sports and Games\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/add-a-comment_18/index.html\">\n                            \n                                Add a comment\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/fantasy_19/index.html\">\n                            \n                                Fantasy\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/new-adult_20/index.html\">\n                            \n                                New Adult\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/young-adult_21/index.html\">\n                            \n                                Young Adult\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/science_22/index.html\">\n                            \n                                Science\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/poetry_23/index.html\">\n                            \n                                Poetry\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/paranormal_24/index.html\">\n                            \n                                Paranormal\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/art_25/index.html\">\n                            \n                                Art\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/psychology_26/index.html\">\n                            \n                                Psychology\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/autobiography_27/index.html\">\n                            \n                                Autobiography\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/parenting_28/index.html\">\n                            \n                                Parenting\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/adult-fiction_29/index.html\">\n                            \n                                Adult Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/humor_30/index.html\">\n                            \n                                Humor\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/horror_31/index.html\">\n                            \n                                Horror\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/history_32/index.html\">\n                            \n                                History\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/food-and-drink_33/index.html\">\n                            \n                                Food and Drink\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/christian-fiction_34/index.html\">\n                            \n                                Christian Fiction\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/business_35/index.html\">\n                            \n                                Business\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/biography_36/index.html\">\n                            \n                                Biography\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/thriller_37/index.html\">\n                            \n                                Thriller\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/contemporary_38/index.html\">\n                            \n                                Contemporary\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/spirituality_39/index.html\">\n                            \n                                Spirituality\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/academic_40/index.html\">\n                            \n                                Academic\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/self-help_41/index.html\">\n                            \n                                Self Help\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/historical_42/index.html\">\n                            \n                                Historical\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/christian_43/index.html\">\n                            \n                                Christian\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/suspense_44/index.html\">\n                            \n                                Suspense\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/short-stories_45/index.html\">\n                            \n                                Short Stories\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/novels_46/index.html\">\n                            \n                                Novels\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/health_47/index.html\">\n                            \n                                Health\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/politics_48/index.html\">\n                            \n                                Politics\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/cultural_49/index.html\">\n                            \n                                Cultural\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/erotica_50/index.html\">\n                            \n                                Erotica\n                            \n                        </a>\n\n                        </li>\n                        \n                \n                    <li>\n                        <a href=\"category/books/crime_51/index.html\">\n                            \n                                Crime\n                            \n                        </a>\n\n                        </li>\n                        \n                            </ul></li>\n                        \n                \n            </ul>\n        </div>\n    \n    \n\n            </aside>\n\n            <div class=\"col-sm-8 col-md-9\">\n                \n                <div class=\"page-header action\">\n                    <h1>All products</h1>\n                </div>\n                \n\n                \n\n\n\n<div id=\"messages\">\n\n</div>\n\n\n                <div id=\"promotions\">\n                    \n                </div>\n\n                \n    <form method=\"get\" class=\"form-horizontal\">\n        \n        <div style=\"display:none\">\n            \n            \n        </div>\n\n        \n            \n                \n                    <strong>1000</strong> results - showing <strong>21</strong> to <strong>40</strong>.\n                \n            \n            \n        \n    </form>\n    \n        <section>\n            <div class=\"alert alert-warning\" role=\"alert\"><strong>Warning!</strong> This is a demo website for web scraping purposes. Prices and ratings here were randomly assigned and have no real meaning.</div>\n\n            <div>\n                <ol class=\"row\">\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"in-her-wake_980/index.html\"><img src=\"../media/cache/5d/72/5d72709c6a7a9584a4d1cf07648bfce1.jpg\" alt=\"In Her Wake\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating One\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"in-her-wake_980/index.html\" title=\"In Her Wake\">In Her Wake</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£12.84</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"how-music-works_979/index.html\"><img src=\"../media/cache/5c/c8/5cc8e107246cb478960d4f0aba1e1c8e.jpg\" alt=\"How Music Works\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Two\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"how-music-works_979/index.html\" title=\"How Music Works\">How Music Works</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£37.32</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"foolproof-preserving-a-guide-to-small-batch-jams-jellies-pickles-condiments-and-more-a-foolproof-guide-to-making-small-batch-jams-jellies-pickles-condiments-and-more_978/index.html\"><img src=\"../media/cache/9f/59/9f59f01fa916a7bb8f0b28a4012179a4.jpg\" alt=\"Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"foolproof-preserving-a-guide-to-small-batch-jams-jellies-pickles-condiments-and-more-a-foolproof-guide-to-making-small-batch-jams-jellies-pickles-condiments-and-more_978/index.html\" title=\"Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More\">Foolproof Preserving: A Guide ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£30.52</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"chase-me-paris-nights-2_977/index.html\"><img src=\"../media/cache/9c/2e/9c2e0eb8866b8e3f3b768994fd3d1c1a.jpg\" alt=\"Chase Me (Paris Nights #2)\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"chase-me-paris-nights-2_977/index.html\" title=\"Chase Me (Paris Nights #2)\">Chase Me (Paris Nights ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£25.27</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"black-dust_976/index.html\"><img src=\"../media/cache/44/cc/44ccc99c8f82c33d4f9d2afa4ef25787.jpg\" alt=\"Black Dust\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"black-dust_976/index.html\" title=\"Black Dust\">Black Dust</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£34.53</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"birdsong-a-story-in-pictures_975/index.html\"><img src=\"../media/cache/af/6e/af6e796160fe63e0cf19d44395c7ddf2.jpg\" alt=\"Birdsong: A Story in Pictures\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"birdsong-a-story-in-pictures_975/index.html\" title=\"Birdsong: A Story in Pictures\">Birdsong: A Story in ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£54.64</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"americas-cradle-of-quarterbacks-western-pennsylvanias-football-factory-from-johnny-unitas-to-joe-montana_974/index.html\"><img src=\"../media/cache/ef/0b/ef0bed08de4e083dba5e20fdb98d9c36.jpg\" alt=\"America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"americas-cradle-of-quarterbacks-western-pennsylvanias-football-factory-from-johnny-unitas-to-joe-montana_974/index.html\" title=\"America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana\">America's Cradle of Quarterbacks: ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£22.50</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"aladdin-and-his-wonderful-lamp_973/index.html\"><img src=\"../media/cache/d6/da/d6da0371958068bbaf39ea9c174275cd.jpg\" alt=\"Aladdin and His Wonderful Lamp\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Three\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"aladdin-and-his-wonderful-lamp_973/index.html\" title=\"Aladdin and His Wonderful Lamp\">Aladdin and His Wonderful ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£53.13</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"worlds-elsewhere-journeys-around-shakespeares-globe_972/index.html\"><img src=\"../media/cache/2e/98/2e98c332bf8563b584784971541c4445.jpg\" alt=\"Worlds Elsewhere: Journeys Around Shakespeare’s Globe\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"worlds-elsewhere-journeys-around-shakespeares-globe_972/index.html\" title=\"Worlds Elsewhere: Journeys Around Shakespeare’s Globe\">Worlds Elsewhere: Journeys Around ...</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£40.30</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"wall-and-piece_971/index.html\"><img src=\"../media/cache/a5/41/a5416b9646aaa7287baa287ec2590270.jpg\" alt=\"Wall and Piece\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Four\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                </p>\n            \n        \n\n        \n            <h3><a href=\"wall-and-piece_971/index.html\" title=\"Wall and Piece\">Wall and Piece</a></h3>\n        \n\n        \n            <div class=\"product_price\">\n                \n\n\n\n\n\n\n    \n        <p class=\"price_color\">£44.18</p>\n    \n\n<p class=\"instock availability\">\n    <i class=\"icon-ok\"></i>\n    \n        In stock\n    \n</p>\n\n                \n                    \n\n\n\n\n\n\n    \n    <form>\n        <button type=\"submit\" class=\"btn btn-primary btn-block\" data-loading-text=\"Adding...\">Add to basket</button>\n    </form>\n\n\n                \n            </div>\n        \n    </article>\n\n</li>\n                    \n                        <li class=\"col-xs-6 col-sm-4 col-md-3 col-lg-3\">\n\n\n\n\n\n\n    <article class=\"product_pod\">\n        \n            <div class=\"image_container\">\n                \n                    \n                    <a href=\"the-four-agreements-a-practical-guide-to-personal-freedom_970/index.html\"><img src=\"../media/cache/0f/7e/0f7ee69495c0df1d35723f012624a9f8.jpg\" alt=\"The Four Agreements: A Practical Guide to Personal Freedom\" class=\"thumbnail\"></a>\n                    \n                \n            </div>\n        \n\n        \n            \n                <p class=\"star-rating Five\">\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n                    <i class=\"icon-star\"></i>\n  

Code examples#

Requirements#

All examples#