Zyte API browser actions#
When extracting browser HTML or a
screenshot, you can use the actions
key in
your API request body to define a sequence of actions to perform during browser
rendering, and hence modify the DOM before the requested output is generated
for you.
Note
Install and configure code example requirements to run the example below.
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using HtmlAgilityPack;
HttpClientHandler handler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.All
};
HttpClient client = new HttpClient(handler);
var apiKey = "YOUR_API_KEY";
var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(apiKey + ":");
var auth = System.Convert.ToBase64String(bytes);
client.DefaultRequestHeaders.Add("Authorization", "Basic " + auth);
client.DefaultRequestHeaders.Add("Accept-Encoding", "br, gzip, deflate");
var input = new Dictionary<string, object>(){
{"url", "https://quotes.toscrape.com/scroll"},
{"browserHtml", true},
{
"actions",
new List<Dictionary<string, object>>()
{
new Dictionary<string, object>()
{
{"action", "scrollBottom"}
}
}
}
};
var inputJson = JsonSerializer.Serialize(input);
var content = new StringContent(inputJson, Encoding.UTF8, "application/json");
HttpResponseMessage response = await client.PostAsync("https://api.zyte.com/v1/extract", content);
var body = await response.Content.ReadAsByteArrayAsync();
var data = JsonDocument.Parse(body);
var browserHtml = data.RootElement.GetProperty("browserHtml").ToString();
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(browserHtml);
var navigator = htmlDocument.CreateNavigator();
var quoteCount = (double)navigator.Evaluate("count(//*[@class=\"quote\"])");
{
"url": "https://quotes.toscrape.com/scroll",
"browserHtml": true,
"actions": [
{
"action": "scrollBottom"
}
]
}
curl \
--user YOUR_API_KEY: \
--header 'Content-Type: application/json' \
--data @input.json \
--compressed \
https://api.zyte.com/v1/extract \
| jq --raw-output .browserHtml \
| xmllint --html --xpath 'count(//*[@class="quote"])' - 2> /dev/null
{"url":"https://quotes.toscrape.com/scroll","browserHtml":true,"actions":[{"action":"scrollBottom"}]}
zyte-api input.jsonl 2> /dev/null \
| jq --raw-output .browserHtml \
| xmllint --html --xpath 'count(//*[@class="quote"])' - 2> /dev/null
import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collections;
import java.util.Map;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
class Example {
private static final String API_KEY = "YOUR_API_KEY";
public static void main(final String[] args)
throws InterruptedException, IOException, ParseException {
Map<String, Object> action = ImmutableMap.of("action", "scrollBottom");
Map<String, Object> parameters =
ImmutableMap.of(
"url",
"https://quotes.toscrape.com/scroll",
"browserHtml",
true,
"actions",
Collections.singletonList(action));
String requestBody = new Gson().toJson(parameters);
HttpPost request = new HttpPost("https://api.zyte.com/v1/extract");
request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON);
request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate");
request.setHeader(HttpHeaders.AUTHORIZATION, buildAuthHeader());
request.setEntity(new StringEntity(requestBody));
try (CloseableHttpClient client = HttpClients.createDefault()) {
try (CloseableHttpResponse response = client.execute(request)) {
HttpEntity entity = response.getEntity();
String apiResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
JsonObject jsonObject = JsonParser.parseString(apiResponse).getAsJsonObject();
String browserHtml = jsonObject.get("browserHtml").getAsString();
Document document = Jsoup.parse(browserHtml);
int quoteCount = document.select(".quote").size();
}
}
}
private static String buildAuthHeader() {
String auth = API_KEY + ":";
String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes());
return "Basic " + encodedAuth;
}
}
const axios = require('axios')
const cheerio = require('cheerio')
axios.post(
'https://api.zyte.com/v1/extract',
{
url: 'https://quotes.toscrape.com/scroll',
browserHtml: true,
actions: [
{
action: 'scrollBottom'
}
]
},
{
auth: { username: 'YOUR_API_KEY' },
headers: { 'Accept-Encoding': 'gzip, deflate' }
}
).then((response) => {
const browserHtml = response.data.browserHtml
const $ = cheerio.load(browserHtml)
const quoteCount = $('.quote').length
})
<?php
$client = new GuzzleHttp\Client();
$response = $client->request('POST', 'https://api.zyte.com/v1/extract', [
'auth' => ['YOUR_API_KEY', ''],
'headers' => ['Accept-Encoding' => 'gzip'],
'json' => [
'url' => 'https://quotes.toscrape.com/scroll',
'browserHtml' => true,
'actions' => [
['action' => 'scrollBottom'],
],
],
]);
$data = json_decode($response->getBody());
$doc = new DOMDocument();
$doc->loadHTML($data->browserHtml);
$xpath = new DOMXPath($doc);
$quote_count = $xpath->query("//*[@class='quote']")->count();
import json
import requests
from parsel import Selector
api_response = requests.post(
'https://api.zyte.com/v1/extract',
auth=('YOUR_API_KEY', ''),
json={
'url': 'https://quotes.toscrape.com/scroll',
'browserHtml': True,
'actions': [
{
'action': 'scrollBottom',
},
],
},
)
browser_html = api_response.json()['browserHtml']
quote_count = len(Selector(browser_html).css('.quote'))
import asyncio
import json
from parsel import Selector
from zyte_api.aio.client import AsyncClient
async def main():
client = AsyncClient()
api_response = await client.request_raw(
{
'url': 'https://quotes.toscrape.com/scroll',
'browserHtml': True,
'actions': [
{
'action': 'scrollBottom',
},
],
},
)
browser_html = api_response['browserHtml']
quote_count = len(Selector(browser_html).css('.quote'))
asyncio.run(main())
from scrapy import Request, Spider
class QuotesToScrapeComSpider(Spider):
name = "quotes_toscrape_com"
def start_requests(self):
yield Request(
"https://quotes.toscrape.com/scroll",
meta={
"zyte_api_automap": {
"browserHtml": True,
"actions": [
{
"action": "scrollBottom",
},
],
},
},
)
def parse(self, response):
quote_count = len(response.css(".quote"))
Look up actions
in the specification for the
complete actions API.
Types#
Zyte API supports 3 types of browser actions:
Generic actions work on every website. They allow you to type text into input fields, perform cursor actions (click, hover, scroll), and wait for certain events or for a given time.
Special actions expose functionality that requires specific knowledge of the target website, such as using their search box or filling a form.
They are only available for certain websites. To find out if an action is available for a given website, send a test request using that action. If the action is not supported, you will get an error API response indicating so.
Custom browser scripts, available for Enterprise plans.
Limits#
You are free to use as many actions as you wish, but total browser execution time is limited to 60 seconds. If your actions are still running by that time, the on-going action is interrupted, follow-up actions are not executed at all, and you get your requested output (browser HTML, screenshot) as it was rendered at that time.
The Zyte API response includes an action
key that provides details about
action execution, including elapsedTime
, error
, and status
fields
to help you debug your actions, e.g. to find out which actions were executed
successfully and which actions were not.
Selectors#
Browser actions that interact with a webpage element all have a selector
key that allows you to define how to find the target webpage element.
You must define a query to find the target webpage element in the
selector.value
field.
You must specify the language of your query in the selector.type
field,
which supports the following values: CSS Selector (css
), XPath 1.0
(xpath
). For information about these query languages, see Learning CSS and
XPath.
Finally, the selector.state
field lets you filter elements by visibility:
visible
(default) filters out invisible elements, hidden
filters out
visible elements, and attached
does not filter out any element based on
visibility.
Waiting#
You can use the following browser actions to introduce wait times in your
browser action sequences or in your custom browser scripts: waitForSelector
, waitForRequest
,
waitForResponse
, and waitForTimeout
.
Whenever you need to wait for something to happen on a webpage, your should
consider using waitForSelector
first. It waits for a given selector to match some element.
Tip
If waitForSelector
does not seem to work as expected for you,
please review your selector. Common issues
include trying to wait for an invisible element without setting
selector.state
to attached
or hidden
.
waitForRequest
and waitForResponse
wait for a request to be sent or for
a response to be received, filtering by URL pattern.
waitForTimeout
pauses your sequence of actions or your custom browser
script for the specified amount of time. Because action run time is
limited, you should avoid using this type of action
when an alternative waiting action can replace it. However, this action can be
necessary for certain scenarios, such as simulating human reaction time.