Using Smart Proxy Manager with PhantomJS, CasperJS, SpookyJS¶
Warning
zyte-smartproxy-ca.crt
should be installed in your OS for the below code to work. You can follow these instructions in order to install it.
Note
All the code in this documentation has been tested with Node.js v14.15.1, PhantomJS 2.1.1, CasperJS 1.1.4 and SpookyJS 0.2.5.
Installation¶
Setup the Zyte SmartProxy (formerly Crawlera) Headless Proxy as described in Using Headless Browsers with Zyte Smart Proxy Manager. It’ll be used in all of the below mentioned code examples.
PhantomJS¶
Install PhantomJS.
Run the following code using
phantomjs --ignore-ssl-errors=true sample.js
.
// sample.js
phantom.setProxy('localhost', '3128', 'http');
var page = require('webpage').create();
page.open('https://toscrape.com', function (status) {
console.log(page.content);
phantom.exit();
});
CasperJS¶
Set environment variable
ENGINE_EXECUTABLE
with thephantomjs
executable path.Run the following code using
casperjs --ignore-ssl-errors=true sample.js
.
// sample.js
phantom.setProxy('localhost', '3128', 'http');
var casper = require('casper').create();
casper.start('https://toscrape.com', function() {
this.echo(this.getHTML());
});
casper.run();
SpookyJS¶
Install SpookyJS using
npm install spooky
.Install SpookyJS dependencies using
npm install --prefix node_modules/spooky
.Set environment variable and value
ENGINE_FLAGS=--ignore-ssl-errors=true
.Run the sample code using
node sample.js
.
// sample.js
var Spooky = require('spooky');
var spooky = new Spooky({
child: {
proxy: 'localhost:3128',
transport: 'http',
},
casper: {
logLevel: 'debug',
verbose: true
}
}, function (err) {
if (err) {
e = new Error('Failed to initialize SpookyJS');
e.details = err;
throw e;
}
spooky.start('https://toscrape.com');
spooky.then(function () {
this.emit('response', this.evaluate(function () {
return document.documentElement.innerHTML;
}));
});
spooky.run();
});
spooky.on('error', function (e, stack) {
console.error(e);
if (stack) {
console.log(stack);
}
});
spooky.on('response', function (response) {
console.log(response);
});