Using Smart Proxy Manager with PhantomJS, CasperJS, SpookyJS

Warning

zyte-smartproxy-ca.crt should be installed in your OS for the below code to work. You can follow these instructions in order to install it.

Note

All the code in this documentation has been tested with Node.js v14.15.1, PhantomJS 2.1.1, CasperJS 1.1.4 and SpookyJS 0.2.5.

Installation

Setup the Zyte SmartProxy (formerly Crawlera) Headless Proxy as described in Using Headless Browsers with Zyte Smart Proxy Manager. It’ll be used in all of the below mentioned code examples.

PhantomJS

1. Install PhantomJS.

2. Run the following code using phantomjs --ignore-ssl-errors=true sample.js.

// sample.js
phantom.setProxy('localhost', '3128', 'http');
var page = require('webpage').create();

page.open('https://toscrape.com', function (status) {
   console.log(page.content);
   phantom.exit();
});

CasperJS

1. Install PhantomJS & CasperJS.

2. Set environment variable ENGINE_EXECUTABLE with the phantomjs executable path.

3. Run the following code using casperjs --ignore-ssl-errors=true sample.js.

// sample.js
phantom.setProxy('localhost', '3128', 'http');
var casper = require('casper').create();

casper.start('https://toscrape.com', function() {
   this.echo(this.getHTML());
});

casper.run();

SpookyJS

1. Make sure to install PhantomJS & CasperJS.

2. Install SpookyJS using npm install spooky.

3. Install SpookyJS dependencies using npm install --prefix node_modules/spooky.

4. Set environment variable and value ENGINE_FLAGS=--ignore-ssl-errors=true.

5. Run the sample code using node sample.js.

// sample.js
var Spooky = require('spooky');
var spooky = new Spooky({
      child: {
            proxy: 'localhost:3128',
            transport: 'http',
      },
      casper: {
            logLevel: 'debug',
            verbose: true
      }
   }, function (err) {
      if (err) {
            e = new Error('Failed to initialize SpookyJS');
            e.details = err;
            throw e;
      }

      spooky.start('https://toscrape.com');
      spooky.then(function () {
            this.emit('response', this.evaluate(function () {
               return document.documentElement.innerHTML;
            }));
      });
      spooky.run();
});

spooky.on('error', function (e, stack) {
   console.error(e);
   if (stack) {
      console.log(stack);
   }
});

spooky.on('response', function (response) {
   console.log(response);
});