Using Multiple page.open in Single Script

Question

My goal is to execute PhantomJS by using:

// adding $op and $er for debugging purposes
exec(\'phantomjs script.js\', $op, $er);
print_r($op);
echo $er;


        

Answer 1

The problem is that the second page.open is being invoked before the first one finishes, which can cause multiple problems. You want logic roughly like the following (assuming the filenames are given as command line arguments):

function handle_page(file){
    page.open(file,function(){
        ...
        page.evaluate(function(){
            ...do stuff...
        });
        page.render(...);
        setTimeout(next_page,100);
    });
}
function next_page(){
    var file=args.shift();
    if(!file){phantom.exit(0);}
    handle_page(file);
}
next_page();

Right, it's recursive. This ensures that the processing of the function passed to page.open finishes, with a little 100ms grace period, before you go to the next file.

By the way, you don't need to keep repeating

page = require('webpage').create();

Answer 2

I've tried the accepted answer suggestions, but it doesn't work (at least not for v2.1.1).

To be accurate the accepted answer worked some of the time, but I still experienced sporadic failed page.open() calls, about 90% of the time on specific data sets.

The simplest answer I found is to instantiate a new page module for each url.

// first page
var urlA = "http://first/url"
var pageA = require('webpage').create()

pageA.open(urlA, function(status){
    if (status){
        setTimeout(openPageB, 100) // open second page call
    } else{
        phantom.exit(1)
    }
})

// second page
var urlB = "http://second/url"
var pageB = require('webpage').create()

function openPageB(){
    pageB.open(urlB, function(){
        // ... 
        // ...
    })
}

The following from the page module api documentation on the close method says:

close() {void}

Close the page and releases the memory heap associated with it. Do not use the page instance after calling this.

Due to some technical limitations, the web page object might not be completely garbage collected. This is often encountered when the same object is used over and over again. Calling this function may stop the increasing heap allocation.

Basically after I tested the close() method I decided using the same web page instance for different open() calls is too unreliable and it needed to be said.

Answer 3

You can use recursion:

var page = require('webpage').create();

// the urls to navigate to
var urls = [
    'http://phantomjs.org/',
    'https://twitter.com/sidanmor',
    'https://github.com/sidanmor'
];

var i = 0;

// the recursion function
var genericCallback = function () {
    return function (status) {
        console.log("URL: " + urls[i]);
        console.log("Status: " + status);
        // exit if there was a problem with the navigation
        if (!status || status === 'fail') phantom.exit();

        i++;

        if (status === "success") {

            //-- YOUR STUFF HERE ---------------------- 
            // do your stuff here... I'm taking a picture of the page
            page.render('example' + i + '.png');
            //-----------------------------------------

            if (i < urls.length) {
                // navigate to the next url and the callback is this function (recursion)
                page.open(urls[i], genericCallback());
            } else {
                // try navigate to the next url (it is undefined because it is the last element) so the callback is exit
                page.open(urls[i], function () {
                    phantom.exit();
                });
            }
        }
    };
};

// start from the first url
page.open(urls[i], genericCallback());

Answer 4

Using Queued Processes, sample:

var page = require('webpage').create();

// Queue Class Helper
var Queue = function() {
    this._tasks = [];
};
Queue.prototype.add = function(fn, scope) {
    this._tasks.push({fn: fn,scope: scope});
    return this;
};
Queue.prototype.process = function() {
    var proxy, self = this;
    task = this._tasks.shift();
    if(!task) {return;}
    proxy = {end: function() {self.process();}};
    task.fn.call(task.scope, proxy);
    return this;        
};
Queue.prototype.clear = function() {
    this._tasks = []; return this;
};

// Init pages .....  
var q = new Queue();       

q.add(function(proxy) {
  page.open(url1, function() {
    // page.evaluate
    proxy.end();
  });            
});

q.add(function(proxy) {
  page.open(url2, function() {
    // page.evaluate
    proxy.end();
  });            
});


q.add(function(proxy) {
  page.open(urln, function() {
    // page.evaluate
    proxy.end();
  });            
});

// .....

q.add(function(proxy) {
  phantom.exit()
  proxy.end();
});

q.process();

I hope this is useful, regards.