Phantomjs 根据Casperjs源码拓展download方法

最近项目在使用Phantomjs作自动化检测时，有一个需求，需要下载检测网站的所有资源，包括css、js和图片资源，方便人工分析时可以把整个page还原。可惜，Phantomjs并没有直接提供download()这样的方法。查找资料后发现Casperjs有一个download的方法，可以把任意url的内容下载为字符串。由于Casperjs是根据Phantomjs开发的，因此从Casperjs的源码上分析，可能会得到一点启发。

目的：根据Casperjs源码，拓展Phantomjs，添加download方法

1. 先测试Casperjs的download方法[1]

var casper = require('casper').create({
pageSettings : {
webSecurityEnabled: false
}
});

casper.start('http://www.baidu.com/', function() {
this.download('http://www.w3school.com.cn/', 'w3school.html');
});

casper.run();

保存为D:/script.js，在命令行执行(casperjs D:/script.js)。Casperjs需要Phantomjs，请确保已安装Phantomjs v1.x版本。

2. 分析Casperjs源码

download方法在casper模块里，打开源码包下modules/casper.js，先找到download这个方法体(#592行)

/**
* Downloads a resource and saves it on the filesystem.
*
* @param  String  url         The url of the resource to download
* @param  String  targetPath  The destination file path
* @param  String  method      The HTTP method to use (default: GET)
* @param  String  data        Optional data to pass performing the request
* @return Casper
*/
Casper.prototype.download = function download(url, targetPath, method, data) {
"use strict";
this.checkStarted();    //在#426行,检查this是否已启动
var cu = require('clientutils').create(utils.mergeObjects({}, this.options));
try {
fs.write(targetPath, cu.decode(this.base64encode(url, method, data)), 'wb');
this.emit('downloaded.file', targetPath);
this.log(f("Downloaded and saved resource in %s", targetPath));
} catch (e) {
this.log(f("Error while downloading %s to %s: %s", url, targetPath, e), "error");
}
return this;
};

上面源码中，cu为'clientutils'模块的实例，用于decode()，具体功能后面再讲述。第#16行中，emit()在events模块中(与this绑定的语句在源码#226行)，功能为发送日志广播之类，与下面的this.log()一样，对download功能没大影响。因此核心语句在fs.write()中，url的内容在this.base64encode中获取。

再找base64encode这个方法，在源码#255行，返回callUtils('getBase64', url, method, data)。callUtils在#283行。

/**
* Invokes a client side utils object method within the remote page, with arguments.
*
* @param  {String}   method  Method name
* @return {...args}          Arguments
* @return {Mixed}
* @throws {CasperError}      If invokation failed.
*/
Casper.prototype.callUtils = function callUtils(method) {
"use strict";
var args = [].slice.call(arguments, 1); //把除method外的其余参数存到args
var result = this.evaluate(function(method, args) {
return __utils__.__call(method, args);
}, method, args);
if (utils.isObject(result) && result.__isCallError) {
throw new CasperError(f("callUtils(%s) with args %s thrown an error: %s",
method, args, result.message));
}
return result;
};

此时的method的值为“getBase64”，估计是一个方法名。这个方法核心语句在this.evaluate()，具体执行为this.evaluate(fn, "getBase64", [url, method, data])。evaluate()在#689行。

/**
* Evaluates an expression in the page context, a bit like what
* WebPage#evaluate does, but the passed function can also accept
* parameters if a context Object is also passed:
*
*     casper.evaluate(function(username, password) {
*         document.querySelector('#username').value = username;
*         document.querySelector('#password').value = password;
*         document.querySelector('#submit').click();
*     }, 'Bazoonga', 'baz00nga');
*
* @param  Function  fn       The function to be evaluated within current page DOM
* @param  Object    context  Object containing the parameters to inject into the function
* @return mixed
* @see    WebPage#evaluate
*/
//实际执行evaluate(fn, 'getBase64', [url, method, data])
//即context='getBase64', arguments.length=3
Casper.prototype.evaluate = function evaluate(fn, context) {
"use strict";
this.checkStarted();
console.log("context:"+context);

if (!utils.isFunction(fn) && !utils.isString(fn)) {
throw new CasperError("evaluate() only accepts functions or strings");
}

this.injectClientUtils();       //注入clientutils.js,稍后再细看

if (arguments.length === 1) {
return utils.clone(this.page.evaluate(fn));
} else if (arguments.length === 2) {
// check for closure signature if it matches context
if (utils.isObject(context) && eval(fn).length === Object.keys(context).length) {
context = utils.objectValues(context);
} else {
context = [context];
}
} else {        //arguments.length==3,实际执行到这里
// phantomjs-style signature
context = [].slice.call(arguments).slice(1);
}
//此时context = ['getBase64', [url, method, data]]
//[fn].concat(context) = [fn, 'getBase64', [url, method, data]]
return utils.clone(this.page.evaluate.apply(this.page, [fn].concat(context)));
};

以上第#28行注入了clientutils.js，具体实现方法下面再分析。第#17和#18行说明调用本方法时的参数情况，根据参数个数，实际执行到#39行，详细说明在#43和#44行的注释。因此，#45行相当于执行this.page.evaluate(fn, 'getBase64', [url, method, data])。fn在callUtils中定义了，最终效果相当于：

this.page.evaluate(function(method, args) {
return __utils__.__call(method, args);
}, 'getBase64', [url, method, data])

其中，function中的method='getBaes64'，args=[url, method, data]。所以最后，这句的意义等于在page中注入脚本执行__utils__.__call('getBase64', [url, method, data])。

再回头看，__utils__对象在以上#28行this.injectClientUtils()中注入的，injectClientUtils在#1256行。

/**
* Injects Client-side utilities in current page context.
*
*/
Casper.prototype.injectClientUtils = function injectClientUtils() {
"use strict";
this.checkStarted();
//保证不重复注入
var clientUtilsInjected = this.page.evaluate(function() {
return typeof __utils__ === "object";
});
if (true === clientUtilsInjected) {
return;
}
var clientUtilsPath = require('fs').pathJoin(phantom.casperPath, 'modules', 'clientutils.js');
if (true === this.page.injectJs(clientUtilsPath)) {
this.log("Successfully injected Casper client-side utilities", "debug");
} else {
this.warn("Failed to inject Casper client-side utilities");
}
// ClientUtils and Casper shares the same options
// These are not the lines I'm the most proud of in my life, but it works.
/*global __options*/
this.page.evaluate(function() {
window.__utils__ = new window.ClientUtils(__options);
}.toString().replace('__options', JSON.stringify(this.options)));
};

以上代码很好解释。先检查有没有__utils__对象，如果有说明已经注入clientutils了。若没有则注入clientutils.js，并新建ClientUtils对象，取名为__utils__。因此，下一步应该看clientutils.js。

在clientutils.js中，找到__call方法，在#70行。

/**
* Calls a method part of the current prototype, with arguments.
*
* @param  {String} method Method name
* @param  {Array}  args   arguments
* @return {Mixed}
*/
this.__call = function __call(method, args) {
if (method === "__call") {
return;
}
try {
return this[method].apply(this, args);
} catch (err) {
err.__isCallError = true;
return err;
}
};

核心在#13行，很好理解，即执行method指定的方法，并返回结果。回顾上面，method为'getBase64'，因此再找到getBase64方法，在#364行，其引用的getBinary()在下一个方法。getBinary()引用this.sendAJAX()。

至此整个下载过程的原理已经很清楚了，就是在page中注入脚本，利用跨域同步AJAX取得指定url的内容，然后再返回给Casperjs。sendAJAX则新建XMLHttpRequest来发出请求，这里不详细讲解。

3. 拓展download模块

首先模仿clientutils封装client模块，保存为modules/client.js。

/*
* 用于phantomjs引用或注入page
*/
(function(exports) {
"use strict";

exports.create = function create() {
return new this.Client();
}

exports.Client = function Client() {
var BASE64_ENCODE_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
var BASE64_DECODE_CHARS = new Array(
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63,
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,
-1,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1,
-1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1
);

/**
* Performs an AJAX request.
*
* @param   String   url      Url.
* @param   String   method   HTTP method (default: GET).
* @param   Object   data     Request parameters.
* @param   Boolean  async    Asynchroneous request? (default: false)
* @param   Object   settings Other settings when perform the ajax request
* @return  String            Response text.
*/
this.sendAJAX = function sendAJAX(url, method, data, async, settings) {
var xhr = new XMLHttpRequest(),
dataString = "",
dataList = [];
method = method && method.toUpperCase() || "GET";
var contentType = settings && settings.contentType || "application/x-www-form-urlencoded";
xhr.open(method, url, !!async);
xhr.overrideMimeType("text/plain; charset=x-user-defined");
if (method === "POST") {
if (typeof data === "object") {
for (var k in data) {
dataList.push(encodeURIComponent(k) + "=" + encodeURIComponent(data[k].toString()));
}
dataString = dataList.join('&');
} else if (typeof data === "string") {
dataString = data;
}
xhr.setRequestHeader("Content-Type", contentType);
}
xhr.send(method === "POST" ? dataString : null);
return this.encode(xhr.responseText);
};

/**
* Base64 encodes a string, even binary ones. Succeeds where
* window.btoa() fails.
*
* @param  String  str  The string content to encode
* @return string
*/
this.encode = function encode(str) {
/*jshint maxstatements:30 */
var out = "", i = 0, len = str.length, c1, c2, c3;
while (i < len) {
c1 = str.charCodeAt(i++) & 0xff;
if (i === len) {
out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4);
out += "==";
break;
}
c2 = str.charCodeAt(i++);
if (i === len) {
out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
out += BASE64_ENCODE_CHARS.charAt(((c1 & 0x3)<< 4) | ((c2 & 0xF0) >> 4));
out += BASE64_ENCODE_CHARS.charAt((c2 & 0xF) << 2);
out += "=";
break;
}
c3 = str.charCodeAt(i++);
out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
out += BASE64_ENCODE_CHARS.charAt(((c1 & 0x3) << 4) | ((c2 & 0xF0) >> 4));
out += BASE64_ENCODE_CHARS.charAt(((c2 & 0xF) << 2) | ((c3 & 0xC0) >> 6));
out += BASE64_ENCODE_CHARS.charAt(c3 & 0x3F);
}
return out;
};

/**
* Decodes a base64 encoded string. Succeeds where window.atob() fails.
*
* @param  String  str  The base64 encoded contents
* @return string
*/
this.decode = function decode(str) {
/*jshint maxstatements:30, maxcomplexity:30 */
var c1, c2, c3, c4, i = 0, len = str.length, out = "";
while (i < len) {
do {
c1 = BASE64_DECODE_CHARS[str.charCodeAt(i++) & 0xff];
} while (i < len && c1 === -1);
if (c1 === -1) {
break;
}
do {
c2 = BASE64_DECODE_CHARS[str.charCodeAt(i++) & 0xff];
} while (i < len && c2 === -1);
if (c2 === -1) {
break;
}
out += String.fromCharCode((c1 << 2) | ((c2 & 0x30) >> 4));
do {
c3 = str.charCodeAt(i++) & 0xff;
if (c3 === 61)
return out;
c3 = BASE64_DECODE_CHARS[c3];
} while (i < len && c3 === -1);
if (c3 === -1) {
break;
}
out += String.fromCharCode(((c2 & 0XF) << 4) | ((c3 & 0x3C) >> 2));
do {
c4 = str.charCodeAt(i++) & 0xff;
if (c4 === 61) {
return out;
}
c4 = BASE64_DECODE_CHARS[c4];
} while (i < len && c4 === -1);
if (c4 === -1) {
break;
}
out += String.fromCharCode(((c3 & 0x03) << 6) | c4);
}
return out;
};
};
})(typeof exports === 'object' ? exports : window);

封装download模块，保存为modules/download.js

/*
* 拓展模块，添加使用GET/POST下载资源的方法
*/
exports.create = function create(page) {
return new this.Casper(page);
}

exports.Casper = function Casper(page) {
this.page = page;
this.fs = require('fs');
//client.js模块所在路径
this.clientPath = this.fs.absolute(require('system').args[0]) + '/../modules/client.js';
this.client = require(this.clientPath).create();

this.get = function get(url, targetPath) {
this.injectClientJs();  //注入client.js
var content = this.page.evaluate(function(url) {
return __utils__.sendAJAX(url);
}, url);
this.fs.write(targetPath, this.client.decode(content), 'wb');
}

this.post = function post(url, data, targetPath) {
this.injectClientJs();  //注入client.js
var content = this.page.evaluate(function(url, data) {
return __utils__.sendAJAX(url, 'POST', data);
}, url, data);
this.fs.write(targetPath, this.client.decode(content), 'wb');
}

this.injectClientJs = function injectClientJs() {
"use strict";
//避免重复注入
var isJsInjected = this.page.evaluate(function() {
return typeof __utils__ === 'object';
});
if (true === isJsInjected) {
return ;
}
if (true !== this.page.injectJs(this.clientPath)) {
console.log('WARNING: Failed to inject client module!');
}
this.page.evaluate(function() {
window.__utils__ = new window.Client(); //新建Client对象
});
};
};

写一份测试脚本保存为script.js。脚本路径与modules文件夹同级，假设分别为D:/script.js和D:/modules/。

var fs = require('fs');
//切换至当前脚本路径下，方便引入自定义模块
var isChangeDirSuccees = fs.changeWorkingDirectory(fs.absolute(require('system').args[0]) + '/../');
if (!isChangeDirSuccees) {
console.log('ERROR: Failed to change working directory!');
phantom.exit();
}

var page = require('webpage').create();
page.open('http://www.w3school.com.cn/', function(status) {
var download = require('./modules/download').create(page);
download.get('http://www.w3school.com.cn/i/site_photoref.jpg', 'photo.jpg');
console.log('LOG: Download Completed!');
phantom.exit();
});

以上代码，先访问w3school主页，再下载site_photoref.jpg图片，保存在photo.jpg中。

经过测试，download可下载所有类型的资源，包括压缩文件、APK。但是注意一点，由于同源策略，当执行跨域请求时(page.open和download的url不在同域下)，要把web-security设为false[2]，在命令行启动时输入：phantomjs --web-security=false script.js。

参考资料及引用：

[1] download方法例子：Casper官网. Casperjs Api.
http://docs.casperjs.org/en/latest/modules/casper.html#download

[2] web-security：Phantomjs官网. 命令行选项.
http://phantomjs.org/api/command-line.html