phantomJS抓取多页会导致执行顺序错误

phantomJS crawling multi pages get misorder execution

本文关键字:执行 顺序 错误 抓取 phantomJS      更新时间:2023-09-26

我正在使用PhantomJS对从输入流生成的页面进行爬网。但是resultjson和Headers总是{},即使状态是"成功"(第一个url可以是好的,但通常第二个开始是{})。有人能告诉我它出了什么问题吗?

var page = require('webpage').create();
var system = require('system');
var args = system.args;
var fs = require('fs');
var resultjson = {};
var Headers = {};
var urlType, url, path;
var isInUse = false;

function pageInit(page){
    page.settings.resourceTimeout = 500000;
    page.onResourceTimeout = function (request) {
        console.log("fail");
    }
    page.onConsoleMessage = function (msg, lineNum, sourceID) {
        //console.log(msg + "at line " + lineNum);
    }

    page.onResourceReceived = function (response) {
        if (response.url === url && response.stage === "end") {
            if (response.status === 301 && response.redirectURL !== null) {
                url = response.redirectURL;
            }
            else {
                resultjson.Id = response.id;
                resultjson.Url = response.url;
                resultjson.Time = response.time;
                resultjson.BodySize = response.bodySize;
                resultjson.ContentType = response.contentType;
                resultjson.RedirectURL = response.redirectURL;
                resultjson.Stage = response.stage;
                resultjson.Status = response.status;
                resultjson.StatusText = response.statusText;
                response.headers.forEach(function (header) {
                    Headers[header.name] = header.value;
                });
            }
        }
    };
}
function GetOtherPage(url, path) {
    page.open(url, function (status) {
        console.log(JSON.stringify(resultjson));
        console.log(JSON.stringify(Headers));
        resultjson = {};
        Headers = {};
        if (status !== "success") {
            //fs.write(path, page.content, 'w');
            console.log("fail");
        }
        else {
            fs.write(path, page.content, 'w');
            console.log("success");
        }
        DoTask();
    });
}
function DoTask() {
    page.close();
    page = require('webpage').create();
    pageInit(page);
    //page.content = "";
    urlType = parseInt(system.stdin.readLine());
    if (urlType === -1)
        phantom.exit();
    url = system.stdin.readLine();
    path = system.stdin.readLine();
    if (urlType === 3) {
        GetOtherPage(url, path);
    }
}
DoTask();

结果:

3(input)
https://www.google.com(input)
output.html(input)
{}
{}
success
3(input)
https://www.google.com(input)
output.html(input)
{}
{}
success

更新:当你试图通过一个phantomJS进程获取相同的url时,phantomJS会在你的磁盘缓存中获取它们,所以它不会第二次请求url,所以resultjson和Headers是{}。

属性response.url是标准化的,因此它在键入的属性上添加了一个尾随的'/'

由于该比较response.url === url为假,因此它从不进入if

使用https://www.google.com/(带有尾随的'/')进行测试,得到以下输出:

3
https://www.google.com/
output.html
{"Id":1,"Url":"https://www.google.com/","Time":"2016-07-29T03:56:24.294Z","ContentType":"text/html; charset=UTF-8","RedirectU
RL":"https://www.google.com.br/?gfe_rd=cr&ei=BNSaV_KoL6KB8QfJrYPoDA","Stage":"end","Status":302,"StatusText":"Found"}
{"Cache-Control":"private","Content-Type":"text/html; charset=UTF-8","Location":"https://www.google.com.br/?gfe_rd=cr&ei=BNSa
V_KoL6KB8QfJrYPoDA","Content-Length":"263","Date":"Fri, 29 Jul 2016 03:56:52 GMT","Alternate-Protocol":"443:quic","Alt-Svc":"
quic='":443'"; ma=2592000; v='"36,35,34,33,32,31,30,29,28,27,26,25'""}
success

使用phantomjs v2.1.1测试