Selebyun爬网程序超时问题C#

Selebiun Crawler Timeout Issues C#

本文关键字:问题 超时 程序 Selebyun      更新时间:2023-09-26

我正在运行以下代码,在执行过程中收到一个错误。

  public void GetCategoriesSelenium() {
            string javascript = System.IO.File.ReadAllText(@"GetCategory.js");
            CrawlerWebSeleniumJS.ExecuteScript("var finished;");
            CrawlerWebSeleniumJS.ExecuteScript("var all_categories;");
            CrawlerWebSeleniumJS.ExecuteScript("finished = false;");
            CrawlerWebSeleniumJS.ExecuteScript("all_categories = [];");
            CrawlerWebSelenium.Manage().Timeouts().SetScriptTimeout(TimeSpan.FromDays(1));
            CrawlerWebSelenium.Manage().Timeouts().SetPageLoadTimeout(TimeSpan.FromDays(1));
            CrawlerWebSelenium.Manage().Timeouts().ImplicitlyWait(TimeSpan.FromDays(1));
            AddToConsole("CRAWLER: GET - Categories");
            try {
                CrawlerWebSeleniumJS.ExecuteScript(javascript);
                }
            catch {
                }
            int ready = 2;
            for (int i = 0; i < ready; i++) {
                try {
                    if (CrawlerWebSeleniumJS.ExecuteScript("return finished").ToString() == "True") {
                        i = i++ + ready++;
                        }
                    else {
                        ready++;
                        }
                    }
                catch {
                    }
                }
            AddToCatsTreeSelenium();
            }
$('.p-pstctgry-lnk-ctgry').each(function (i) {
    var idBits = this.id.split('_');
    var theId = idBits[1];
    var theTitle = this.text;
    var subcategories = [];
    //initiate ajax request for json results
    $.ajax({
        async: false,
        type: 'GET',
        dataType: 'json',
        url: 'URL REMOVED',
        data: {
            nodeType: 'cat',
            level1id: theId
        }
    }).done(function (theJSON1) {
        var thelength1 = Object.keys(theJSON1['items']).length;
        //loop through found subs
        for (var i = 0; i < thelength1; i++) {
            //start of next recursive block to copy and paste inside
            var subsubcategories = [];
            //initiate ajax request for sub json results
            $.ajax({
                async: false,
                type: 'GET',
                dataType: 'json',
                url: 'URL REMOVED',
                data: {
                    nodeType: 'cat',
                    level1id: theId,
                    level2id: theJSON1['items'][i]['id']
                }
            }).done(function (theJSON2) {
                var thelength2 = Object.keys(theJSON2['items']).length;
                for (var k = 0; k < thelength2; k++) {
                    //start of next recursive block to copy and paste inside
                    var subsubsubcategories = [];
                    //initiate ajax request for sub json results
                    if ((theJSON2['items'][k]['id'] != 'OFFER') && (theJSON2['items'][k]['id'] != 'WANTED')) {
                        $.ajax({
                            async: false,
                            type: 'GET',
                            dataType: 'json',
                            url: 'URL REMOVED',
                            data: {
                                nodeType: 'cat',
                                level1id: theId,
                                level2id: theJSON1['items'][i]['id'],
                                level3id: theJSON2['items'][k]['id']
                            }
                        }).done(function (theJSON3) {
                            var thelength3 = Object.keys(theJSON3['items']).length;
                            for (var l = 0; l < thelength3; l++) {
                                console.log('---' + theJSON3['items'][l]['value'] + ' ' + theJSON3['items'][l]['id']);
                                //store this subsub
                                subsubsubcategories.push({
                                    title: theJSON3['items'][l]['value'],
                                    id: theJSON3['items'][l]['id'],
                                    sub: ''
                                });
                            }
                            //end done theJSON
                        });
                    }
                    //end of next recursive block to copy and paste inside
                    console.log('--' + theJSON2['items'][k]['value'] + ' ' + theJSON2['items'][k]['id']);
                    //store this subsub
                    subsubcategories.push({
                        title: theJSON2['items'][k]['value'],
                        id: theJSON2['items'][k]['id'],
                        sub: subsubsubcategories
                    });
                }
                //end done theJSON
            });
            console.log('-' + theJSON1['items'][i]['value'] + ' ' + theJSON1['items'][i]['id']);
            //store this sub with -> subsub
            subcategories.push({
                title: theJSON1['items'][i]['value'],
                id: theJSON1['items'][i]['id'],
                sub: subsubcategories
            });
            //end of next recursive block to copy and paste inside
            //end sub loop
        }
        console.log('' + theTitle + ' ' + theId);
        //store this cat with -> sub -> subsub
        all_categories.push({
            title: theTitle,
            id: theId,
            sub: subcategories
        });
        console.log(all_categories);
        //end first json subcat loop
    });
    //end main cat scan loop
});
finished = true;

上面的代码是我运行的方法,它下面的代码是通过selenium运行的纯javascript。

因此,问题一,当代码运行时,selenium锁定。我能理解。这个过程大约需要4分钟。60秒后超时,出现错误

向远程WebDriver服务器请求URL的HTTP请求在60秒后超时。

这真的很烦人,而且会锁住系统。我知道一个非常快速简单的方法来解决这个问题。(线程。睡眠(300000)这太恶心了。。。

我的想法是,也许它正在运行一个javascript查询并等待它完成,而我不断地用更多的javascript请求来冲击Selenium,这些请求如预期的那样超时。

还有其他想法吗?

驱动程序的构造函数应该有一个重载,其中包括一个TimeSpan,指示.NET绑定用于与远程端通信的HTTP客户端的超时。将其设置为适当大的值应该足以完成操作。