同时运行多个 Selenium 任务并防止系统崩溃-解网

问：

我将通过 Selenium 和 HtmlAgilityPack（在 C#、.NET 7 中）从网站获取每个国家/地区的人口排名。此代码适用于 10 个国家/地区，但是当我想请求所有国家/地区时，由于许多任务，系统崩溃并且我遇到缓慢。这是什么方法？

static async void GetData()
{
    string website = "......";
    List<string> Countries = new List<string>()
    {
        // 195 Countries
    };

    List<Task<JObject>> Tasks = new List<Task<JObject>>();
    foreach (string countryName in Countries)
    {
        Tasks.Add(FetchData(website + "/" + countryName));
    }
    await Task.WhenAll(Tasks);
    foreach (JObject populationRank in Tasks.Select(task => task.Result))
    {
        WriteLine(populationRank);
    }
}

static Task<JObject> FetchData(string URL)
{
    return Task.Run(async () =>
    {
        ChromeDriver myDriver = new CustomizedDriver();
        myDriver.Navigate().GoToUrl(URL);
        HtmlDocument Document = new HtmlDocument();
        Document.LoadHtml(await myDriver.GetPageSourceAsync());
        JObject Object = new JObject()
        {
            ["PopulationRank"] = Document.DocumentNode.SelectSingleNode("//div[@id='popRank']").InnerText
        };
        myDriver.Quit();
        myDriver.Dispose();
        return Object;
    });
}
static Task<string> GetPageSourceAsync(this IWebDriver driver)
{
    return Task.Run(() =>
    {
        while (true)
        {
            string PageState = (string)( (IJavaScriptExecutor)driver ).ExecuteScript("return document.readyState");
            if (PageState == "interactive" || PageState == "complete")
                return driver.PageSource;
        }
    });
}
static ChromeDriver CustomizedDriver()
{
    ChromeDriverService chromeService = ChromeDriverService.CreateDefaultService();
    chromeService.HideCommandPromptWindow = true;
    ChromeOptions options = new ChromeOptions();
    options.PageLoadStrategy = PageLoadStrategy.None;
    options.AddArgument("--headless --disable-cookies --blink-settings=imagesEnabled=false");
    return new ChromeDriver(chromeService, options);
}

C# selenium-webdriver 异步 async-await 任务

    static async void GetData()
{
    string website = "......";
    List<string> Countries = new List<string>()
    {
        // 195 Countries
    };

    var results = new ConcurrentBag<JObject>();
    
    var parallelOptions = new ParallelOptions()
    {
        MaxDegreeOfParallelism = 10 // Here you control how many countries in parallel to process.
    };

    Parallel.ForEachAsync(Countries, parallelOptions, async (country, token) => 
    {
        var result = await FetchData(website + "/" + countryName);
        results.Add(result);
    }

    foreach (JObject populationRank in results))
    {
        WriteLine(populationRank);
    }
}

static async Task<JObject> FetchData(string URL)
{
        ChromeDriver myDriver = new CustomizedDriver();
        myDriver.Navigate().GoToUrl(URL);
        HtmlDocument Document = new HtmlDocument();
        Document.LoadHtml(await myDriver.GetPageSourceAsync());
        JObject Object = new JObject()
        {
            ["PopulationRank"] = Document.DocumentNode.SelectSingleNode("//div[@id='popRank']").InnerText
        };
        myDriver.Quit();
        myDriver.Dispose();
        return Object;
}
static Task<string> GetPageSourceAsync(this IWebDriver driver)
{
    return Task.Run(() =>
    {
        while (true)
        {
            string PageState = (string)( (IJavaScriptExecutor)driver ).ExecuteScript("return document.readyState");
            if (PageState == "interactive" || PageState == "complete")
                return driver.PageSource;
        }
    });
}
static ChromeDriver CustomizedDriver()
{
    ChromeDriverService chromeService = ChromeDriverService.CreateDefaultService();
    chromeService.HideCommandPromptWindow = true;
    ChromeOptions options = new ChromeOptions();
    options.PageLoadStrategy = PageLoadStrategy.None;
    options.AddArgument("--headless --disable-cookies --blink-settings=imagesEnabled=false");
    return new ChromeDriver(chromeService, options);
}

同时运行多个 Selenium 任务并防止系统崩溃

Running many Selenium tasks concurrently and preventing system crashes

评论

评论