Newsbot.Collector/Newsbot.Collector.Services/HtmlParser/BrowserClient.cs

using OpenQA.Selenium;
using OpenQA.Selenium.Firefox;

namespace Newsbot.Collector.Services.HtmlParser;

public class BrowserClient : IDisposable
{
    private readonly IWebDriver _driver;

    public BrowserClient()
    {
        _driver = new FirefoxDriver();
    }

    public void Dispose()
    {
        _driver.Close();
        _driver.Quit();
        _driver.Dispose();
    }

    public string GetPageSource(string url, int sleep = 5000)
    {
        _driver.Navigate().GoToUrl(url);

        // Give the page some time to finish loading js
        Thread.Sleep(sleep);
        return _driver.PageSource;
    }
}