using System.Collections; using System.ServiceModel.Syndication; using System.Xml; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.HtmlParser; using Serilog; namespace Newsbot.Collector.Services.Jobs; public class CodeProjectWatcherJobOptions { public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } //public string ConnectionString { get; set; } = ""; public bool FeaturePullReleases { get; set; } = false; public bool FeaturePullCommits { get; set; } = false; //public bool PullIssues { get; set; } = false; } public class CodeProjectWatcherJob { private const string JobName = "CodeProjectWatcher"; private IArticlesRepository _articles; private ILogger _logger; private IDiscordQueueRepository _queue; private ISourcesRepository _source; public CodeProjectWatcherJob() { _articles = new ArticlesTable(""); _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); _logger = JobLogger.GetLogger("", JobName); } public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options) { options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); } public void InitAndExecute(CodeProjectWatcherJobOptions options) { options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); Execute(); } private void Execute() { var sources = _source.ListByType(SourceTypes.CodeProject); // query sources for things to pull var items = new List(); foreach (var source in sources) { items.AddRange(CheckForReleases(source)); //items.AddRange(CheckForCommits(source)); } foreach (var item in items) { _articles.New(item); _queue.New(new DiscordQueueModel { ArticleID = item.ID }); } } public IEnumerable CheckForReleases(SourceModel source) { var url = new Uri(source.Url); var links = new List { $"{url.AbsoluteUri}/releases.atom", $"{url.AbsoluteUri}/tags.atom" //github converts tags as releases }; foreach (var link in links) try { using var reader = XmlReader.Create(link); var client = SyndicationFeed.Load(reader); return ProcessFeed(client.Items, source, true, false); //if (link.EndsWith("tags.atom")) //{ // return ProcessFeed(client.Items, source, false, true, false); //} } catch { _logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}. Might not have anything", JobName, url.AbsoluteUri); } return new List(); } public IEnumerable CheckForCommits(SourceModel source) { var url = new Uri(source.Url); var links = new List { $"{url.AbsoluteUri}/commits/main.atom", $"{url.AbsoluteUri}/commits/master.atom" }; foreach (var link in links) try { using var reader = XmlReader.Create(link); var client = SyndicationFeed.Load(reader); return ProcessFeed(client.Items, source, false, true); } catch { _logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}. Might not have anything", JobName, url.AbsoluteUri); } return new List(); } private IEnumerable ProcessFeed(IEnumerable feed, SourceModel source, bool isRelease, bool isCommit) { var items = new List(); foreach (var item in feed) { var itemUrl = item.Links[0].Uri.AbsoluteUri; var exits = _articles.GetByUrl(itemUrl); if (exits.ID != Guid.Empty) continue; var parser = new HtmlPageReader(new HtmlPageReaderOptions { Url = itemUrl }); parser.Parse(); var a = new ArticlesModel { SourceID = source.ID, Tags = source.Tags, Title = item.Title.Text, URL = itemUrl, PubDate = item.LastUpdatedTime.DateTime, Thumbnail = parser.Data.Header.Image, Description = item.Title.Text, AuthorName = item.Authors[0].Name ?? "", AuthorImage = item.Authors[0].Uri ?? "", CodeIsRelease = isRelease, CodeIsCommit = isCommit, }; items.Add(a); } return items; } }