From 0d751cc571830c441684ed7705be18125d3732d9 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 4 Aug 2023 23:04:32 -0700 Subject: [PATCH] Collector jobs have been updated to handle authors --- .../Jobs/CodeProjectWatcherJob.cs | 20 ++++- .../Jobs/DiscordNotificationJob.cs | 18 ++-- .../Jobs/YoutubeWatcherJob.cs | 86 ++++++++++++------- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs index 1414a08..1551cbd 100644 --- a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs @@ -30,6 +30,7 @@ public class CodeProjectWatcherJob private ILogger _logger; private IDiscordQueueRepository _queue; private ISourcesRepository _source; + private IAuthorTable _author; public CodeProjectWatcherJob() { @@ -37,12 +38,14 @@ public class CodeProjectWatcherJob _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); _logger = JobLogger.GetLogger("", JobName); + _author = new AuthorsTable(""); } public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options) { options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); + _author = new AuthorsTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); @@ -52,6 +55,7 @@ public class CodeProjectWatcherJob { options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); + _author = new AuthorsTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); @@ -157,17 +161,29 @@ public class CodeProjectWatcherJob }); parser.Parse(); + if (item.Authors[0].Name is null) + { + _logger.Warning("Author was missing from the record and will continue with a missing author"); + } + + var authorExists = _author.CreateIfMissingAsync(new AuthorEntity + { + Name = item.Authors[0].Name, + SourceId = source.Id, + Image = "", + }); + authorExists.Wait(); + var a = new ArticlesEntity { SourceId = source.Id, + AuthorId = authorExists.Result.Id, Tags = source.Tags, Title = item.Title.Text, Url = itemUrl, PubDate = item.LastUpdatedTime.DateTime, Thumbnail = parser.Data.Header.Image, Description = item.Title.Text, - AuthorName = item.Authors[0].Name ?? "", - AuthorImage = item.Authors[0].Uri ?? "", CodeIsRelease = isRelease, CodeIsCommit = isCommit, }; diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index 9544c63..c2a80fa 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -36,6 +36,7 @@ public class DiscordNotificationJob private ILogger _logger; //private DatabaseContext _databaseContext; private IArticlesRepository _article; + private IAuthorTable _author; private IIconsRepository _icons; private IDiscordQueueRepository _queue; private ISourcesRepository _sources; @@ -46,6 +47,7 @@ public class DiscordNotificationJob { _queue = new DiscordQueueTable(""); _article = new ArticlesTable(""); + _author = new AuthorsTable(""); _webhook = new DiscordWebhooksTable(""); _sources = new SourcesTable(""); _subs = new DiscordNotificationTable(""); @@ -92,7 +94,6 @@ public class DiscordNotificationJob _logger.Debug($"{JobName} - Processing {request.Id}"); // Get all details on the article in the queue var articleDetails = _article.GetById(request.ArticleId); - // Get the details of the source var sourceDetails = _sources.GetById(articleDetails.SourceId); if (sourceDetails.Id == Guid.Empty) @@ -103,6 +104,9 @@ public class DiscordNotificationJob return; } + var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name); + author.Wait(); + var sourceIcon = new IconEntity(); try { @@ -118,13 +122,13 @@ public class DiscordNotificationJob var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id); foreach (var sub in allSubscriptions) - SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); + SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity()); _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id); _queue.Delete(request.Id); } - public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub) + public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity) { // Check if the subscription code flags // If the article is a code commit and the subscription does not want them, skip. @@ -140,7 +144,7 @@ public class DiscordNotificationJob var client = new DiscordClient(discordDetails.Url); try { - client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon)); + client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity)); } catch (Exception e) { @@ -154,7 +158,7 @@ public class DiscordNotificationJob Thread.Sleep(3000); } - public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon) + public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author) { var embed = new DiscordMessageEmbed { @@ -163,7 +167,7 @@ public class DiscordNotificationJob Description = MessageValidation.ConvertHtmlCodes(article.Description), Author = new DiscordMessageEmbedAuthor { - Name = article.AuthorName, + Name = author.Name, IconUrl = icon.FileName }, Footer = new DiscordMessageEmbedFooter @@ -189,7 +193,7 @@ public class DiscordNotificationJob Url = article.Thumbnail }; - if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage; + embed.Author.IconUrl = author.Image ?? ""; return new DiscordMessage { diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index 2374fe5..325847c 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -25,6 +25,7 @@ public class YoutubeWatcherJob private readonly YoutubeWatcherJobOptions _options; private IArticlesRepository _articles; + private IAuthorTable _author; private IIconsRepository _icons; private ILogger _logger; private IDiscordQueueRepository _queue; @@ -34,6 +35,7 @@ public class YoutubeWatcherJob { _options = new YoutubeWatcherJobOptions(); _articles = new ArticlesTable(""); + _author = new AuthorsTable(""); _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); _icons = new IconsTable(""); @@ -43,6 +45,7 @@ public class YoutubeWatcherJob public void InitAndExecute(YoutubeWatcherJobOptions options) { _articles = new ArticlesTable(options.DatabaseConnectionString ?? ""); + _author = new AuthorsTable(options.DatabaseConnectionString ?? ""); _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? ""); _source = new SourcesTable(options.DatabaseConnectionString ?? ""); _icons = new IconsTable(options.DatabaseConnectionString ?? ""); @@ -77,11 +80,23 @@ public class YoutubeWatcherJob _logger.Information($"{JobName} - Checking '{source.Name}'"); var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}"; - var newVideos = CheckFeed(url, source); + var newVideos = FindMissingPosts(url, source); _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos"); + foreach (var video in newVideos) { - _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found"); + var author = _author.GetById(video.AuthorId); + author.Wait(); + + if (author.Result is null) + { + _logger.Warning("Missing author record for article id {VideoId}", video.Id); + } + else + { + _logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title); + } + _articles.New(video); _queue.New(new DiscordQueueEntity { @@ -104,13 +119,12 @@ public class YoutubeWatcherJob var id = pageReader.Data.Header.YoutubeChannelID ?? ""; if (id == "") - _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."), - url); + _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), ""); return id; } - private List CheckFeed(string url, SourceEntity source) + private List FindMissingPosts(string url, SourceEntity source) { var videos = new List(); @@ -118,37 +132,49 @@ public class YoutubeWatcherJob var feed = SyndicationFeed.Load(reader); foreach (var post in feed.Items.ToList()) { - var articleUrl = post.Links[0].Uri.AbsoluteUri; - if (IsThisUrlKnown(articleUrl)) continue; - - var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions - { - Url = articleUrl - }); - videoDetails.Parse(); - - var article = new ArticlesEntity - { - //Todo add the icon - AuthorName = post.Authors[0].Name, - Title = post.Title.Text, - Tags = FetchTags(post), - Url = articleUrl, - PubDate = post.PublishDate.DateTime, - Thumbnail = videoDetails.Data.Header.Image, - Description = videoDetails.Data.Header.Description, - SourceId = source.Id, - Video = "true" - }; - + var article = CheckFeedItem(post, source.Id); + if (article is null) continue; videos.Add(article); - - Thread.Sleep(_options.SleepTimer); } return videos; } + private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId) + { + var articleUrl = post.Links[0].Uri.AbsoluteUri; + if (IsThisUrlKnown(articleUrl)) return null; + + var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions + { + Url = articleUrl + }); + videoDetails.Parse(); + + var author = _author.CreateIfMissingAsync(new AuthorEntity + { + Image = post.Authors[0].Uri, + Name = post.Authors[0].Name + }); + author.Wait(); + + var article = new ArticlesEntity + { + //Todo add the icon + AuthorId = author.Result.Id, + Title = post.Title.Text, + Tags = FetchTags(post), + Url = articleUrl, + PubDate = post.PublishDate.DateTime, + Thumbnail = videoDetails.Data.Header.Image, + Description = videoDetails.Data.Header.Description, + SourceId = sourceId, + Video = "true" + }; + + return article; + } + private bool IsThisUrlKnown(string url) { var isKnown = _articles.GetByUrl(url);