Collector jobs have been updated to handle authors

Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs

@@ -30,6 +30,7 @@ public class CodeProjectWatcherJob
     private ILogger _logger;
     private IDiscordQueueRepository _queue;
     private ISourcesRepository _source;
+    private IAuthorTable _author;
 
     public CodeProjectWatcherJob()
     {
@@ -37,12 +38,14 @@ public class CodeProjectWatcherJob
         _queue = new DiscordQueueTable("");
         _source = new SourcesTable("");
         _logger = JobLogger.GetLogger("", JobName);
+        _author = new AuthorsTable("");
     }
 
     public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
     {
         options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
         _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
+        _author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
         _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
         _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
         _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@@ -52,6 +55,7 @@ public class CodeProjectWatcherJob
     {
         options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
         _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
+        _author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
         _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
         _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
         _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@@ -157,17 +161,29 @@ public class CodeProjectWatcherJob
             });
             parser.Parse();
 
+            if (item.Authors[0].Name is null)
+            {
+                _logger.Warning("Author was missing from the record and will continue with a missing author");
+            }
+
+            var authorExists = _author.CreateIfMissingAsync(new AuthorEntity
+            {
+                Name = item.Authors[0].Name,
+                SourceId = source.Id,
+                Image = "",
+            });
+            authorExists.Wait();
+            
             var a = new ArticlesEntity
             {
                 SourceId = source.Id,
+                AuthorId = authorExists.Result.Id,
                 Tags = source.Tags,
                 Title = item.Title.Text,
                 Url = itemUrl,
                 PubDate = item.LastUpdatedTime.DateTime,
                 Thumbnail = parser.Data.Header.Image,
                 Description = item.Title.Text,
-                AuthorName = item.Authors[0].Name ?? "",
-                AuthorImage = item.Authors[0].Uri ?? "",
                 CodeIsRelease = isRelease,
                 CodeIsCommit = isCommit,
             };

Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs

@@ -36,6 +36,7 @@ public class DiscordNotificationJob
     private ILogger _logger;
     //private DatabaseContext _databaseContext;
     private IArticlesRepository _article;
+    private IAuthorTable _author;
     private IIconsRepository _icons;
     private IDiscordQueueRepository _queue;
     private ISourcesRepository _sources;
@@ -46,6 +47,7 @@ public class DiscordNotificationJob
     {
         _queue = new DiscordQueueTable("");
         _article = new ArticlesTable("");
+        _author = new AuthorsTable("");
         _webhook = new DiscordWebhooksTable("");
         _sources = new SourcesTable("");
         _subs = new DiscordNotificationTable("");
@@ -92,7 +94,6 @@ public class DiscordNotificationJob
         _logger.Debug($"{JobName} - Processing {request.Id}");
         // Get all details on the article in the queue
         var articleDetails = _article.GetById(request.ArticleId);
-
         // Get the details of the source
         var sourceDetails = _sources.GetById(articleDetails.SourceId);
         if (sourceDetails.Id == Guid.Empty)
@@ -103,6 +104,9 @@ public class DiscordNotificationJob
             return;
         }
 
+        var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name);
+        author.Wait();
+        
         var sourceIcon = new IconEntity();
         try
         {
@@ -118,13 +122,13 @@ public class DiscordNotificationJob
         var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
 
         foreach (var sub in allSubscriptions)
-            SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub);
+            SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity());
 
         _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id);
         _queue.Delete(request.Id);
     }
 
-    public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub)
+    public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity)
     {
         // Check if the subscription code flags
         // If the article is a code commit and the subscription does not want them, skip.
@@ -140,7 +144,7 @@ public class DiscordNotificationJob
         var client = new DiscordClient(discordDetails.Url);
         try
         {
-            client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon));
+            client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity));
         }
         catch (Exception e)
         {
@@ -154,7 +158,7 @@ public class DiscordNotificationJob
         Thread.Sleep(3000);
     }
 
-    public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon)
+    public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author)
     {
         var embed = new DiscordMessageEmbed
         {
@@ -163,7 +167,7 @@ public class DiscordNotificationJob
             Description = MessageValidation.ConvertHtmlCodes(article.Description),
             Author = new DiscordMessageEmbedAuthor
             {
-                Name = article.AuthorName,
+                Name = author.Name,
                 IconUrl = icon.FileName
             },
             Footer = new DiscordMessageEmbedFooter
@@ -189,7 +193,7 @@ public class DiscordNotificationJob
                 Url = article.Thumbnail
             };
 
-        if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage;
+        embed.Author.IconUrl = author.Image ?? "";
 
         return new DiscordMessage
         {

Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs

@@ -25,6 +25,7 @@ public class YoutubeWatcherJob
     
     private readonly YoutubeWatcherJobOptions _options;
     private IArticlesRepository _articles;
+    private IAuthorTable _author;
     private IIconsRepository _icons;
     private ILogger _logger;
     private IDiscordQueueRepository _queue;
@@ -34,6 +35,7 @@ public class YoutubeWatcherJob
     {
         _options = new YoutubeWatcherJobOptions();
         _articles = new ArticlesTable("");
+        _author = new AuthorsTable("");
         _queue = new DiscordQueueTable("");
         _source = new SourcesTable("");
         _icons = new IconsTable("");
@@ -43,6 +45,7 @@ public class YoutubeWatcherJob
     public void InitAndExecute(YoutubeWatcherJobOptions options)
     {
         _articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
+        _author = new AuthorsTable(options.DatabaseConnectionString ?? "");
         _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
         _source = new SourcesTable(options.DatabaseConnectionString ?? "");
         _icons = new IconsTable(options.DatabaseConnectionString ?? "");
@@ -77,11 +80,23 @@ public class YoutubeWatcherJob
             _logger.Information($"{JobName} - Checking '{source.Name}'");
             var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
 
-            var newVideos = CheckFeed(url, source);
+            var newVideos = FindMissingPosts(url, source);
             _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
+            
             foreach (var video in newVideos)
             {
-                _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
+                var author = _author.GetById(video.AuthorId);
+                author.Wait();
+
+                if (author.Result is null)
+                {
+                    _logger.Warning("Missing author record for article id {VideoId}", video.Id);
+                }
+                else
+                {
+                    _logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title);
+                }
+                
                 _articles.New(video);
                 _queue.New(new DiscordQueueEntity
                 {
@@ -104,13 +119,12 @@ public class YoutubeWatcherJob
 
         var id = pageReader.Data.Header.YoutubeChannelID ?? "";
         if (id == "")
-            _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
+            _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), "");
-                url);
 
         return id;
     }
 
-    private List<ArticlesEntity> CheckFeed(string url, SourceEntity source)
+    private List<ArticlesEntity> FindMissingPosts(string url, SourceEntity source)
     {
         var videos = new List<ArticlesEntity>();
 
@@ -118,37 +132,49 @@ public class YoutubeWatcherJob
         var feed = SyndicationFeed.Load(reader);
         foreach (var post in feed.Items.ToList())
         {
-            var articleUrl = post.Links[0].Uri.AbsoluteUri;
+            var article = CheckFeedItem(post, source.Id);
-            if (IsThisUrlKnown(articleUrl)) continue;
+            if (article is null) continue;
-
-            var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
-            {
-                Url = articleUrl
-            });
-            videoDetails.Parse();
-
-            var article = new ArticlesEntity
-            {
-                //Todo add the icon 
-                AuthorName = post.Authors[0].Name,
-                Title = post.Title.Text,
-                Tags = FetchTags(post),
-                Url = articleUrl,
-                PubDate = post.PublishDate.DateTime,
-                Thumbnail = videoDetails.Data.Header.Image,
-                Description = videoDetails.Data.Header.Description,
-                SourceId = source.Id,
-                Video = "true"
-            };
-
             videos.Add(article);
-
-            Thread.Sleep(_options.SleepTimer);
         }
 
         return videos;
     }
 
+    private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId)
+    {
+        var articleUrl = post.Links[0].Uri.AbsoluteUri;
+        if (IsThisUrlKnown(articleUrl)) return null;
+
+        var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
+        {
+            Url = articleUrl
+        });
+        videoDetails.Parse();
+
+        var author = _author.CreateIfMissingAsync(new AuthorEntity
+        {
+            Image = post.Authors[0].Uri,
+            Name = post.Authors[0].Name
+        });
+        author.Wait();
+
+        var article = new ArticlesEntity
+        {
+            //Todo add the icon 
+            AuthorId = author.Result.Id,
+            Title = post.Title.Text,
+            Tags = FetchTags(post),
+            Url = articleUrl,
+            PubDate = post.PublishDate.DateTime,
+            Thumbnail = videoDetails.Data.Header.Image,
+            Description = videoDetails.Data.Header.Description,
+            SourceId = sourceId,
+            Video = "true"
+        };
+
+        return article;
+    }
+    
     private bool IsThisUrlKnown(string url)
     {
         var isKnown = _articles.GetByUrl(url);