Features/testing discord queue (#8)

* new seed and corrected some issues talking to the db

* adding some prep work for adding youtube back again

* small updates for rss and getting results from discord job

* new command to reload db

* added discord alerts job to hangfire

Newsbot.Collector.Api/Program.cs

@@ -1,9 +1,9 @@
 using Hangfire;
 using Hangfire.MemoryStorage;
-using Serilog;
-using Newsbot.Collector.Services.Jobs;
-using Newsbot.Collector.Domain.Models;
 using Newsbot.Collector.Domain.Consts;
+using Newsbot.Collector.Domain.Models;
+using Newsbot.Collector.Services.Jobs;
+using Serilog;
 
 Log.Logger = new LoggerConfiguration()
     .WriteTo.Console()
@@ -63,12 +63,18 @@ static IConfiguration GetConfiguration()
 
 static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
 {
-    RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
-    {
-        Message = "Hello from the background!"
-    }), "0/1 * * * *");
+    //RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
+    //{
+    //    Message = "Hello from the background!"
+    //}), "0/1 * * * *");
+
     RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions
     {
         ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
     }), "15 0-23 * * *");
+
+    RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions
+    {
+        DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
+    }), "5/10 * * * *");
 }

Newsbot.Collector.Database/Migrations/20230305204112_seed.sql

@@ -0,0 +1,20 @@
+-- +goose Up
+-- +goose StatementBegin
+INSERT INTO sources VALUES (
+    uuid_generate_v4(), 
+    'rss', 
+    'Let''s Mosley', 
+    'feed', 
+    'rss', 
+    'podcast', 
+    TRUE, 
+    'https://anchor.fm/s/6c7aa4c4/podcast/rss', 
+    'rss,let''s mosley,fitnes,coach',
+    FALSE);
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DELETE FROM sources Where type = 'rss' And Name = 'Let''s Mosley'
+-- +goose StatementEnd

Newsbot.Collector.Database/Repositories/ArticlesTable.cs

@@ -95,7 +95,7 @@ public class ArticlesTable : IArticlesRepository
         var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
         var res = conn.Execute(q, new
         {
-            id = Guid.NewGuid(),
+            id = model.ID,
             sourceid = model.SourceID,
             tags = model.Tags,
             title = model.Title,

Newsbot.Collector.Database/Repositories/DiscordQueue.cs

@@ -46,7 +46,7 @@ public class DiscordQueueTable : IDiscordQueueRepository
     public List<DiscordQueueModel> List(int limit = 25)
     {
         using var conn = OpenConnection(_connectionString);
-        var query = "Select * from DiscordQueue LIMIT @id;";
+        var query = "Select * from DiscordQueue LIMIT @limit;";
         return conn.Query<DiscordQueueModel>(query, new {
             limit = limit
         }).ToList();

Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs

@@ -59,13 +59,12 @@ public class SubscriptionsTable : ISubscriptionRepository
         }).ToList();
     }
 
-    // todo add paging
     public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25)
     {
         using var conn = OpenConnection(_connectionString);
         var query = @"Select * From subscriptions 
-        Offset @page Fetch Next @count Rows Only
-        Where sourceid = @sourceid";
+        Where sourceid = @sourceid
+        Offset @page Fetch Next @count Rows Only";
         return conn.Query<SubscriptionModel>(query, new
         {
             page = page * count,
@@ -78,8 +77,8 @@ public class SubscriptionsTable : ISubscriptionRepository
     {
         using var conn = OpenConnection(_connectionString);
         var query = @"Select * From subscriptions 
-        Offset @page Fetch Next @count Rows Only
-        Where discordwebhookid = @webhookid";
+        Where discordwebhookid = @webhookid
+        Offset @page Fetch Next @count Rows Only";
         return conn.Query<SubscriptionModel>(query, new
         {
             page = page * count,

Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs

@@ -12,7 +12,7 @@ public class HeadParserClient
 
     private string _htmlContent;
 
-    public HeadParserClient(string htmlContent)
+    public HeadParserClient(string htmlContent, bool useBrowser = false)
     {
         _htmlContent = htmlContent;
         Data = new HeadParserModel();
@@ -28,6 +28,7 @@ public class HeadParserClient
         Data.ColorTheme = GetMetaColorTheme();
 
         Data.FeedUri = GetSiteFeed();
+        Data.YoutubeChannelID = GetYouTubeChannelId();
     }
 
     private List<HtmlNode> CollectMetaTags()
@@ -93,7 +94,7 @@ public class HeadParserClient
     public string GetMetaImage()
     {
         var htmlTags = CollectMetaTags();
-        string[] tags = new string[] { "twitter:image", "og:image", "image"  };
+        string[] tags = new string[] { "twitter:image", "og:image", "image" };
         return FindFirstResult(tags, htmlTags);
     }
 
@@ -118,6 +119,13 @@ public class HeadParserClient
         return FindFirstResult(tags, htmlTags);
     }
 
+    public string GetYouTubeChannelId()
+    {
+        var htmlTags = CollectMetaTags();
+        string[] tags = new string[] { "channelId" };
+        return FindFirstResult(tags, htmlTags);
+    }
+
     /// <summary>
     /// This will parse the headers looking for known keys that will contain a RSS feed link.
     /// If the feed is not found, this will throw an exception (MissingHeaderValueException).

Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs

@@ -10,6 +10,7 @@ public class HeadParserModel
     public string ColorTheme { get; set; } = "";
 
     public string? FeedUri { get; set; }
+    public string? YoutubeChannelID { get; set; }
 }
 
 public class HtmlData

Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs

@@ -2,15 +2,16 @@ using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
 using Newsbot.Collector.Services.Notifications.Discord;
+using Serilog;
 
 namespace Newsbot.Collector.Services.Jobs;
 
 public class DiscordNotificationJobOptions
 {
-
+    public string? DatabaseConnectionString { get; set; }
 }
 
-public class DiscordNotifificationJob
+public class DiscordNotificationJob
 {
 
     private IDiscordQueueRepository _queue;
@@ -19,21 +20,24 @@ public class DiscordNotifificationJob
     private ISourcesRepository _sources;
     private ISubscriptionRepository _subs;
 
-    private IDiscordNotificatioClient _webhookClient;
-
-    public DiscordNotifificationJob()
+    public DiscordNotificationJob()
     {
         _queue = new DiscordQueueTable("");
         _article = new ArticlesTable("");
         _webhook = new DiscordWebhooksTable("");
         _sources = new SourcesTable("");
         _subs = new SubscriptionsTable("");
-        _webhookClient = new DiscordWebhookClient("");
     }
 
-    public void InitAndExecute()
+    public void InitAndExecute(DiscordNotificationJobOptions options)
     {
+        _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
+        _article = new ArticlesTable(options.DatabaseConnectionString ?? "");
+        _webhook = new DiscordWebhooksTable(options.DatabaseConnectionString ?? "");
+        _sources = new SourcesTable(options.DatabaseConnectionString ?? "");
+        _subs = new SubscriptionsTable(options.DatabaseConnectionString ?? "");
 
+        Execute();
     }
 
     private void Execute()
@@ -48,6 +52,12 @@ public class DiscordNotifificationJob
 
             // Get the deatils of the source
             var sourceDetails = _sources.GetByID(articleDetails.SourceID);
+            if (sourceDetails.ID == Guid.Empty)
+            {
+                Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID.  Removing from the queue");
+                _queue.Delete(request.ID);
+                continue;
+            }
 
             // Find all the subscriptions for that source
             var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID);
@@ -56,10 +66,26 @@ public class DiscordNotifificationJob
             {
                 // find the discord webhooks we need to post to
                 var discordDetails = _webhook.GetByID(sub.DiscordWebHookID);
+                if (discordDetails.Enabled == false)
+                {
+                    continue;
+                }
 
                 var client = new DiscordWebhookClient(discordDetails.Url);
-                client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails));
+                try
+                {
+                    client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails));
+                }
+                catch (Exception e)
+                {
+                    Log.Error($"Failed to post message to Discord. {e}");
+                    continue;
+                }
+
+                Thread.Sleep(3000);
             }
+
+            _queue.Delete(request.ID);
         }
     }
 
@@ -77,6 +103,15 @@ public class DiscordNotifificationJob
             Footer = new DiscordMessageEmbedFooter
             {
                 Text = "Brought to you by Newsbot",
+            },
+            Fields = new DiscordMessageEmbedField[]
+            {
+                new DiscordMessageEmbedField
+                {
+                    Name = "Link",
+                    Value = article.URL,
+                    Inline = false,
+                }
             }
         };
 

Newsbot.Collector.Services/Jobs/RssWatcherJob.cs

@@ -72,6 +72,11 @@ public class RssWatcherJob : IHangfireJob
         var sources = _source.ListByType(SourceTypes.Rss);
         foreach (var source in sources)
         {
+            if (source.Enabled == false)
+            {
+                continue;
+            }
+
             var results = Collect(source.Url, source.ID);
 
             articles.AddRange(results);
@@ -137,11 +142,6 @@ public class RssWatcherJob : IHangfireJob
                 continue;
             }
 
-            if (IsThisUrlKnown(item.URL) == true)
-            {
-                continue;
-            }
-
             var p = _articles.New(item);
             _queue.New(new DiscordQueueModel
             {

makefile

@@ -23,3 +23,6 @@ migrate-dev: ## Apply sql migrations to dev db
 
 migrate-dev-down: ## revert sql migrations to dev db
 	goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down 
+
+migrate-refresh: ## Rolls back all migrations
+	goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset