From aa53b1eeebcdef102511c1f3988721327962c328 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 5 Mar 2023 22:33:41 -0800 Subject: [PATCH] Features/testing discord queue (#8) * new seed and corrected some issues talking to the db * adding some prep work for adding youtube back again * small updates for rss and getting results from discord job * new command to reload db * added discord alerts job to hangfire --- Newsbot.Collector.Api/Program.cs | 20 +++++--- .../Migrations/20230305204112_seed.sql | 20 ++++++++ .../Repositories/ArticlesTable.cs | 2 +- .../Repositories/DiscordQueue.cs | 2 +- .../Repositories/SubscriptionsTable.cs | 9 ++-- .../HtmlParser/HeadParserClient.cs | 12 ++++- .../HtmlParser/HtmlParserModels.cs | 1 + .../Jobs/DiscordNotificationJob.cs | 51 ++++++++++++++++--- .../Jobs/RssWatcherJob.cs | 10 ++-- makefile | 3 ++ 10 files changed, 101 insertions(+), 29 deletions(-) create mode 100644 Newsbot.Collector.Database/Migrations/20230305204112_seed.sql diff --git a/Newsbot.Collector.Api/Program.cs b/Newsbot.Collector.Api/Program.cs index 4d766d0..75f0334 100644 --- a/Newsbot.Collector.Api/Program.cs +++ b/Newsbot.Collector.Api/Program.cs @@ -1,9 +1,9 @@ using Hangfire; using Hangfire.MemoryStorage; -using Serilog; -using Newsbot.Collector.Services.Jobs; -using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Services.Jobs; +using Serilog; Log.Logger = new LoggerConfiguration() .WriteTo.Console() @@ -63,12 +63,18 @@ static IConfiguration GetConfiguration() static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger) { - RecurringJob.AddOrUpdate("Example", x => x.InitAndExecute(new HelloWorldJobOptions - { - Message = "Hello from the background!" - }), "0/1 * * * *"); + //RecurringJob.AddOrUpdate("Example", x => x.InitAndExecute(new HelloWorldJobOptions + //{ + // Message = "Hello from the background!" + //}), "0/1 * * * *"); + RecurringJob.AddOrUpdate("RSS", x => x.InitAndExecute(new RssWatcherJobOptions { ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" }), "15 0-23 * * *"); + + RecurringJob.AddOrUpdate("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions + { + DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" + }), "5/10 * * * *"); } diff --git a/Newsbot.Collector.Database/Migrations/20230305204112_seed.sql b/Newsbot.Collector.Database/Migrations/20230305204112_seed.sql new file mode 100644 index 0000000..2429119 --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230305204112_seed.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- +goose StatementBegin +INSERT INTO sources VALUES ( + uuid_generate_v4(), + 'rss', + 'Let''s Mosley', + 'feed', + 'rss', + 'podcast', + TRUE, + 'https://anchor.fm/s/6c7aa4c4/podcast/rss', + 'rss,let''s mosley,fitnes,coach', + FALSE); + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DELETE FROM sources Where type = 'rss' And Name = 'Let''s Mosley' +-- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs index d0821db..8cc2edb 100644 --- a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs +++ b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs @@ -95,7 +95,7 @@ public class ArticlesTable : IArticlesRepository var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; var res = conn.Execute(q, new { - id = Guid.NewGuid(), + id = model.ID, sourceid = model.SourceID, tags = model.Tags, title = model.Title, diff --git a/Newsbot.Collector.Database/Repositories/DiscordQueue.cs b/Newsbot.Collector.Database/Repositories/DiscordQueue.cs index bf41b38..233d8ac 100644 --- a/Newsbot.Collector.Database/Repositories/DiscordQueue.cs +++ b/Newsbot.Collector.Database/Repositories/DiscordQueue.cs @@ -46,7 +46,7 @@ public class DiscordQueueTable : IDiscordQueueRepository public List List(int limit = 25) { using var conn = OpenConnection(_connectionString); - var query = "Select * from DiscordQueue LIMIT @id;"; + var query = "Select * from DiscordQueue LIMIT @limit;"; return conn.Query(query, new { limit = limit }).ToList(); diff --git a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs index 5679a7b..5974677 100644 --- a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs +++ b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs @@ -59,13 +59,12 @@ public class SubscriptionsTable : ISubscriptionRepository }).ToList(); } - // todo add paging public List ListBySourceID(Guid id, int page = 0, int count = 25) { using var conn = OpenConnection(_connectionString); var query = @"Select * From subscriptions - Offset @page Fetch Next @count Rows Only - Where sourceid = @sourceid"; + Where sourceid = @sourceid + Offset @page Fetch Next @count Rows Only"; return conn.Query(query, new { page = page * count, @@ -78,8 +77,8 @@ public class SubscriptionsTable : ISubscriptionRepository { using var conn = OpenConnection(_connectionString); var query = @"Select * From subscriptions - Offset @page Fetch Next @count Rows Only - Where discordwebhookid = @webhookid"; + Where discordwebhookid = @webhookid + Offset @page Fetch Next @count Rows Only"; return conn.Query(query, new { page = page * count, diff --git a/Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs b/Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs index 9b0f4a4..9224c23 100644 --- a/Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs +++ b/Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs @@ -12,7 +12,7 @@ public class HeadParserClient private string _htmlContent; - public HeadParserClient(string htmlContent) + public HeadParserClient(string htmlContent, bool useBrowser = false) { _htmlContent = htmlContent; Data = new HeadParserModel(); @@ -28,6 +28,7 @@ public class HeadParserClient Data.ColorTheme = GetMetaColorTheme(); Data.FeedUri = GetSiteFeed(); + Data.YoutubeChannelID = GetYouTubeChannelId(); } private List CollectMetaTags() @@ -93,7 +94,7 @@ public class HeadParserClient public string GetMetaImage() { var htmlTags = CollectMetaTags(); - string[] tags = new string[] { "twitter:image", "og:image", "image" }; + string[] tags = new string[] { "twitter:image", "og:image", "image" }; return FindFirstResult(tags, htmlTags); } @@ -118,6 +119,13 @@ public class HeadParserClient return FindFirstResult(tags, htmlTags); } + public string GetYouTubeChannelId() + { + var htmlTags = CollectMetaTags(); + string[] tags = new string[] { "channelId" }; + return FindFirstResult(tags, htmlTags); + } + /// /// This will parse the headers looking for known keys that will contain a RSS feed link. /// If the feed is not found, this will throw an exception (MissingHeaderValueException). diff --git a/Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs b/Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs index 3823bc5..69f66b1 100644 --- a/Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs +++ b/Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs @@ -10,6 +10,7 @@ public class HeadParserModel public string ColorTheme { get; set; } = ""; public string? FeedUri { get; set; } + public string? YoutubeChannelID { get; set; } } public class HtmlData diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index 0bded1f..f7ab662 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -2,15 +2,16 @@ using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.Notifications.Discord; +using Serilog; namespace Newsbot.Collector.Services.Jobs; public class DiscordNotificationJobOptions { - + public string? DatabaseConnectionString { get; set; } } -public class DiscordNotifificationJob +public class DiscordNotificationJob { private IDiscordQueueRepository _queue; @@ -19,21 +20,24 @@ public class DiscordNotifificationJob private ISourcesRepository _sources; private ISubscriptionRepository _subs; - private IDiscordNotificatioClient _webhookClient; - - public DiscordNotifificationJob() + public DiscordNotificationJob() { _queue = new DiscordQueueTable(""); _article = new ArticlesTable(""); _webhook = new DiscordWebhooksTable(""); _sources = new SourcesTable(""); _subs = new SubscriptionsTable(""); - _webhookClient = new DiscordWebhookClient(""); } - public void InitAndExecute() + public void InitAndExecute(DiscordNotificationJobOptions options) { + _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? ""); + _article = new ArticlesTable(options.DatabaseConnectionString ?? ""); + _webhook = new DiscordWebhooksTable(options.DatabaseConnectionString ?? ""); + _sources = new SourcesTable(options.DatabaseConnectionString ?? ""); + _subs = new SubscriptionsTable(options.DatabaseConnectionString ?? ""); + Execute(); } private void Execute() @@ -48,6 +52,12 @@ public class DiscordNotifificationJob // Get the deatils of the source var sourceDetails = _sources.GetByID(articleDetails.SourceID); + if (sourceDetails.ID == Guid.Empty) + { + Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue"); + _queue.Delete(request.ID); + continue; + } // Find all the subscriptions for that source var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID); @@ -56,10 +66,26 @@ public class DiscordNotifificationJob { // find the discord webhooks we need to post to var discordDetails = _webhook.GetByID(sub.DiscordWebHookID); + if (discordDetails.Enabled == false) + { + continue; + } var client = new DiscordWebhookClient(discordDetails.Url); - client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails)); + try + { + client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails)); + } + catch (Exception e) + { + Log.Error($"Failed to post message to Discord. {e}"); + continue; + } + + Thread.Sleep(3000); } + + _queue.Delete(request.ID); } } @@ -77,6 +103,15 @@ public class DiscordNotifificationJob Footer = new DiscordMessageEmbedFooter { Text = "Brought to you by Newsbot", + }, + Fields = new DiscordMessageEmbedField[] + { + new DiscordMessageEmbedField + { + Name = "Link", + Value = article.URL, + Inline = false, + } } }; diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index 8d6ce4e..77bda4d 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -72,6 +72,11 @@ public class RssWatcherJob : IHangfireJob var sources = _source.ListByType(SourceTypes.Rss); foreach (var source in sources) { + if (source.Enabled == false) + { + continue; + } + var results = Collect(source.Url, source.ID); articles.AddRange(results); @@ -137,11 +142,6 @@ public class RssWatcherJob : IHangfireJob continue; } - if (IsThisUrlKnown(item.URL) == true) - { - continue; - } - var p = _articles.New(item); _queue.New(new DiscordQueueModel { diff --git a/makefile b/makefile index a22c8a3..22a8877 100644 --- a/makefile +++ b/makefile @@ -23,3 +23,6 @@ migrate-dev: ## Apply sql migrations to dev db migrate-dev-down: ## revert sql migrations to dev db goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down + +migrate-refresh: ## Rolls back all migrations + goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset