From 09fe274752f5a18150a94a022d156d01f207cf48 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sat, 1 Apr 2023 08:53:34 -0700 Subject: [PATCH] Features/less comples config managment (#16) * New single const file for all config info * seed was updated to reflect route params * Adjusting config loads to make it easier to follow * test updates for config loading --- Newsbot.Collector.Api/BackgroundJobs.cs | 18 ++++---- .../Controllers/JobsController.cs | 5 ++- .../Consts/ConfigConst.cs | 33 ++++++++++++++ .../Jobs/DiscordNotificationJob.cs | 43 +++++++++++-------- .../Jobs/RssWatcherJob.cs | 25 +++++------ .../Jobs/RssWatcherJobTest.cs | 14 +++--- seed.ps1 | 9 ++-- 7 files changed, 92 insertions(+), 55 deletions(-) create mode 100644 Newsbot.Collector.Domain/Consts/ConfigConst.cs diff --git a/Newsbot.Collector.Api/BackgroundJobs.cs b/Newsbot.Collector.Api/BackgroundJobs.cs index 95312d4..23689b3 100644 --- a/Newsbot.Collector.Api/BackgroundJobs.cs +++ b/Newsbot.Collector.Api/BackgroundJobs.cs @@ -5,22 +5,24 @@ using Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Api; -public class BackgroundJobs +public static class BackgroundJobs { public static void SetupRecurringJobs(IConfiguration configuration) { - RecurringJob.AddOrUpdate("RSS", x => x.InitAndExecute(new RssWatcherJobOptions + RecurringJob.AddOrUpdate("RSS", x => + x.InitAndExecute(new RssWatcherJobOptions { - ConnectionStrings = - configuration.GetValue(ConfigSectionsConst.ConnectionStrings), - Config = configuration.GetValue(ConfigSectionsConst.Rss) + ConnectionString = configuration.GetValue(ConfigConst.ConnectionStringDatabase), + OpenTelemetry = configuration.GetValue(ConfigConst.ConnectionStringOpenTelemetry), + IsEnabled = configuration.GetValue(ConfigConst.RedditIsEnabled) }), "15 0-23 * * *"); - + RecurringJob.AddOrUpdate("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions { - ConnectionStrings = configuration.GetValue(ConfigSectionsConst.ConnectionStrings), - Config = configuration.GetValue(ConfigSectionsConst.NotificationsDiscord) + ConnectionString = configuration.GetValue(ConfigConst.ConnectionStringDatabase), + OpenTelemetry = configuration.GetValue(ConfigConst.ConnectionStringOpenTelemetry), + IsEnabled = configuration.GetValue(ConfigConst.DiscordNotificationsEnabled), }), "5/10 * * * *"); } } \ No newline at end of file diff --git a/Newsbot.Collector.Api/Controllers/JobsController.cs b/Newsbot.Collector.Api/Controllers/JobsController.cs index e53f989..4bfa455 100644 --- a/Newsbot.Collector.Api/Controllers/JobsController.cs +++ b/Newsbot.Collector.Api/Controllers/JobsController.cs @@ -31,8 +31,9 @@ public class JobsController { BackgroundJob.Enqueue(x => x.InitAndExecute(new RssWatcherJobOptions { - ConnectionStrings = _connectionStrings, - Config = _rssConfig + ConnectionString = _connectionStrings.Database, + OpenTelemetry = _connectionStrings.OpenTelemetry, + IsEnabled = _rssConfig.IsEnabled })); } diff --git a/Newsbot.Collector.Domain/Consts/ConfigConst.cs b/Newsbot.Collector.Domain/Consts/ConfigConst.cs new file mode 100644 index 0000000..3d5ad17 --- /dev/null +++ b/Newsbot.Collector.Domain/Consts/ConfigConst.cs @@ -0,0 +1,33 @@ +namespace Newsbot.Collector.Domain.Consts; + +public class ConfigConst +{ + public const string SectionConnectionStrings = "ConnectionStrings"; + public const string SectionFinalFantasyXiv = "FinalFantasyXiv"; + public const string SectionReddit = "Reddit"; + public const string SectionRss = "Rss"; + public const string SectionTwitch = "Twitch"; + public const string SectionYoutube = "Youtube"; + public const string SectionNotificationsDiscord = "Notifications:Discord"; + + public const string ConnectionStringDatabase = "ConnectionStrings:Database"; + public const string ConnectionStringOpenTelemetry = "ConnectionStrings:OpenTelemetry"; + + public const string RedditIsEnabled = "Reddit:IsEnabled"; + public const string RedditPullHot = "Reddit:PullHot"; + public const string RedditPullNsfw = "Reddit:PullNsfw"; + public const string RedditPullTop = "Reddit:PullTop"; + + public const string RssIsEnabled = "Rss:IsEnabled"; + + public const string TwitchIsEnabled = "Twitch:IsEnabled"; + public const string TwitchClientId = "Twitch:ClientID"; + public const string TwitchClientSecret = "Twitch:ClientSecret"; + + public const string YoutubeIsEnable = "Youtube:IsEnabled"; + public const string YoutubeDebug = "Youtube:Debug"; + + public const string EnableSwagger = "EnableSwagger"; + + public const string DiscordNotificationsEnabled = "Notifications:Discord:IsEnabled"; +} \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index de59554..80f1035 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -3,26 +3,29 @@ using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.Notifications.Discord; +using Newtonsoft.Json.Linq; using Serilog; namespace Newsbot.Collector.Services.Jobs; public class DiscordNotificationJobOptions { - public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } - public ConfigSectionNotificationsDiscord? Config { get; set; } - + public string? ConnectionString { get; init; } + public string? OpenTelemetry { get; init; } + public bool IsEnabled { get; init; } } public class DiscordNotificationJob { - + private const string JobName = "DiscordNotifications"; + private IDiscordQueueRepository _queue; private IArticlesRepository _article; private IDiscordWebHooksRepository _webhook; private ISourcesRepository _sources; private ISubscriptionRepository _subs; private IIconsRepository _icons; + private ILogger _logger; public DiscordNotificationJob() { @@ -32,21 +35,27 @@ public class DiscordNotificationJob _sources = new SourcesTable(""); _subs = new SubscriptionsTable(""); _icons = new IconsTable(""); + _logger = JobLogger.GetLogger("", JobName); } public void InitAndExecute(DiscordNotificationJobOptions options) { - options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); - options.Config ??= new ConfigSectionNotificationsDiscord(); - - _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); - _article = new ArticlesTable(options.ConnectionStrings.Database ?? ""); - _webhook = new DiscordWebhooksTable(options.ConnectionStrings.Database ?? ""); - _sources = new SourcesTable(options.ConnectionStrings.Database ?? ""); - _subs = new SubscriptionsTable(options.ConnectionStrings.Database ?? ""); - _icons = new IconsTable(options.ConnectionStrings.Database ?? ""); + _queue = new DiscordQueueTable(options.ConnectionString ?? ""); + _article = new ArticlesTable(options.ConnectionString ?? ""); + _webhook = new DiscordWebhooksTable(options.ConnectionString ?? ""); + _sources = new SourcesTable(options.ConnectionString ?? ""); + _subs = new SubscriptionsTable(options.ConnectionString ?? ""); + _icons = new IconsTable(options.ConnectionString ?? ""); + _logger = JobLogger.GetLogger(options.OpenTelemetry ?? "", JobName); + + if (!options.IsEnabled) + { + _logger.Warning($"{JobName} - Going to exit because feature flag is off."); + return; + } + _logger.Information($"{JobName} - Starting up the job."); Execute(); } @@ -60,11 +69,11 @@ public class DiscordNotificationJob // Get all details on the article in the queue var articleDetails = _article.GetById(request.ArticleID); - // Get the deatils of the source + // Get the details of the source var sourceDetails = _sources.GetByID(articleDetails.SourceID); if (sourceDetails.ID == Guid.Empty) { - Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue"); + _logger.Error($"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue"); _queue.Delete(request.ID); continue; } @@ -90,7 +99,7 @@ public class DiscordNotificationJob } catch (Exception e) { - Log.Error($"Failed to post message to Discord. {e}"); + _logger.Error($"Failed to post message to Discord. {e}"); continue; } @@ -133,7 +142,7 @@ public class DiscordNotificationJob embed.Url = article.URL; } - if (article.Thumbnail is not null && article.Thumbnail != "") + if (article.Thumbnail != "") { embed.Image = new DiscordMessageEmbedImage { diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index 65cff03..0e7e512 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -12,11 +12,9 @@ namespace Newsbot.Collector.Services.Jobs; public class RssWatcherJobOptions { - //public string? ConnectionString { get; init; } - //public string? OpenTelemetry { get; init; } - - public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } - public ConfigSectionRssModel? Config { get; set; } + public string? ConnectionString { get; init; } + public string? OpenTelemetry { get; init; } + public bool IsEnabled { get; init; } } // This class was made to work with Hangfire and it does not support constructors. @@ -39,22 +37,19 @@ public class RssWatcherJob public void InitAndExecute(RssWatcherJobOptions options) { - options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); - options.Config ??= new ConfigSectionRssModel(); + _articles = new ArticlesTable(options.ConnectionString ?? ""); + _queue = new DiscordQueueTable(options.ConnectionString ?? ""); + _source = new SourcesTable(options.ConnectionString ?? ""); + _logger = JobLogger.GetLogger(options.OpenTelemetry ?? "", JobName); - _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); - _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); - _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); - _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); - - _logger.Information($"{JobName} - Job was triggered"); - if (!options.Config.IsEnabled) + _logger.Information($"{JobName} - Job was triggered."); + if (!options.IsEnabled) { _logger.Information($"{JobName} - Going to exit because feature flag is off."); return; } - _logger.Information($"{JobName} - Setting up the job"); + _logger.Information($"{JobName} - Setting up the job."); Execute(); } diff --git a/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs b/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs index 7981329..b0575bc 100644 --- a/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs @@ -50,10 +50,9 @@ public class RssWatcherJobTest var client = new RssWatcherJob(); client.InitAndExecute(new RssWatcherJobOptions { - ConnectionStrings = new ConfigSectionConnectionStrings - { - Database = ConnectionString() - } + ConnectionString = ConnectionString(), + OpenTelemetry = "", + IsEnabled = true }); var items = client.Collect(url, Guid.NewGuid(), 0); client.UpdateDatabase(items); @@ -66,10 +65,9 @@ public class RssWatcherJobTest var client = new RssWatcherJob(); client.InitAndExecute(new RssWatcherJobOptions { - ConnectionStrings = new ConfigSectionConnectionStrings - { - Database = ConnectionString() - } + ConnectionString = ConnectionString(), + OpenTelemetry = "", + IsEnabled = true }); } } \ No newline at end of file diff --git a/seed.ps1 b/seed.ps1 index 2afcaf9..2f67e2a 100644 --- a/seed.ps1 +++ b/seed.ps1 @@ -8,11 +8,10 @@ $ErrorActionPreference = 'Stop' function New-RedditSource { param ( - [string] $Url + [string] $Name ) - $urlEncoded = [uri]::EscapeDataString($Url) - $param = "url=$urlEncoded" + $param = "name=$Name" $uri = "$ApiServer/api/sources/new/reddit?$param" $res = Invoke-RestMethod -Method Post -Uri $uri return $res @@ -32,7 +31,7 @@ function New-RssSource { function New-YoutubeSource { param ( - [Parameter(Required)][string] $Url + [string] $Url ) $urlEncoded = [uri]::EscapeDataString($Url) [string] $param = "url=$urlEncoded" @@ -90,7 +89,7 @@ $youtubeGameGrumps = New-YoutubeSource -Url "https://www.youtube.com/user/GameGr $youtubeCityPlannerPlays = New-YoutubeSource -Url "https://www.youtube.com/c/cityplannerplays" $youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTechTips" -$twitchNintendo = NewTwitchSource -Name "Nintendo" +$twitchNintendo = New-TwitchSource -Name "Nintendo" $miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01