Features/less comples config managment (#16)

* New single const file for all config info

* seed was updated to reflect route params

* Adjusting config loads to make it easier to follow

* test updates for config loading
This commit is contained in:
James Tombleson 2023-04-01 08:53:34 -07:00 committed by GitHub
parent b7b50f5124
commit 09fe274752
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 92 additions and 55 deletions

View File

@ -5,22 +5,24 @@ using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api; namespace Newsbot.Collector.Api;
public class BackgroundJobs public static class BackgroundJobs
{ {
public static void SetupRecurringJobs(IConfiguration configuration) public static void SetupRecurringJobs(IConfiguration configuration)
{ {
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x =>
x.InitAndExecute(new RssWatcherJobOptions
{ {
ConnectionStrings = ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
configuration.GetValue<ConfigSectionConnectionStrings>(ConfigSectionsConst.ConnectionStrings), OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
Config = configuration.GetValue<ConfigSectionRssModel>(ConfigSectionsConst.Rss) IsEnabled = configuration.GetValue<bool>(ConfigConst.RedditIsEnabled)
}), "15 0-23 * * *"); }), "15 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
x.InitAndExecute(new DiscordNotificationJobOptions x.InitAndExecute(new DiscordNotificationJobOptions
{ {
ConnectionStrings = configuration.GetValue<ConfigSectionConnectionStrings>(ConfigSectionsConst.ConnectionStrings), ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
Config = configuration.GetValue<ConfigSectionNotificationsDiscord>(ConfigSectionsConst.NotificationsDiscord) OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled),
}), "5/10 * * * *"); }), "5/10 * * * *");
} }
} }

View File

@ -31,8 +31,9 @@ public class JobsController
{ {
BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions
{ {
ConnectionStrings = _connectionStrings, ConnectionString = _connectionStrings.Database,
Config = _rssConfig OpenTelemetry = _connectionStrings.OpenTelemetry,
IsEnabled = _rssConfig.IsEnabled
})); }));
} }

View File

@ -0,0 +1,33 @@
namespace Newsbot.Collector.Domain.Consts;
public class ConfigConst
{
public const string SectionConnectionStrings = "ConnectionStrings";
public const string SectionFinalFantasyXiv = "FinalFantasyXiv";
public const string SectionReddit = "Reddit";
public const string SectionRss = "Rss";
public const string SectionTwitch = "Twitch";
public const string SectionYoutube = "Youtube";
public const string SectionNotificationsDiscord = "Notifications:Discord";
public const string ConnectionStringDatabase = "ConnectionStrings:Database";
public const string ConnectionStringOpenTelemetry = "ConnectionStrings:OpenTelemetry";
public const string RedditIsEnabled = "Reddit:IsEnabled";
public const string RedditPullHot = "Reddit:PullHot";
public const string RedditPullNsfw = "Reddit:PullNsfw";
public const string RedditPullTop = "Reddit:PullTop";
public const string RssIsEnabled = "Rss:IsEnabled";
public const string TwitchIsEnabled = "Twitch:IsEnabled";
public const string TwitchClientId = "Twitch:ClientID";
public const string TwitchClientSecret = "Twitch:ClientSecret";
public const string YoutubeIsEnable = "Youtube:IsEnabled";
public const string YoutubeDebug = "Youtube:Debug";
public const string EnableSwagger = "EnableSwagger";
public const string DiscordNotificationsEnabled = "Notifications:Discord:IsEnabled";
}

View File

@ -3,26 +3,29 @@ using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
using Newtonsoft.Json.Linq;
using Serilog; using Serilog;
namespace Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Services.Jobs;
public class DiscordNotificationJobOptions public class DiscordNotificationJobOptions
{ {
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } public string? ConnectionString { get; init; }
public ConfigSectionNotificationsDiscord? Config { get; set; } public string? OpenTelemetry { get; init; }
public bool IsEnabled { get; init; }
} }
public class DiscordNotificationJob public class DiscordNotificationJob
{ {
private const string JobName = "DiscordNotifications";
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
private IArticlesRepository _article; private IArticlesRepository _article;
private IDiscordWebHooksRepository _webhook; private IDiscordWebHooksRepository _webhook;
private ISourcesRepository _sources; private ISourcesRepository _sources;
private ISubscriptionRepository _subs; private ISubscriptionRepository _subs;
private IIconsRepository _icons; private IIconsRepository _icons;
private ILogger _logger;
public DiscordNotificationJob() public DiscordNotificationJob()
{ {
@ -32,21 +35,27 @@ public class DiscordNotificationJob
_sources = new SourcesTable(""); _sources = new SourcesTable("");
_subs = new SubscriptionsTable(""); _subs = new SubscriptionsTable("");
_icons = new IconsTable(""); _icons = new IconsTable("");
_logger = JobLogger.GetLogger("", JobName);
} }
public void InitAndExecute(DiscordNotificationJobOptions options) public void InitAndExecute(DiscordNotificationJobOptions options)
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _queue = new DiscordQueueTable(options.ConnectionString ?? "");
options.Config ??= new ConfigSectionNotificationsDiscord(); _article = new ArticlesTable(options.ConnectionString ?? "");
_webhook = new DiscordWebhooksTable(options.ConnectionString ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _sources = new SourcesTable(options.ConnectionString ?? "");
_article = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _subs = new SubscriptionsTable(options.ConnectionString ?? "");
_webhook = new DiscordWebhooksTable(options.ConnectionStrings.Database ?? ""); _icons = new IconsTable(options.ConnectionString ?? "");
_sources = new SourcesTable(options.ConnectionStrings.Database ?? "");
_subs = new SubscriptionsTable(options.ConnectionStrings.Database ?? "");
_icons = new IconsTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.OpenTelemetry ?? "", JobName);
if (!options.IsEnabled)
{
_logger.Warning($"{JobName} - Going to exit because feature flag is off.");
return;
}
_logger.Information($"{JobName} - Starting up the job.");
Execute(); Execute();
} }
@ -60,11 +69,11 @@ public class DiscordNotificationJob
// Get all details on the article in the queue // Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleID); var articleDetails = _article.GetById(request.ArticleID);
// Get the deatils of the source // Get the details of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID); var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty) if (sourceDetails.ID == Guid.Empty)
{ {
Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue"); _logger.Error($"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue");
_queue.Delete(request.ID); _queue.Delete(request.ID);
continue; continue;
} }
@ -90,7 +99,7 @@ public class DiscordNotificationJob
} }
catch (Exception e) catch (Exception e)
{ {
Log.Error($"Failed to post message to Discord. {e}"); _logger.Error($"Failed to post message to Discord. {e}");
continue; continue;
} }
@ -133,7 +142,7 @@ public class DiscordNotificationJob
embed.Url = article.URL; embed.Url = article.URL;
} }
if (article.Thumbnail is not null && article.Thumbnail != "") if (article.Thumbnail != "")
{ {
embed.Image = new DiscordMessageEmbedImage embed.Image = new DiscordMessageEmbedImage
{ {

View File

@ -12,11 +12,9 @@ namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJobOptions public class RssWatcherJobOptions
{ {
//public string? ConnectionString { get; init; } public string? ConnectionString { get; init; }
//public string? OpenTelemetry { get; init; } public string? OpenTelemetry { get; init; }
public bool IsEnabled { get; init; }
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
public ConfigSectionRssModel? Config { get; set; }
} }
// This class was made to work with Hangfire and it does not support constructors. // This class was made to work with Hangfire and it does not support constructors.
@ -39,22 +37,19 @@ public class RssWatcherJob
public void InitAndExecute(RssWatcherJobOptions options) public void InitAndExecute(RssWatcherJobOptions options)
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionString ?? "");
options.Config ??= new ConfigSectionRssModel(); _queue = new DiscordQueueTable(options.ConnectionString ?? "");
_source = new SourcesTable(options.ConnectionString ?? "");
_logger = JobLogger.GetLogger(options.OpenTelemetry ?? "", JobName);
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _logger.Information($"{JobName} - Job was triggered.");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); if (!options.IsEnabled)
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
_logger.Information($"{JobName} - Job was triggered");
if (!options.Config.IsEnabled)
{ {
_logger.Information($"{JobName} - Going to exit because feature flag is off."); _logger.Information($"{JobName} - Going to exit because feature flag is off.");
return; return;
} }
_logger.Information($"{JobName} - Setting up the job"); _logger.Information($"{JobName} - Setting up the job.");
Execute(); Execute();
} }

View File

@ -50,10 +50,9 @@ public class RssWatcherJobTest
var client = new RssWatcherJob(); var client = new RssWatcherJob();
client.InitAndExecute(new RssWatcherJobOptions client.InitAndExecute(new RssWatcherJobOptions
{ {
ConnectionStrings = new ConfigSectionConnectionStrings ConnectionString = ConnectionString(),
{ OpenTelemetry = "",
Database = ConnectionString() IsEnabled = true
}
}); });
var items = client.Collect(url, Guid.NewGuid(), 0); var items = client.Collect(url, Guid.NewGuid(), 0);
client.UpdateDatabase(items); client.UpdateDatabase(items);
@ -66,10 +65,9 @@ public class RssWatcherJobTest
var client = new RssWatcherJob(); var client = new RssWatcherJob();
client.InitAndExecute(new RssWatcherJobOptions client.InitAndExecute(new RssWatcherJobOptions
{ {
ConnectionStrings = new ConfigSectionConnectionStrings ConnectionString = ConnectionString(),
{ OpenTelemetry = "",
Database = ConnectionString() IsEnabled = true
}
}); });
} }
} }

View File

@ -8,11 +8,10 @@ $ErrorActionPreference = 'Stop'
function New-RedditSource { function New-RedditSource {
param ( param (
[string] $Url [string] $Name
) )
$urlEncoded = [uri]::EscapeDataString($Url)
$param = "url=$urlEncoded" $param = "name=$Name"
$uri = "$ApiServer/api/sources/new/reddit?$param" $uri = "$ApiServer/api/sources/new/reddit?$param"
$res = Invoke-RestMethod -Method Post -Uri $uri $res = Invoke-RestMethod -Method Post -Uri $uri
return $res return $res
@ -32,7 +31,7 @@ function New-RssSource {
function New-YoutubeSource { function New-YoutubeSource {
param ( param (
[Parameter(Required)][string] $Url [string] $Url
) )
$urlEncoded = [uri]::EscapeDataString($Url) $urlEncoded = [uri]::EscapeDataString($Url)
[string] $param = "url=$urlEncoded" [string] $param = "url=$urlEncoded"
@ -90,7 +89,7 @@ $youtubeGameGrumps = New-YoutubeSource -Url "https://www.youtube.com/user/GameGr
$youtubeCityPlannerPlays = New-YoutubeSource -Url "https://www.youtube.com/c/cityplannerplays" $youtubeCityPlannerPlays = New-YoutubeSource -Url "https://www.youtube.com/c/cityplannerplays"
$youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTechTips" $youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTechTips"
$twitchNintendo = NewTwitchSource -Name "Nintendo" $twitchNintendo = New-TwitchSource -Name "Nintendo"
$miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01 $miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01