jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Features/less comples config managment (#16)

Browse Source 
* New single const file for all config info

* seed was updated to reflect route params

* Adjusting config loads to make it easier to follow

* test updates for config loading
This commit is contained in:
James Tombleson 2023-04-01 08:53:34 -07:00 committed by GitHub
parent b7b50f5124
commit 09fe274752
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 92 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Newsbot.Collector.Api/BackgroundJobs.cs
View File

@ -5,22 +5,24 @@ using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api;
public class BackgroundJobs
public static class BackgroundJobs
{
public static void SetupRecurringJobs(IConfiguration configuration)
{
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x =>
x.InitAndExecute(new RssWatcherJobOptions
{
ConnectionStrings =
configuration.GetValue<ConfigSectionConnectionStrings>(ConfigSectionsConst.ConnectionStrings),
Config = configuration.GetValue<ConfigSectionRssModel>(ConfigSectionsConst.Rss)
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.RedditIsEnabled)
}), "15 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
x.InitAndExecute(new DiscordNotificationJobOptions
{
ConnectionStrings = configuration.GetValue<ConfigSectionConnectionStrings>(ConfigSectionsConst.ConnectionStrings),
Config = configuration.GetValue<ConfigSectionNotificationsDiscord>(ConfigSectionsConst.NotificationsDiscord)
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled),
}), "5/10 * * * *");
}
}

5
Newsbot.Collector.Api/Controllers/JobsController.cs
View File

@ -31,8 +31,9 @@ public class JobsController
{
BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions
{
ConnectionStrings = _connectionStrings,
Config = _rssConfig
ConnectionString = _connectionStrings.Database,
OpenTelemetry = _connectionStrings.OpenTelemetry,
IsEnabled = _rssConfig.IsEnabled
}));
}

33
Newsbot.Collector.Domain/Consts/ConfigConst.cs Normal file
View File

@ -0,0 +1,33 @@
namespace Newsbot.Collector.Domain.Consts;
public class ConfigConst
{
public const string SectionConnectionStrings = "ConnectionStrings";
public const string SectionFinalFantasyXiv = "FinalFantasyXiv";
public const string SectionReddit = "Reddit";
public const string SectionRss = "Rss";
public const string SectionTwitch = "Twitch";
public const string SectionYoutube = "Youtube";
public const string SectionNotificationsDiscord = "Notifications:Discord";
public const string ConnectionStringDatabase = "ConnectionStrings:Database";
public const string ConnectionStringOpenTelemetry = "ConnectionStrings:OpenTelemetry";
public const string RedditIsEnabled = "Reddit:IsEnabled";
public const string RedditPullHot = "Reddit:PullHot";
public const string RedditPullNsfw = "Reddit:PullNsfw";
public const string RedditPullTop = "Reddit:PullTop";
public const string RssIsEnabled = "Rss:IsEnabled";
public const string TwitchIsEnabled = "Twitch:IsEnabled";
public const string TwitchClientId = "Twitch:ClientID";
public const string TwitchClientSecret = "Twitch:ClientSecret";
public const string YoutubeIsEnable = "Youtube:IsEnabled";
public const string YoutubeDebug = "Youtube:Debug";
public const string EnableSwagger = "EnableSwagger";
public const string DiscordNotificationsEnabled = "Notifications:Discord:IsEnabled";
}

43
Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs
View File

@ -3,26 +3,29 @@ using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Notifications.Discord;
using Newtonsoft.Json.Linq;
using Serilog;
namespace Newsbot.Collector.Services.Jobs;
public class DiscordNotificationJobOptions
{
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
public ConfigSectionNotificationsDiscord? Config { get; set; }
public string? ConnectionString { get; init; }
public string? OpenTelemetry { get; init; }
public bool IsEnabled { get; init; }
}
public class DiscordNotificationJob
{
private const string JobName = "DiscordNotifications";
private IDiscordQueueRepository _queue;
private IArticlesRepository _article;
private IDiscordWebHooksRepository _webhook;
private ISourcesRepository _sources;
private ISubscriptionRepository _subs;
private IIconsRepository _icons;
private ILogger _logger;
public DiscordNotificationJob()
{
@ -32,21 +35,27 @@ public class DiscordNotificationJob
_sources = new SourcesTable("");
_subs = new SubscriptionsTable("");
_icons = new IconsTable("");
_logger = JobLogger.GetLogger("", JobName);
}
public void InitAndExecute(DiscordNotificationJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
options.Config ??= new ConfigSectionNotificationsDiscord();
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_article = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_webhook = new DiscordWebhooksTable(options.ConnectionStrings.Database ?? "");
_sources = new SourcesTable(options.ConnectionStrings.Database ?? "");
_subs = new SubscriptionsTable(options.ConnectionStrings.Database ?? "");
_icons = new IconsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionString ?? "");
_article = new ArticlesTable(options.ConnectionString ?? "");
_webhook = new DiscordWebhooksTable(options.ConnectionString ?? "");
_sources = new SourcesTable(options.ConnectionString ?? "");
_subs = new SubscriptionsTable(options.ConnectionString ?? "");
_icons = new IconsTable(options.ConnectionString ?? "");
_logger = JobLogger.GetLogger(options.OpenTelemetry ?? "", JobName);
if (!options.IsEnabled)
{
_logger.Warning($"{JobName} - Going to exit because feature flag is off.");
return;
}
_logger.Information($"{JobName} - Starting up the job.");
Execute();
}
@ -60,11 +69,11 @@ public class DiscordNotificationJob
// Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleID);
// Get the deatils of the source
// Get the details of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty)
{
Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue");
_logger.Error($"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue");
_queue.Delete(request.ID);
continue;
}
@ -90,7 +99,7 @@ public class DiscordNotificationJob
}
catch (Exception e)
{
Log.Error($"Failed to post message to Discord. {e}");
_logger.Error($"Failed to post message to Discord. {e}");
continue;
}
@ -133,7 +142,7 @@ public class DiscordNotificationJob
embed.Url = article.URL;
}
if (article.Thumbnail is not null && article.Thumbnail != "")
if (article.Thumbnail != "")
{
embed.Image = new DiscordMessageEmbedImage
{

25
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -12,11 +12,9 @@ namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJobOptions
{
//public string? ConnectionString { get; init; }
//public string? OpenTelemetry { get; init; }
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
public ConfigSectionRssModel? Config { get; set; }
public string? ConnectionString { get; init; }
public string? OpenTelemetry { get; init; }
public bool IsEnabled { get; init; }
}
// This class was made to work with Hangfire and it does not support constructors.
@ -39,22 +37,19 @@ public class RssWatcherJob
public void InitAndExecute(RssWatcherJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
options.Config ??= new ConfigSectionRssModel();
_articles = new ArticlesTable(options.ConnectionString ?? "");
_queue = new DiscordQueueTable(options.ConnectionString ?? "");
_source = new SourcesTable(options.ConnectionString ?? "");
_logger = JobLogger.GetLogger(options.OpenTelemetry ?? "", JobName);
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
_logger.Information($"{JobName} - Job was triggered");
if (!options.Config.IsEnabled)
_logger.Information($"{JobName} - Job was triggered.");
if (!options.IsEnabled)
{
_logger.Information($"{JobName} - Going to exit because feature flag is off.");
return;
}
_logger.Information($"{JobName} - Setting up the job");
_logger.Information($"{JobName} - Setting up the job.");
Execute();
}

14
Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs
View File

@ -50,10 +50,9 @@ public class RssWatcherJobTest
var client = new RssWatcherJob();
client.InitAndExecute(new RssWatcherJobOptions
{
ConnectionStrings = new ConfigSectionConnectionStrings
{
Database = ConnectionString()
}
ConnectionString = ConnectionString(),
OpenTelemetry = "",
IsEnabled = true
});
var items = client.Collect(url, Guid.NewGuid(), 0);
client.UpdateDatabase(items);
@ -66,10 +65,9 @@ public class RssWatcherJobTest
var client = new RssWatcherJob();
client.InitAndExecute(new RssWatcherJobOptions
{
ConnectionStrings = new ConfigSectionConnectionStrings
{
Database = ConnectionString()
}
ConnectionString = ConnectionString(),
OpenTelemetry = "",
IsEnabled = true
});
}
}

9
seed.ps1
View File

@ -8,11 +8,10 @@ $ErrorActionPreference = 'Stop'
function New-RedditSource {
param (
[string] $Url
[string] $Name
)
$urlEncoded = [uri]::EscapeDataString($Url)
$param = "url=$urlEncoded"
$param = "name=$Name"
$uri = "$ApiServer/api/sources/new/reddit?$param"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
@ -32,7 +31,7 @@ function New-RssSource {
function New-YoutubeSource {
param (
[Parameter(Required)][string] $Url
[string] $Url
)
$urlEncoded = [uri]::EscapeDataString($Url)
[string] $param = "url=$urlEncoded"
@ -90,7 +89,7 @@ $youtubeGameGrumps = New-YoutubeSource -Url "https://www.youtube.com/user/GameGr
$youtubeCityPlannerPlays = New-YoutubeSource -Url "https://www.youtube.com/c/cityplannerplays"
$youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTechTips"
$twitchNintendo = NewTwitchSource -Name "Nintendo"
$twitchNintendo = New-TwitchSource -Name "Nintendo"
$miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01