From 84b4137bdd2d268e7c05f31817ee702266267c28 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 13 Apr 2023 22:13:06 -0700 Subject: [PATCH] Features/codeproject/subscription options (#28) * Updated migrations to add new columns to subscriptions * repos updated with new columns * dto updated with new columns * subscription model was updated * DiscordNotificationJob.cs was updated to reflect subscription options * updated seed for codeproject subscriptions --- .../Controllers/SubscriptionsController.cs | 78 ++++++++++++------- .../20230326223545_icon_sourceid.sql | 9 --- ...230413203006_subscription_code_options.sql | 13 ++++ .../20230413212643_icon_sourceid.sql | 11 +++ .../Repositories/ArticlesTable.cs | 6 +- .../Repositories/SubscriptionsTable.cs | 13 ++-- .../Dto/SubscriptionDetailsDto.cs | 7 +- .../Dto/SubscriptionDto.cs | 16 ++-- .../Models/DatabaseModel.cs | 9 ++- .../Jobs/DiscordNotificationJob.cs | 56 +++++++------ .../Jobs/YoutubeWatcherJob.cs | 1 - seed.ps1 | 25 +++++- 12 files changed, 158 insertions(+), 86 deletions(-) create mode 100644 Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql create mode 100644 Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql diff --git a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs index 46a969c..733e9f5 100644 --- a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs +++ b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs @@ -11,11 +11,12 @@ namespace Newsbot.Collector.Api.Controllers; [Route("api/subscriptions")] public class SubscriptionsController : ControllerBase { - private readonly ILogger _logger; - private readonly ISubscriptionRepository _subscription; private readonly IDiscordWebHooksRepository _discord; + private readonly ILogger _logger; private readonly ISourcesRepository _sources; - public SubscriptionsController(ILogger logger, IOptions settings) + private readonly ISubscriptionRepository _subscription; + + public SubscriptionsController(ILogger logger, IOptions settings) { _logger = logger; _subscription = new SubscriptionsTable(settings.Value.Database); @@ -28,10 +29,7 @@ public class SubscriptionsController : ControllerBase { var res = new List(); var items = _subscription.List(page); - foreach (var item in items) - { - res.Add(SubscriptionDto.Convert(item)); - } + foreach (var item in items) res.Add(SubscriptionDto.Convert(item)); return res; } @@ -45,8 +43,8 @@ public class SubscriptionsController : ControllerBase public SubscriptionDetailsDto GetDetailsById(Guid id) { var sub = _subscription.GetById(id); - var webhook = _discord.GetByID(sub.DiscordWebHookID); - var source = _sources.GetByID(sub.SourceID); + var webhook = _discord.GetByID(sub.DiscordWebHookId); + var source = _sources.GetByID(sub.SourceId); return SubscriptionDetailsDto.Convert(sub, source, webhook); } @@ -62,10 +60,7 @@ public class SubscriptionsController : ControllerBase { var res = new List(); var items = _subscription.ListByWebhook(id); - foreach (var item in items) - { - res.Add(SubscriptionDto.Convert(item)); - } + foreach (var item in items) res.Add(SubscriptionDto.Convert(item)); return res; } @@ -74,30 +69,61 @@ public class SubscriptionsController : ControllerBase { var res = new List(); var items = _subscription.ListBySourceID(id); - foreach (var item in items) - { - res.Add(SubscriptionDto.Convert(item)); - } + foreach (var item in items) res.Add(SubscriptionDto.Convert(item)); return res; } [HttpPost(Name = "New Subscription")] - public SubscriptionDto New(Guid sourceId, Guid discordId) + public ActionResult New(Guid sourceId, Guid discordId) { + if (sourceId == Guid.Empty) return new BadRequestResult(); + if (discordId == Guid.Empty) return new BadRequestResult(); + var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); - // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - if (exists.ID != Guid.Empty) - { - return SubscriptionDto.Convert(exists); - } + if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists); + + var discord = _discord.GetByID(discordId); + if (discord.ID == Guid.Empty) return new BadRequestResult(); + + var source = _sources.GetByID(sourceId); + if (source.ID == Guid.Empty) return new BadRequestResult(); var item = _subscription.New(new SubscriptionModel { - ID = Guid.NewGuid(), - SourceID = sourceId, - DiscordWebHookID = discordId + Id = Guid.NewGuid(), + SourceId = sourceId, + DiscordWebHookId = discordId, + CodeAllowCommits = false, + CodeAllowReleases = false }); return SubscriptionDto.Convert(item); } + + [HttpPost("new/codeproject")] + public ActionResult NewCodeProjectSubscription(Guid sourceId, Guid discordId, bool allowReleases, + bool allowCommits) + { + if (sourceId == Guid.Empty) return new BadRequestResult(); + if (discordId == Guid.Empty) return new BadRequestResult(); + + var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); + if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists); + + var discord = _discord.GetByID(discordId); + if (discord.ID == Guid.Empty) return new BadRequestResult(); + + var source = _sources.GetByID(sourceId); + if (source.ID == Guid.Empty) return new BadRequestResult(); + + var sub = _subscription.New(new SubscriptionModel + { + DiscordWebHookId = discordId, + SourceId = sourceId, + CodeAllowCommits = allowCommits, + CodeAllowReleases = allowReleases + }); + + return new SubscriptionDto(); + } } \ No newline at end of file diff --git a/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql b/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql index 2a6228f..d1580b9 100644 --- a/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql +++ b/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql @@ -1,21 +1,12 @@ -- +goose Up -- +goose StatementBegin -SELECT 'up SQL query'; ALTER TABLE sources ADD COLUMN IconUri uuid; - -ALTER TABLE articles - ADD COLUMN CodeIsRelease bool, - ADD COLUMN CodeIsCommit bool; -- +goose StatementEnd -- +goose Down -- +goose StatementBegin -SELECT 'down SQL query'; ALTER TABLE sources DROP COLUMN IconUri; -Alter TABLE articles - DROP COLUMN CodeIsRelease, - DROP COLUMN CodeIsCommit -- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql b/Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql new file mode 100644 index 0000000..920c4e6 --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE subscriptions + ADD COLUMN CodeAllowReleases bool, + ADD COLUMN CodeAllowCommits bool; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE subscriptions + DROP COLUMN CodeAllowReleases, + DROP COLUMN CodeAllowCommits; +-- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql b/Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql new file mode 100644 index 0000000..1abc397 --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE icons + ADD COLUMN SourceId uuid; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE icons + Drop COLUMN SourceId; +-- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs index 376e091..168ce32 100644 --- a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs +++ b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs @@ -76,7 +76,7 @@ public class ArticlesTable : IArticlesRepository using var conn = OpenConnection(_connectionString); var q = - "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; + "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage, codeiscommit, codeisrelease) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage, @codeiscommit, @codeisrelease);"; var res = conn.Execute(q, new { id = model.ID, @@ -91,7 +91,9 @@ public class ArticlesTable : IArticlesRepository thumbnail = model.Thumbnail, description = model.Description, authorname = model.AuthorName, - authorimage = model.AuthorImage + authorimage = model.AuthorImage, + codeiscommit = model.CodeIsCommit, + codeisrelease = model.CodeIsRelease }); return model; } diff --git a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs index a492f79..61b30a6 100644 --- a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs +++ b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs @@ -25,16 +25,19 @@ public class SubscriptionsTable : ISubscriptionRepository public SubscriptionModel New(SubscriptionModel model) { - model.ID = Guid.NewGuid(); + model.Id = Guid.NewGuid(); using var conn = OpenConnection(_connectionString); - var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);"; + var query = + "Insert Into subscriptions (ID, DiscordWebHookId, SourceId, codeAllowCommits, codeAllowReleases) Values (@id, @webhookId, @sourceId, @codeAllowCommits, @codeAllowReleases);"; try { conn.Execute(query, new { - id = model.ID, - webhookid = model.DiscordWebHookID, - sourceid = model.SourceID + id = model.Id, + webhookId = model.DiscordWebHookId, + sourceId = model.SourceId, + codeAllowCommits = model.CodeAllowCommits, + codeAllowReleases = model.CodeAllowReleases }); return model; } diff --git a/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs b/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs index 70c8665..816a9ae 100644 --- a/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs +++ b/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs @@ -4,15 +4,16 @@ namespace Newsbot.Collector.Domain.Dto; public class SubscriptionDetailsDto { - public Guid ID { get; set; } + public Guid Id { get; set; } public SourceDto? Source { get; set; } public DiscordWebHookDto? DiscordWebHook { get; set; } - public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source, DiscordWebHookModel discord) + public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source, + DiscordWebHookModel discord) { return new SubscriptionDetailsDto { - ID = subscription.ID, + Id = subscription.Id, Source = SourceDto.Convert(source), DiscordWebHook = DiscordWebHookDto.Convert(discord) }; diff --git a/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs b/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs index e0132ce..1b6f1c2 100644 --- a/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs +++ b/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs @@ -4,17 +4,21 @@ namespace Newsbot.Collector.Domain.Dto; public class SubscriptionDto { - public Guid ID { get; set; } - public Guid SourceID { get; set; } - public Guid DiscordWebHookID { get; set; } + public Guid Id { get; set; } + public Guid SourceId { get; set; } + public Guid DiscordWebHookId { get; set; } + public bool CodeAllowReleases { get; set; } + public bool CodeAllowCommits { get; set; } public static SubscriptionDto Convert(SubscriptionModel model) { return new SubscriptionDto { - ID = model.ID, - SourceID = model.SourceID, - DiscordWebHookID = model.DiscordWebHookID + Id = model.Id, + SourceId = model.SourceId, + DiscordWebHookId = model.DiscordWebHookId, + CodeAllowCommits = model.CodeAllowCommits, + CodeAllowReleases = model.CodeAllowReleases }; } } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Models/DatabaseModel.cs b/Newsbot.Collector.Domain/Models/DatabaseModel.cs index 8a1a46e..f186ad2 100644 --- a/Newsbot.Collector.Domain/Models/DatabaseModel.cs +++ b/Newsbot.Collector.Domain/Models/DatabaseModel.cs @@ -73,11 +73,14 @@ public class SourceModel public string Tags { get; set; } = ""; public bool Deleted { get; set; } public string YoutubeId { get; set; } = ""; + } public class SubscriptionModel { - public Guid ID { get; set; } - public Guid DiscordWebHookID { get; set; } - public Guid SourceID { get; set; } + public Guid Id { get; set; } + public Guid DiscordWebHookId { get; set; } + public Guid SourceId { get; set; } + public bool CodeAllowReleases { get; set; } + public bool CodeAllowCommits { get; set; } } \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index 74b0722..0a4dbb5 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -1,9 +1,7 @@ using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; -using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.Notifications.Discord; -using Newtonsoft.Json.Linq; using Serilog; namespace Newsbot.Collector.Services.Jobs; @@ -18,15 +16,15 @@ public class DiscordNotificationJobOptions public class DiscordNotificationJob { private const string JobName = "DiscordNotifications"; - - private IDiscordQueueRepository _queue; private IArticlesRepository _article; - private IDiscordWebHooksRepository _webhook; - private ISourcesRepository _sources; - private ISubscriptionRepository _subs; private IIconsRepository _icons; private ILogger _logger; + private IDiscordQueueRepository _queue; + private ISourcesRepository _sources; + private ISubscriptionRepository _subs; + private IDiscordWebHooksRepository _webhook; + public DiscordNotificationJob() { _queue = new DiscordQueueTable(""); @@ -54,7 +52,7 @@ public class DiscordNotificationJob _logger.Warning($"{JobName} - Going to exit because feature flag is off."); return; } - + _logger.Information($"{JobName} - Starting up the job."); Execute(); } @@ -62,7 +60,7 @@ public class DiscordNotificationJob private void Execute() { //collect all the new requests - var requests = _queue.List(25); + var requests = _queue.List(100); _logger.Debug($"{JobName} - Collected {requests.Count} items to send"); foreach (var request in requests) @@ -75,7 +73,8 @@ public class DiscordNotificationJob var sourceDetails = _sources.GetByID(articleDetails.SourceID); if (sourceDetails.ID == Guid.Empty) { - _logger.Error($"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue."); + _logger.Error( + $"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue."); _queue.Delete(request.ID); continue; } @@ -95,12 +94,16 @@ public class DiscordNotificationJob foreach (var sub in allSubscriptions) { + // Check if the subscription code flags + // If the article is a code commit and the subscription does not want them, skip. + if (articleDetails.CodeIsCommit && sub.CodeAllowCommits == false) continue; + + // same for releases + if (articleDetails.CodeIsRelease && sub.CodeAllowReleases == false) continue; + // find the discord webhooks we need to post to - var discordDetails = _webhook.GetByID(sub.DiscordWebHookID); - if (discordDetails.Enabled == false) - { - continue; - } + var discordDetails = _webhook.GetByID(sub.DiscordWebHookId); + if (discordDetails.Enabled == false) continue; var client = new DiscordWebhookClient(discordDetails.Url); try @@ -113,7 +116,7 @@ public class DiscordNotificationJob _logger.Debug($"Queue Record: {request.ID}"); _logger.Debug($"Article: {articleDetails.ID}"); _logger.Debug($"Source: {sourceDetails.ID}"); - _logger.Debug($"Subscription: {sub.ID}"); + _logger.Debug($"Subscription: {sub.Id}"); } Thread.Sleep(3000); @@ -122,6 +125,7 @@ public class DiscordNotificationJob _logger.Debug($"{JobName} - Removing {request.ID} from the queue."); _queue.Delete(request.ID); } + _logger.Information($"{JobName} - Loop has been completed."); } @@ -139,40 +143,32 @@ public class DiscordNotificationJob }, Footer = new DiscordMessageEmbedFooter { - Text = "Brought to you by Newsbot", + Text = "Brought to you by Newsbot" }, Fields = new DiscordMessageEmbedField[] { - new DiscordMessageEmbedField + new() { Name = "Link", Value = article.URL, - Inline = false, + Inline = false } } }; - if (article.URL is not null && article.URL != "") - { - embed.Url = article.URL; - } + if (article.URL is not null && article.URL != "") embed.Url = article.URL; if (article.Thumbnail != "") - { embed.Image = new DiscordMessageEmbedImage { Url = article.Thumbnail }; - } - if (article.AuthorImage is not null && article.AuthorImage != "") - { - embed.Author.IconUrl = article.AuthorImage; - } + if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage; return new DiscordMessage { - Embeds = new DiscordMessageEmbed[] + Embeds = new[] { embed } diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index 984a6f3..3e742e5 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -69,7 +69,6 @@ public class YoutubeWatcherJob _source.UpdateYoutubeId(source.ID, channelId); } - // Make sure we have a Icon for the channel var icon = _icons.GetBySourceId(source.ID); if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V"); diff --git a/seed.ps1 b/seed.ps1 index 980603f..e6156e9 100644 --- a/seed.ps1 +++ b/seed.ps1 @@ -13,6 +13,8 @@ function New-RedditSource { $param = "name=$Name" $uri = "$ApiServer/api/sources/new/reddit?$param" + + Write-Host "Adding Reddit: $Name" $res = Invoke-RestMethod -Method Post -Uri $uri return $res } @@ -24,6 +26,8 @@ function New-RssSource { ) $urlEncoded = [uri]::EscapeDataString($Url) $param = "name=$Name&url=$urlEncoded" + + Write-Host "Adding RSS: $Name" [string] $uri = "$ApiServer/api/sources/new/rss?$param" $res = Invoke-RestMethod -Method Post -Uri $uri return $res @@ -36,6 +40,8 @@ function New-YoutubeSource { $urlEncoded = [uri]::EscapeDataString($Url) [string] $param = "url=$urlEncoded" [string] $uri = "$ApiServer/api/sources/new/youtube?$param" + + Write-Host = "Adding YouTube: $Url" $res = Invoke-RestMethod -Method Post -Uri $uri return $res } @@ -46,6 +52,8 @@ function New-TwitchSource { ) [string] $param = "name=$Name" [string] $uri = "$ApiServer/api/sources/new/twitch?$param" + + Write-Host "Adding Twitch: $Name" $res = Invoke-RestMethod -Method Post -Uri $uri return $res } @@ -78,6 +86,8 @@ function New-DiscordWebhook { return $res } + + function New-Subscription { param ( [string] $SourceId, @@ -89,6 +99,19 @@ function New-Subscription { return $res } +function New-CodeProjectSubscription { + param ( + [string] $SourceId, + [string] $DiscordWebhookId, + [switch] $AllowReleases = $false, + [switch] $AllowCommits = $false + ) + [string] $param = "sourceId=$SourceId&discordId=$DiscordWebhookId&allowReleases=$AllowReleases&allowCommits=$AllowCommits" + [string] $uri = "$ApiServer/api/subscriptions/new/codeproject?$param" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + # Load Secrets file $secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json @@ -130,7 +153,7 @@ New-Subscription -SourceId $rssOmgLinux.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id -New-Subscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id +New-CodeProjectSubscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id -AllowReleases New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id