Features/codeproject/subscription options (#28)

* Updated migrations to add new columns to subscriptions

* repos updated with new columns

* dto updated with new columns

* subscription model was updated

* DiscordNotificationJob.cs was updated to reflect subscription options

* updated seed for codeproject subscriptions
This commit is contained in:
James Tombleson 2023-04-13 22:13:06 -07:00 committed by GitHub
parent 02c94d442e
commit 84b4137bdd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 158 additions and 86 deletions

View File

@ -11,11 +11,12 @@ namespace Newsbot.Collector.Api.Controllers;
[Route("api/subscriptions")] [Route("api/subscriptions")]
public class SubscriptionsController : ControllerBase public class SubscriptionsController : ControllerBase
{ {
private readonly ILogger<ArticlesController> _logger;
private readonly ISubscriptionRepository _subscription;
private readonly IDiscordWebHooksRepository _discord; private readonly IDiscordWebHooksRepository _discord;
private readonly ILogger<SubscriptionsController> _logger;
private readonly ISourcesRepository _sources; private readonly ISourcesRepository _sources;
public SubscriptionsController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings) private readonly ISubscriptionRepository _subscription;
public SubscriptionsController(ILogger<SubscriptionsController> logger, IOptions<ConnectionStrings> settings)
{ {
_logger = logger; _logger = logger;
_subscription = new SubscriptionsTable(settings.Value.Database); _subscription = new SubscriptionsTable(settings.Value.Database);
@ -28,10 +29,7 @@ public class SubscriptionsController : ControllerBase
{ {
var res = new List<SubscriptionDto>(); var res = new List<SubscriptionDto>();
var items = _subscription.List(page); var items = _subscription.List(page);
foreach (var item in items) foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
{
res.Add(SubscriptionDto.Convert(item));
}
return res; return res;
} }
@ -45,8 +43,8 @@ public class SubscriptionsController : ControllerBase
public SubscriptionDetailsDto GetDetailsById(Guid id) public SubscriptionDetailsDto GetDetailsById(Guid id)
{ {
var sub = _subscription.GetById(id); var sub = _subscription.GetById(id);
var webhook = _discord.GetByID(sub.DiscordWebHookID); var webhook = _discord.GetByID(sub.DiscordWebHookId);
var source = _sources.GetByID(sub.SourceID); var source = _sources.GetByID(sub.SourceId);
return SubscriptionDetailsDto.Convert(sub, source, webhook); return SubscriptionDetailsDto.Convert(sub, source, webhook);
} }
@ -62,10 +60,7 @@ public class SubscriptionsController : ControllerBase
{ {
var res = new List<SubscriptionDto>(); var res = new List<SubscriptionDto>();
var items = _subscription.ListByWebhook(id); var items = _subscription.ListByWebhook(id);
foreach (var item in items) foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
{
res.Add(SubscriptionDto.Convert(item));
}
return res; return res;
} }
@ -74,30 +69,61 @@ public class SubscriptionsController : ControllerBase
{ {
var res = new List<SubscriptionDto>(); var res = new List<SubscriptionDto>();
var items = _subscription.ListBySourceID(id); var items = _subscription.ListBySourceID(id);
foreach (var item in items) foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
{
res.Add(SubscriptionDto.Convert(item));
}
return res; return res;
} }
[HttpPost(Name = "New Subscription")] [HttpPost(Name = "New Subscription")]
public SubscriptionDto New(Guid sourceId, Guid discordId) public ActionResult<SubscriptionDto> New(Guid sourceId, Guid discordId)
{ {
if (sourceId == Guid.Empty) return new BadRequestResult();
if (discordId == Guid.Empty) return new BadRequestResult();
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
if (exists.ID != Guid.Empty)
{ var discord = _discord.GetByID(discordId);
return SubscriptionDto.Convert(exists); if (discord.ID == Guid.Empty) return new BadRequestResult();
}
var source = _sources.GetByID(sourceId);
if (source.ID == Guid.Empty) return new BadRequestResult();
var item = _subscription.New(new SubscriptionModel var item = _subscription.New(new SubscriptionModel
{ {
ID = Guid.NewGuid(), Id = Guid.NewGuid(),
SourceID = sourceId, SourceId = sourceId,
DiscordWebHookID = discordId DiscordWebHookId = discordId,
CodeAllowCommits = false,
CodeAllowReleases = false
}); });
return SubscriptionDto.Convert(item); return SubscriptionDto.Convert(item);
} }
[HttpPost("new/codeproject")]
public ActionResult<SubscriptionDto> NewCodeProjectSubscription(Guid sourceId, Guid discordId, bool allowReleases,
bool allowCommits)
{
if (sourceId == Guid.Empty) return new BadRequestResult();
if (discordId == Guid.Empty) return new BadRequestResult();
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetByID(discordId);
if (discord.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetByID(sourceId);
if (source.ID == Guid.Empty) return new BadRequestResult();
var sub = _subscription.New(new SubscriptionModel
{
DiscordWebHookId = discordId,
SourceId = sourceId,
CodeAllowCommits = allowCommits,
CodeAllowReleases = allowReleases
});
return new SubscriptionDto();
}
} }

View File

@ -1,21 +1,12 @@
-- +goose Up -- +goose Up
-- +goose StatementBegin -- +goose StatementBegin
SELECT 'up SQL query';
ALTER TABLE sources ALTER TABLE sources
ADD COLUMN IconUri uuid; ADD COLUMN IconUri uuid;
ALTER TABLE articles
ADD COLUMN CodeIsRelease bool,
ADD COLUMN CodeIsCommit bool;
-- +goose StatementEnd -- +goose StatementEnd
-- +goose Down -- +goose Down
-- +goose StatementBegin -- +goose StatementBegin
SELECT 'down SQL query';
ALTER TABLE sources ALTER TABLE sources
DROP COLUMN IconUri; DROP COLUMN IconUri;
Alter TABLE articles
DROP COLUMN CodeIsRelease,
DROP COLUMN CodeIsCommit
-- +goose StatementEnd -- +goose StatementEnd

View File

@ -0,0 +1,13 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE subscriptions
ADD COLUMN CodeAllowReleases bool,
ADD COLUMN CodeAllowCommits bool;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE subscriptions
DROP COLUMN CodeAllowReleases,
DROP COLUMN CodeAllowCommits;
-- +goose StatementEnd

View File

@ -0,0 +1,11 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE icons
ADD COLUMN SourceId uuid;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE icons
Drop COLUMN SourceId;
-- +goose StatementEnd

View File

@ -76,7 +76,7 @@ public class ArticlesTable : IArticlesRepository
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var q = var q =
"INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage, codeiscommit, codeisrelease) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage, @codeiscommit, @codeisrelease);";
var res = conn.Execute(q, new var res = conn.Execute(q, new
{ {
id = model.ID, id = model.ID,
@ -91,7 +91,9 @@ public class ArticlesTable : IArticlesRepository
thumbnail = model.Thumbnail, thumbnail = model.Thumbnail,
description = model.Description, description = model.Description,
authorname = model.AuthorName, authorname = model.AuthorName,
authorimage = model.AuthorImage authorimage = model.AuthorImage,
codeiscommit = model.CodeIsCommit,
codeisrelease = model.CodeIsRelease
}); });
return model; return model;
} }

View File

@ -25,16 +25,19 @@ public class SubscriptionsTable : ISubscriptionRepository
public SubscriptionModel New(SubscriptionModel model) public SubscriptionModel New(SubscriptionModel model)
{ {
model.ID = Guid.NewGuid(); model.Id = Guid.NewGuid();
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);"; var query =
"Insert Into subscriptions (ID, DiscordWebHookId, SourceId, codeAllowCommits, codeAllowReleases) Values (@id, @webhookId, @sourceId, @codeAllowCommits, @codeAllowReleases);";
try try
{ {
conn.Execute(query, new conn.Execute(query, new
{ {
id = model.ID, id = model.Id,
webhookid = model.DiscordWebHookID, webhookId = model.DiscordWebHookId,
sourceid = model.SourceID sourceId = model.SourceId,
codeAllowCommits = model.CodeAllowCommits,
codeAllowReleases = model.CodeAllowReleases
}); });
return model; return model;
} }

View File

@ -4,15 +4,16 @@ namespace Newsbot.Collector.Domain.Dto;
public class SubscriptionDetailsDto public class SubscriptionDetailsDto
{ {
public Guid ID { get; set; } public Guid Id { get; set; }
public SourceDto? Source { get; set; } public SourceDto? Source { get; set; }
public DiscordWebHookDto? DiscordWebHook { get; set; } public DiscordWebHookDto? DiscordWebHook { get; set; }
public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source, DiscordWebHookModel discord) public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source,
DiscordWebHookModel discord)
{ {
return new SubscriptionDetailsDto return new SubscriptionDetailsDto
{ {
ID = subscription.ID, Id = subscription.Id,
Source = SourceDto.Convert(source), Source = SourceDto.Convert(source),
DiscordWebHook = DiscordWebHookDto.Convert(discord) DiscordWebHook = DiscordWebHookDto.Convert(discord)
}; };

View File

@ -4,17 +4,21 @@ namespace Newsbot.Collector.Domain.Dto;
public class SubscriptionDto public class SubscriptionDto
{ {
public Guid ID { get; set; } public Guid Id { get; set; }
public Guid SourceID { get; set; } public Guid SourceId { get; set; }
public Guid DiscordWebHookID { get; set; } public Guid DiscordWebHookId { get; set; }
public bool CodeAllowReleases { get; set; }
public bool CodeAllowCommits { get; set; }
public static SubscriptionDto Convert(SubscriptionModel model) public static SubscriptionDto Convert(SubscriptionModel model)
{ {
return new SubscriptionDto return new SubscriptionDto
{ {
ID = model.ID, Id = model.Id,
SourceID = model.SourceID, SourceId = model.SourceId,
DiscordWebHookID = model.DiscordWebHookID DiscordWebHookId = model.DiscordWebHookId,
CodeAllowCommits = model.CodeAllowCommits,
CodeAllowReleases = model.CodeAllowReleases
}; };
} }
} }

View File

@ -73,11 +73,14 @@ public class SourceModel
public string Tags { get; set; } = ""; public string Tags { get; set; } = "";
public bool Deleted { get; set; } public bool Deleted { get; set; }
public string YoutubeId { get; set; } = ""; public string YoutubeId { get; set; } = "";
} }
public class SubscriptionModel public class SubscriptionModel
{ {
public Guid ID { get; set; } public Guid Id { get; set; }
public Guid DiscordWebHookID { get; set; } public Guid DiscordWebHookId { get; set; }
public Guid SourceID { get; set; } public Guid SourceId { get; set; }
public bool CodeAllowReleases { get; set; }
public bool CodeAllowCommits { get; set; }
} }

View File

@ -1,9 +1,7 @@
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
using Newtonsoft.Json.Linq;
using Serilog; using Serilog;
namespace Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Services.Jobs;
@ -18,15 +16,15 @@ public class DiscordNotificationJobOptions
public class DiscordNotificationJob public class DiscordNotificationJob
{ {
private const string JobName = "DiscordNotifications"; private const string JobName = "DiscordNotifications";
private IDiscordQueueRepository _queue;
private IArticlesRepository _article; private IArticlesRepository _article;
private IDiscordWebHooksRepository _webhook;
private ISourcesRepository _sources;
private ISubscriptionRepository _subs;
private IIconsRepository _icons; private IIconsRepository _icons;
private ILogger _logger; private ILogger _logger;
private IDiscordQueueRepository _queue;
private ISourcesRepository _sources;
private ISubscriptionRepository _subs;
private IDiscordWebHooksRepository _webhook;
public DiscordNotificationJob() public DiscordNotificationJob()
{ {
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
@ -54,7 +52,7 @@ public class DiscordNotificationJob
_logger.Warning($"{JobName} - Going to exit because feature flag is off."); _logger.Warning($"{JobName} - Going to exit because feature flag is off.");
return; return;
} }
_logger.Information($"{JobName} - Starting up the job."); _logger.Information($"{JobName} - Starting up the job.");
Execute(); Execute();
} }
@ -62,7 +60,7 @@ public class DiscordNotificationJob
private void Execute() private void Execute()
{ {
//collect all the new requests //collect all the new requests
var requests = _queue.List(25); var requests = _queue.List(100);
_logger.Debug($"{JobName} - Collected {requests.Count} items to send"); _logger.Debug($"{JobName} - Collected {requests.Count} items to send");
foreach (var request in requests) foreach (var request in requests)
@ -75,7 +73,8 @@ public class DiscordNotificationJob
var sourceDetails = _sources.GetByID(articleDetails.SourceID); var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty) if (sourceDetails.ID == Guid.Empty)
{ {
_logger.Error($"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue."); _logger.Error(
$"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.ID); _queue.Delete(request.ID);
continue; continue;
} }
@ -95,12 +94,16 @@ public class DiscordNotificationJob
foreach (var sub in allSubscriptions) foreach (var sub in allSubscriptions)
{ {
// Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip.
if (articleDetails.CodeIsCommit && sub.CodeAllowCommits == false) continue;
// same for releases
if (articleDetails.CodeIsRelease && sub.CodeAllowReleases == false) continue;
// find the discord webhooks we need to post to // find the discord webhooks we need to post to
var discordDetails = _webhook.GetByID(sub.DiscordWebHookID); var discordDetails = _webhook.GetByID(sub.DiscordWebHookId);
if (discordDetails.Enabled == false) if (discordDetails.Enabled == false) continue;
{
continue;
}
var client = new DiscordWebhookClient(discordDetails.Url); var client = new DiscordWebhookClient(discordDetails.Url);
try try
@ -113,7 +116,7 @@ public class DiscordNotificationJob
_logger.Debug($"Queue Record: {request.ID}"); _logger.Debug($"Queue Record: {request.ID}");
_logger.Debug($"Article: {articleDetails.ID}"); _logger.Debug($"Article: {articleDetails.ID}");
_logger.Debug($"Source: {sourceDetails.ID}"); _logger.Debug($"Source: {sourceDetails.ID}");
_logger.Debug($"Subscription: {sub.ID}"); _logger.Debug($"Subscription: {sub.Id}");
} }
Thread.Sleep(3000); Thread.Sleep(3000);
@ -122,6 +125,7 @@ public class DiscordNotificationJob
_logger.Debug($"{JobName} - Removing {request.ID} from the queue."); _logger.Debug($"{JobName} - Removing {request.ID} from the queue.");
_queue.Delete(request.ID); _queue.Delete(request.ID);
} }
_logger.Information($"{JobName} - Loop has been completed."); _logger.Information($"{JobName} - Loop has been completed.");
} }
@ -139,40 +143,32 @@ public class DiscordNotificationJob
}, },
Footer = new DiscordMessageEmbedFooter Footer = new DiscordMessageEmbedFooter
{ {
Text = "Brought to you by Newsbot", Text = "Brought to you by Newsbot"
}, },
Fields = new DiscordMessageEmbedField[] Fields = new DiscordMessageEmbedField[]
{ {
new DiscordMessageEmbedField new()
{ {
Name = "Link", Name = "Link",
Value = article.URL, Value = article.URL,
Inline = false, Inline = false
} }
} }
}; };
if (article.URL is not null && article.URL != "") if (article.URL is not null && article.URL != "") embed.Url = article.URL;
{
embed.Url = article.URL;
}
if (article.Thumbnail != "") if (article.Thumbnail != "")
{
embed.Image = new DiscordMessageEmbedImage embed.Image = new DiscordMessageEmbedImage
{ {
Url = article.Thumbnail Url = article.Thumbnail
}; };
}
if (article.AuthorImage is not null && article.AuthorImage != "") if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage;
{
embed.Author.IconUrl = article.AuthorImage;
}
return new DiscordMessage return new DiscordMessage
{ {
Embeds = new DiscordMessageEmbed[] Embeds = new[]
{ {
embed embed
} }

View File

@ -69,7 +69,6 @@ public class YoutubeWatcherJob
_source.UpdateYoutubeId(source.ID, channelId); _source.UpdateYoutubeId(source.ID, channelId);
} }
// Make sure we have a Icon for the channel // Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.ID); var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V"); if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");

View File

@ -13,6 +13,8 @@ function New-RedditSource {
$param = "name=$Name" $param = "name=$Name"
$uri = "$ApiServer/api/sources/new/reddit?$param" $uri = "$ApiServer/api/sources/new/reddit?$param"
Write-Host "Adding Reddit: $Name"
$res = Invoke-RestMethod -Method Post -Uri $uri $res = Invoke-RestMethod -Method Post -Uri $uri
return $res return $res
} }
@ -24,6 +26,8 @@ function New-RssSource {
) )
$urlEncoded = [uri]::EscapeDataString($Url) $urlEncoded = [uri]::EscapeDataString($Url)
$param = "name=$Name&url=$urlEncoded" $param = "name=$Name&url=$urlEncoded"
Write-Host "Adding RSS: $Name"
[string] $uri = "$ApiServer/api/sources/new/rss?$param" [string] $uri = "$ApiServer/api/sources/new/rss?$param"
$res = Invoke-RestMethod -Method Post -Uri $uri $res = Invoke-RestMethod -Method Post -Uri $uri
return $res return $res
@ -36,6 +40,8 @@ function New-YoutubeSource {
$urlEncoded = [uri]::EscapeDataString($Url) $urlEncoded = [uri]::EscapeDataString($Url)
[string] $param = "url=$urlEncoded" [string] $param = "url=$urlEncoded"
[string] $uri = "$ApiServer/api/sources/new/youtube?$param" [string] $uri = "$ApiServer/api/sources/new/youtube?$param"
Write-Host = "Adding YouTube: $Url"
$res = Invoke-RestMethod -Method Post -Uri $uri $res = Invoke-RestMethod -Method Post -Uri $uri
return $res return $res
} }
@ -46,6 +52,8 @@ function New-TwitchSource {
) )
[string] $param = "name=$Name" [string] $param = "name=$Name"
[string] $uri = "$ApiServer/api/sources/new/twitch?$param" [string] $uri = "$ApiServer/api/sources/new/twitch?$param"
Write-Host "Adding Twitch: $Name"
$res = Invoke-RestMethod -Method Post -Uri $uri $res = Invoke-RestMethod -Method Post -Uri $uri
return $res return $res
} }
@ -78,6 +86,8 @@ function New-DiscordWebhook {
return $res return $res
} }
function New-Subscription { function New-Subscription {
param ( param (
[string] $SourceId, [string] $SourceId,
@ -89,6 +99,19 @@ function New-Subscription {
return $res return $res
} }
function New-CodeProjectSubscription {
param (
[string] $SourceId,
[string] $DiscordWebhookId,
[switch] $AllowReleases = $false,
[switch] $AllowCommits = $false
)
[string] $param = "sourceId=$SourceId&discordId=$DiscordWebhookId&allowReleases=$AllowReleases&allowCommits=$AllowCommits"
[string] $uri = "$ApiServer/api/subscriptions/new/codeproject?$param"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
}
# Load Secrets file # Load Secrets file
$secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json $secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json
@ -130,7 +153,7 @@ New-Subscription -SourceId $rssOmgLinux.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id New-CodeProjectSubscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id -AllowReleases
New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id