jtom38
/
Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Features/codeproject/subscription options (#28) 

* Updated migrations to add new columns to subscriptions

* repos updated with new columns

* dto updated with new columns

* subscription model was updated

* DiscordNotificationJob.cs was updated to reflect subscription options

* updated seed for codeproject subscriptions
This commit is contained in:
James Tombleson 2023-04-13 22:13:06 -07:00 committed by GitHub
parent 02c94d442e
commit 84b4137bdd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 158 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
Newsbot.Collector.Api/Controllers/SubscriptionsController.cs
View File

@ -11,11 +11,12 @@ namespace Newsbot.Collector.Api.Controllers;
[Route("api/subscriptions")]
public class SubscriptionsController : ControllerBase
{
private readonly ILogger<ArticlesController> _logger;
private readonly ISubscriptionRepository _subscription;
private readonly IDiscordWebHooksRepository _discord;
private readonly ILogger<SubscriptionsController> _logger;
private readonly ISourcesRepository _sources;
public SubscriptionsController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
private readonly ISubscriptionRepository _subscription;
public SubscriptionsController(ILogger<SubscriptionsController> logger, IOptions<ConnectionStrings> settings)
{
_logger = logger;
_subscription = new SubscriptionsTable(settings.Value.Database);
@ -28,10 +29,7 @@ public class SubscriptionsController : ControllerBase
{
var res = new List<SubscriptionDto>();
var items = _subscription.List(page);
foreach (var item in items)
{
res.Add(SubscriptionDto.Convert(item));
}
foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
return res;
}
@ -45,8 +43,8 @@ public class SubscriptionsController : ControllerBase
public SubscriptionDetailsDto GetDetailsById(Guid id)
{
var sub = _subscription.GetById(id);
var webhook = _discord.GetByID(sub.DiscordWebHookID);
var source = _sources.GetByID(sub.SourceID);
var webhook = _discord.GetByID(sub.DiscordWebHookId);
var source = _sources.GetByID(sub.SourceId);
return SubscriptionDetailsDto.Convert(sub, source, webhook);
}
@ -62,10 +60,7 @@ public class SubscriptionsController : ControllerBase
{
var res = new List<SubscriptionDto>();
var items = _subscription.ListByWebhook(id);
foreach (var item in items)
{
res.Add(SubscriptionDto.Convert(item));
}
foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
return res;
}
@ -74,30 +69,61 @@ public class SubscriptionsController : ControllerBase
{
var res = new List<SubscriptionDto>();
var items = _subscription.ListBySourceID(id);
foreach (var item in items)
{
res.Add(SubscriptionDto.Convert(item));
}
foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
return res;
}
[HttpPost(Name = "New Subscription")]
public SubscriptionDto New(Guid sourceId, Guid discordId)
public ActionResult<SubscriptionDto> New(Guid sourceId, Guid discordId)
{
if (sourceId == Guid.Empty) return new BadRequestResult();
if (discordId == Guid.Empty) return new BadRequestResult();
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (exists.ID != Guid.Empty)
{
return SubscriptionDto.Convert(exists);
}
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetByID(discordId);
if (discord.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetByID(sourceId);
if (source.ID == Guid.Empty) return new BadRequestResult();
var item = _subscription.New(new SubscriptionModel
{
ID = Guid.NewGuid(),
SourceID = sourceId,
DiscordWebHookID = discordId
Id = Guid.NewGuid(),
SourceId = sourceId,
DiscordWebHookId = discordId,
CodeAllowCommits = false,
CodeAllowReleases = false
});
return SubscriptionDto.Convert(item);
}
[HttpPost("new/codeproject")]
public ActionResult<SubscriptionDto> NewCodeProjectSubscription(Guid sourceId, Guid discordId, bool allowReleases,
bool allowCommits)
{
if (sourceId == Guid.Empty) return new BadRequestResult();
if (discordId == Guid.Empty) return new BadRequestResult();
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetByID(discordId);
if (discord.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetByID(sourceId);
if (source.ID == Guid.Empty) return new BadRequestResult();
var sub = _subscription.New(new SubscriptionModel
{
DiscordWebHookId = discordId,
SourceId = sourceId,
CodeAllowCommits = allowCommits,
CodeAllowReleases = allowReleases
});
return new SubscriptionDto();
}
}

9
Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql
View File

@ -1,21 +1,12 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
ALTER TABLE sources
ADD COLUMN IconUri uuid;
ALTER TABLE articles
ADD COLUMN CodeIsRelease bool,
ADD COLUMN CodeIsCommit bool;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
SELECT 'down SQL query';
ALTER TABLE sources
DROP COLUMN IconUri;
Alter TABLE articles
DROP COLUMN CodeIsRelease,
DROP COLUMN CodeIsCommit
-- +goose StatementEnd

13
Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql Normal file
View File

@ -0,0 +1,13 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE subscriptions
ADD COLUMN CodeAllowReleases bool,
ADD COLUMN CodeAllowCommits bool;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE subscriptions
DROP COLUMN CodeAllowReleases,
DROP COLUMN CodeAllowCommits;
-- +goose StatementEnd

11
Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql Normal file
View File

@ -0,0 +1,11 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE icons
ADD COLUMN SourceId uuid;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE icons
Drop COLUMN SourceId;
-- +goose StatementEnd

6
Newsbot.Collector.Database/Repositories/ArticlesTable.cs
View File

@ -76,7 +76,7 @@ public class ArticlesTable : IArticlesRepository
using var conn = OpenConnection(_connectionString);
var q =
"INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
"INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage, codeiscommit, codeisrelease) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage, @codeiscommit, @codeisrelease);";
var res = conn.Execute(q, new
{
id = model.ID,
@ -91,7 +91,9 @@ public class ArticlesTable : IArticlesRepository
thumbnail = model.Thumbnail,
description = model.Description,
authorname = model.AuthorName,
authorimage = model.AuthorImage
authorimage = model.AuthorImage,
codeiscommit = model.CodeIsCommit,
codeisrelease = model.CodeIsRelease
});
return model;
}

13
Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs
View File

@ -25,16 +25,19 @@ public class SubscriptionsTable : ISubscriptionRepository
public SubscriptionModel New(SubscriptionModel model)
{
model.ID = Guid.NewGuid();
model.Id = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);";
var query =
"Insert Into subscriptions (ID, DiscordWebHookId, SourceId, codeAllowCommits, codeAllowReleases) Values (@id, @webhookId, @sourceId, @codeAllowCommits, @codeAllowReleases);";
try
{
conn.Execute(query, new
{
id = model.ID,
webhookid = model.DiscordWebHookID,
sourceid = model.SourceID
id = model.Id,
webhookId = model.DiscordWebHookId,
sourceId = model.SourceId,
codeAllowCommits = model.CodeAllowCommits,
codeAllowReleases = model.CodeAllowReleases
});
return model;
}

7
Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs
View File

@ -4,15 +4,16 @@ namespace Newsbot.Collector.Domain.Dto;
public class SubscriptionDetailsDto
{
public Guid ID { get; set; }
public Guid Id { get; set; }
public SourceDto? Source { get; set; }
public DiscordWebHookDto? DiscordWebHook { get; set; }
public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source, DiscordWebHookModel discord)
public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source,
DiscordWebHookModel discord)
{
return new SubscriptionDetailsDto
{
ID = subscription.ID,
Id = subscription.Id,
Source = SourceDto.Convert(source),
DiscordWebHook = DiscordWebHookDto.Convert(discord)
};

16
Newsbot.Collector.Domain/Dto/SubscriptionDto.cs
View File

@ -4,17 +4,21 @@ namespace Newsbot.Collector.Domain.Dto;
public class SubscriptionDto
{
public Guid ID { get; set; }
public Guid SourceID { get; set; }
public Guid DiscordWebHookID { get; set; }
public Guid Id { get; set; }
public Guid SourceId { get; set; }
public Guid DiscordWebHookId { get; set; }
public bool CodeAllowReleases { get; set; }
public bool CodeAllowCommits { get; set; }
public static SubscriptionDto Convert(SubscriptionModel model)
{
return new SubscriptionDto
{
ID = model.ID,
SourceID = model.SourceID,
DiscordWebHookID = model.DiscordWebHookID
Id = model.Id,
SourceId = model.SourceId,
DiscordWebHookId = model.DiscordWebHookId,
CodeAllowCommits = model.CodeAllowCommits,
CodeAllowReleases = model.CodeAllowReleases
};
}
}

9
Newsbot.Collector.Domain/Models/DatabaseModel.cs
View File

@ -73,11 +73,14 @@ public class SourceModel
public string Tags { get; set; } = "";
public bool Deleted { get; set; }
public string YoutubeId { get; set; } = "";
}
public class SubscriptionModel
{
public Guid ID { get; set; }
public Guid DiscordWebHookID { get; set; }
public Guid SourceID { get; set; }
public Guid Id { get; set; }
public Guid DiscordWebHookId { get; set; }
public Guid SourceId { get; set; }
public bool CodeAllowReleases { get; set; }
public bool CodeAllowCommits { get; set; }
}

56
Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs
View File

@ -1,9 +1,7 @@
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Notifications.Discord;
using Newtonsoft.Json.Linq;
using Serilog;
namespace Newsbot.Collector.Services.Jobs;
@ -18,15 +16,15 @@ public class DiscordNotificationJobOptions
public class DiscordNotificationJob
{
private const string JobName = "DiscordNotifications";
private IDiscordQueueRepository _queue;
private IArticlesRepository _article;
private IDiscordWebHooksRepository _webhook;
private ISourcesRepository _sources;
private ISubscriptionRepository _subs;
private IIconsRepository _icons;
private ILogger _logger;
private IDiscordQueueRepository _queue;
private ISourcesRepository _sources;
private ISubscriptionRepository _subs;
private IDiscordWebHooksRepository _webhook;
public DiscordNotificationJob()
{
_queue = new DiscordQueueTable("");
@ -54,7 +52,7 @@ public class DiscordNotificationJob
_logger.Warning($"{JobName} - Going to exit because feature flag is off.");
return;
}
_logger.Information($"{JobName} - Starting up the job.");
Execute();
}
@ -62,7 +60,7 @@ public class DiscordNotificationJob
private void Execute()
{
//collect all the new requests
var requests = _queue.List(25);
var requests = _queue.List(100);
_logger.Debug($"{JobName} - Collected {requests.Count} items to send");
foreach (var request in requests)
@ -75,7 +73,8 @@ public class DiscordNotificationJob
var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty)
{
_logger.Error($"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
_logger.Error(
$"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.ID);
continue;
}
@ -95,12 +94,16 @@ public class DiscordNotificationJob
foreach (var sub in allSubscriptions)
{
// Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip.
if (articleDetails.CodeIsCommit && sub.CodeAllowCommits == false) continue;
// same for releases
if (articleDetails.CodeIsRelease && sub.CodeAllowReleases == false) continue;
// find the discord webhooks we need to post to
var discordDetails = _webhook.GetByID(sub.DiscordWebHookID);
if (discordDetails.Enabled == false)
{
continue;
}
var discordDetails = _webhook.GetByID(sub.DiscordWebHookId);
if (discordDetails.Enabled == false) continue;
var client = new DiscordWebhookClient(discordDetails.Url);
try
@ -113,7 +116,7 @@ public class DiscordNotificationJob
_logger.Debug($"Queue Record: {request.ID}");
_logger.Debug($"Article: {articleDetails.ID}");
_logger.Debug($"Source: {sourceDetails.ID}");
_logger.Debug($"Subscription: {sub.ID}");
_logger.Debug($"Subscription: {sub.Id}");
}
Thread.Sleep(3000);
@ -122,6 +125,7 @@ public class DiscordNotificationJob
_logger.Debug($"{JobName} - Removing {request.ID} from the queue.");
_queue.Delete(request.ID);
}
_logger.Information($"{JobName} - Loop has been completed.");
}
@ -139,40 +143,32 @@ public class DiscordNotificationJob
},
Footer = new DiscordMessageEmbedFooter
{
Text = "Brought to you by Newsbot",
Text = "Brought to you by Newsbot"
},
Fields = new DiscordMessageEmbedField[]
{
new DiscordMessageEmbedField
new()
{
Name = "Link",
Value = article.URL,
Inline = false,
Inline = false
}
}
};
if (article.URL is not null && article.URL != "")
{
embed.Url = article.URL;
}
if (article.URL is not null && article.URL != "") embed.Url = article.URL;
if (article.Thumbnail != "")
{
embed.Image = new DiscordMessageEmbedImage
{
Url = article.Thumbnail
};
}
if (article.AuthorImage is not null && article.AuthorImage != "")
{
embed.Author.IconUrl = article.AuthorImage;
}
if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage;
return new DiscordMessage
{
Embeds = new DiscordMessageEmbed[]
Embeds = new[]
{
embed
}

1
Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs
View File

@ -69,7 +69,6 @@ public class YoutubeWatcherJob
_source.UpdateYoutubeId(source.ID, channelId);
}
// Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");

25
seed.ps1
View File

@ -13,6 +13,8 @@ function New-RedditSource {
$param = "name=$Name"
$uri = "$ApiServer/api/sources/new/reddit?$param"
Write-Host "Adding Reddit: $Name"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
}
@ -24,6 +26,8 @@ function New-RssSource {
)
$urlEncoded = [uri]::EscapeDataString($Url)
$param = "name=$Name&url=$urlEncoded"
Write-Host "Adding RSS: $Name"
[string] $uri = "$ApiServer/api/sources/new/rss?$param"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
@ -36,6 +40,8 @@ function New-YoutubeSource {
$urlEncoded = [uri]::EscapeDataString($Url)
[string] $param = "url=$urlEncoded"
[string] $uri = "$ApiServer/api/sources/new/youtube?$param"
Write-Host = "Adding YouTube: $Url"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
}
@ -46,6 +52,8 @@ function New-TwitchSource {
)
[string] $param = "name=$Name"
[string] $uri = "$ApiServer/api/sources/new/twitch?$param"
Write-Host "Adding Twitch: $Name"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
}
@ -78,6 +86,8 @@ function New-DiscordWebhook {
return $res
}
function New-Subscription {
param (
[string] $SourceId,
@ -89,6 +99,19 @@ function New-Subscription {
return $res
}
function New-CodeProjectSubscription {
param (
[string] $SourceId,
[string] $DiscordWebhookId,
[switch] $AllowReleases = $false,
[switch] $AllowCommits = $false
)
[string] $param = "sourceId=$SourceId&discordId=$DiscordWebhookId&allowReleases=$AllowReleases&allowCommits=$AllowCommits"
[string] $uri = "$ApiServer/api/subscriptions/new/codeproject?$param"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
}
# Load Secrets file
$secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json
@ -130,7 +153,7 @@ New-Subscription -SourceId $rssOmgLinux.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id
New-CodeProjectSubscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id -AllowReleases
New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id