Features/testing discord queue (#8)

* new seed and corrected some issues talking to the db

* adding some prep work for adding youtube back again

* small updates for rss and getting results from discord job

* new command to reload db

* added discord alerts job to hangfire
This commit is contained in:
James Tombleson 2023-03-05 22:33:41 -08:00 committed by GitHub
parent bb832ed441
commit aa53b1eeeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 101 additions and 29 deletions

View File

@ -1,9 +1,9 @@
using Hangfire; using Hangfire;
using Hangfire.MemoryStorage; using Hangfire.MemoryStorage;
using Serilog;
using Newsbot.Collector.Services.Jobs;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs;
using Serilog;
Log.Logger = new LoggerConfiguration() Log.Logger = new LoggerConfiguration()
.WriteTo.Console() .WriteTo.Console()
@ -63,12 +63,18 @@ static IConfiguration GetConfiguration()
static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger) static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
{ {
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions //RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
{ //{
Message = "Hello from the background!" // Message = "Hello from the background!"
}), "0/1 * * * *"); //}), "0/1 * * * *");
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions
{ {
ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "15 0-23 * * *"); }), "15 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions
{
DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "5/10 * * * *");
} }

View File

@ -0,0 +1,20 @@
-- +goose Up
-- +goose StatementBegin
INSERT INTO sources VALUES (
uuid_generate_v4(),
'rss',
'Let''s Mosley',
'feed',
'rss',
'podcast',
TRUE,
'https://anchor.fm/s/6c7aa4c4/podcast/rss',
'rss,let''s mosley,fitnes,coach',
FALSE);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DELETE FROM sources Where type = 'rss' And Name = 'Let''s Mosley'
-- +goose StatementEnd

View File

@ -95,7 +95,7 @@ public class ArticlesTable : IArticlesRepository
var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
var res = conn.Execute(q, new var res = conn.Execute(q, new
{ {
id = Guid.NewGuid(), id = model.ID,
sourceid = model.SourceID, sourceid = model.SourceID,
tags = model.Tags, tags = model.Tags,
title = model.Title, title = model.Title,

View File

@ -46,7 +46,7 @@ public class DiscordQueueTable : IDiscordQueueRepository
public List<DiscordQueueModel> List(int limit = 25) public List<DiscordQueueModel> List(int limit = 25)
{ {
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordQueue LIMIT @id;"; var query = "Select * from DiscordQueue LIMIT @limit;";
return conn.Query<DiscordQueueModel>(query, new { return conn.Query<DiscordQueueModel>(query, new {
limit = limit limit = limit
}).ToList(); }).ToList();

View File

@ -59,13 +59,12 @@ public class SubscriptionsTable : ISubscriptionRepository
}).ToList(); }).ToList();
} }
// todo add paging
public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25) public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25)
{ {
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions var query = @"Select * From subscriptions
Offset @page Fetch Next @count Rows Only Where sourceid = @sourceid
Where sourceid = @sourceid"; Offset @page Fetch Next @count Rows Only";
return conn.Query<SubscriptionModel>(query, new return conn.Query<SubscriptionModel>(query, new
{ {
page = page * count, page = page * count,
@ -78,8 +77,8 @@ public class SubscriptionsTable : ISubscriptionRepository
{ {
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions var query = @"Select * From subscriptions
Offset @page Fetch Next @count Rows Only Where discordwebhookid = @webhookid
Where discordwebhookid = @webhookid"; Offset @page Fetch Next @count Rows Only";
return conn.Query<SubscriptionModel>(query, new return conn.Query<SubscriptionModel>(query, new
{ {
page = page * count, page = page * count,

View File

@ -12,7 +12,7 @@ public class HeadParserClient
private string _htmlContent; private string _htmlContent;
public HeadParserClient(string htmlContent) public HeadParserClient(string htmlContent, bool useBrowser = false)
{ {
_htmlContent = htmlContent; _htmlContent = htmlContent;
Data = new HeadParserModel(); Data = new HeadParserModel();
@ -28,6 +28,7 @@ public class HeadParserClient
Data.ColorTheme = GetMetaColorTheme(); Data.ColorTheme = GetMetaColorTheme();
Data.FeedUri = GetSiteFeed(); Data.FeedUri = GetSiteFeed();
Data.YoutubeChannelID = GetYouTubeChannelId();
} }
private List<HtmlNode> CollectMetaTags() private List<HtmlNode> CollectMetaTags()
@ -93,7 +94,7 @@ public class HeadParserClient
public string GetMetaImage() public string GetMetaImage()
{ {
var htmlTags = CollectMetaTags(); var htmlTags = CollectMetaTags();
string[] tags = new string[] { "twitter:image", "og:image", "image" }; string[] tags = new string[] { "twitter:image", "og:image", "image" };
return FindFirstResult(tags, htmlTags); return FindFirstResult(tags, htmlTags);
} }
@ -118,6 +119,13 @@ public class HeadParserClient
return FindFirstResult(tags, htmlTags); return FindFirstResult(tags, htmlTags);
} }
public string GetYouTubeChannelId()
{
var htmlTags = CollectMetaTags();
string[] tags = new string[] { "channelId" };
return FindFirstResult(tags, htmlTags);
}
/// <summary> /// <summary>
/// This will parse the headers looking for known keys that will contain a RSS feed link. /// This will parse the headers looking for known keys that will contain a RSS feed link.
/// If the feed is not found, this will throw an exception (MissingHeaderValueException). /// If the feed is not found, this will throw an exception (MissingHeaderValueException).

View File

@ -10,6 +10,7 @@ public class HeadParserModel
public string ColorTheme { get; set; } = ""; public string ColorTheme { get; set; } = "";
public string? FeedUri { get; set; } public string? FeedUri { get; set; }
public string? YoutubeChannelID { get; set; }
} }
public class HtmlData public class HtmlData

View File

@ -2,15 +2,16 @@ using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
using Serilog;
namespace Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Services.Jobs;
public class DiscordNotificationJobOptions public class DiscordNotificationJobOptions
{ {
public string? DatabaseConnectionString { get; set; }
} }
public class DiscordNotifificationJob public class DiscordNotificationJob
{ {
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
@ -19,21 +20,24 @@ public class DiscordNotifificationJob
private ISourcesRepository _sources; private ISourcesRepository _sources;
private ISubscriptionRepository _subs; private ISubscriptionRepository _subs;
private IDiscordNotificatioClient _webhookClient; public DiscordNotificationJob()
public DiscordNotifificationJob()
{ {
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_article = new ArticlesTable(""); _article = new ArticlesTable("");
_webhook = new DiscordWebhooksTable(""); _webhook = new DiscordWebhooksTable("");
_sources = new SourcesTable(""); _sources = new SourcesTable("");
_subs = new SubscriptionsTable(""); _subs = new SubscriptionsTable("");
_webhookClient = new DiscordWebhookClient("");
} }
public void InitAndExecute() public void InitAndExecute(DiscordNotificationJobOptions options)
{ {
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_article = new ArticlesTable(options.DatabaseConnectionString ?? "");
_webhook = new DiscordWebhooksTable(options.DatabaseConnectionString ?? "");
_sources = new SourcesTable(options.DatabaseConnectionString ?? "");
_subs = new SubscriptionsTable(options.DatabaseConnectionString ?? "");
Execute();
} }
private void Execute() private void Execute()
@ -48,6 +52,12 @@ public class DiscordNotifificationJob
// Get the deatils of the source // Get the deatils of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID); var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty)
{
Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue");
_queue.Delete(request.ID);
continue;
}
// Find all the subscriptions for that source // Find all the subscriptions for that source
var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID); var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID);
@ -56,10 +66,26 @@ public class DiscordNotifificationJob
{ {
// find the discord webhooks we need to post to // find the discord webhooks we need to post to
var discordDetails = _webhook.GetByID(sub.DiscordWebHookID); var discordDetails = _webhook.GetByID(sub.DiscordWebHookID);
if (discordDetails.Enabled == false)
{
continue;
}
var client = new DiscordWebhookClient(discordDetails.Url); var client = new DiscordWebhookClient(discordDetails.Url);
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails)); try
{
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails));
}
catch (Exception e)
{
Log.Error($"Failed to post message to Discord. {e}");
continue;
}
Thread.Sleep(3000);
} }
_queue.Delete(request.ID);
} }
} }
@ -77,6 +103,15 @@ public class DiscordNotifificationJob
Footer = new DiscordMessageEmbedFooter Footer = new DiscordMessageEmbedFooter
{ {
Text = "Brought to you by Newsbot", Text = "Brought to you by Newsbot",
},
Fields = new DiscordMessageEmbedField[]
{
new DiscordMessageEmbedField
{
Name = "Link",
Value = article.URL,
Inline = false,
}
} }
}; };

View File

@ -72,6 +72,11 @@ public class RssWatcherJob : IHangfireJob
var sources = _source.ListByType(SourceTypes.Rss); var sources = _source.ListByType(SourceTypes.Rss);
foreach (var source in sources) foreach (var source in sources)
{ {
if (source.Enabled == false)
{
continue;
}
var results = Collect(source.Url, source.ID); var results = Collect(source.Url, source.ID);
articles.AddRange(results); articles.AddRange(results);
@ -137,11 +142,6 @@ public class RssWatcherJob : IHangfireJob
continue; continue;
} }
if (IsThisUrlKnown(item.URL) == true)
{
continue;
}
var p = _articles.New(item); var p = _articles.New(item);
_queue.New(new DiscordQueueModel _queue.New(new DiscordQueueModel
{ {

View File

@ -23,3 +23,6 @@ migrate-dev: ## Apply sql migrations to dev db
migrate-dev-down: ## revert sql migrations to dev db migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down
migrate-refresh: ## Rolls back all migrations
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset