Features/testing discord queue (#8)

* new seed and corrected some issues talking to the db

* adding some prep work for adding youtube back again

* small updates for rss and getting results from discord job

* new command to reload db

* added discord alerts job to hangfire
This commit is contained in:
James Tombleson 2023-03-05 22:33:41 -08:00 committed by GitHub
parent bb832ed441
commit aa53b1eeeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 101 additions and 29 deletions

View File

@ -1,9 +1,9 @@
using Hangfire;
using Hangfire.MemoryStorage;
using Serilog;
using Newsbot.Collector.Services.Jobs;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs;
using Serilog;
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
@ -63,12 +63,18 @@ static IConfiguration GetConfiguration()
static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
{
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
{
Message = "Hello from the background!"
}), "0/1 * * * *");
//RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
//{
// Message = "Hello from the background!"
//}), "0/1 * * * *");
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions
{
ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "15 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions
{
DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "5/10 * * * *");
}

View File

@ -0,0 +1,20 @@
-- +goose Up
-- +goose StatementBegin
INSERT INTO sources VALUES (
uuid_generate_v4(),
'rss',
'Let''s Mosley',
'feed',
'rss',
'podcast',
TRUE,
'https://anchor.fm/s/6c7aa4c4/podcast/rss',
'rss,let''s mosley,fitnes,coach',
FALSE);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DELETE FROM sources Where type = 'rss' And Name = 'Let''s Mosley'
-- +goose StatementEnd

View File

@ -95,7 +95,7 @@ public class ArticlesTable : IArticlesRepository
var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
var res = conn.Execute(q, new
{
id = Guid.NewGuid(),
id = model.ID,
sourceid = model.SourceID,
tags = model.Tags,
title = model.Title,

View File

@ -46,7 +46,7 @@ public class DiscordQueueTable : IDiscordQueueRepository
public List<DiscordQueueModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordQueue LIMIT @id;";
var query = "Select * from DiscordQueue LIMIT @limit;";
return conn.Query<DiscordQueueModel>(query, new {
limit = limit
}).ToList();

View File

@ -59,13 +59,12 @@ public class SubscriptionsTable : ISubscriptionRepository
}).ToList();
}
// todo add paging
public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions
Offset @page Fetch Next @count Rows Only
Where sourceid = @sourceid";
Where sourceid = @sourceid
Offset @page Fetch Next @count Rows Only";
return conn.Query<SubscriptionModel>(query, new
{
page = page * count,
@ -78,8 +77,8 @@ public class SubscriptionsTable : ISubscriptionRepository
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions
Offset @page Fetch Next @count Rows Only
Where discordwebhookid = @webhookid";
Where discordwebhookid = @webhookid
Offset @page Fetch Next @count Rows Only";
return conn.Query<SubscriptionModel>(query, new
{
page = page * count,

View File

@ -12,7 +12,7 @@ public class HeadParserClient
private string _htmlContent;
public HeadParserClient(string htmlContent)
public HeadParserClient(string htmlContent, bool useBrowser = false)
{
_htmlContent = htmlContent;
Data = new HeadParserModel();
@ -28,6 +28,7 @@ public class HeadParserClient
Data.ColorTheme = GetMetaColorTheme();
Data.FeedUri = GetSiteFeed();
Data.YoutubeChannelID = GetYouTubeChannelId();
}
private List<HtmlNode> CollectMetaTags()
@ -118,6 +119,13 @@ public class HeadParserClient
return FindFirstResult(tags, htmlTags);
}
public string GetYouTubeChannelId()
{
var htmlTags = CollectMetaTags();
string[] tags = new string[] { "channelId" };
return FindFirstResult(tags, htmlTags);
}
/// <summary>
/// This will parse the headers looking for known keys that will contain a RSS feed link.
/// If the feed is not found, this will throw an exception (MissingHeaderValueException).

View File

@ -10,6 +10,7 @@ public class HeadParserModel
public string ColorTheme { get; set; } = "";
public string? FeedUri { get; set; }
public string? YoutubeChannelID { get; set; }
}
public class HtmlData

View File

@ -2,15 +2,16 @@ using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Notifications.Discord;
using Serilog;
namespace Newsbot.Collector.Services.Jobs;
public class DiscordNotificationJobOptions
{
public string? DatabaseConnectionString { get; set; }
}
public class DiscordNotifificationJob
public class DiscordNotificationJob
{
private IDiscordQueueRepository _queue;
@ -19,21 +20,24 @@ public class DiscordNotifificationJob
private ISourcesRepository _sources;
private ISubscriptionRepository _subs;
private IDiscordNotificatioClient _webhookClient;
public DiscordNotifificationJob()
public DiscordNotificationJob()
{
_queue = new DiscordQueueTable("");
_article = new ArticlesTable("");
_webhook = new DiscordWebhooksTable("");
_sources = new SourcesTable("");
_subs = new SubscriptionsTable("");
_webhookClient = new DiscordWebhookClient("");
}
public void InitAndExecute()
public void InitAndExecute(DiscordNotificationJobOptions options)
{
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_article = new ArticlesTable(options.DatabaseConnectionString ?? "");
_webhook = new DiscordWebhooksTable(options.DatabaseConnectionString ?? "");
_sources = new SourcesTable(options.DatabaseConnectionString ?? "");
_subs = new SubscriptionsTable(options.DatabaseConnectionString ?? "");
Execute();
}
private void Execute()
@ -48,6 +52,12 @@ public class DiscordNotifificationJob
// Get the deatils of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty)
{
Log.Error($"DiscordNotificationJob - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue");
_queue.Delete(request.ID);
continue;
}
// Find all the subscriptions for that source
var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID);
@ -56,10 +66,26 @@ public class DiscordNotifificationJob
{
// find the discord webhooks we need to post to
var discordDetails = _webhook.GetByID(sub.DiscordWebHookID);
if (discordDetails.Enabled == false)
{
continue;
}
var client = new DiscordWebhookClient(discordDetails.Url);
try
{
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails));
}
catch (Exception e)
{
Log.Error($"Failed to post message to Discord. {e}");
continue;
}
Thread.Sleep(3000);
}
_queue.Delete(request.ID);
}
}
@ -77,6 +103,15 @@ public class DiscordNotifificationJob
Footer = new DiscordMessageEmbedFooter
{
Text = "Brought to you by Newsbot",
},
Fields = new DiscordMessageEmbedField[]
{
new DiscordMessageEmbedField
{
Name = "Link",
Value = article.URL,
Inline = false,
}
}
};

View File

@ -72,6 +72,11 @@ public class RssWatcherJob : IHangfireJob
var sources = _source.ListByType(SourceTypes.Rss);
foreach (var source in sources)
{
if (source.Enabled == false)
{
continue;
}
var results = Collect(source.Url, source.ID);
articles.AddRange(results);
@ -137,11 +142,6 @@ public class RssWatcherJob : IHangfireJob
continue;
}
if (IsThisUrlKnown(item.URL) == true)
{
continue;
}
var p = _articles.New(item);
_queue.New(new DiscordQueueModel
{

View File

@ -23,3 +23,6 @@ migrate-dev: ## Apply sql migrations to dev db
migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down
migrate-refresh: ## Rolls back all migrations
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset