From bacc04ad7d50fb910a194b5b04a876e11aa5f0de Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sat, 8 Apr 2023 09:30:59 -0700 Subject: [PATCH] Features/enabling youtube (#24) * Adding a youtube controller to trigger the job * Renamed jobs controller to rss * cleaned up background jobs and added youtube to the startup. * Handled merge issues and validated things are still working. --- Newsbot.Collector.Api/BackgroundJobs.cs | 22 +++-- .../{JobsController.cs => RssController.cs} | 19 +--- .../Controllers/YoutubeController.cs | 35 ++++++++ .../Jobs/YoutubeWatcherJob.cs | 87 +++++++++++-------- .../Jobs/YoutubeWatcherJobTests.cs | 19 ++++ .../Newsbot.Collector.Tests.csproj | 8 ++ Newsbot.Collector.Tests/TestHelper.cs | 9 ++ 7 files changed, 139 insertions(+), 60 deletions(-) rename Newsbot.Collector.Api/Controllers/{JobsController.cs => RssController.cs} (63%) create mode 100644 Newsbot.Collector.Api/Controllers/YoutubeController.cs create mode 100644 Newsbot.Collector.Tests/Jobs/YoutubeWatcherJobTests.cs diff --git a/Newsbot.Collector.Api/BackgroundJobs.cs b/Newsbot.Collector.Api/BackgroundJobs.cs index 8f126b3..819d8f3 100644 --- a/Newsbot.Collector.Api/BackgroundJobs.cs +++ b/Newsbot.Collector.Api/BackgroundJobs.cs @@ -1,6 +1,5 @@ using Hangfire; using Newsbot.Collector.Domain.Consts; -using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Api; @@ -9,20 +8,27 @@ public static class BackgroundJobs { public static void SetupRecurringJobs(IConfiguration configuration) { - RecurringJob.AddOrUpdate("RSS", x => + RecurringJob.AddOrUpdate("RSS", x => x.InitAndExecute(new RssWatcherJobOptions + { + ConnectionString = configuration.GetValue(ConfigConst.ConnectionStringDatabase), + OpenTelemetry = configuration.GetValue(ConfigConst.ConnectionStringOpenTelemetry), + IsEnabled = configuration.GetValue(ConfigConst.RssIsEnabled) + }), "15 0-23 * * *"); + + RecurringJob.AddOrUpdate("Youtube", x => x.InitAndExecute(new YoutubeWatcherJobOptions { - ConnectionString = configuration.GetValue(ConfigConst.ConnectionStringDatabase), - OpenTelemetry = configuration.GetValue(ConfigConst.ConnectionStringOpenTelemetry), - IsEnabled = configuration.GetValue(ConfigConst.RssIsEnabled) - }), "15 0-23 * * *"); - + DatabaseConnectionString = configuration.GetValue(ConfigConst.ConnectionStringDatabase), + OpenTelemetryConnectionString = configuration.GetValue(ConfigConst.ConnectionStringOpenTelemetry), + IsEnabled = configuration.GetValue(ConfigConst.YoutubeIsEnable) + }), "20 0-23 * * *"); + RecurringJob.AddOrUpdate("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions { ConnectionString = configuration.GetValue(ConfigConst.ConnectionStringDatabase), OpenTelemetry = configuration.GetValue(ConfigConst.ConnectionStringOpenTelemetry), - IsEnabled = configuration.GetValue(ConfigConst.DiscordNotificationsEnabled), + IsEnabled = configuration.GetValue(ConfigConst.DiscordNotificationsEnabled) }), "5/10 * * * *"); } } \ No newline at end of file diff --git a/Newsbot.Collector.Api/Controllers/JobsController.cs b/Newsbot.Collector.Api/Controllers/RssController.cs similarity index 63% rename from Newsbot.Collector.Api/Controllers/JobsController.cs rename to Newsbot.Collector.Api/Controllers/RssController.cs index 4bfa455..cbf34de 100644 --- a/Newsbot.Collector.Api/Controllers/JobsController.cs +++ b/Newsbot.Collector.Api/Controllers/RssController.cs @@ -9,24 +9,22 @@ using Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Api.Controllers; [ApiController] -[Route("api/jobs")] -public class JobsController +[Route("api/rss")] +public class RssController { private readonly ConfigSectionConnectionStrings _connectionStrings; private readonly ConfigSectionRssModel _rssConfig; private readonly ILogger _logger; - private readonly ISourcesRepository _sources; - public JobsController(ILogger logger, IOptions connectionStrings, + public RssController(ILogger logger, IOptions connectionStrings, IOptions rss) { _logger = logger; _connectionStrings = connectionStrings.Value; _rssConfig = rss.Value; - _sources = new SourcesTable(connectionStrings.Value.Database ?? ""); } - [HttpPost("check/rss")] + [HttpPost("check")] public void CheckReddit() { BackgroundJob.Enqueue(x => x.InitAndExecute(new RssWatcherJobOptions @@ -36,13 +34,4 @@ public class JobsController IsEnabled = _rssConfig.IsEnabled })); } - - [HttpPost("check/youtube")] - public void CheckYoutube() - { - BackgroundJob.Enqueue(x => x.InitAndExecute(new YoutubeWatcherJobOptions - { - ConnectionStrings = _connectionStrings - })); - } } \ No newline at end of file diff --git a/Newsbot.Collector.Api/Controllers/YoutubeController.cs b/Newsbot.Collector.Api/Controllers/YoutubeController.cs new file mode 100644 index 0000000..b569481 --- /dev/null +++ b/Newsbot.Collector.Api/Controllers/YoutubeController.cs @@ -0,0 +1,35 @@ +using Hangfire; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using Newsbot.Collector.Domain.Models.Config; +using Newsbot.Collector.Services.Jobs; +using ILogger = Grpc.Core.Logging.ILogger; + +namespace Newsbot.Collector.Api.Controllers; + +[ApiController] +[Route("api/youtube")] +public class YoutubeController +{ + private readonly ILogger _logger; + private readonly ConfigSectionConnectionStrings _connectionStrings; + private readonly ConfigSectionYoutubeModel _config; + + public YoutubeController(ILogger logger, IOptions config, IOptions connectionStrings) + { + _logger = logger; + _connectionStrings = connectionStrings.Value; + _config = config.Value; + } + + [HttpPost("check")] + public void CheckYoutube() + { + BackgroundJob.Enqueue(x => x.InitAndExecute(new YoutubeWatcherJobOptions + { + DatabaseConnectionString = _connectionStrings.Database, + OpenTelemetryConnectionString = _connectionStrings.OpenTelemetry, + IsEnabled = _config.IsEnabled + })); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index 315e2fc..984a6f3 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -4,7 +4,6 @@ using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; -using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.HtmlParser; using Serilog; @@ -12,12 +11,17 @@ namespace Newsbot.Collector.Services.Jobs; public class YoutubeWatcherJobOptions { - public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } + public string? DatabaseConnectionString { get; set; } + public string? OpenTelemetryConnectionString { get; set; } + public int SleepTimer { get; set; } = 3000; + public bool IsEnabled { get; set; } = true; } public class YoutubeWatcherJob { + private const string JobName = "YoutubeWatcherJob"; + private readonly YoutubeWatcherJobOptions _options; private IArticlesRepository _articles; private IIconsRepository _icons; @@ -32,55 +36,61 @@ public class YoutubeWatcherJob _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); _icons = new IconsTable(""); - _logger = JobLogger.GetLogger("", "YoutubeWatcherJob"); + _logger = JobLogger.GetLogger("", JobName); } public void InitAndExecute(YoutubeWatcherJobOptions options) { - options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); - - _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); - _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); - _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); - _icons = new IconsTable(options.ConnectionStrings.Database ?? ""); - _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", "YoutubeWatcherJob"); + _articles = new ArticlesTable(options.DatabaseConnectionString ?? ""); + _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? ""); + _source = new SourcesTable(options.DatabaseConnectionString ?? ""); + _icons = new IconsTable(options.DatabaseConnectionString ?? ""); + _logger = JobLogger.GetLogger(options.OpenTelemetryConnectionString ?? "", JobName); Execute(); } private void Execute() { - var videos = new List(); - var sources = _source.ListByType(SourceTypes.YouTube, 100); - foreach (var source in sources) CheckSource(source); - } - - private void CheckSource(SourceModel source) - { - var channelId = ""; - - if (source.YoutubeId == "") + foreach (var source in sources) { - channelId = GetChannelId(source.Url); - _source.UpdateYoutubeId(source.ID, channelId); - } - else - { - channelId = source.YoutubeId; - } + if (!source.Enabled) + { + _logger.Debug($"{JobName} - {source.Name} was disabled and will be skipped."); + continue; + } + + var channelId = source.YoutubeId; + if (channelId == "") + { + channelId = GetChannelId(source.Url); + _source.UpdateYoutubeId(source.ID, channelId); + } + + + // Make sure we have a Icon for the channel + var icon = _icons.GetBySourceId(source.ID); + if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V"); - // Make sure we have a Icon for the channel - var icon = _icons.GetBySourceId(source.ID); - if (icon.Id == Guid.Empty) - { + _logger.Information($"{JobName} - Checking '{source.Name}'"); + var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}"; + + var newVideos = CheckFeed(url, source); + _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos"); + foreach (var video in newVideos) + { + _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found"); + _articles.New(video); + _queue.New(new DiscordQueueModel + { + ArticleID = video.ID + }); + } } - - var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}"; - - var newVideos = CheckFeed(url, source); - foreach (var video in newVideos) _articles.New(video); + + _logger.Information($"{JobName} - Done"); } private string GetChannelId(string url) @@ -93,7 +103,9 @@ public class YoutubeWatcherJob pageReader.Parse(); var id = pageReader.Data.Header.YoutubeChannelID ?? ""; - if (id == "") _logger.Error(new Exception("Unable to find the Youtube Channel ID for the requested url."), url); + if (id == "") + _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."), + url); return id; } @@ -130,6 +142,7 @@ public class YoutubeWatcherJob }; videos.Add(article); + Thread.Sleep(_options.SleepTimer); } diff --git a/Newsbot.Collector.Tests/Jobs/YoutubeWatcherJobTests.cs b/Newsbot.Collector.Tests/Jobs/YoutubeWatcherJobTests.cs new file mode 100644 index 0000000..9d73e15 --- /dev/null +++ b/Newsbot.Collector.Tests/Jobs/YoutubeWatcherJobTests.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Services.Jobs; + +namespace Newsbot.Collector.Tests.Jobs; + +public class YoutubeWatcherJobTests +{ + [Fact] + public void InitTest() + { + var client = new YoutubeWatcherJob(); + client.InitAndExecute(new YoutubeWatcherJobOptions + { + DatabaseConnectionString = TestHelper.LoadConfig().GetValue(ConfigConst.ConnectionStringDatabase), + SleepTimer = 1000 + }); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj b/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj index e34e404..3fde038 100644 --- a/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj +++ b/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj @@ -10,6 +10,8 @@ + + @@ -35,4 +37,10 @@ + + + PreserveNewest + + + diff --git a/Newsbot.Collector.Tests/TestHelper.cs b/Newsbot.Collector.Tests/TestHelper.cs index 1b666fb..37caea1 100644 --- a/Newsbot.Collector.Tests/TestHelper.cs +++ b/Newsbot.Collector.Tests/TestHelper.cs @@ -1,5 +1,14 @@ +using System.Net.Sockets; +using Microsoft.Extensions.Configuration; + namespace Newsbot.Collector.Tests; public static class TestHelper { + public static IConfiguration LoadConfig() + { + return new ConfigurationBuilder() + .AddJsonFile("appsettings.json", false) + .Build(); + } } \ No newline at end of file