Features/enabling youtube (#24)

* Adding a youtube controller to trigger the job

* Renamed jobs controller to rss

* cleaned up background jobs and added youtube to the startup.

* Handled merge issues and validated things are still working.

Newsbot.Collector.Api/BackgroundJobs.cs

@@ -1,6 +1,5 @@
 using Hangfire;
 using Newsbot.Collector.Domain.Consts;
-using Newsbot.Collector.Domain.Models.Config;
 using Newsbot.Collector.Services.Jobs;
 
 namespace Newsbot.Collector.Api;
@@ -11,18 +10,25 @@ public static class BackgroundJobs
     {
         RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x =>
             x.InitAndExecute(new RssWatcherJobOptions
+            {
+                ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
+                OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
+                IsEnabled = configuration.GetValue<bool>(ConfigConst.RssIsEnabled)
+            }), "15 0-23 * * *");
+
+        RecurringJob.AddOrUpdate<YoutubeWatcherJob>("Youtube", x => x.InitAndExecute(new YoutubeWatcherJobOptions
         {
-            ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
+            DatabaseConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
-            OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
+            OpenTelemetryConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
-            IsEnabled = configuration.GetValue<bool>(ConfigConst.RssIsEnabled)
+            IsEnabled = configuration.GetValue<bool>(ConfigConst.YoutubeIsEnable)
-        }), "15 0-23 * * *");
+        }), "20 0-23 * * *");
 
         RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
             x.InitAndExecute(new DiscordNotificationJobOptions
             {
                 ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
                 OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
-                IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled),
+                IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled)
             }), "5/10 * * * *");
     }
 }

Newsbot.Collector.Api/Controllers/RssController.cs

@@ -9,24 +9,22 @@ using Newsbot.Collector.Services.Jobs;
 namespace Newsbot.Collector.Api.Controllers;
 
 [ApiController]
-[Route("api/jobs")]
+[Route("api/rss")]
-public class JobsController
+public class RssController
 {
     private readonly ConfigSectionConnectionStrings _connectionStrings;
     private readonly ConfigSectionRssModel _rssConfig;
     private readonly ILogger<SourcesController> _logger;
-    private readonly ISourcesRepository _sources;
 
-    public JobsController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
+    public RssController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
         IOptions<ConfigSectionRssModel> rss)
     {
         _logger = logger;
         _connectionStrings = connectionStrings.Value;
         _rssConfig = rss.Value;
-        _sources = new SourcesTable(connectionStrings.Value.Database ?? "");
     }
 
-    [HttpPost("check/rss")]
+    [HttpPost("check")]
     public void CheckReddit()
     {
         BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions
@@ -36,13 +34,4 @@ public class JobsController
             IsEnabled = _rssConfig.IsEnabled
         }));
     }
-
-    [HttpPost("check/youtube")]
-    public void CheckYoutube()
-    {
-        BackgroundJob.Enqueue<YoutubeWatcherJob>(x => x.InitAndExecute(new YoutubeWatcherJobOptions
-        {
-            ConnectionStrings = _connectionStrings
-        }));
-    }
 }

Newsbot.Collector.Api/Controllers/YoutubeController.cs

@@ -0,0 +1,35 @@
+using Hangfire;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;
+using Newsbot.Collector.Domain.Models.Config;
+using Newsbot.Collector.Services.Jobs;
+using ILogger = Grpc.Core.Logging.ILogger;
+
+namespace Newsbot.Collector.Api.Controllers;
+
+[ApiController]
+[Route("api/youtube")]
+public class YoutubeController
+{
+    private readonly ILogger<YoutubeController> _logger;
+    private readonly ConfigSectionConnectionStrings _connectionStrings;
+    private readonly ConfigSectionYoutubeModel _config;
+    
+    public YoutubeController(ILogger<YoutubeController> logger, IOptions<ConfigSectionYoutubeModel> config, IOptions<ConfigSectionConnectionStrings> connectionStrings)
+    {
+        _logger = logger;
+        _connectionStrings = connectionStrings.Value;
+        _config = config.Value;
+    }
+    
+    [HttpPost("check")]
+    public void CheckYoutube()
+    {
+        BackgroundJob.Enqueue<YoutubeWatcherJob>(x => x.InitAndExecute(new YoutubeWatcherJobOptions
+        {
+            DatabaseConnectionString = _connectionStrings.Database,
+            OpenTelemetryConnectionString = _connectionStrings.OpenTelemetry,
+            IsEnabled = _config.IsEnabled
+        }));
+    }
+}

Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs

@@ -4,7 +4,6 @@ using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Consts;
 using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
-using Newsbot.Collector.Domain.Models.Config;
 using Newsbot.Collector.Services.HtmlParser;
 using Serilog;
 
@@ -12,12 +11,17 @@ namespace Newsbot.Collector.Services.Jobs;
 
 public class YoutubeWatcherJobOptions
 {
-    public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
+    public string? DatabaseConnectionString { get; set; }
+    public string? OpenTelemetryConnectionString { get; set; }
+
     public int SleepTimer { get; set; } = 3000;
+    public bool IsEnabled { get; set; } = true;
 }
 
 public class YoutubeWatcherJob
 {
+    private const string JobName = "YoutubeWatcherJob";
+    
     private readonly YoutubeWatcherJobOptions _options;
     private IArticlesRepository _articles;
     private IIconsRepository _icons;
@@ -32,55 +36,61 @@ public class YoutubeWatcherJob
         _queue = new DiscordQueueTable("");
         _source = new SourcesTable("");
         _icons = new IconsTable("");
-        _logger = JobLogger.GetLogger("", "YoutubeWatcherJob");
+        _logger = JobLogger.GetLogger("", JobName);
     }
 
     public void InitAndExecute(YoutubeWatcherJobOptions options)
     {
-        options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
+        _articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
-
+        _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
-        _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
+        _source = new SourcesTable(options.DatabaseConnectionString ?? "");
-        _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
+        _icons = new IconsTable(options.DatabaseConnectionString ?? "");
-        _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
+        _logger = JobLogger.GetLogger(options.OpenTelemetryConnectionString ?? "", JobName);
-        _icons = new IconsTable(options.ConnectionStrings.Database ?? "");
-        _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", "YoutubeWatcherJob");
 
         Execute();
     }
 
     private void Execute()
     {
-        var videos = new List<ArticlesModel>();
-
         var sources = _source.ListByType(SourceTypes.YouTube, 100);
 
-        foreach (var source in sources) CheckSource(source);
+        foreach (var source in sources)
-    }
-
-    private void CheckSource(SourceModel source)
-    {
-        var channelId = "";
-
-        if (source.YoutubeId == "")
         {
-            channelId = GetChannelId(source.Url);
+            if (!source.Enabled)
-            _source.UpdateYoutubeId(source.ID, channelId);
+            {
-        }
+                _logger.Debug($"{JobName} - {source.Name} was disabled and will be skipped.");
-        else
+                continue;
-        {
+            }
-            channelId = source.YoutubeId;
+            
+            var channelId = source.YoutubeId;
+            if (channelId == "")
+            {
+                channelId = GetChannelId(source.Url);
+                _source.UpdateYoutubeId(source.ID, channelId);
+            }
+            
+            
+            // Make sure we have a Icon for the channel
+            var icon = _icons.GetBySourceId(source.ID);
+            if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");
+
+            _logger.Information($"{JobName} - Checking '{source.Name}'");
+            var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
+
+            var newVideos = CheckFeed(url, source);
+            _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
+            foreach (var video in newVideos)
+            {
+                _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
+                _articles.New(video);
+                _queue.New(new DiscordQueueModel
+                {
+                    ArticleID = video.ID
+                });
+            }
         }
         
-        // Make sure we have a Icon for the channel
+        _logger.Information($"{JobName} - Done");
-        var icon = _icons.GetBySourceId(source.ID);
-        if (icon.Id == Guid.Empty)
-        {
-        }
-
-        var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
-
-        var newVideos = CheckFeed(url, source);
-        foreach (var video in newVideos) _articles.New(video);
     }
 
     private string GetChannelId(string url)
@@ -93,7 +103,9 @@ public class YoutubeWatcherJob
         pageReader.Parse();
 
         var id = pageReader.Data.Header.YoutubeChannelID ?? "";
-        if (id == "") _logger.Error(new Exception("Unable to find the Youtube Channel ID for the requested url."), url);
+        if (id == "")
+            _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
+                url);
 
         return id;
     }
@@ -130,6 +142,7 @@ public class YoutubeWatcherJob
             };
 
             videos.Add(article);
+
             Thread.Sleep(_options.SleepTimer);
         }
 

Newsbot.Collector.Tests/Jobs/YoutubeWatcherJobTests.cs

@@ -0,0 +1,19 @@
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Consts;
+using Newsbot.Collector.Services.Jobs;
+
+namespace Newsbot.Collector.Tests.Jobs;
+
+public class YoutubeWatcherJobTests
+{
+    [Fact]
+    public void InitTest()
+    {
+        var client = new YoutubeWatcherJob();
+        client.InitAndExecute(new YoutubeWatcherJobOptions
+        {
+            DatabaseConnectionString = TestHelper.LoadConfig().GetValue<string>(ConfigConst.ConnectionStringDatabase),
+            SleepTimer = 1000
+        });
+    }
+}

Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj

@@ -10,6 +10,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
     <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
@@ -35,4 +37,10 @@
     </Content>
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

Newsbot.Collector.Tests/TestHelper.cs

@@ -1,5 +1,14 @@
+using System.Net.Sockets;
+using Microsoft.Extensions.Configuration;
+
 namespace Newsbot.Collector.Tests;
 
 public static class TestHelper
 {
+    public static IConfiguration LoadConfig()
+    {
+        return new ConfigurationBuilder()
+            .AddJsonFile("appsettings.json", false)
+            .Build();
+    }
 }