Features/enabling youtube (#24)

* Adding a youtube controller to trigger the job

* Renamed jobs controller to rss

* cleaned up background jobs and added youtube to the startup.

* Handled merge issues and validated things are still working.
This commit is contained in:
James Tombleson 2023-04-08 09:30:59 -07:00 committed by GitHub
parent 4b6ba97a87
commit bacc04ad7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 139 additions and 60 deletions

View File

@ -1,6 +1,5 @@
using Hangfire; using Hangfire;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api; namespace Newsbot.Collector.Api;
@ -11,18 +10,25 @@ public static class BackgroundJobs
{ {
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x =>
x.InitAndExecute(new RssWatcherJobOptions x.InitAndExecute(new RssWatcherJobOptions
{
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.RssIsEnabled)
}), "15 0-23 * * *");
RecurringJob.AddOrUpdate<YoutubeWatcherJob>("Youtube", x => x.InitAndExecute(new YoutubeWatcherJobOptions
{ {
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase), DatabaseConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry), OpenTelemetryConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.RssIsEnabled) IsEnabled = configuration.GetValue<bool>(ConfigConst.YoutubeIsEnable)
}), "15 0-23 * * *"); }), "20 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
x.InitAndExecute(new DiscordNotificationJobOptions x.InitAndExecute(new DiscordNotificationJobOptions
{ {
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase), ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry), OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled), IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled)
}), "5/10 * * * *"); }), "5/10 * * * *");
} }
} }

View File

@ -9,24 +9,22 @@ using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api.Controllers; namespace Newsbot.Collector.Api.Controllers;
[ApiController] [ApiController]
[Route("api/jobs")] [Route("api/rss")]
public class JobsController public class RssController
{ {
private readonly ConfigSectionConnectionStrings _connectionStrings; private readonly ConfigSectionConnectionStrings _connectionStrings;
private readonly ConfigSectionRssModel _rssConfig; private readonly ConfigSectionRssModel _rssConfig;
private readonly ILogger<SourcesController> _logger; private readonly ILogger<SourcesController> _logger;
private readonly ISourcesRepository _sources;
public JobsController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings, public RssController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
IOptions<ConfigSectionRssModel> rss) IOptions<ConfigSectionRssModel> rss)
{ {
_logger = logger; _logger = logger;
_connectionStrings = connectionStrings.Value; _connectionStrings = connectionStrings.Value;
_rssConfig = rss.Value; _rssConfig = rss.Value;
_sources = new SourcesTable(connectionStrings.Value.Database ?? "");
} }
[HttpPost("check/rss")] [HttpPost("check")]
public void CheckReddit() public void CheckReddit()
{ {
BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions
@ -36,13 +34,4 @@ public class JobsController
IsEnabled = _rssConfig.IsEnabled IsEnabled = _rssConfig.IsEnabled
})); }));
} }
[HttpPost("check/youtube")]
public void CheckYoutube()
{
BackgroundJob.Enqueue<YoutubeWatcherJob>(x => x.InitAndExecute(new YoutubeWatcherJobOptions
{
ConnectionStrings = _connectionStrings
}));
}
} }

View File

@ -0,0 +1,35 @@
using Hangfire;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
using ILogger = Grpc.Core.Logging.ILogger;
namespace Newsbot.Collector.Api.Controllers;
[ApiController]
[Route("api/youtube")]
public class YoutubeController
{
private readonly ILogger<YoutubeController> _logger;
private readonly ConfigSectionConnectionStrings _connectionStrings;
private readonly ConfigSectionYoutubeModel _config;
public YoutubeController(ILogger<YoutubeController> logger, IOptions<ConfigSectionYoutubeModel> config, IOptions<ConfigSectionConnectionStrings> connectionStrings)
{
_logger = logger;
_connectionStrings = connectionStrings.Value;
_config = config.Value;
}
[HttpPost("check")]
public void CheckYoutube()
{
BackgroundJob.Enqueue<YoutubeWatcherJob>(x => x.InitAndExecute(new YoutubeWatcherJobOptions
{
DatabaseConnectionString = _connectionStrings.Database,
OpenTelemetryConnectionString = _connectionStrings.OpenTelemetry,
IsEnabled = _config.IsEnabled
}));
}
}

View File

@ -4,7 +4,6 @@ using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.HtmlParser; using Newsbot.Collector.Services.HtmlParser;
using Serilog; using Serilog;
@ -12,12 +11,17 @@ namespace Newsbot.Collector.Services.Jobs;
public class YoutubeWatcherJobOptions public class YoutubeWatcherJobOptions
{ {
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } public string? DatabaseConnectionString { get; set; }
public string? OpenTelemetryConnectionString { get; set; }
public int SleepTimer { get; set; } = 3000; public int SleepTimer { get; set; } = 3000;
public bool IsEnabled { get; set; } = true;
} }
public class YoutubeWatcherJob public class YoutubeWatcherJob
{ {
private const string JobName = "YoutubeWatcherJob";
private readonly YoutubeWatcherJobOptions _options; private readonly YoutubeWatcherJobOptions _options;
private IArticlesRepository _articles; private IArticlesRepository _articles;
private IIconsRepository _icons; private IIconsRepository _icons;
@ -32,55 +36,61 @@ public class YoutubeWatcherJob
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_source = new SourcesTable(""); _source = new SourcesTable("");
_icons = new IconsTable(""); _icons = new IconsTable("");
_logger = JobLogger.GetLogger("", "YoutubeWatcherJob"); _logger = JobLogger.GetLogger("", JobName);
} }
public void InitAndExecute(YoutubeWatcherJobOptions options) public void InitAndExecute(YoutubeWatcherJobOptions options)
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.DatabaseConnectionString ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _icons = new IconsTable(options.DatabaseConnectionString ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.OpenTelemetryConnectionString ?? "", JobName);
_icons = new IconsTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", "YoutubeWatcherJob");
Execute(); Execute();
} }
private void Execute() private void Execute()
{ {
var videos = new List<ArticlesModel>();
var sources = _source.ListByType(SourceTypes.YouTube, 100); var sources = _source.ListByType(SourceTypes.YouTube, 100);
foreach (var source in sources) CheckSource(source); foreach (var source in sources)
}
private void CheckSource(SourceModel source)
{
var channelId = "";
if (source.YoutubeId == "")
{ {
channelId = GetChannelId(source.Url); if (!source.Enabled)
_source.UpdateYoutubeId(source.ID, channelId); {
} _logger.Debug($"{JobName} - {source.Name} was disabled and will be skipped.");
else continue;
{ }
channelId = source.YoutubeId;
var channelId = source.YoutubeId;
if (channelId == "")
{
channelId = GetChannelId(source.Url);
_source.UpdateYoutubeId(source.ID, channelId);
}
// Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");
_logger.Information($"{JobName} - Checking '{source.Name}'");
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = CheckFeed(url, source);
_logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
foreach (var video in newVideos)
{
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
_articles.New(video);
_queue.New(new DiscordQueueModel
{
ArticleID = video.ID
});
}
} }
// Make sure we have a Icon for the channel _logger.Information($"{JobName} - Done");
var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty)
{
}
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = CheckFeed(url, source);
foreach (var video in newVideos) _articles.New(video);
} }
private string GetChannelId(string url) private string GetChannelId(string url)
@ -93,7 +103,9 @@ public class YoutubeWatcherJob
pageReader.Parse(); pageReader.Parse();
var id = pageReader.Data.Header.YoutubeChannelID ?? ""; var id = pageReader.Data.Header.YoutubeChannelID ?? "";
if (id == "") _logger.Error(new Exception("Unable to find the Youtube Channel ID for the requested url."), url); if (id == "")
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
url);
return id; return id;
} }
@ -130,6 +142,7 @@ public class YoutubeWatcherJob
}; };
videos.Add(article); videos.Add(article);
Thread.Sleep(_options.SleepTimer); Thread.Sleep(_options.SleepTimer);
} }

View File

@ -0,0 +1,19 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Tests.Jobs;
public class YoutubeWatcherJobTests
{
[Fact]
public void InitTest()
{
var client = new YoutubeWatcherJob();
client.InitAndExecute(new YoutubeWatcherJobOptions
{
DatabaseConnectionString = TestHelper.LoadConfig().GetValue<string>(ConfigConst.ConnectionStringDatabase),
SleepTimer = 1000
});
}
}

View File

@ -10,6 +10,8 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" /> <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
<PackageReference Include="xunit" Version="2.4.2" /> <PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5"> <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
@ -35,4 +37,10 @@
</Content> </Content>
</ItemGroup> </ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project> </Project>

View File

@ -1,5 +1,14 @@
using System.Net.Sockets;
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Tests; namespace Newsbot.Collector.Tests;
public static class TestHelper public static class TestHelper
{ {
public static IConfiguration LoadConfig()
{
return new ConfigurationBuilder()
.AddJsonFile("appsettings.json", false)
.Build();
}
} }