jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Features/enabling youtube (#24)

Browse Source 
* Adding a youtube controller to trigger the job

* Renamed jobs controller to rss

* cleaned up background jobs and added youtube to the startup.

* Handled merge issues and validated things are still working.
This commit is contained in:
James Tombleson 2023-04-08 09:30:59 -07:00 committed by GitHub
parent 4b6ba97a87
commit bacc04ad7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 139 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Newsbot.Collector.Api/BackgroundJobs.cs
View File

@ -1,6 +1,5 @@
using Hangfire;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api;
@ -9,20 +8,27 @@ public static class BackgroundJobs
{
public static void SetupRecurringJobs(IConfiguration configuration)
{
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x =>
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x =>
x.InitAndExecute(new RssWatcherJobOptions
{
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.RssIsEnabled)
}), "15 0-23 * * *");
RecurringJob.AddOrUpdate<YoutubeWatcherJob>("Youtube", x => x.InitAndExecute(new YoutubeWatcherJobOptions
{
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.RssIsEnabled)
}), "15 0-23 * * *");
DatabaseConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetryConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.YoutubeIsEnable)
}), "20 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
x.InitAndExecute(new DiscordNotificationJobOptions
{
ConnectionString = configuration.GetValue<string>(ConfigConst.ConnectionStringDatabase),
OpenTelemetry = configuration.GetValue<string>(ConfigConst.ConnectionStringOpenTelemetry),
IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled),
IsEnabled = configuration.GetValue<bool>(ConfigConst.DiscordNotificationsEnabled)
}), "5/10 * * * *");
}
}

19
Newsbot.Collector.Api/Controllers/JobsController.cs → Newsbot.Collector.Api/Controllers/RssController.cs
View File

@ -9,24 +9,22 @@ using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api.Controllers;
[ApiController]
[Route("api/jobs")]
public class JobsController
[Route("api/rss")]
public class RssController
{
private readonly ConfigSectionConnectionStrings _connectionStrings;
private readonly ConfigSectionRssModel _rssConfig;
private readonly ILogger<SourcesController> _logger;
private readonly ISourcesRepository _sources;
public JobsController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
public RssController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
IOptions<ConfigSectionRssModel> rss)
{
_logger = logger;
_connectionStrings = connectionStrings.Value;
_rssConfig = rss.Value;
_sources = new SourcesTable(connectionStrings.Value.Database ?? "");
}
[HttpPost("check/rss")]
[HttpPost("check")]
public void CheckReddit()
{
BackgroundJob.Enqueue<RssWatcherJob>(x => x.InitAndExecute(new RssWatcherJobOptions
@ -36,13 +34,4 @@ public class JobsController
IsEnabled = _rssConfig.IsEnabled
}));
}
[HttpPost("check/youtube")]
public void CheckYoutube()
{
BackgroundJob.Enqueue<YoutubeWatcherJob>(x => x.InitAndExecute(new YoutubeWatcherJobOptions
{
ConnectionStrings = _connectionStrings
}));
}
}

35
Newsbot.Collector.Api/Controllers/YoutubeController.cs Normal file
View File

@ -0,0 +1,35 @@
using Hangfire;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
using ILogger = Grpc.Core.Logging.ILogger;
namespace Newsbot.Collector.Api.Controllers;
[ApiController]
[Route("api/youtube")]
public class YoutubeController
{
private readonly ILogger<YoutubeController> _logger;
private readonly ConfigSectionConnectionStrings _connectionStrings;
private readonly ConfigSectionYoutubeModel _config;
public YoutubeController(ILogger<YoutubeController> logger, IOptions<ConfigSectionYoutubeModel> config, IOptions<ConfigSectionConnectionStrings> connectionStrings)
{
_logger = logger;
_connectionStrings = connectionStrings.Value;
_config = config.Value;
}
[HttpPost("check")]
public void CheckYoutube()
{
BackgroundJob.Enqueue<YoutubeWatcherJob>(x => x.InitAndExecute(new YoutubeWatcherJobOptions
{
DatabaseConnectionString = _connectionStrings.Database,
OpenTelemetryConnectionString = _connectionStrings.OpenTelemetry,
IsEnabled = _config.IsEnabled
}));
}
}

87
Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs
View File

@ -4,7 +4,6 @@ using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.HtmlParser;
using Serilog;
@ -12,12 +11,17 @@ namespace Newsbot.Collector.Services.Jobs;
public class YoutubeWatcherJobOptions
{
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
public string? DatabaseConnectionString { get; set; }
public string? OpenTelemetryConnectionString { get; set; }
public int SleepTimer { get; set; } = 3000;
public bool IsEnabled { get; set; } = true;
}
public class YoutubeWatcherJob
{
private const string JobName = "YoutubeWatcherJob";
private readonly YoutubeWatcherJobOptions _options;
private IArticlesRepository _articles;
private IIconsRepository _icons;
@ -32,55 +36,61 @@ public class YoutubeWatcherJob
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
_icons = new IconsTable("");
_logger = JobLogger.GetLogger("", "YoutubeWatcherJob");
_logger = JobLogger.GetLogger("", JobName);
}
public void InitAndExecute(YoutubeWatcherJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_icons = new IconsTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", "YoutubeWatcherJob");
_articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_source = new SourcesTable(options.DatabaseConnectionString ?? "");
_icons = new IconsTable(options.DatabaseConnectionString ?? "");
_logger = JobLogger.GetLogger(options.OpenTelemetryConnectionString ?? "", JobName);
Execute();
}
private void Execute()
{
var videos = new List<ArticlesModel>();
var sources = _source.ListByType(SourceTypes.YouTube, 100);
foreach (var source in sources) CheckSource(source);
}
private void CheckSource(SourceModel source)
{
var channelId = "";
if (source.YoutubeId == "")
foreach (var source in sources)
{
channelId = GetChannelId(source.Url);
_source.UpdateYoutubeId(source.ID, channelId);
}
else
{
channelId = source.YoutubeId;
}
if (!source.Enabled)
{
_logger.Debug($"{JobName} - {source.Name} was disabled and will be skipped.");
continue;
}
var channelId = source.YoutubeId;
if (channelId == "")
{
channelId = GetChannelId(source.Url);
_source.UpdateYoutubeId(source.ID, channelId);
}
// Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");
// Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty)
{
_logger.Information($"{JobName} - Checking '{source.Name}'");
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = CheckFeed(url, source);
_logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
foreach (var video in newVideos)
{
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
_articles.New(video);
_queue.New(new DiscordQueueModel
{
ArticleID = video.ID
});
}
}
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = CheckFeed(url, source);
foreach (var video in newVideos) _articles.New(video);
_logger.Information($"{JobName} - Done");
}
private string GetChannelId(string url)
@ -93,7 +103,9 @@ public class YoutubeWatcherJob
pageReader.Parse();
var id = pageReader.Data.Header.YoutubeChannelID ?? "";
if (id == "") _logger.Error(new Exception("Unable to find the Youtube Channel ID for the requested url."), url);
if (id == "")
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
url);
return id;
}
@ -130,6 +142,7 @@ public class YoutubeWatcherJob
};
videos.Add(article);
Thread.Sleep(_options.SleepTimer);
}

19
Newsbot.Collector.Tests/Jobs/YoutubeWatcherJobTests.cs Normal file
View File

@ -0,0 +1,19 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Tests.Jobs;
public class YoutubeWatcherJobTests
{
[Fact]
public void InitTest()
{
var client = new YoutubeWatcherJob();
client.InitAndExecute(new YoutubeWatcherJobOptions
{
DatabaseConnectionString = TestHelper.LoadConfig().GetValue<string>(ConfigConst.ConnectionStringDatabase),
SleepTimer = 1000
});
}
}

8
Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj
View File

@ -10,6 +10,8 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
<PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
@ -35,4 +37,10 @@
</Content>
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

9
Newsbot.Collector.Tests/TestHelper.cs
View File

@ -1,5 +1,14 @@
using System.Net.Sockets;
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Tests;
public static class TestHelper
{
public static IConfiguration LoadConfig()
{
return new ConfigurationBuilder()
.AddJsonFile("appsettings.json", false)
.Build();
}
}