From 117653c0015736c1db17553fa2c152b6ecccc69b Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 10 Apr 2023 22:59:13 -0700 Subject: [PATCH] Features/add code projects watcher (#26) * Migration added to update Articles to define if Release or Commit * CodeProjectWatcher Job was created from GithubWatcher as this will target services like gitlab and also gitea. * article model was updated to reflect migration changes * Added CodeProjects to startup * Seed was updated with CodeProjects and some new defaults * Added Delete call for Sources * Added a route to cleanup all records based on SourceId * Added CodeProject const values to load from config * minor changes to the rss controller * Added codeprojects to the routes to trigger the job --- Newsbot.Collector.Api/BackgroundJobs.cs | 10 + .../Controllers/ArticlesController.cs | 13 +- .../Controllers/CodeProjectController.cs | 33 ++++ .../Controllers/RssController.cs | 6 +- .../Controllers/SourcesController.cs | 22 ++- .../20230326223545_icon_sourceid.sql | 16 +- .../Repositories/ArticlesTable.cs | 58 +++--- .../Repositories/SourcesTable.cs | 11 ++ .../Consts/ConfigConst.cs | 13 +- .../Consts/SourcesTypes.cs | 3 +- .../Interfaces/IArticlesRepository.cs | 5 +- .../Interfaces/ISourcesRepository.cs | 1 + .../Models/DatabaseModel.cs | 2 + .../Jobs/CodeProjectWatcherJob.cs | 178 ++++++++++++++++++ .../Jobs/GithubWatcherJob.cs | 115 ----------- .../Jobs/CodeProjectWatcherJobTests.cs | 88 +++++++++ .../Jobs/GithubWatcherJobTests.cs | 46 ----- seed.ps1 | 24 +++ 18 files changed, 421 insertions(+), 223 deletions(-) create mode 100644 Newsbot.Collector.Api/Controllers/CodeProjectController.cs create mode 100644 Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs delete mode 100644 Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs create mode 100644 Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs delete mode 100644 Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs diff --git a/Newsbot.Collector.Api/BackgroundJobs.cs b/Newsbot.Collector.Api/BackgroundJobs.cs index 819d8f3..e1e6892 100644 --- a/Newsbot.Collector.Api/BackgroundJobs.cs +++ b/Newsbot.Collector.Api/BackgroundJobs.cs @@ -1,5 +1,6 @@ using Hangfire; using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Api; @@ -23,6 +24,15 @@ public static class BackgroundJobs IsEnabled = configuration.GetValue(ConfigConst.YoutubeIsEnable) }), "20 0-23 * * *"); + RecurringJob.AddOrUpdate("CodeProjects", x => + x.InitAndExecute(new CodeProjectWatcherJobOptions + { + ConnectionStrings = configuration.GetSection(ConfigConst.SectionConnectionStrings) + .Get(), + FeaturePullCommits = true, + FeaturePullReleases = true + }), "25 0-23 * * *"); + RecurringJob.AddOrUpdate("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions { diff --git a/Newsbot.Collector.Api/Controllers/ArticlesController.cs b/Newsbot.Collector.Api/Controllers/ArticlesController.cs index 0867e01..a8962c9 100644 --- a/Newsbot.Collector.Api/Controllers/ArticlesController.cs +++ b/Newsbot.Collector.Api/Controllers/ArticlesController.cs @@ -11,8 +11,8 @@ namespace Newsbot.Collector.Api.Controllers; [Route("api/articles")] public class ArticlesController : ControllerBase { - private readonly ILogger _logger; private readonly IArticlesRepository _articles; + private readonly ILogger _logger; private readonly ISourcesRepository _sources; public ArticlesController(ILogger logger, IOptions settings) @@ -27,14 +27,12 @@ public class ArticlesController : ControllerBase { var res = new List(); var items = _articles.List(0, 25); - foreach (var item in items) - { - res.Add(ArticleDto.Convert(item)); - } + foreach (var item in items) res.Add(ArticleDto.Convert(item)); return res; } [HttpGet("{id:guid}")] + [EndpointDescription("Returns the article based on the Id value given.")] public ArticleDto GetById(Guid id) { var item = _articles.GetById(id); @@ -54,10 +52,7 @@ public class ArticlesController : ControllerBase { var res = new List(); var items = _articles.ListBySourceId(sourceId, page, count); - foreach (var item in items) - { - res.Add(ArticleDto.Convert(item)); - } + foreach (var item in items) res.Add(ArticleDto.Convert(item)); return res; } } \ No newline at end of file diff --git a/Newsbot.Collector.Api/Controllers/CodeProjectController.cs b/Newsbot.Collector.Api/Controllers/CodeProjectController.cs new file mode 100644 index 0000000..491d5ef --- /dev/null +++ b/Newsbot.Collector.Api/Controllers/CodeProjectController.cs @@ -0,0 +1,33 @@ +using Hangfire; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using Newsbot.Collector.Domain.Models.Config; +using Newsbot.Collector.Services.Jobs; + +namespace Newsbot.Collector.Api.Controllers; + +[ApiController] +[Route("api/codeprojects")] +public class CodeProjectController +{ + private readonly ConfigSectionConnectionStrings _connectionStrings; + private readonly ILogger _logger; + + public CodeProjectController(ILogger logger, + IOptions settings) + { + _logger = logger; + _connectionStrings = settings.Value; + } + + [HttpPost("check")] + public void PullNow() + { + BackgroundJob.Enqueue(x => x.InitAndExecute(new CodeProjectWatcherJobOptions + { + ConnectionStrings = _connectionStrings, + FeaturePullReleases = true, + FeaturePullCommits = true + })); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Api/Controllers/RssController.cs b/Newsbot.Collector.Api/Controllers/RssController.cs index cbf34de..68ab76d 100644 --- a/Newsbot.Collector.Api/Controllers/RssController.cs +++ b/Newsbot.Collector.Api/Controllers/RssController.cs @@ -1,8 +1,6 @@ using Hangfire; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; -using Newsbot.Collector.Database.Repositories; -using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.Jobs; @@ -13,10 +11,10 @@ namespace Newsbot.Collector.Api.Controllers; public class RssController { private readonly ConfigSectionConnectionStrings _connectionStrings; + private readonly ILogger _logger; private readonly ConfigSectionRssModel _rssConfig; - private readonly ILogger _logger; - public RssController(ILogger logger, IOptions connectionStrings, + public RssController(ILogger logger, IOptions connectionStrings, IOptions rss) { _logger = logger; diff --git a/Newsbot.Collector.Api/Controllers/SourcesController.cs b/Newsbot.Collector.Api/Controllers/SourcesController.cs index 7c98819..57e244d 100644 --- a/Newsbot.Collector.Api/Controllers/SourcesController.cs +++ b/Newsbot.Collector.Api/Controllers/SourcesController.cs @@ -13,10 +13,11 @@ namespace Newsbot.Collector.Api.Controllers; [Route("api/sources")] public class SourcesController : ControllerBase { - private readonly IIconsRepository _icons; - private readonly ILogger _logger; + private readonly IArticlesRepository _articles; //private readonly ConnectionStrings _settings; + private readonly IIconsRepository _icons; + private readonly ILogger _logger; private readonly ISourcesRepository _sources; public SourcesController(ILogger logger, IOptions settings) @@ -25,6 +26,7 @@ public class SourcesController : ControllerBase //_settings = settings.Value; _sources = new SourcesTable(settings.Value.Database); _icons = new IconsTable(settings.Value.Database); + _articles = new ArticlesTable(settings.Value.Database); } [HttpGet(Name = "GetSources")] @@ -154,10 +156,10 @@ public class SourcesController : ControllerBase return SourceDto.Convert(item); } - [HttpPost("new/github")] + [HttpPost("new/codeproject")] public SourceDto NewGithub(string url) { - if (!url.Contains("github.com")) return new SourceDto(); + //if (!url.Contains("github.com")) return new SourceDto(); var res = _sources.GetByUrl(url); if (res.ID != Guid.Empty) return SourceDto.Convert(res); @@ -172,13 +174,13 @@ public class SourcesController : ControllerBase var item = _sources.New(new SourceModel { - Site = SourceTypes.GitHub, - Type = SourceTypes.GitHub, + Site = SourceTypes.CodeProject, + Type = SourceTypes.CodeProject, Name = $"{slice[3]}/{slice[4]}", Url = url, Source = "feed", Enabled = true, - Tags = $"{SourceTypes.GitHub}, {slice[3]}, {slice[4]}" + Tags = $"{slice[2]},{slice[3]},{slice[4]}" }); _icons.New(new IconModel @@ -209,4 +211,10 @@ public class SourcesController : ControllerBase { _sources.Enable(id); } + + [HttpDelete("{id}")] + public void Delete(Guid id, bool purgeOrphanedRecords) + { + _sources.Delete(id); + } } \ No newline at end of file diff --git a/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql b/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql index 54f220f..2a6228f 100644 --- a/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql +++ b/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql @@ -1,13 +1,21 @@ -- +goose Up -- +goose StatementBegin SELECT 'up SQL query'; -ALTER TABLE icons - ADD COLUMN SourceId uuid; +ALTER TABLE sources + ADD COLUMN IconUri uuid; + +ALTER TABLE articles + ADD COLUMN CodeIsRelease bool, + ADD COLUMN CodeIsCommit bool; -- +goose StatementEnd -- +goose Down -- +goose StatementBegin SELECT 'down SQL query'; -ALTER TABLE icons - DROP COLUMN SourceId; +ALTER TABLE sources + DROP COLUMN IconUri; + +Alter TABLE articles + DROP COLUMN CodeIsRelease, + DROP COLUMN CodeIsCommit -- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs index 8cc2edb..376e091 100644 --- a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs +++ b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs @@ -9,8 +9,7 @@ namespace Newsbot.Collector.Database.Repositories; public class ArticlesTable : IArticlesRepository { - - private string _connectionString; + private readonly string _connectionString; public ArticlesTable(string connectionString) { @@ -20,21 +19,11 @@ public class ArticlesTable : IArticlesRepository public ArticlesTable(IConfiguration configuration) { var conn = configuration.GetConnectionString("database"); - if (conn is null) - { - conn = ""; - } + if (conn is null) conn = ""; _connectionString = conn; } - private IDbConnection OpenConnection(string connectionString) - { - var conn = new NpgsqlConnection(_connectionString); - conn.Open(); - return conn; - } - public List List(int page = 0, int count = 25) { using var conn = OpenConnection(_connectionString); @@ -42,22 +31,19 @@ public class ArticlesTable : IArticlesRepository Order By PubDate Desc Offset @Page Fetch Next @Count Rows Only", new - { - Page = page * count, - Count = count - }) - .ToList(); + { + Page = page * count, + Count = count + }) + .ToList(); return res; } public ArticlesModel GetById(Guid ID) { using var conn = OpenConnection(_connectionString); - var res = conn.Query("select * from articles where ID = @ID", new { ID = ID }); - if (res.Count() == 0) - { - return new ArticlesModel(); - } + var res = conn.Query("select * from articles where ID = @ID", new { ID }); + if (res.Count() == 0) return new ArticlesModel(); return res.First(); } @@ -65,10 +51,7 @@ public class ArticlesTable : IArticlesRepository { using var conn = OpenConnection(_connectionString); var res = conn.Query("select * from articles where Url = @Url Limit 1", new { Url = url }); - if (res.Count() == 0) - { - return new ArticlesModel(); - } + if (res.Count() == 0) return new ArticlesModel(); return res.First(); } @@ -83,7 +66,7 @@ public class ArticlesTable : IArticlesRepository { sourceid = id, page = page * count, - count = count + count }).ToList(); } @@ -92,7 +75,8 @@ public class ArticlesTable : IArticlesRepository model.ID = Guid.NewGuid(); using var conn = OpenConnection(_connectionString); - var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; + var q = + "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; var res = conn.Execute(q, new { id = model.ID, @@ -112,4 +96,20 @@ public class ArticlesTable : IArticlesRepository return model; } + public void DeleteAllBySourceId(Guid sourceId) + { + using var conn = OpenConnection(_connectionString); + var res = conn.Execute("Delete from articles where sourceid = '@id'", new + { + sourceId + }); + if (res == 0) throw new Exception($"No records where deleted that linked to SourceId = '{sourceId}'"); + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); + conn.Open(); + return conn; + } } \ No newline at end of file diff --git a/Newsbot.Collector.Database/Repositories/SourcesTable.cs b/Newsbot.Collector.Database/Repositories/SourcesTable.cs index 62e5e69..61348cf 100644 --- a/Newsbot.Collector.Database/Repositories/SourcesTable.cs +++ b/Newsbot.Collector.Database/Repositories/SourcesTable.cs @@ -155,6 +155,17 @@ public class SourcesTable : ISourcesRepository }); } + public void Delete(Guid id) + { + using var conn = OpenConnection(_connectionString); + var query = "Delete From sources where id = @id;"; + var res = conn.Execute(query, new + { + id + }); + if (res == 0) throw new Exception("Nothing was deleted"); + } + public int UpdateYoutubeId(Guid id, string youtubeId) { using var conn = OpenConnection(_connectionString); diff --git a/Newsbot.Collector.Domain/Consts/ConfigConst.cs b/Newsbot.Collector.Domain/Consts/ConfigConst.cs index 2f4922e..7aa8227 100644 --- a/Newsbot.Collector.Domain/Consts/ConfigConst.cs +++ b/Newsbot.Collector.Domain/Consts/ConfigConst.cs @@ -3,32 +3,35 @@ namespace Newsbot.Collector.Domain.Consts; public class ConfigConst { public const string LoggingDefault = "Logging:LogLevel:Default"; - + public const string SectionConnectionStrings = "ConnectionStrings"; public const string SectionFinalFantasyXiv = "FinalFantasyXiv"; public const string SectionReddit = "Reddit"; public const string SectionRss = "Rss"; public const string SectionTwitch = "Twitch"; public const string SectionYoutube = "Youtube"; + public const string SectionCodeProjects = "CodeProjects"; public const string SectionNotificationsDiscord = "Notifications:Discord"; - + public const string ConnectionStringDatabase = "ConnectionStrings:Database"; public const string ConnectionStringOpenTelemetry = "ConnectionStrings:OpenTelemetry"; - + public const string RedditIsEnabled = "Reddit:IsEnabled"; public const string RedditPullHot = "Reddit:PullHot"; public const string RedditPullNsfw = "Reddit:PullNsfw"; public const string RedditPullTop = "Reddit:PullTop"; public const string RssIsEnabled = "Rss:IsEnabled"; - + public const string TwitchIsEnabled = "Twitch:IsEnabled"; public const string TwitchClientId = "Twitch:ClientID"; public const string TwitchClientSecret = "Twitch:ClientSecret"; - + public const string YoutubeIsEnable = "Youtube:IsEnabled"; public const string YoutubeDebug = "Youtube:Debug"; + public const string CodeProjectsIsEnabled = "CodeProjects:IsEnabled"; + public const string EnableSwagger = "EnableSwagger"; public const string DiscordNotificationsEnabled = "Notifications:Discord:IsEnabled"; diff --git a/Newsbot.Collector.Domain/Consts/SourcesTypes.cs b/Newsbot.Collector.Domain/Consts/SourcesTypes.cs index 7c1befd..cac8193 100644 --- a/Newsbot.Collector.Domain/Consts/SourcesTypes.cs +++ b/Newsbot.Collector.Domain/Consts/SourcesTypes.cs @@ -7,6 +7,5 @@ public class SourceTypes public const string YouTube = "youtube"; public const string Twitch = "twitch"; public const string FinalFantasyXiv = "ffxiv"; - public const string GitHub = "github"; - + public const string CodeProject = "code"; } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs index 9be0fb4..c17e62c 100644 --- a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs @@ -4,9 +4,10 @@ namespace Newsbot.Collector.Domain.Interfaces; public interface IArticlesRepository : ITableRepository { - ListList(int page, int count); - ListListBySourceId(Guid id, int page = 0, int count = 25); + List List(int page, int count); + List ListBySourceId(Guid id, int page = 0, int count = 25); ArticlesModel GetById(Guid ID); ArticlesModel GetByUrl(string url); ArticlesModel New(ArticlesModel model); + void DeleteAllBySourceId(Guid sourceId); } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs index 8db6a82..c528230 100644 --- a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs @@ -15,5 +15,6 @@ public interface ISourcesRepository public List ListByType(string type, int limit = 25); public int Disable(Guid id); public int Enable(Guid id); + public void Delete(Guid id); public int UpdateYoutubeId(Guid id, string youtubeId); } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Models/DatabaseModel.cs b/Newsbot.Collector.Domain/Models/DatabaseModel.cs index e8c5080..8a1a46e 100644 --- a/Newsbot.Collector.Domain/Models/DatabaseModel.cs +++ b/Newsbot.Collector.Domain/Models/DatabaseModel.cs @@ -15,6 +15,8 @@ public class ArticlesModel public string Description { get; set; } = ""; public string AuthorName { get; set; } = ""; public string? AuthorImage { get; set; } + public bool CodeIsRelease { get; set; } + public bool CodeIsCommit { get; set; } } public class AuthorModel diff --git a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs new file mode 100644 index 0000000..9b78281 --- /dev/null +++ b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs @@ -0,0 +1,178 @@ +using System.Collections; +using System.ServiceModel.Syndication; +using System.Xml; +using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Interfaces; +using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Models.Config; +using Newsbot.Collector.Services.HtmlParser; +using Serilog; + +namespace Newsbot.Collector.Services.Jobs; + +public class CodeProjectWatcherJobOptions +{ + public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } + + //public string ConnectionString { get; set; } = ""; + public bool FeaturePullReleases { get; set; } = false; + + public bool FeaturePullCommits { get; set; } = false; + //public bool PullIssues { get; set; } = false; +} + +public class CodeProjectWatcherJob +{ + private const string JobName = "CodeProjectWatcher"; + private IArticlesRepository _articles; + private ILogger _logger; + private IDiscordQueueRepository _queue; + private ISourcesRepository _source; + + public CodeProjectWatcherJob() + { + _articles = new ArticlesTable(""); + _queue = new DiscordQueueTable(""); + _source = new SourcesTable(""); + _logger = JobLogger.GetLogger("", JobName); + } + + public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options) + { + options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); + _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); + _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); + _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); + _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); + } + + public void InitAndExecute(CodeProjectWatcherJobOptions options) + { + options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); + _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); + _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); + _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); + _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); + + Execute(); + } + + private void Execute() + { + var sources = _source.ListByType(SourceTypes.CodeProject); + + // query sources for things to pull + var items = new List(); + + foreach (var source in sources) + { + items.AddRange(CheckForReleases(source)); + items.AddRange(CheckForCommits(source)); + } + + foreach (var item in items) + { + _articles.New(item); + _queue.New(new DiscordQueueModel + { + ArticleID = item.ID + }); + } + } + + public IEnumerable CheckForReleases(SourceModel source) + { + var url = new Uri(source.Url); + var links = new List + { + $"{url.AbsoluteUri}/releases.atom", + $"{url.AbsoluteUri}/tags.atom" //github converts tags as releases + }; + + foreach (var link in links) + try + { + using var reader = XmlReader.Create(link); + var client = SyndicationFeed.Load(reader); + + return ProcessFeed(client.Items, source, true, false); + + + //if (link.EndsWith("tags.atom")) + //{ + // return ProcessFeed(client.Items, source, false, true, false); + //} + } + catch + { + _logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}. Might not have anything", JobName, + url.AbsoluteUri); + } + + return new List(); + } + + public IEnumerable CheckForCommits(SourceModel source) + { + var url = new Uri(source.Url); + var links = new List + { + $"{url.AbsoluteUri}/commits/main.atom", + $"{url.AbsoluteUri}/commits/master.atom" + }; + + foreach (var link in links) + try + { + using var reader = XmlReader.Create(link); + var client = SyndicationFeed.Load(reader); + + return ProcessFeed(client.Items, source, false, true); + } + catch + { + _logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}. Might not have anything", JobName, + url.AbsoluteUri); + } + + return new List(); + } + + private IEnumerable ProcessFeed(IEnumerable feed, SourceModel source, + bool isRelease, bool isCommit) + { + var items = new List(); + + foreach (var item in feed) + { + var itemUrl = item.Links[0].Uri.AbsoluteUri; + var exits = _articles.GetByUrl(itemUrl); + if (exits.ID != Guid.Empty) continue; + + var parser = new HtmlPageReader(new HtmlPageReaderOptions + { + Url = itemUrl + }); + parser.Parse(); + + var a = new ArticlesModel + { + SourceID = source.ID, + Tags = source.Tags, + Title = item.Title.Text, + URL = itemUrl, + PubDate = item.LastUpdatedTime.DateTime, + Thumbnail = parser.Data.Header.Image, + Description = item.Title.Text, + AuthorName = item.Authors[0].Name ?? "", + AuthorImage = item.Authors[0].Uri ?? "", + CodeIsRelease = isRelease, + CodeIsCommit = isCommit, + }; + items.Add(a); + } + + return items; + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs b/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs deleted file mode 100644 index 4ae819f..0000000 --- a/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System.ServiceModel.Syndication; -using System.Xml; -using Newsbot.Collector.Database.Repositories; -using Newsbot.Collector.Domain.Consts; -using Newsbot.Collector.Domain.Interfaces; -using Newsbot.Collector.Domain.Models; -using Newsbot.Collector.Domain.Models.Config; -using Newsbot.Collector.Services.HtmlParser; - -namespace Newsbot.Collector.Services.Jobs; - -public class GithubWatcherJobOptions -{ - public ConfigSectionConnectionStrings? ConnectionStrings { get; set; } - - //public string ConnectionString { get; set; } = ""; - public bool FeaturePullReleases { get; set; } = false; - - public bool FeaturePullCommits { get; set; } = false; - //public bool PullIssues { get; set; } = false; -} - -public class GithubWatcherJob -{ - private IArticlesRepository _articles; - private IDiscordQueueRepository _queue; - private ISourcesRepository _source; - - public GithubWatcherJob() - { - _articles = new ArticlesTable(""); - _queue = new DiscordQueueTable(""); - _source = new SourcesTable(""); - } - - public void InitAndExecute(GithubWatcherJobOptions options) - { - options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); - _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); - _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); - _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); - - Execute(); - } - - private void Execute() - { - _source.ListBySource(SourceTypes.GitHub, 25); - - // query sources for things to pull - var items = new List(); - - - items.AddRange(Collect(new Uri("https://github.com/jtom38/dvb"))); - - // query */commits/master.atom - // query */commits/main.atom - } - - public List Collect(Uri url) - { - var items = new List(); - - var placeHolderId = Guid.NewGuid(); - // query */release.atom - items.AddRange(CollectItems($"{url.AbsoluteUri}/releases.atom", placeHolderId)); - items.AddRange(CollectItems($"{url.AbsoluteUri}/master.atom", placeHolderId)); - - return items; - } - - private List CollectItems(string baseUrl, Guid sourceId) - { - var items = new List(); - - using var reader = XmlReader.Create(baseUrl); - var client = SyndicationFeed.Load(reader); - - foreach (var item in client.Items) - { - var itemUrl = item.Links[0].Uri.AbsoluteUri; - var exits = _articles.GetByUrl(itemUrl); - if (exits.ID != Guid.Empty) continue; - - var parser = new HtmlPageReader(new HtmlPageReaderOptions - { - Url = itemUrl - }); - parser.Parse(); - - try - { - var a = new ArticlesModel - { - SourceID = sourceId, - Tags = "github", - Title = item.Title.Text, - URL = itemUrl, - //PubDate = item.LastUpdatedTime.DateTime, - Thumbnail = parser.Data.Header.Image, - Description = $"'dvb' has released '{item.Title.Text}'!", - AuthorName = item.Authors[0].Name ?? "", - AuthorImage = item.Authors[0].Uri ?? "" - }; - items.Add(a); - } - catch (Exception e) - { - Console.WriteLine(e); - } - } - - return items; - } -} \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs b/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs new file mode 100644 index 0000000..36ecb04 --- /dev/null +++ b/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs @@ -0,0 +1,88 @@ +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Models.Config; +using Newsbot.Collector.Services.Jobs; + +namespace Newsbot.Collector.Tests.Jobs; + +public class CodeProjectWatcherJobTests +{ + [Fact] + public void CanReturnReleases() + { + var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions + { + ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings) + .Get(), + FeaturePullCommits = true, + FeaturePullReleases = true + }); + var results = client.CheckForReleases(new SourceModel + { + ID = Guid.NewGuid(), + Url = "https://github.com/jtom38/dvb", + Type = SourceTypes.CodeProject, + Site = SourceTypes.CodeProject, + Name = "jtom38/dvb", + Source = "feed", + Enabled = true + }); + if (!results.Any()) + { + Assert.Fail("Expected at least one item"); + } + } + + [Fact] + public void CollectsTagsButNoReleases() + { + var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions + { + ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings) + .Get(), + FeaturePullCommits = true, + FeaturePullReleases = true + }); + var results = client.CheckForReleases(new SourceModel + { + ID = Guid.NewGuid(), + Url = "https://github.com/python/cpython", + Type = SourceTypes.CodeProject, + Site = SourceTypes.CodeProject, + Name = "python.cpython", + Source = "feed", + Enabled = true + }); + if (!results.Any()) + { + Assert.Fail("Expected at least one item"); + } + } + + [Fact] + public void CollectsCommits() + { + var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions + { + ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings) + .Get(), + FeaturePullCommits = true, + FeaturePullReleases = true + }); + var results = client.CheckForCommits(new SourceModel + { + ID = Guid.NewGuid(), + Url = "https://github.com/jtom38/dvb", + Type = SourceTypes.CodeProject, + Site = SourceTypes.CodeProject, + Name = "jtom38/dvb", + Source = "feed", + Enabled = true + }); + if (!results.Any()) + { + Assert.Fail("Expected at least one item"); + } + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs b/Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs deleted file mode 100644 index 445bae9..0000000 --- a/Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Newsbot.Collector.Domain.Models; -using Newsbot.Collector.Domain.Models.Config; -using Newsbot.Collector.Services.Jobs; - -namespace Newsbot.Collector.Tests.Jobs; - -public class GithubWatcherJobTests -{ - private IConfiguration GetConfiguration() - { - var inMemorySettings = new Dictionary - { - { - "ConnectionStrings:database", - "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable" - } - }; - - IConfiguration configuration = new ConfigurationBuilder() - .AddInMemoryCollection(inMemorySettings) - .Build(); - return configuration; - } - - private string ConnectionString() - { - return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; - } - - [Fact] - public void CanPullAFeed() - { - var client = new GithubWatcherJob(); - client.InitAndExecute(new GithubWatcherJobOptions - { - ConnectionStrings = new ConfigSectionConnectionStrings - { - Database = ConnectionString() - }, - FeaturePullCommits = true, - FeaturePullReleases = true - }); - client.Collect(new Uri("https://github.com/jtom38/dvb")); - } -} \ No newline at end of file diff --git a/seed.ps1 b/seed.ps1 index e4552c8..980603f 100644 --- a/seed.ps1 +++ b/seed.ps1 @@ -50,6 +50,20 @@ function New-TwitchSource { return $res } +function New-CodeProject { + param ( + [string] $Url + ) + + $urlEncoded = [uri]::EscapeDataString($Url) + [string] $param = "url=$urlEncoded" + [string] $uri = "$ApiServer/api/sources/new/codeproject?$param" + + Write-Host "Adding CodeProject '$url'" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + function New-DiscordWebhook { param ( [string] $Server, @@ -94,6 +108,11 @@ $youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTe $youtubeFireship = New-YoutubeSource -Url "https://www.youtube.com/@Fireship" $youtubeClimateTown = New-YoutubeSource -Url "https://www.youtube.com/c/ClimateTown" +$codeDotnet = New-CodeProject -Url "https://github.com/dotnet/runtime" +#$codePython = New-CodeProject -Url "https://github.com/python/cpython" +#$codeGolang = New-CodeProject -Url "https://github.com/golang/go" +#$codePowerShell = New-CodeProject -Url "https://github.com/PowerShell/PowerShell" +#$codeLinux = New-CodeProject -Url "https://github.com/torvalds/linux" $twitchNintendo = New-TwitchSource -Name "Nintendo" @@ -102,6 +121,7 @@ $sky = New-DiscordWebhook -Server "Let's Mosley" -Channel "bot test" -url $secre New-Subscription -SourceId $redditDadJokes.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $redditSteamDeck.id -DiscordWebhookId $miharuMonitor.id + New-Subscription -SourceId $rssSteamDeck.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssFaysHaremporium.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $miharuMonitor.id @@ -109,9 +129,13 @@ New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $sky.id New-Subscription -SourceId $rssOmgLinux.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id + +New-Subscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id + New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeLinusTechTips.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeFireship.id -DiscordWebhookId $miharuMonitor.id New-Subscription -SourceId $youtubeClimateTown.id -DiscordWebhookId $miharuMonitor.id + New-Subscription -SourceId $twitchNintendo.id -DiscordWebhookId $miharuMonitor.id