Features/add code projects watcher (#26)

* Migration added to update Articles to define if Release or Commit

* CodeProjectWatcher Job was created from GithubWatcher as this will target services like gitlab and also gitea.

* article model was updated to reflect migration changes

* Added CodeProjects to startup

* Seed was updated with CodeProjects and some new defaults

* Added Delete call for Sources

* Added a route to cleanup all records based on SourceId

* Added CodeProject const values to load from config

* minor changes to the rss controller

* Added codeprojects to the routes to trigger the job

Newsbot.Collector.Api/BackgroundJobs.cs

@@ -1,5 +1,6 @@
 using Hangfire;
 using Newsbot.Collector.Domain.Consts;
+using Newsbot.Collector.Domain.Models.Config;
 using Newsbot.Collector.Services.Jobs;
 
 namespace Newsbot.Collector.Api;
@@ -23,6 +24,15 @@ public static class BackgroundJobs
             IsEnabled = configuration.GetValue<bool>(ConfigConst.YoutubeIsEnable)
         }), "20 0-23 * * *");
 
+        RecurringJob.AddOrUpdate<CodeProjectWatcherJob>("CodeProjects", x =>
+            x.InitAndExecute(new CodeProjectWatcherJobOptions
+            {
+                ConnectionStrings = configuration.GetSection(ConfigConst.SectionConnectionStrings)
+                    .Get<ConfigSectionConnectionStrings>(),
+                FeaturePullCommits = true,
+                FeaturePullReleases = true
+            }), "25 0-23 * * *");
+
         RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
             x.InitAndExecute(new DiscordNotificationJobOptions
             {

Newsbot.Collector.Api/Controllers/ArticlesController.cs

@@ -11,8 +11,8 @@ namespace Newsbot.Collector.Api.Controllers;
 [Route("api/articles")]
 public class ArticlesController : ControllerBase
 {
-    private readonly ILogger<ArticlesController> _logger;
     private readonly IArticlesRepository _articles;
+    private readonly ILogger<ArticlesController> _logger;
     private readonly ISourcesRepository _sources;
 
     public ArticlesController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
@@ -27,14 +27,12 @@ public class ArticlesController : ControllerBase
     {
         var res = new List<ArticleDto>();
         var items = _articles.List(0, 25);
-        foreach (var item in items)
-        {
-            res.Add(ArticleDto.Convert(item));
-        }
+        foreach (var item in items) res.Add(ArticleDto.Convert(item));
         return res;
     }
 
     [HttpGet("{id:guid}")]
+    [EndpointDescription("Returns the article based on the Id value given.")]
     public ArticleDto GetById(Guid id)
     {
         var item = _articles.GetById(id);
@@ -54,10 +52,7 @@ public class ArticlesController : ControllerBase
     {
         var res = new List<ArticleDto>();
         var items = _articles.ListBySourceId(sourceId, page, count);
-        foreach (var item in items)
-        {
-            res.Add(ArticleDto.Convert(item));
-        }
+        foreach (var item in items) res.Add(ArticleDto.Convert(item));
         return res;
     }
 }

Newsbot.Collector.Api/Controllers/CodeProjectController.cs

@@ -0,0 +1,33 @@
+using Hangfire;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;
+using Newsbot.Collector.Domain.Models.Config;
+using Newsbot.Collector.Services.Jobs;
+
+namespace Newsbot.Collector.Api.Controllers;
+
+[ApiController]
+[Route("api/codeprojects")]
+public class CodeProjectController
+{
+    private readonly ConfigSectionConnectionStrings _connectionStrings;
+    private readonly ILogger<CodeProjectController> _logger;
+
+    public CodeProjectController(ILogger<CodeProjectController> logger,
+        IOptions<ConfigSectionConnectionStrings> settings)
+    {
+        _logger = logger;
+        _connectionStrings = settings.Value;
+    }
+
+    [HttpPost("check")]
+    public void PullNow()
+    {
+        BackgroundJob.Enqueue<CodeProjectWatcherJob>(x => x.InitAndExecute(new CodeProjectWatcherJobOptions
+        {
+            ConnectionStrings = _connectionStrings,
+            FeaturePullReleases = true,
+            FeaturePullCommits = true
+        }));
+    }
+}

Newsbot.Collector.Api/Controllers/RssController.cs

@@ -1,8 +1,6 @@
 using Hangfire;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
-using Newsbot.Collector.Database.Repositories;
-using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models.Config;
 using Newsbot.Collector.Services.Jobs;
 
@@ -13,10 +11,10 @@ namespace Newsbot.Collector.Api.Controllers;
 public class RssController
 {
     private readonly ConfigSectionConnectionStrings _connectionStrings;
+    private readonly ILogger<RssController> _logger;
     private readonly ConfigSectionRssModel _rssConfig;
-    private readonly ILogger<SourcesController> _logger;
 
-    public RssController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
+    public RssController(ILogger<RssController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
         IOptions<ConfigSectionRssModel> rss)
     {
         _logger = logger;

Newsbot.Collector.Api/Controllers/SourcesController.cs

@@ -13,10 +13,11 @@ namespace Newsbot.Collector.Api.Controllers;
 [Route("api/sources")]
 public class SourcesController : ControllerBase
 {
-    private readonly IIconsRepository _icons;
-    private readonly ILogger<SourcesController> _logger;
+    private readonly IArticlesRepository _articles;
 
     //private readonly ConnectionStrings _settings;
+    private readonly IIconsRepository _icons;
+    private readonly ILogger<SourcesController> _logger;
     private readonly ISourcesRepository _sources;
 
     public SourcesController(ILogger<SourcesController> logger, IOptions<ConnectionStrings> settings)
@@ -25,6 +26,7 @@ public class SourcesController : ControllerBase
         //_settings = settings.Value;
         _sources = new SourcesTable(settings.Value.Database);
         _icons = new IconsTable(settings.Value.Database);
+        _articles = new ArticlesTable(settings.Value.Database);
     }
 
     [HttpGet(Name = "GetSources")]
@@ -154,10 +156,10 @@ public class SourcesController : ControllerBase
         return SourceDto.Convert(item);
     }
 
-    [HttpPost("new/github")]
+    [HttpPost("new/codeproject")]
     public SourceDto NewGithub(string url)
     {
-        if (!url.Contains("github.com")) return new SourceDto();
+        //if (!url.Contains("github.com")) return new SourceDto();
 
         var res = _sources.GetByUrl(url);
         if (res.ID != Guid.Empty) return SourceDto.Convert(res);
@@ -172,13 +174,13 @@ public class SourcesController : ControllerBase
 
         var item = _sources.New(new SourceModel
         {
-            Site = SourceTypes.GitHub,
-            Type = SourceTypes.GitHub,
+            Site = SourceTypes.CodeProject,
+            Type = SourceTypes.CodeProject,
             Name = $"{slice[3]}/{slice[4]}",
             Url = url,
             Source = "feed",
             Enabled = true,
-            Tags = $"{SourceTypes.GitHub}, {slice[3]}, {slice[4]}"
+            Tags = $"{slice[2]},{slice[3]},{slice[4]}"
         });
 
         _icons.New(new IconModel
@@ -209,4 +211,10 @@ public class SourcesController : ControllerBase
     {
         _sources.Enable(id);
     }
+
+    [HttpDelete("{id}")]
+    public void Delete(Guid id, bool purgeOrphanedRecords)
+    {
+        _sources.Delete(id);
+    }
 }

Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql

@@ -1,13 +1,21 @@
 -- +goose Up
 -- +goose StatementBegin
 SELECT 'up SQL query';
-ALTER TABLE icons
-    ADD COLUMN SourceId uuid;
+ALTER TABLE sources
+    ADD COLUMN IconUri uuid;
+
+ALTER TABLE articles
+    ADD COLUMN CodeIsRelease bool,
+    ADD COLUMN CodeIsCommit  bool;
 -- +goose StatementEnd
 
 -- +goose Down
 -- +goose StatementBegin
 SELECT 'down SQL query';
-ALTER TABLE icons
-    DROP COLUMN SourceId;
+ALTER TABLE sources
+    DROP COLUMN IconUri;
+
+Alter TABLE articles
+    DROP COLUMN CodeIsRelease,
+    DROP COLUMN CodeIsCommit
 -- +goose StatementEnd

Newsbot.Collector.Database/Repositories/ArticlesTable.cs

@@ -9,8 +9,7 @@ namespace Newsbot.Collector.Database.Repositories;
 
 public class ArticlesTable : IArticlesRepository
 {
-
-    private string _connectionString;
+    private readonly string _connectionString;
 
     public ArticlesTable(string connectionString)
     {
@@ -20,21 +19,11 @@ public class ArticlesTable : IArticlesRepository
     public ArticlesTable(IConfiguration configuration)
     {
         var conn = configuration.GetConnectionString("database");
-        if (conn is null)
-        {
-            conn = "";
-        }
+        if (conn is null) conn = "";
 
         _connectionString = conn;
     }
 
-    private IDbConnection OpenConnection(string connectionString)
-    {
-        var conn = new NpgsqlConnection(_connectionString);
-        conn.Open();
-        return conn;
-    }
-
     public List<ArticlesModel> List(int page = 0, int count = 25)
     {
         using var conn = OpenConnection(_connectionString);
@@ -42,22 +31,19 @@ public class ArticlesTable : IArticlesRepository
         Order By PubDate Desc 
         Offset @Page
         Fetch Next @Count Rows Only", new
-        {
-            Page = page * count,
-            Count = count
-        })
-        .ToList();
+            {
+                Page = page * count,
+                Count = count
+            })
+            .ToList();
         return res;
     }
 
     public ArticlesModel GetById(Guid ID)
     {
         using var conn = OpenConnection(_connectionString);
-        var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID = ID });
-        if (res.Count() == 0)
-        {
-            return new ArticlesModel();
-        }
+        var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID });
+        if (res.Count() == 0) return new ArticlesModel();
         return res.First();
     }
 
@@ -65,10 +51,7 @@ public class ArticlesTable : IArticlesRepository
     {
         using var conn = OpenConnection(_connectionString);
         var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
-        if (res.Count() == 0)
-        {
-            return new ArticlesModel();
-        }
+        if (res.Count() == 0) return new ArticlesModel();
         return res.First();
     }
 
@@ -83,7 +66,7 @@ public class ArticlesTable : IArticlesRepository
         {
             sourceid = id,
             page = page * count,
-            count = count
+            count
         }).ToList();
     }
 
@@ -92,7 +75,8 @@ public class ArticlesTable : IArticlesRepository
         model.ID = Guid.NewGuid();
 
         using var conn = OpenConnection(_connectionString);
-        var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
+        var q =
+            "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
         var res = conn.Execute(q, new
         {
             id = model.ID,
@@ -112,4 +96,20 @@ public class ArticlesTable : IArticlesRepository
         return model;
     }
 
+    public void DeleteAllBySourceId(Guid sourceId)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var res = conn.Execute("Delete from articles where sourceid = '@id'", new
+        {
+            sourceId
+        });
+        if (res == 0) throw new Exception($"No records where deleted that linked to SourceId = '{sourceId}'");
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
+        conn.Open();
+        return conn;
+    }
 }

Newsbot.Collector.Database/Repositories/SourcesTable.cs

@@ -155,6 +155,17 @@ public class SourcesTable : ISourcesRepository
         });
     }
 
+    public void Delete(Guid id)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Delete From sources where id = @id;";
+        var res = conn.Execute(query, new
+        {
+            id
+        });
+        if (res == 0) throw new Exception("Nothing was deleted");
+    }
+
     public int UpdateYoutubeId(Guid id, string youtubeId)
     {
         using var conn = OpenConnection(_connectionString);

Newsbot.Collector.Domain/Consts/ConfigConst.cs

@@ -3,32 +3,35 @@ namespace Newsbot.Collector.Domain.Consts;
 public class ConfigConst
 {
     public const string LoggingDefault = "Logging:LogLevel:Default";
-    
+
     public const string SectionConnectionStrings = "ConnectionStrings";
     public const string SectionFinalFantasyXiv = "FinalFantasyXiv";
     public const string SectionReddit = "Reddit";
     public const string SectionRss = "Rss";
     public const string SectionTwitch = "Twitch";
     public const string SectionYoutube = "Youtube";
+    public const string SectionCodeProjects = "CodeProjects";
     public const string SectionNotificationsDiscord = "Notifications:Discord";
-    
+
     public const string ConnectionStringDatabase = "ConnectionStrings:Database";
     public const string ConnectionStringOpenTelemetry = "ConnectionStrings:OpenTelemetry";
-    
+
     public const string RedditIsEnabled = "Reddit:IsEnabled";
     public const string RedditPullHot = "Reddit:PullHot";
     public const string RedditPullNsfw = "Reddit:PullNsfw";
     public const string RedditPullTop = "Reddit:PullTop";
 
     public const string RssIsEnabled = "Rss:IsEnabled";
-    
+
     public const string TwitchIsEnabled = "Twitch:IsEnabled";
     public const string TwitchClientId = "Twitch:ClientID";
     public const string TwitchClientSecret = "Twitch:ClientSecret";
-    
+
     public const string YoutubeIsEnable = "Youtube:IsEnabled";
     public const string YoutubeDebug = "Youtube:Debug";
 
+    public const string CodeProjectsIsEnabled = "CodeProjects:IsEnabled";
+
     public const string EnableSwagger = "EnableSwagger";
 
     public const string DiscordNotificationsEnabled = "Notifications:Discord:IsEnabled";

Newsbot.Collector.Domain/Consts/SourcesTypes.cs

@@ -7,6 +7,5 @@ public class SourceTypes
     public const string YouTube = "youtube";
     public const string Twitch = "twitch";
     public const string FinalFantasyXiv = "ffxiv";
-    public const string GitHub = "github";
-
+    public const string CodeProject = "code";
 }

Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs

@@ -4,9 +4,10 @@ namespace Newsbot.Collector.Domain.Interfaces;
 
 public interface IArticlesRepository : ITableRepository
 {
-    List<ArticlesModel>List(int page, int count);
-    List<ArticlesModel>ListBySourceId(Guid id, int page = 0, int count = 25);
+    List<ArticlesModel> List(int page, int count);
+    List<ArticlesModel> ListBySourceId(Guid id, int page = 0, int count = 25);
     ArticlesModel GetById(Guid ID);
     ArticlesModel GetByUrl(string url);
     ArticlesModel New(ArticlesModel model);
+    void DeleteAllBySourceId(Guid sourceId);
 }

Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs

@@ -15,5 +15,6 @@ public interface ISourcesRepository
     public List<SourceModel> ListByType(string type, int limit = 25);
     public int Disable(Guid id);
     public int Enable(Guid id);
+    public void Delete(Guid id);
     public int UpdateYoutubeId(Guid id, string youtubeId);
 }

Newsbot.Collector.Domain/Models/DatabaseModel.cs

@@ -15,6 +15,8 @@ public class ArticlesModel
     public string Description { get; set; } = "";
     public string AuthorName { get; set; } = "";
     public string? AuthorImage { get; set; }
+    public bool CodeIsRelease { get; set; }
+    public bool CodeIsCommit { get; set; }
 }
 
 public class AuthorModel

Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs

@@ -0,0 +1,178 @@
+using System.Collections;
+using System.ServiceModel.Syndication;
+using System.Xml;
+using Newsbot.Collector.Database.Repositories;
+using Newsbot.Collector.Domain.Consts;
+using Newsbot.Collector.Domain.Interfaces;
+using Newsbot.Collector.Domain.Models;
+using Newsbot.Collector.Domain.Models.Config;
+using Newsbot.Collector.Services.HtmlParser;
+using Serilog;
+
+namespace Newsbot.Collector.Services.Jobs;
+
+public class CodeProjectWatcherJobOptions
+{
+    public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
+
+    //public string ConnectionString { get; set; } = "";
+    public bool FeaturePullReleases { get; set; } = false;
+
+    public bool FeaturePullCommits { get; set; } = false;
+    //public bool PullIssues { get; set; } = false;
+}
+
+public class CodeProjectWatcherJob
+{
+    private const string JobName = "CodeProjectWatcher";
+    private IArticlesRepository _articles;
+    private ILogger _logger;
+    private IDiscordQueueRepository _queue;
+    private ISourcesRepository _source;
+
+    public CodeProjectWatcherJob()
+    {
+        _articles = new ArticlesTable("");
+        _queue = new DiscordQueueTable("");
+        _source = new SourcesTable("");
+        _logger = JobLogger.GetLogger("", JobName);
+    }
+
+    public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
+    {
+        options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
+        _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
+        _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
+        _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
+        _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
+    }
+
+    public void InitAndExecute(CodeProjectWatcherJobOptions options)
+    {
+        options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
+        _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
+        _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
+        _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
+        _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
+
+        Execute();
+    }
+
+    private void Execute()
+    {
+        var sources = _source.ListByType(SourceTypes.CodeProject);
+
+        // query sources for things to pull
+        var items = new List<ArticlesModel>();
+
+        foreach (var source in sources)
+        {
+            items.AddRange(CheckForReleases(source));
+            items.AddRange(CheckForCommits(source));
+        }
+
+        foreach (var item in items)
+        {
+            _articles.New(item);
+            _queue.New(new DiscordQueueModel
+            {
+                ArticleID = item.ID
+            });
+        }
+    }
+
+    public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source)
+    {
+        var url = new Uri(source.Url);
+        var links = new List<string>
+        {
+            $"{url.AbsoluteUri}/releases.atom",
+            $"{url.AbsoluteUri}/tags.atom" //github converts tags as releases
+        };
+
+        foreach (var link in links)
+            try
+            {
+                using var reader = XmlReader.Create(link);
+                var client = SyndicationFeed.Load(reader);
+                
+                return ProcessFeed(client.Items, source, true, false);
+                
+
+                //if (link.EndsWith("tags.atom"))
+                //{
+                //    return ProcessFeed(client.Items, source, false, true, false);
+                //}
+            }
+            catch
+            {
+                _logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}.  Might not have anything", JobName,
+                    url.AbsoluteUri);
+            }
+
+        return new List<ArticlesModel>();
+    }
+    
+    public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source)
+    {
+        var url = new Uri(source.Url);
+        var links = new List<string>
+        {
+            $"{url.AbsoluteUri}/commits/main.atom",
+            $"{url.AbsoluteUri}/commits/master.atom"
+        };
+
+        foreach (var link in links)
+            try
+            {
+                using var reader = XmlReader.Create(link);
+                var client = SyndicationFeed.Load(reader);
+                
+                return ProcessFeed(client.Items, source, false, true);
+            }
+            catch
+            {
+                _logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}.  Might not have anything", JobName,
+                    url.AbsoluteUri);
+            }
+
+        return new List<ArticlesModel>();
+    }
+
+    private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
+        bool isRelease, bool isCommit)
+    {
+        var items = new List<ArticlesModel>();
+
+        foreach (var item in feed)
+        {
+            var itemUrl = item.Links[0].Uri.AbsoluteUri;
+            var exits = _articles.GetByUrl(itemUrl);
+            if (exits.ID != Guid.Empty) continue;
+
+            var parser = new HtmlPageReader(new HtmlPageReaderOptions
+            {
+                Url = itemUrl
+            });
+            parser.Parse();
+
+            var a = new ArticlesModel
+            {
+                SourceID = source.ID,
+                Tags = source.Tags,
+                Title = item.Title.Text,
+                URL = itemUrl,
+                PubDate = item.LastUpdatedTime.DateTime,
+                Thumbnail = parser.Data.Header.Image,
+                Description = item.Title.Text,
+                AuthorName = item.Authors[0].Name ?? "",
+                AuthorImage = item.Authors[0].Uri ?? "",
+                CodeIsRelease = isRelease,
+                CodeIsCommit = isCommit,
+            };
+            items.Add(a);
+        }
+
+        return items;
+    }
+}

Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs

@@ -1,115 +0,0 @@
-using System.ServiceModel.Syndication;
-using System.Xml;
-using Newsbot.Collector.Database.Repositories;
-using Newsbot.Collector.Domain.Consts;
-using Newsbot.Collector.Domain.Interfaces;
-using Newsbot.Collector.Domain.Models;
-using Newsbot.Collector.Domain.Models.Config;
-using Newsbot.Collector.Services.HtmlParser;
-
-namespace Newsbot.Collector.Services.Jobs;
-
-public class GithubWatcherJobOptions
-{
-    public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
-
-    //public string ConnectionString { get; set; } = "";
-    public bool FeaturePullReleases { get; set; } = false;
-
-    public bool FeaturePullCommits { get; set; } = false;
-    //public bool PullIssues { get; set; } = false;
-}
-
-public class GithubWatcherJob
-{
-    private IArticlesRepository _articles;
-    private IDiscordQueueRepository _queue;
-    private ISourcesRepository _source;
-
-    public GithubWatcherJob()
-    {
-        _articles = new ArticlesTable("");
-        _queue = new DiscordQueueTable("");
-        _source = new SourcesTable("");
-    }
-
-    public void InitAndExecute(GithubWatcherJobOptions options)
-    {
-        options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
-        _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
-        _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
-        _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
-
-        Execute();
-    }
-
-    private void Execute()
-    {
-        _source.ListBySource(SourceTypes.GitHub, 25);
-
-        // query sources for things to pull
-        var items = new List<ArticlesModel>();
-
-
-        items.AddRange(Collect(new Uri("https://github.com/jtom38/dvb")));
-
-        // query */commits/master.atom
-        // query */commits/main.atom
-    }
-
-    public List<ArticlesModel> Collect(Uri url)
-    {
-        var items = new List<ArticlesModel>();
-
-        var placeHolderId = Guid.NewGuid();
-        // query */release.atom
-        items.AddRange(CollectItems($"{url.AbsoluteUri}/releases.atom", placeHolderId));
-        items.AddRange(CollectItems($"{url.AbsoluteUri}/master.atom", placeHolderId));
-
-        return items;
-    }
-
-    private List<ArticlesModel> CollectItems(string baseUrl, Guid sourceId)
-    {
-        var items = new List<ArticlesModel>();
-
-        using var reader = XmlReader.Create(baseUrl);
-        var client = SyndicationFeed.Load(reader);
-
-        foreach (var item in client.Items)
-        {
-            var itemUrl = item.Links[0].Uri.AbsoluteUri;
-            var exits = _articles.GetByUrl(itemUrl);
-            if (exits.ID != Guid.Empty) continue;
-
-            var parser = new HtmlPageReader(new HtmlPageReaderOptions
-            {
-                Url = itemUrl
-            });
-            parser.Parse();
-
-            try
-            {
-                var a = new ArticlesModel
-                {
-                    SourceID = sourceId,
-                    Tags = "github",
-                    Title = item.Title.Text,
-                    URL = itemUrl,
-                    //PubDate = item.LastUpdatedTime.DateTime,
-                    Thumbnail = parser.Data.Header.Image,
-                    Description = $"'dvb' has released '{item.Title.Text}'!",
-                    AuthorName = item.Authors[0].Name ?? "",
-                    AuthorImage = item.Authors[0].Uri ?? ""
-                };
-                items.Add(a);
-            }
-            catch (Exception e)
-            {
-                Console.WriteLine(e);
-            }
-        }
-
-        return items;
-    }
-}

Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs

@@ -0,0 +1,88 @@
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Consts;
+using Newsbot.Collector.Domain.Models;
+using Newsbot.Collector.Domain.Models.Config;
+using Newsbot.Collector.Services.Jobs;
+
+namespace Newsbot.Collector.Tests.Jobs;
+
+public class CodeProjectWatcherJobTests
+{
+    [Fact]
+    public void CanReturnReleases()
+    {
+        var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions
+        {
+            ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings)
+                .Get<ConfigSectionConnectionStrings>(),
+            FeaturePullCommits = true,
+            FeaturePullReleases = true
+        });
+        var results = client.CheckForReleases(new SourceModel
+        {
+            ID = Guid.NewGuid(),
+            Url = "https://github.com/jtom38/dvb",
+            Type = SourceTypes.CodeProject,
+            Site = SourceTypes.CodeProject,
+            Name = "jtom38/dvb",
+            Source = "feed",
+            Enabled = true
+        });
+        if (!results.Any())
+        {
+            Assert.Fail("Expected at least one item");
+        }
+    }
+
+    [Fact]
+    public void CollectsTagsButNoReleases()
+    {
+        var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions
+        {
+            ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings)
+                .Get<ConfigSectionConnectionStrings>(),
+            FeaturePullCommits = true,
+            FeaturePullReleases = true
+        });
+        var results = client.CheckForReleases(new SourceModel
+        {
+            ID = Guid.NewGuid(),
+            Url = "https://github.com/python/cpython",
+            Type = SourceTypes.CodeProject,
+            Site = SourceTypes.CodeProject,
+            Name = "python.cpython",
+            Source = "feed",
+            Enabled = true
+        });
+        if (!results.Any())
+        {
+            Assert.Fail("Expected at least one item");
+        }
+    }
+
+    [Fact]
+    public void CollectsCommits()
+    {
+        var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions
+        {
+            ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings)
+                .Get<ConfigSectionConnectionStrings>(),
+            FeaturePullCommits = true,
+            FeaturePullReleases = true
+        });
+        var results = client.CheckForCommits(new SourceModel
+        {
+            ID = Guid.NewGuid(),
+            Url = "https://github.com/jtom38/dvb",
+            Type = SourceTypes.CodeProject,
+            Site = SourceTypes.CodeProject,
+            Name = "jtom38/dvb",
+            Source = "feed",
+            Enabled = true
+        });
+        if (!results.Any())
+        {
+            Assert.Fail("Expected at least one item");
+        }
+    }
+}

Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs

@@ -1,46 +0,0 @@
-using Microsoft.Extensions.Configuration;
-using Newsbot.Collector.Domain.Models;
-using Newsbot.Collector.Domain.Models.Config;
-using Newsbot.Collector.Services.Jobs;
-
-namespace Newsbot.Collector.Tests.Jobs;
-
-public class GithubWatcherJobTests
-{
-    private IConfiguration GetConfiguration()
-    {
-        var inMemorySettings = new Dictionary<string, string>
-        {
-            {
-                "ConnectionStrings:database",
-                "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"
-            }
-        };
-
-        IConfiguration configuration = new ConfigurationBuilder()
-            .AddInMemoryCollection(inMemorySettings)
-            .Build();
-        return configuration;
-    }
-
-    private string ConnectionString()
-    {
-        return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
-    }
-
-    [Fact]
-    public void CanPullAFeed()
-    {
-        var client = new GithubWatcherJob();
-        client.InitAndExecute(new GithubWatcherJobOptions
-        {
-            ConnectionStrings = new ConfigSectionConnectionStrings
-            {
-                Database = ConnectionString()
-            },
-            FeaturePullCommits = true,
-            FeaturePullReleases = true
-        });
-        client.Collect(new Uri("https://github.com/jtom38/dvb"));
-    }
-}

seed.ps1

@@ -50,6 +50,20 @@ function New-TwitchSource {
     return $res
 }
 
+function New-CodeProject {
+    param (
+        [string] $Url
+    )
+    
+    $urlEncoded = [uri]::EscapeDataString($Url)
+    [string] $param = "url=$urlEncoded"
+    [string] $uri = "$ApiServer/api/sources/new/codeproject?$param"
+    
+    Write-Host "Adding CodeProject '$url'"
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
 function New-DiscordWebhook {
     param (
         [string] $Server,
@@ -94,6 +108,11 @@ $youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTe
 $youtubeFireship = New-YoutubeSource -Url "https://www.youtube.com/@Fireship"
 $youtubeClimateTown = New-YoutubeSource -Url "https://www.youtube.com/c/ClimateTown"
 
+$codeDotnet = New-CodeProject -Url "https://github.com/dotnet/runtime"
+#$codePython = New-CodeProject -Url "https://github.com/python/cpython"
+#$codeGolang = New-CodeProject -Url "https://github.com/golang/go"
+#$codePowerShell = New-CodeProject -Url "https://github.com/PowerShell/PowerShell"
+#$codeLinux = New-CodeProject -Url "https://github.com/torvalds/linux"
 
 $twitchNintendo = New-TwitchSource -Name "Nintendo"
 
@@ -102,6 +121,7 @@ $sky = New-DiscordWebhook -Server "Let's Mosley" -Channel "bot test" -url $secre
 
 New-Subscription -SourceId $redditDadJokes.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $redditSteamDeck.id -DiscordWebhookId $miharuMonitor.id
+
 New-Subscription -SourceId $rssSteamDeck.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $rssFaysHaremporium.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $miharuMonitor.id
@@ -109,9 +129,13 @@ New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $sky.id
 New-Subscription -SourceId $rssOmgLinux.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id
+
+New-Subscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id
+
 New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $youtubeLinusTechTips.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $youtubeFireship.id -DiscordWebhookId $miharuMonitor.id
 New-Subscription -SourceId $youtubeClimateTown.id -DiscordWebhookId $miharuMonitor.id
+
 New-Subscription -SourceId $twitchNintendo.id -DiscordWebhookId $miharuMonitor.id