Features/table clients (#5)

* inital table clients are almost ready

* adding new repo interfaces

* adding MS DI to the tables

* adding tables and MS DI to jobs

* updated how the config builder works and pass Iconfig to jobs

* Updated how articles interface returns a value

* updated constructors to support DI and removed static call

* added source consts and model notes

* updated sources.type to contain the value to link what job collects it

* added RssWatcherJob to hangfire

* DI and hangfire jobs wont work.  Defering to options classes

* Services updated to have options exposed over DI

* Tests have been updated.. more to come

Newsbot.Collector.Api/Newsbot.Collector.Api.csproj

@@ -10,6 +10,7 @@
     <PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
     <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
     <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
   </ItemGroup>
 

Newsbot.Collector.Api/Program.cs

@@ -8,11 +8,7 @@ var builder = WebApplication.CreateBuilder(args);
 // Add services to the container.
 
 // Build the conifg
-var config = new ConfigurationBuilder()
+var config = GetConfiguration();
-    .AddJsonFile("appsettings.json")
-    .AddEnvironmentVariables()
-    .Build();
-var cfg = config.GetRequiredSection("Config").Get<ConfigModel>();
 builder.Configuration.AddConfiguration(config);
 
 builder.Services.AddHangfire(f => f.UseMemoryStorage());
@@ -35,10 +31,40 @@ if (app.Environment.IsDevelopment())
 app.UseHttpsRedirection();
 
 app.UseHangfireDashboard();
-RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.Execute(), "0/2 * * * *");
+SetupRecurringJobs(config);
 
 app.UseAuthorization();
 
 app.MapControllers();
 
 app.Run();
+
+static IConfiguration GetConfiguration()
+{
+    return new ConfigurationBuilder()
+        .AddJsonFile("appsettings.json", true)
+        .AddEnvironmentVariables()
+        .Build();
+}
+
+static void SetupRecurringJobs(IConfiguration configuration)
+{
+    var databaseConnectionString = configuration.GetConnectionString("database");
+    if (databaseConnectionString is null)
+    {
+        databaseConnectionString = "";
+    }
+
+    RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
+    {
+        Message = "Hello from the background!"
+    }), "0/2 * * * *");
+    //RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(config), "15 0-23 * * *");
+    
+    var c = new RssWatcherJob();
+    BackgroundJob.Enqueue(() => c.InitAndExecute(new RssWatcherJobOptions
+    {
+        ConnectionString = databaseConnectionString
+    }));
+
+}

Newsbot.Collector.Database/Migrations/20220529082459_seed.sql

@@ -7,33 +7,33 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
 
 -- Final Fantasy XIV Entries
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'scrape', 'ffxiv', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'scrape', 'ffxiv', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'scrape', 'ffxiv', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'scrape', 'ffxiv', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'scrape', 'ffxiv', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
 
 -- Reddit Entries
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
+(uuid_generate_v4(), 'reddit', 'dadjokes', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
+(uuid_generate_v4(), 'reddit', 'steamdeck', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
 
 -- Youtube Entries
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
+(uuid_generate_v4(), 'youtube', 'Game Grumps', 'feed', 'youtube', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
 
 -- RSS Entries
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
+(uuid_generate_v4(), 'steampowered', 'steam deck', 'feed', 'rss', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
 
 -- Twitch Entries 
 INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
+(uuid_generate_v4(), 'twitch', 'Nintendo', 'api', 'twitch', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
 
 -- +goose StatementEnd
 

Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql

@@ -6,6 +6,6 @@ ALTER TABLE sources Add COLUMN Deleted BOOLEAN;
 
 -- +goose Down
 -- +goose StatementBegin
-SELECT 'down SQL query';
+--SELECT 'down SQL query';
-ALTER TABLE sources Drop Deleted Deleted BOOLEAN;
+ALTER TABLE sources Drop Column Deleted;
 -- +goose StatementEnd

Newsbot.Collector.Database/Newsbot.Collector.Database.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="dapper" Version="2.0.123" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
     <PackageReference Include="Npgsql" Version="7.0.2" />
   </ItemGroup>
 

Newsbot.Collector.Database/Repositories/ArticlesTable.cs

@@ -1,11 +1,13 @@
 using System.Data;
 using Dapper;
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
 using Npgsql;
 
 namespace Newsbot.Collector.Database.Repositories;
 
-public class ArticlesTable
+public class ArticlesTable : IArticlesRepository
 {
 
     private string _connectionString;
@@ -15,21 +17,31 @@ public class ArticlesTable
         _connectionString = connectionString;
     }
 
-    public static IDbConnection OpenConnection(string connectionString)
+    public ArticlesTable(IConfiguration configuration)
     {
-        var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
+        var conn = configuration.GetConnectionString("database");
-        var conn = new NpgsqlConnection(cs);
+        if (conn is null)
+        {
+            conn = "";
+        }
+
+        _connectionString = conn;
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
         conn.Open();
         return conn;
     }
 
-    public List<ArticlesModel> List(int Page = 0, int Count = 25)
+    public List<ArticlesModel> List(int page = 0, int count = 25)
     {
         using var conn = OpenConnection(_connectionString);
         var res = conn.Query<ArticlesModel>(@"select * from articles 
         Order By PubDate Desc 
         Offset @Page
-        Fetch Next @Count Rows Only", new { Page = Page * Count, Count = Count }).ToList();
+        Fetch Next @Count Rows Only", new { Page = page * count, Count = count }).ToList();
         return res;
     }
 
@@ -37,6 +49,10 @@ public class ArticlesTable
     {
         using var conn = OpenConnection(_connectionString);
         var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID = ID });
+        if (res.Count() == 0)
+        {
+            return new ArticlesModel();
+        }
         return res.First();
     }
 
@@ -44,36 +60,36 @@ public class ArticlesTable
     {
         using var conn = OpenConnection(_connectionString);
         var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
+        if (res.Count() == 0)
+        {
+            return new ArticlesModel();
+        }
         return res.First();
     }
 
-    public void New(ArticlesModel model)
+    public ArticlesModel New(ArticlesModel model)
     {
         model.ID = Guid.NewGuid();
 
         using var conn = OpenConnection(_connectionString);
-        var q = @"INSERT INTO Articles 
+        var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
-        (ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
+        var res = conn.Execute(q, new
-        Values
+        {
-        (@Id, @SourceId, @Tags, @Title, @Url, @PubDate, @Video, @VideoHeight, @VideoWidth, @Thumbnail, @Description, @AuthorName, @AuthorImage);
+            id = Guid.NewGuid(),
-        ";
+            sourceid = model.SourceID,
-        var res = conn.Execute(q, model);
+            tags = model.Tags,
-        //new{
+            title = model.Title,
-        //    Id = Guid.NewGuid(),
+            url = model.URL,
-        //    SourceId = model.SourceID,
+            pubdate = model.PubDate,
-        //    Tags = model.Tags,
+            video = model.Video,
-        //    Title = model.Title,
+            videoheight = model.VideoHeight,
-        //    Url = model.URL,
+            videowidth = model.VideoWidth,
-        //    PubDate = model.PubDate,
+            thumbnail = model.Thumbnail,
-        //    Video = model.Video,
+            description = model.Description,
-        //    VideoHeight = model.VideoHeight,
+            authorname = model.AuthorName,
-        //    VideoWidth = model.VideoWidth,
+            authorimage = model.AuthorImage
-        //    Thumbnail = model.Thumbnail,
+        });
-        //    Description = model.Description,
+        return model;
-        //    AuthorName = model.AuthorName,
-        //    AuthorImage = model.AuthorImage
-        //});
-        Console.WriteLine(res);
     }
 
 }

Newsbot.Collector.Database/Repositories/DiscordQueue.cs

@@ -0,0 +1,54 @@
+using System.Data;
+using Dapper;
+using Newsbot.Collector.Domain.Interfaces;
+using Newsbot.Collector.Domain.Models;
+using Npgsql;
+
+namespace Newsbot.Collector.Database.Repositories;
+
+public class DiscordQueueTable : IDiscordQueueRepository
+{
+    private string _connectionString;
+
+    public DiscordQueueTable(string connectionString)
+    {
+        _connectionString = connectionString;
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
+        conn.Open();
+        return conn;
+    }
+
+    public void New(DiscordQueueModel model)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);";
+        conn.Execute(query, new
+        {
+            id = Guid.NewGuid(),
+            articleid = model.ArticleID
+        });
+    }
+
+    public void Delete(Guid id)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Delete From DiscordQueue Where ID = @id;";
+        conn.Execute(query, new
+        {
+            id = id
+        });
+    }
+
+    public List<DiscordQueueModel> List(int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * from DiscordQueue LIMIT @id;";
+        return conn.Query<DiscordQueueModel>(query, new {
+            limit = limit
+        }).ToList();
+    }
+}

Newsbot.Collector.Database/Repositories/SettingsTable.cs

@@ -1,5 +1,6 @@
 using System.Data;
 using Dapper;
+using Microsoft.Extensions.Configuration;
 using Newsbot.Collector.Domain.Models;
 using Npgsql;
 
@@ -15,10 +16,19 @@ public class SettingsTable
         _connectionString = connectionString;
     }
 
-    public static IDbConnection OpenConnection(string connectionString)
+    public SettingsTable(IConfiguration configuration)
     {
-        var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
+        var connstr = configuration.GetConnectionString("database");
-        var conn = new NpgsqlConnection(cs);
+        if (connstr is null)
+        {
+            connstr = "";
+        }
+        _connectionString = connstr;
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
         conn.Open();
         return conn;
     }

Newsbot.Collector.Database/Repositories/SourcesTable.cs

@@ -0,0 +1,161 @@
+using System.Data;
+using Dapper;
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Interfaces;
+using Newsbot.Collector.Domain.Models;
+using Npgsql;
+
+namespace Newsbot.Collector.Database.Repositories;
+
+public class SourcesTable : ISourcesRepository
+{
+    private string _connectionString;
+
+    public SourcesTable(string connectionString)
+    {
+        _connectionString = connectionString;
+    }
+
+    public SourcesTable(IConfiguration configuration)
+    {
+        var connstr = configuration.GetConnectionString("database");
+        if (connstr is null)
+        {
+            connstr = "";
+        }
+        _connectionString = connstr;
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
+        conn.Open();
+        return conn;
+    }
+
+    public SourceModel New(SourceModel model)
+    {
+        model.ID = Guid.NewGuid();
+        using var conn = OpenConnection(_connectionString);
+        var query = "Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags);";
+        conn.Execute(query, new
+        {
+            id = model.ID,
+            model.Site,
+            model.Name,
+            model.Source,
+            model.Type,
+            model.Value,
+            model.Enabled,
+            model.Url,
+            model.Tags
+        });
+        return model;
+    }
+
+    public SourceModel GetByID(Guid ID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From Sources where ID = @id Limit 1;";
+        var res = conn.Query<SourceModel>(query, new
+        {
+            id = ID
+        });
+        if (res.Count() == 0)
+        {
+            return new SourceModel();
+        }
+        return res.First();
+    }
+
+    public SourceModel GetByID(string ID)
+    {
+        var uid = Guid.Parse(ID);
+        return GetByID(uid);
+    }
+
+    public SourceModel GetByName(string Name)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * from Sources where name = @name Limit 1;";
+        var res = conn.Query<SourceModel>(query, new
+        {
+            name = Name
+        });
+
+        if (res.Count() == 0)
+        {
+            return new SourceModel();
+        }
+        return res.First();
+    }
+
+    public SourceModel GetByNameAndSource(string name, string source)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * from Sources WHERE name = @name and source = @source;";
+        var res = conn.Query<SourceModel>(query, new
+        {
+            name = name,
+            source = source
+        });
+
+        if (res.Count() == 0)
+        {
+            return new SourceModel();
+        }
+        return res.First();
+    }
+
+    public List<SourceModel> List(int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From Sources Limit @limit;";
+        return conn.Query<SourceModel>(query, new
+        {
+            limit = 25
+        }).ToList();
+    }
+
+    public List<SourceModel> ListBySource(string source, int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From Sources where Source = @source Limit @limit;";
+        return conn.Query<SourceModel>(query, new
+        {
+            source = source,
+            limit = limit
+        }).ToList();
+    }
+
+    public List<SourceModel> ListByType(string type, int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From Sources where Type = @type Limit @limit;";
+        return conn.Query<SourceModel>(query, new
+        {
+            type = type,
+            limit = limit
+        }).ToList();
+    }
+    public int Disable(Guid ID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Update Sources Set Enabled = FALSE where ID = @id;";
+        return conn.Execute(query, new
+        {
+            id = ID
+        });
+    }
+    
+    public int Enable(Guid ID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Update Sources Set Enabled = TRUE where ID = @id;";
+        return conn.Execute(query, new
+        {
+            id = ID
+        });
+    }
+
+}

Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs

@@ -0,0 +1,97 @@
+using System.Data;
+using Dapper;
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Models;
+using Npgsql;
+
+namespace Newsbot.Collector.Database.Repositories;
+
+public class SubscriptionsTable
+{
+    private string _connectionString;
+
+    public SubscriptionsTable(string connectionString)
+    {
+        _connectionString = connectionString;
+    }
+
+    public SubscriptionsTable(IConfiguration configuration)
+    {
+        var connstr = configuration.GetConnectionString("database");
+        if (connstr is null)
+        {
+            connstr = "";
+        }
+        _connectionString = connstr;
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
+        conn.Open();
+        return conn;
+    }
+
+    public void New(SubscriptionModel model)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);";
+        conn.Execute(query, new
+        {
+            id = Guid.NewGuid(),
+            webhookid = model.DiscordWebHookID,
+            sourceid = model.SourceID
+        });
+    }
+
+    public List<SubscriptionModel> List(int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From subscriptions Limit @limit;";
+        return conn.Query<SubscriptionModel>(query, new
+        {
+            limit = limit,
+        }).ToList();
+    }
+
+    // todo add paging
+    public List<SubscriptionModel> ListBySourceID(Guid sourceID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From subscriptions where sourceid = @sourceid";
+        return conn.Query<SubscriptionModel>(query, new
+        {
+            sourceid = sourceID
+        }).ToList();
+    }
+
+    public List<SubscriptionModel> GetByWebhookAndSource(Guid webhookId, Guid sourceId)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From subscriptions Where discordwebhookid = @webhookid and sourceid = @sourceid;";
+        return conn.Query<SubscriptionModel>(query, new
+        {
+            webhookid = webhookId,
+            sourceid = sourceId,
+        }).ToList();
+    }
+
+    public List<SubscriptionModel> ListByWebhook(Guid webhookId)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From subscriptions Where discordwebhookid = @webhookid";
+        return conn.Query<SubscriptionModel>(query, new
+        {
+            webhookid = webhookId,
+        }).ToList();
+    }
+
+    public void Delete(Guid id)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Delete From subscriptions Where id = @id;";
+        conn.Execute(query, new {
+            id = id
+        });
+    }
+}

Newsbot.Collector.Database/Repositories/WebhooksTable.cs

@@ -0,0 +1,121 @@
+using System.Data;
+using Dapper;
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Models;
+using Npgsql;
+
+namespace Newsbot.Collector.Database.Repositories;
+
+public class WebhooksTable
+{
+    private string _connectionString;
+
+    public WebhooksTable(string connectionString)
+    {
+        _connectionString = connectionString;
+    }
+
+    public WebhooksTable(IConfiguration configuration)
+    {
+        var connstr = configuration.GetConnectionString("database");
+        if (connstr is null)
+        {
+            connstr = "";
+        }
+        _connectionString = connstr;
+    }
+
+    private IDbConnection OpenConnection(string connectionString)
+    {
+        var conn = new NpgsqlConnection(_connectionString);
+        conn.Open();
+        return conn;
+    }
+
+    public void New(DiscordWebHook model)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);";
+        conn.Execute(query, new
+        {
+            id = model.ID,
+            url = model.Url,
+            server = model.Server,
+            channel = model.Channel,
+            enabled = model.Enabled
+        });
+    }
+
+    public DiscordWebHook GetByID(Guid ID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;";
+        return conn.Query<DiscordWebHook>(query, new
+        {
+            id = ID
+        }).First();
+    }
+
+    public DiscordWebHook GetByUrl(string url)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From DiscordWebHooks Where url = @url;";
+        return conn.QueryFirst<DiscordWebHook>(query, new
+        {
+            url = url
+        });
+    }
+
+    public List<DiscordWebHook> List(int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From DiscordWebHooks @limit;";
+        return conn.Query<DiscordWebHook>(query, new
+        {
+            limit = limit
+        }).ToList();
+    }
+
+    public List<DiscordWebHook> ListByServer(string server, int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;";
+        return conn.Query<DiscordWebHook>(query, new
+        {
+            server = server,
+            limit = limit
+        }).ToList();
+    }
+
+    public List<DiscordWebHook> ListByServerAndChannel(string server, string channel, int limit = 25)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;";
+        return conn.Query<DiscordWebHook>(query, new
+        {
+            server = server,
+            channel = channel,
+            limit = limit
+        }).ToList();
+    }
+
+    public int Disable(Guid ID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;";
+        return conn.Execute(query, new
+        {
+            id = ID
+        });
+    }
+
+    public int Enable(Guid ID)
+    {
+        using var conn = OpenConnection(_connectionString);
+        var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;";
+        return conn.Execute(query, new
+        {
+            id = ID
+        });
+    }
+}

Newsbot.Collector.Domain/Consts/SourcesConst.cs

@@ -0,0 +1,12 @@
+namespace Newsbot.Collector.Domain.Consts;
+
+public class SourceTypes
+{
+    public const string Reddit = "reddit";
+    public const string Rss = "rss";
+    public const string YouTube = "youtube";
+    public const string Twitch = "twitch";
+    public const string FinalFantasyXiv = "ffxiv";
+    public const string GitHub = "github";
+
+}

Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs

@@ -0,0 +1,11 @@
+using Newsbot.Collector.Domain.Models;
+
+namespace Newsbot.Collector.Domain.Interfaces;
+
+public interface IArticlesRepository : ITableRepository
+{
+    List<ArticlesModel>List(int age, int count);
+    ArticlesModel GetById(Guid ID);
+    ArticlesModel GetByUrl(string url);
+    ArticlesModel New(ArticlesModel model);
+}

Newsbot.Collector.Domain/Interfaces/ICollector.cs

@@ -2,7 +2,8 @@ using Newsbot.Collector.Domain.Models;
 
 namespace Newsbot.Collector.Domain.Interfaces;
 
-public interface ICollector
+///
+public interface ICollector : IHangfireJob
 {
     List<ArticlesModel> Collect();
 }

Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs

@@ -0,0 +1,10 @@
+using Newsbot.Collector.Domain.Models;
+
+namespace Newsbot.Collector.Domain.Interfaces;
+
+public interface IDiscordQueueRepository
+{
+    void New(DiscordQueueModel model);
+    void Delete(Guid id);
+    List<DiscordQueueModel> List(int limit);
+}

Newsbot.Collector.Domain/Interfaces/IHangfireJob.cs

@@ -0,0 +1,8 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Newsbot.Collector.Domain.Interfaces;
+
+public interface IHangfireJob
+{
+    void InitAndExecute(IConfiguration config);
+}

Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs

@@ -0,0 +1,18 @@
+using System.Globalization;
+using Newsbot.Collector.Domain.Models;
+
+namespace Newsbot.Collector.Domain.Interfaces;
+
+public interface ISourcesRepository
+{
+    public SourceModel New(SourceModel model);
+    public SourceModel GetByID(Guid ID);
+    public SourceModel GetByID(string ID);
+    public SourceModel GetByName(string name);
+    public SourceModel GetByNameAndSource(string name, string source);
+    public List<SourceModel> List(int limit);
+    public List<SourceModel> ListBySource(string source, int limit);
+    public List<SourceModel> ListByType(string type, int limit = 25);
+    public int Disable(Guid ID);
+    public int Enable(Guid ID);
+}

Newsbot.Collector.Domain/Interfaces/ITableRepository.cs

@@ -0,0 +1,8 @@
+using Newsbot.Collector.Domain.Models;
+
+namespace Newsbot.Collector.Domain.Interfaces;
+
+public interface ITableRepository
+{
+
+}

Newsbot.Collector.Domain/Models/DatabaseModel.cs

@@ -7,10 +7,10 @@ public class ArticlesModel
     public string Tags { get; set; } = "";
     public string Title { get; set; } = "";
     public string URL { get; set; } = "";
-    public DateTime PubDate { get; set; }
+    public DateTime PubDate { get; set; } = DateTime.Now;
     public string Video { get; set; } = "";
-    public int VideoHeight { get; set; }
+    public int VideoHeight { get; set; } = 0;
-    public int VideoWidth { get; set; }
+    public int VideoWidth { get; set; } = 0;
     public string Thumbnail { get; set; } = "";
     public string Description { get; set; } = "";
     public string AuthorName { get; set; } = "";
@@ -60,6 +60,8 @@ public class SourceModel
     public Guid ID { get; set; }
     public string Site { get; set; } = "";
     public string Name { get; set; } = "";
+
+    // Source use to deinfe the worker to query with but moving to Type as it was not used really.
     public string Source { get; set; } = "";
     public string Type { get; set; } = "";
     public string Value { get; set; } = "";

Newsbot.Collector.Domain/Newsbot.Collector.Domain.csproj

@@ -6,4 +6,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+  </ItemGroup>
+
 </Project>

Newsbot.Collector.Services/Jobs/HelloWorldJob.cs

@@ -1,23 +1,30 @@
 
+using Microsoft.Extensions.Configuration;
+
 namespace Newsbot.Collector.Services.Jobs;
 
+public class HelloWorldJobOptions
+{
+    public string Message { get; set; } = "";
+}
+
 public class HelloWorldJob
 {
+    private HelloWorldJobOptions _options;
     
-    public string _message { get; set; }
+    public HelloWorldJob(HelloWorldJobOptions options)
-
-    public HelloWorldJob(string message)
     {
-        _message = message;
+        _options = options;
     }
 
-    public void SetMessage(string message)
+    public void InitAndExecute(HelloWorldJobOptions options)
     {
-        _message = message;
+        _options = options;
+        Execute();
     }
 
-    public void Execute()
+    private void Execute()
     {
-        Console.WriteLine(_message);
+        Console.WriteLine(_options.Message);
     }
 }

Newsbot.Collector.Services/Jobs/RssWatcherJob.cs

@@ -1,59 +1,151 @@
+using System.Runtime.InteropServices;
 using System.ServiceModel.Syndication;
 using System.Xml;
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Database.Repositories;
+using Newsbot.Collector.Domain.Consts;
 using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
 
 namespace Newsbot.Collector.Services.Jobs;
 
-public class RssWatcherJob : ICollector
+public class RssWatcherJobOptions
 {
-
+    public string ConnectionString { get; set; } = "";
-    private string? _url;
-
-    public RssWatcherJob(string url)
-    {
-        _url = url;
 }
 
-    public List<ArticlesModel> Collect()
+// This class was made to work with Hangfire and it does not support constructors.
+public class RssWatcherJob : IHangfireJob
+{
+
+    private IArticlesRepository _articles;
+    private IDiscordQueueRepository _queue;
+    private ISourcesRepository _source;
+
+    public RssWatcherJob()
+    {
+        _articles = new ArticlesTable("");
+        _queue = new DiscordQueueTable("");
+        _source = new SourcesTable("");
+    }
+
+    public void InitAndExecute(RssWatcherJobOptions options)
+    {
+        Console.WriteLine("Job was triggered");
+        Console.WriteLine("Setting up the job");
+        Init(options.ConnectionString);
+
+        var articles = new List<ArticlesModel>();
+
+        Console.WriteLine("Requesting sources");
+        var sources = _source.ListByType(SourceTypes.Rss);
+        Console.WriteLine($"Got {sources.Count()} back");
+        foreach (var source in sources)
+        {
+            Console.WriteLine("Starting to request feed to be processed");
+            var results = Collect(source.Url);
+
+            articles.AddRange(results);
+        }
+
+        UpdateDatabase(articles);
+    }
+
+    public void InitAndExecute(IConfiguration config)
+    {
+        // reach out to the db and find all the rss feeds
+        var connectionString = config.GetConnectionString("database");
+        if (connectionString is null)
+        {
+            connectionString = "";
+        }
+        Init(connectionString);
+
+        var articles = new List<ArticlesModel>();
+
+        var sources = _source.ListByType(SourceTypes.Rss);
+        foreach (var source in sources)
+        {
+            var results = Collect(source.Url);
+
+            articles.AddRange(results);
+        }
+
+        UpdateDatabase(articles);
+    }
+
+    public void Init(string connectionString)
+    {
+        _articles = new ArticlesTable(connectionString);
+        _queue = new DiscordQueueTable(connectionString);
+        _source = new SourcesTable(connectionString);
+    }
+
+    public List<ArticlesModel> Collect(string url, int sleep = 3000)
     {
         var CollectedPosts = new List<ArticlesModel>();
 
-        if (_url is null)
+        using var reader = XmlReader.Create(url);
-        {
-            _url = "";
-        }
-
-        using var reader = XmlReader.Create(_url);
         var feed = SyndicationFeed.Load(reader);
-        var posts = feed.Items.ToList();
 
-        foreach (var post in posts)
+        foreach (var post in feed.Items.ToList())
         {
-            var url = post.Links[0].Uri.AbsoluteUri;
+            var articleUrl = post.Links[0].Uri.AbsoluteUri;
 
             // Check if we have seen the url before
             // If we have, skip and save the site bandwidth
+            if (IsThisUrlKnown(articleUrl) == true)
+            {
+                continue;
+            }
 
-            var meta = new HtmlPageReader(url);
+            var meta = new HtmlPageReader(articleUrl);
 
             var article = new ArticlesModel
             {
                 Title = post.Title.Text,
                 Tags = FetchTags(post),
-                URL = post.Links[0].Uri.ToString(),
+                URL = articleUrl,
                 PubDate = post.PublishDate.DateTime,
                 Thumbnail = meta.Data.Header.Meta.Image,
                 Description = meta.Data.Header.Meta.Description,
             };
+
             CollectedPosts.Add(article);
 
             // try to not be too greedy
-            Thread.Sleep(3000);
+            Thread.Sleep(sleep);
         }
         return CollectedPosts;
     }
 
+    public void UpdateDatabase(List<ArticlesModel> items)
+    {
+        foreach (var item in items)
+        {
+            if (IsThisUrlKnown(item.URL) == false)
+            {
+                continue;
+            }
+
+            var p = _articles.New(item);
+            _queue.New(new DiscordQueueModel
+            {
+                ArticleID = p.ID
+            });
+        }
+    }
+
+    private bool IsThisUrlKnown(string url)
+    {
+        var isKnown = _articles.GetByUrl(url);
+        if (isKnown.URL == url)
+        {
+            return true;
+        }
+        return false;
+    }
+
     private string FetchTags(SyndicationItem post)
     {
         string result = "";

Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs

@@ -1,14 +1,41 @@
+using Microsoft.Extensions.Configuration;
 using Newsbot.Collector.Services.Jobs;
 
 namespace Newsbot.Collector.Tests.Jobs;
 
 public class RssWatcherJobTest
 {
+    private IConfiguration GetConfiguration()
+    {
+        var inMemorySettings = new Dictionary<string, string> {
+            {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
+        };
+
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(inMemorySettings)
+            .Build();
+        return configuration;
+    }
+
+    private string ConnectionString()
+    {
+        return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
+    }
+
     [Fact]
-    public void CanFindItems()
+    public void CanFindItemsNoDb()
     {
         var url = "https://www.engadget.com/rss.xml";
-        var client = new RssWatcherJob(url);
+        var client = new RssWatcherJob();
-        var items = client.Collect();
+        var items = client.Collect(url);
+    }
+
+    [Fact]
+    public void CanAddItemsToDb()
+    {
+        var url = "https://www.engadget.com/rss.xml";
+        var client = new RssWatcherJob();
+        client.Init(ConnectionString());
+        client.Collect(url, 0);
     }
 }

Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj

@@ -3,12 +3,13 @@
   <PropertyGroup>
     <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
+    <Nullable>disable</Nullable>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
     <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">

Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs

@@ -1,3 +1,4 @@
+using Microsoft.Extensions.Configuration;
 using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Models;
 
@@ -5,11 +6,23 @@ namespace Newsbot.Collector.Tests.Tables;
 
 public class ArticlesTableTests
 {
+    private IConfiguration GetConfiguration()
+    {
+        var inMemorySettings = new Dictionary<string, string> {
+            {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
+        };
+
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(inMemorySettings)
+            .Build();
+        return configuration;
+    }
 
     [Fact]
     public void ArticlesListTest()
     {
-        var client = new ArticlesTable("");
+        var cfg = GetConfiguration();
+        var client = new ArticlesTable(cfg);
         client.List();
     }
 
@@ -18,7 +31,8 @@ public class ArticlesTableTests
     {
         var uid = Guid.Parse("4ac46772-253c-4c3d-8a2c-29239abd2ad4");
 
-        var client = new ArticlesTable("");
+        var cfg = GetConfiguration();
+        var client = new ArticlesTable(cfg);
         var res = client.GetById(uid);
         if (!res.ID.Equals(uid))
         {
@@ -29,12 +43,17 @@ public class ArticlesTableTests
     [Fact]
     public void NewRecordTest()
     {
-        var client = new ArticlesTable("");
+        var cfg = GetConfiguration();
-        client.New(new ArticlesModel
+        var client = new ArticlesTable(cfg);
+        var m = new ArticlesModel
         {
-            Title = "Unit Testing!",
+            ID = Guid.NewGuid(),
             SourceID = Guid.NewGuid(),
-            PubDate = DateTime.Now
+            Tags = "thing, thing2",
-        });
+            Title = "Unit Testing!",
+            URL = "https://google.com",
+            PubDate = DateTime.Now.ToLocalTime(),
+        };
+        client.New(m);
     }
 }

Newsbot.Collector.Tests/Tables/SettingsTableTests.cs

@@ -1,3 +1,4 @@
+using Microsoft.Extensions.Configuration;
 using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Models;
 
@@ -5,10 +6,22 @@ namespace Newsbot.Collector.Tests.Tables;
 
 public class SettingsTableTests
 {
+    private IConfiguration GetConfiguration()
+    {
+        var inMemorySettings = new Dictionary<string, string> {
+            {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
+        };
+
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(inMemorySettings)
+            .Build();
+        return configuration;
+    }
     [Fact]
     public void New()
     {
-        var client = new SettingsTable("");
+        var cfg = GetConfiguration();
+        var client = new SettingsTable(cfg);
         client.New(new SettingModel
         {
             Key = "Unit Testing",

Newsbot.Collector.Tests/Tables/SourcesTableTests.cs

@@ -0,0 +1,27 @@
+
+
+using Newsbot.Collector.Database.Repositories;
+using Newsbot.Collector.Domain.Models;
+
+public class SourcesTableTests
+{
+    [Fact]
+    public void NewRecordTest()
+    {
+        var client = new SourcesTable("");
+        var m = new SourceModel
+        {
+            ID = Guid.NewGuid(),
+            Site = "Testing",
+            Name = "Testing",
+            Source = "Testing",
+            Type = "Testing",
+            Value = "Testing",
+            Enabled = false,
+            Url = "Testing",
+            Tags = "Unit, Testing",
+            Deleted = false
+        };
+        client.New(m);
+    }
+}