diff --git a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj index 0c9a799..f0697ff 100644 --- a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj +++ b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj @@ -10,6 +10,7 @@ + diff --git a/Newsbot.Collector.Api/Program.cs b/Newsbot.Collector.Api/Program.cs index 3f0af26..d0b2e00 100644 --- a/Newsbot.Collector.Api/Program.cs +++ b/Newsbot.Collector.Api/Program.cs @@ -8,11 +8,7 @@ var builder = WebApplication.CreateBuilder(args); // Add services to the container. // Build the conifg -var config = new ConfigurationBuilder() - .AddJsonFile("appsettings.json") - .AddEnvironmentVariables() - .Build(); -var cfg = config.GetRequiredSection("Config").Get(); +var config = GetConfiguration(); builder.Configuration.AddConfiguration(config); builder.Services.AddHangfire(f => f.UseMemoryStorage()); @@ -35,10 +31,40 @@ if (app.Environment.IsDevelopment()) app.UseHttpsRedirection(); app.UseHangfireDashboard(); -RecurringJob.AddOrUpdate("Example", x => x.Execute(), "0/2 * * * *"); +SetupRecurringJobs(config); app.UseAuthorization(); app.MapControllers(); app.Run(); + +static IConfiguration GetConfiguration() +{ + return new ConfigurationBuilder() + .AddJsonFile("appsettings.json", true) + .AddEnvironmentVariables() + .Build(); +} + +static void SetupRecurringJobs(IConfiguration configuration) +{ + var databaseConnectionString = configuration.GetConnectionString("database"); + if (databaseConnectionString is null) + { + databaseConnectionString = ""; + } + + RecurringJob.AddOrUpdate("Example", x => x.InitAndExecute(new HelloWorldJobOptions + { + Message = "Hello from the background!" + }), "0/2 * * * *"); + //RecurringJob.AddOrUpdate("RSS", x => x.InitAndExecute(config), "15 0-23 * * *"); + + var c = new RssWatcherJob(); + BackgroundJob.Enqueue(() => c.InitAndExecute(new RssWatcherJobOptions + { + ConnectionString = databaseConnectionString + })); + +} diff --git a/Newsbot.Collector.Database/Migrations/20220529082459_seed.sql b/Newsbot.Collector.Database/Migrations/20220529082459_seed.sql index 29da683..279f8cd 100644 --- a/Newsbot.Collector.Database/Migrations/20220529082459_seed.sql +++ b/Newsbot.Collector.Database/Migrations/20220529082459_seed.sql @@ -7,33 +7,33 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -- Final Fantasy XIV Entries INSERT INTO sources VALUES -(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'scrape', 'ffxiv', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); INSERT INTO sources VALUES -(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'scrape', 'ffxiv', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); INSERT INTO sources VALUES -(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'scrape', 'ffxiv', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); INSERT INTO sources VALUES -(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'scrape', 'ffxiv', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); INSERT INTO sources VALUES -(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'scrape', 'ffxiv', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); -- Reddit Entries INSERT INTO sources VALUES -(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); +(uuid_generate_v4(), 'reddit', 'dadjokes', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); INSERT INTO sources VALUES -(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); +(uuid_generate_v4(), 'reddit', 'steamdeck', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); -- Youtube Entries INSERT INTO sources VALUES -(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); +(uuid_generate_v4(), 'youtube', 'Game Grumps', 'feed', 'youtube', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); -- RSS Entries INSERT INTO sources VALUES -(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); +(uuid_generate_v4(), 'steampowered', 'steam deck', 'feed', 'rss', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); -- Twitch Entries INSERT INTO sources VALUES -(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo'); +(uuid_generate_v4(), 'twitch', 'Nintendo', 'api', 'twitch', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo'); -- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql b/Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql index 3130bda..bd79148 100644 --- a/Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql +++ b/Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql @@ -6,6 +6,6 @@ ALTER TABLE sources Add COLUMN Deleted BOOLEAN; -- +goose Down -- +goose StatementBegin -SELECT 'down SQL query'; -ALTER TABLE sources Drop Deleted Deleted BOOLEAN; +--SELECT 'down SQL query'; +ALTER TABLE sources Drop Column Deleted; -- +goose StatementEnd diff --git a/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj b/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj index 876eb77..8f50477 100644 --- a/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj +++ b/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj @@ -6,6 +6,7 @@ + diff --git a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs index a7a0344..8c8e318 100644 --- a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs +++ b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs @@ -1,11 +1,13 @@ using System.Data; using Dapper; +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Npgsql; namespace Newsbot.Collector.Database.Repositories; -public class ArticlesTable +public class ArticlesTable : IArticlesRepository { private string _connectionString; @@ -15,21 +17,31 @@ public class ArticlesTable _connectionString = connectionString; } - public static IDbConnection OpenConnection(string connectionString) + public ArticlesTable(IConfiguration configuration) { - var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; - var conn = new NpgsqlConnection(cs); + var conn = configuration.GetConnectionString("database"); + if (conn is null) + { + conn = ""; + } + + _connectionString = conn; + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); conn.Open(); return conn; } - public List List(int Page = 0, int Count = 25) + public List List(int page = 0, int count = 25) { using var conn = OpenConnection(_connectionString); var res = conn.Query(@"select * from articles Order By PubDate Desc Offset @Page - Fetch Next @Count Rows Only", new { Page = Page * Count, Count = Count }).ToList(); + Fetch Next @Count Rows Only", new { Page = page * count, Count = count }).ToList(); return res; } @@ -37,6 +49,10 @@ public class ArticlesTable { using var conn = OpenConnection(_connectionString); var res = conn.Query("select * from articles where ID = @ID", new { ID = ID }); + if (res.Count() == 0) + { + return new ArticlesModel(); + } return res.First(); } @@ -44,36 +60,36 @@ public class ArticlesTable { using var conn = OpenConnection(_connectionString); var res = conn.Query("select * from articles where Url = @Url Limit 1", new { Url = url }); + if (res.Count() == 0) + { + return new ArticlesModel(); + } return res.First(); } - public void New(ArticlesModel model) + public ArticlesModel New(ArticlesModel model) { model.ID = Guid.NewGuid(); using var conn = OpenConnection(_connectionString); - var q = @"INSERT INTO Articles - (ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage) - Values - (@Id, @SourceId, @Tags, @Title, @Url, @PubDate, @Video, @VideoHeight, @VideoWidth, @Thumbnail, @Description, @AuthorName, @AuthorImage); - "; - var res = conn.Execute(q, model); - //new{ - // Id = Guid.NewGuid(), - // SourceId = model.SourceID, - // Tags = model.Tags, - // Title = model.Title, - // Url = model.URL, - // PubDate = model.PubDate, - // Video = model.Video, - // VideoHeight = model.VideoHeight, - // VideoWidth = model.VideoWidth, - // Thumbnail = model.Thumbnail, - // Description = model.Description, - // AuthorName = model.AuthorName, - // AuthorImage = model.AuthorImage - //}); - Console.WriteLine(res); + var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);"; + var res = conn.Execute(q, new + { + id = Guid.NewGuid(), + sourceid = model.SourceID, + tags = model.Tags, + title = model.Title, + url = model.URL, + pubdate = model.PubDate, + video = model.Video, + videoheight = model.VideoHeight, + videowidth = model.VideoWidth, + thumbnail = model.Thumbnail, + description = model.Description, + authorname = model.AuthorName, + authorimage = model.AuthorImage + }); + return model; } } \ No newline at end of file diff --git a/Newsbot.Collector.Database/Repositories/DiscordQueue.cs b/Newsbot.Collector.Database/Repositories/DiscordQueue.cs new file mode 100644 index 0000000..bf41b38 --- /dev/null +++ b/Newsbot.Collector.Database/Repositories/DiscordQueue.cs @@ -0,0 +1,54 @@ +using System.Data; +using Dapper; +using Newsbot.Collector.Domain.Interfaces; +using Newsbot.Collector.Domain.Models; +using Npgsql; + +namespace Newsbot.Collector.Database.Repositories; + +public class DiscordQueueTable : IDiscordQueueRepository +{ + private string _connectionString; + + public DiscordQueueTable(string connectionString) + { + _connectionString = connectionString; + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); + conn.Open(); + return conn; + } + + public void New(DiscordQueueModel model) + { + using var conn = OpenConnection(_connectionString); + var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);"; + conn.Execute(query, new + { + id = Guid.NewGuid(), + articleid = model.ArticleID + }); + } + + public void Delete(Guid id) + { + using var conn = OpenConnection(_connectionString); + var query = "Delete From DiscordQueue Where ID = @id;"; + conn.Execute(query, new + { + id = id + }); + } + + public List List(int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * from DiscordQueue LIMIT @id;"; + return conn.Query(query, new { + limit = limit + }).ToList(); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Database/Repositories/SettingsTable.cs b/Newsbot.Collector.Database/Repositories/SettingsTable.cs index 1cd784f..255bab1 100644 --- a/Newsbot.Collector.Database/Repositories/SettingsTable.cs +++ b/Newsbot.Collector.Database/Repositories/SettingsTable.cs @@ -1,5 +1,6 @@ using System.Data; using Dapper; +using Microsoft.Extensions.Configuration; using Newsbot.Collector.Domain.Models; using Npgsql; @@ -15,10 +16,19 @@ public class SettingsTable _connectionString = connectionString; } - public static IDbConnection OpenConnection(string connectionString) + public SettingsTable(IConfiguration configuration) { - var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; - var conn = new NpgsqlConnection(cs); + var connstr = configuration.GetConnectionString("database"); + if (connstr is null) + { + connstr = ""; + } + _connectionString = connstr; + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); conn.Open(); return conn; } diff --git a/Newsbot.Collector.Database/Repositories/SourcesTable.cs b/Newsbot.Collector.Database/Repositories/SourcesTable.cs new file mode 100644 index 0000000..a47b734 --- /dev/null +++ b/Newsbot.Collector.Database/Repositories/SourcesTable.cs @@ -0,0 +1,161 @@ +using System.Data; +using Dapper; +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Interfaces; +using Newsbot.Collector.Domain.Models; +using Npgsql; + +namespace Newsbot.Collector.Database.Repositories; + +public class SourcesTable : ISourcesRepository +{ + private string _connectionString; + + public SourcesTable(string connectionString) + { + _connectionString = connectionString; + } + + public SourcesTable(IConfiguration configuration) + { + var connstr = configuration.GetConnectionString("database"); + if (connstr is null) + { + connstr = ""; + } + _connectionString = connstr; + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); + conn.Open(); + return conn; + } + + public SourceModel New(SourceModel model) + { + model.ID = Guid.NewGuid(); + using var conn = OpenConnection(_connectionString); + var query = "Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags);"; + conn.Execute(query, new + { + id = model.ID, + model.Site, + model.Name, + model.Source, + model.Type, + model.Value, + model.Enabled, + model.Url, + model.Tags + }); + return model; + } + + public SourceModel GetByID(Guid ID) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From Sources where ID = @id Limit 1;"; + var res = conn.Query(query, new + { + id = ID + }); + if (res.Count() == 0) + { + return new SourceModel(); + } + return res.First(); + } + + public SourceModel GetByID(string ID) + { + var uid = Guid.Parse(ID); + return GetByID(uid); + } + + public SourceModel GetByName(string Name) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * from Sources where name = @name Limit 1;"; + var res = conn.Query(query, new + { + name = Name + }); + + if (res.Count() == 0) + { + return new SourceModel(); + } + return res.First(); + } + + public SourceModel GetByNameAndSource(string name, string source) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * from Sources WHERE name = @name and source = @source;"; + var res = conn.Query(query, new + { + name = name, + source = source + }); + + if (res.Count() == 0) + { + return new SourceModel(); + } + return res.First(); + } + + public List List(int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From Sources Limit @limit;"; + return conn.Query(query, new + { + limit = 25 + }).ToList(); + } + + public List ListBySource(string source, int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From Sources where Source = @source Limit @limit;"; + return conn.Query(query, new + { + source = source, + limit = limit + }).ToList(); + } + + public List ListByType(string type, int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From Sources where Type = @type Limit @limit;"; + return conn.Query(query, new + { + type = type, + limit = limit + }).ToList(); + } + public int Disable(Guid ID) + { + using var conn = OpenConnection(_connectionString); + var query = "Update Sources Set Enabled = FALSE where ID = @id;"; + return conn.Execute(query, new + { + id = ID + }); + } + + public int Enable(Guid ID) + { + using var conn = OpenConnection(_connectionString); + var query = "Update Sources Set Enabled = TRUE where ID = @id;"; + return conn.Execute(query, new + { + id = ID + }); + } + +} \ No newline at end of file diff --git a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs new file mode 100644 index 0000000..bf48c05 --- /dev/null +++ b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs @@ -0,0 +1,97 @@ +using System.Data; +using Dapper; +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Models; +using Npgsql; + +namespace Newsbot.Collector.Database.Repositories; + +public class SubscriptionsTable +{ + private string _connectionString; + + public SubscriptionsTable(string connectionString) + { + _connectionString = connectionString; + } + + public SubscriptionsTable(IConfiguration configuration) + { + var connstr = configuration.GetConnectionString("database"); + if (connstr is null) + { + connstr = ""; + } + _connectionString = connstr; + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); + conn.Open(); + return conn; + } + + public void New(SubscriptionModel model) + { + using var conn = OpenConnection(_connectionString); + var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);"; + conn.Execute(query, new + { + id = Guid.NewGuid(), + webhookid = model.DiscordWebHookID, + sourceid = model.SourceID + }); + } + + public List List(int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From subscriptions Limit @limit;"; + return conn.Query(query, new + { + limit = limit, + }).ToList(); + } + + // todo add paging + public List ListBySourceID(Guid sourceID) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From subscriptions where sourceid = @sourceid"; + return conn.Query(query, new + { + sourceid = sourceID + }).ToList(); + } + + public List GetByWebhookAndSource(Guid webhookId, Guid sourceId) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From subscriptions Where discordwebhookid = @webhookid and sourceid = @sourceid;"; + return conn.Query(query, new + { + webhookid = webhookId, + sourceid = sourceId, + }).ToList(); + } + + public List ListByWebhook(Guid webhookId) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From subscriptions Where discordwebhookid = @webhookid"; + return conn.Query(query, new + { + webhookid = webhookId, + }).ToList(); + } + + public void Delete(Guid id) + { + using var conn = OpenConnection(_connectionString); + var query = "Delete From subscriptions Where id = @id;"; + conn.Execute(query, new { + id = id + }); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Database/Repositories/WebhooksTable.cs b/Newsbot.Collector.Database/Repositories/WebhooksTable.cs new file mode 100644 index 0000000..22d34ce --- /dev/null +++ b/Newsbot.Collector.Database/Repositories/WebhooksTable.cs @@ -0,0 +1,121 @@ +using System.Data; +using Dapper; +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Models; +using Npgsql; + +namespace Newsbot.Collector.Database.Repositories; + +public class WebhooksTable +{ + private string _connectionString; + + public WebhooksTable(string connectionString) + { + _connectionString = connectionString; + } + + public WebhooksTable(IConfiguration configuration) + { + var connstr = configuration.GetConnectionString("database"); + if (connstr is null) + { + connstr = ""; + } + _connectionString = connstr; + } + + private IDbConnection OpenConnection(string connectionString) + { + var conn = new NpgsqlConnection(_connectionString); + conn.Open(); + return conn; + } + + public void New(DiscordWebHook model) + { + using var conn = OpenConnection(_connectionString); + var query = "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);"; + conn.Execute(query, new + { + id = model.ID, + url = model.Url, + server = model.Server, + channel = model.Channel, + enabled = model.Enabled + }); + } + + public DiscordWebHook GetByID(Guid ID) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;"; + return conn.Query(query, new + { + id = ID + }).First(); + } + + public DiscordWebHook GetByUrl(string url) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From DiscordWebHooks Where url = @url;"; + return conn.QueryFirst(query, new + { + url = url + }); + } + + public List List(int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From DiscordWebHooks @limit;"; + return conn.Query(query, new + { + limit = limit + }).ToList(); + } + + public List ListByServer(string server, int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;"; + return conn.Query(query, new + { + server = server, + limit = limit + }).ToList(); + } + + public List ListByServerAndChannel(string server, string channel, int limit = 25) + { + using var conn = OpenConnection(_connectionString); + var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;"; + return conn.Query(query, new + { + server = server, + channel = channel, + limit = limit + }).ToList(); + } + + public int Disable(Guid ID) + { + using var conn = OpenConnection(_connectionString); + var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;"; + return conn.Execute(query, new + { + id = ID + }); + } + + public int Enable(Guid ID) + { + using var conn = OpenConnection(_connectionString); + var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;"; + return conn.Execute(query, new + { + id = ID + }); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Consts/SourcesConst.cs b/Newsbot.Collector.Domain/Consts/SourcesConst.cs new file mode 100644 index 0000000..7c1befd --- /dev/null +++ b/Newsbot.Collector.Domain/Consts/SourcesConst.cs @@ -0,0 +1,12 @@ +namespace Newsbot.Collector.Domain.Consts; + +public class SourceTypes +{ + public const string Reddit = "reddit"; + public const string Rss = "rss"; + public const string YouTube = "youtube"; + public const string Twitch = "twitch"; + public const string FinalFantasyXiv = "ffxiv"; + public const string GitHub = "github"; + +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs new file mode 100644 index 0000000..9c1493d --- /dev/null +++ b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs @@ -0,0 +1,11 @@ +using Newsbot.Collector.Domain.Models; + +namespace Newsbot.Collector.Domain.Interfaces; + +public interface IArticlesRepository : ITableRepository +{ + ListList(int age, int count); + ArticlesModel GetById(Guid ID); + ArticlesModel GetByUrl(string url); + ArticlesModel New(ArticlesModel model); +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/ICollector.cs b/Newsbot.Collector.Domain/Interfaces/ICollector.cs index bca8bfd..ac2b55b 100644 --- a/Newsbot.Collector.Domain/Interfaces/ICollector.cs +++ b/Newsbot.Collector.Domain/Interfaces/ICollector.cs @@ -2,7 +2,8 @@ using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Interfaces; -public interface ICollector +/// +public interface ICollector : IHangfireJob { List Collect(); } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs b/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs new file mode 100644 index 0000000..2af0c6d --- /dev/null +++ b/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs @@ -0,0 +1,10 @@ +using Newsbot.Collector.Domain.Models; + +namespace Newsbot.Collector.Domain.Interfaces; + +public interface IDiscordQueueRepository +{ + void New(DiscordQueueModel model); + void Delete(Guid id); + List List(int limit); +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/IHangfireJob.cs b/Newsbot.Collector.Domain/Interfaces/IHangfireJob.cs new file mode 100644 index 0000000..fca0dab --- /dev/null +++ b/Newsbot.Collector.Domain/Interfaces/IHangfireJob.cs @@ -0,0 +1,8 @@ +using Microsoft.Extensions.Configuration; + +namespace Newsbot.Collector.Domain.Interfaces; + +public interface IHangfireJob +{ + void InitAndExecute(IConfiguration config); +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs new file mode 100644 index 0000000..61cbd92 --- /dev/null +++ b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs @@ -0,0 +1,18 @@ +using System.Globalization; +using Newsbot.Collector.Domain.Models; + +namespace Newsbot.Collector.Domain.Interfaces; + +public interface ISourcesRepository +{ + public SourceModel New(SourceModel model); + public SourceModel GetByID(Guid ID); + public SourceModel GetByID(string ID); + public SourceModel GetByName(string name); + public SourceModel GetByNameAndSource(string name, string source); + public List List(int limit); + public List ListBySource(string source, int limit); + public List ListByType(string type, int limit = 25); + public int Disable(Guid ID); + public int Enable(Guid ID); +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/ITableRepository.cs b/Newsbot.Collector.Domain/Interfaces/ITableRepository.cs new file mode 100644 index 0000000..68a01ed --- /dev/null +++ b/Newsbot.Collector.Domain/Interfaces/ITableRepository.cs @@ -0,0 +1,8 @@ +using Newsbot.Collector.Domain.Models; + +namespace Newsbot.Collector.Domain.Interfaces; + +public interface ITableRepository +{ + +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Models/DatabaseModel.cs b/Newsbot.Collector.Domain/Models/DatabaseModel.cs index c3af5cb..730f81a 100644 --- a/Newsbot.Collector.Domain/Models/DatabaseModel.cs +++ b/Newsbot.Collector.Domain/Models/DatabaseModel.cs @@ -7,10 +7,10 @@ public class ArticlesModel public string Tags { get; set; } = ""; public string Title { get; set; } = ""; public string URL { get; set; } = ""; - public DateTime PubDate { get; set; } + public DateTime PubDate { get; set; } = DateTime.Now; public string Video { get; set; } = ""; - public int VideoHeight { get; set; } - public int VideoWidth { get; set; } + public int VideoHeight { get; set; } = 0; + public int VideoWidth { get; set; } = 0; public string Thumbnail { get; set; } = ""; public string Description { get; set; } = ""; public string AuthorName { get; set; } = ""; @@ -60,6 +60,8 @@ public class SourceModel public Guid ID { get; set; } public string Site { get; set; } = ""; public string Name { get; set; } = ""; + + // Source use to deinfe the worker to query with but moving to Type as it was not used really. public string Source { get; set; } = ""; public string Type { get; set; } = ""; public string Value { get; set; } = ""; diff --git a/Newsbot.Collector.Domain/Newsbot.Collector.Domain.csproj b/Newsbot.Collector.Domain/Newsbot.Collector.Domain.csproj index cfadb03..5177e08 100644 --- a/Newsbot.Collector.Domain/Newsbot.Collector.Domain.csproj +++ b/Newsbot.Collector.Domain/Newsbot.Collector.Domain.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/Newsbot.Collector.Services/Jobs/HelloWorldJob.cs b/Newsbot.Collector.Services/Jobs/HelloWorldJob.cs index e305568..8f2f2e3 100644 --- a/Newsbot.Collector.Services/Jobs/HelloWorldJob.cs +++ b/Newsbot.Collector.Services/Jobs/HelloWorldJob.cs @@ -1,23 +1,30 @@ +using Microsoft.Extensions.Configuration; + namespace Newsbot.Collector.Services.Jobs; +public class HelloWorldJobOptions +{ + public string Message { get; set; } = ""; +} + public class HelloWorldJob { - - public string _message { get; set; } - - public HelloWorldJob(string message) + private HelloWorldJobOptions _options; + + public HelloWorldJob(HelloWorldJobOptions options) { - _message = message; + _options = options; } - public void SetMessage(string message) + public void InitAndExecute(HelloWorldJobOptions options) { - _message = message; + _options = options; + Execute(); } - public void Execute() + private void Execute() { - Console.WriteLine(_message); + Console.WriteLine(_options.Message); } } \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index fcbbc74..80173c9 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -1,59 +1,151 @@ +using System.Runtime.InteropServices; using System.ServiceModel.Syndication; using System.Xml; +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Services.Jobs; -public class RssWatcherJob : ICollector +public class RssWatcherJobOptions +{ + public string ConnectionString { get; set; } = ""; +} + +// This class was made to work with Hangfire and it does not support constructors. +public class RssWatcherJob : IHangfireJob { - private string? _url; + private IArticlesRepository _articles; + private IDiscordQueueRepository _queue; + private ISourcesRepository _source; - public RssWatcherJob(string url) + public RssWatcherJob() { - _url = url; + _articles = new ArticlesTable(""); + _queue = new DiscordQueueTable(""); + _source = new SourcesTable(""); } - public List Collect() + public void InitAndExecute(RssWatcherJobOptions options) + { + Console.WriteLine("Job was triggered"); + Console.WriteLine("Setting up the job"); + Init(options.ConnectionString); + + var articles = new List(); + + Console.WriteLine("Requesting sources"); + var sources = _source.ListByType(SourceTypes.Rss); + Console.WriteLine($"Got {sources.Count()} back"); + foreach (var source in sources) + { + Console.WriteLine("Starting to request feed to be processed"); + var results = Collect(source.Url); + + articles.AddRange(results); + } + + UpdateDatabase(articles); + } + + public void InitAndExecute(IConfiguration config) + { + // reach out to the db and find all the rss feeds + var connectionString = config.GetConnectionString("database"); + if (connectionString is null) + { + connectionString = ""; + } + Init(connectionString); + + var articles = new List(); + + var sources = _source.ListByType(SourceTypes.Rss); + foreach (var source in sources) + { + var results = Collect(source.Url); + + articles.AddRange(results); + } + + UpdateDatabase(articles); + } + + public void Init(string connectionString) + { + _articles = new ArticlesTable(connectionString); + _queue = new DiscordQueueTable(connectionString); + _source = new SourcesTable(connectionString); + } + + public List Collect(string url, int sleep = 3000) { var CollectedPosts = new List(); - if (_url is null) - { - _url = ""; - } - - using var reader = XmlReader.Create(_url); + using var reader = XmlReader.Create(url); var feed = SyndicationFeed.Load(reader); - var posts = feed.Items.ToList(); - foreach (var post in posts) + foreach (var post in feed.Items.ToList()) { - var url = post.Links[0].Uri.AbsoluteUri; + var articleUrl = post.Links[0].Uri.AbsoluteUri; // Check if we have seen the url before // If we have, skip and save the site bandwidth + if (IsThisUrlKnown(articleUrl) == true) + { + continue; + } - var meta = new HtmlPageReader(url); + var meta = new HtmlPageReader(articleUrl); var article = new ArticlesModel { Title = post.Title.Text, Tags = FetchTags(post), - URL = post.Links[0].Uri.ToString(), + URL = articleUrl, PubDate = post.PublishDate.DateTime, Thumbnail = meta.Data.Header.Meta.Image, Description = meta.Data.Header.Meta.Description, }; + CollectedPosts.Add(article); // try to not be too greedy - Thread.Sleep(3000); + Thread.Sleep(sleep); } return CollectedPosts; } + public void UpdateDatabase(List items) + { + foreach (var item in items) + { + if (IsThisUrlKnown(item.URL) == false) + { + continue; + } + + var p = _articles.New(item); + _queue.New(new DiscordQueueModel + { + ArticleID = p.ID + }); + } + } + + private bool IsThisUrlKnown(string url) + { + var isKnown = _articles.GetByUrl(url); + if (isKnown.URL == url) + { + return true; + } + return false; + } + private string FetchTags(SyndicationItem post) { string result = ""; diff --git a/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs b/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs index 1f24394..e17f15e 100644 --- a/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs @@ -1,14 +1,41 @@ +using Microsoft.Extensions.Configuration; using Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Tests.Jobs; public class RssWatcherJobTest { + private IConfiguration GetConfiguration() + { + var inMemorySettings = new Dictionary { + {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"} + }; + + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + return configuration; + } + + private string ConnectionString() + { + return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; + } + [Fact] - public void CanFindItems() + public void CanFindItemsNoDb() { var url = "https://www.engadget.com/rss.xml"; - var client = new RssWatcherJob(url); - var items = client.Collect(); + var client = new RssWatcherJob(); + var items = client.Collect(url); + } + + [Fact] + public void CanAddItemsToDb() + { + var url = "https://www.engadget.com/rss.xml"; + var client = new RssWatcherJob(); + client.Init(ConnectionString()); + client.Collect(url, 0); } } \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj b/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj index 759ae6c..f8c1e88 100644 --- a/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj +++ b/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj @@ -3,12 +3,13 @@ net7.0 enable - enable + disable false + @@ -21,9 +22,9 @@ - - - + + + diff --git a/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs b/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs index 80de049..2884600 100644 --- a/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs +++ b/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Models; @@ -5,11 +6,23 @@ namespace Newsbot.Collector.Tests.Tables; public class ArticlesTableTests { + private IConfiguration GetConfiguration() + { + var inMemorySettings = new Dictionary { + {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"} + }; + + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + return configuration; + } [Fact] public void ArticlesListTest() { - var client = new ArticlesTable(""); + var cfg = GetConfiguration(); + var client = new ArticlesTable(cfg); client.List(); } @@ -18,7 +31,8 @@ public class ArticlesTableTests { var uid = Guid.Parse("4ac46772-253c-4c3d-8a2c-29239abd2ad4"); - var client = new ArticlesTable(""); + var cfg = GetConfiguration(); + var client = new ArticlesTable(cfg); var res = client.GetById(uid); if (!res.ID.Equals(uid)) { @@ -29,12 +43,17 @@ public class ArticlesTableTests [Fact] public void NewRecordTest() { - var client = new ArticlesTable(""); - client.New(new ArticlesModel + var cfg = GetConfiguration(); + var client = new ArticlesTable(cfg); + var m = new ArticlesModel { - Title = "Unit Testing!", + ID = Guid.NewGuid(), SourceID = Guid.NewGuid(), - PubDate = DateTime.Now - }); + Tags = "thing, thing2", + Title = "Unit Testing!", + URL = "https://google.com", + PubDate = DateTime.Now.ToLocalTime(), + }; + client.New(m); } } \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Tables/SettingsTableTests.cs b/Newsbot.Collector.Tests/Tables/SettingsTableTests.cs index 1f49ee5..223f63a 100644 --- a/Newsbot.Collector.Tests/Tables/SettingsTableTests.cs +++ b/Newsbot.Collector.Tests/Tables/SettingsTableTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Models; @@ -5,10 +6,22 @@ namespace Newsbot.Collector.Tests.Tables; public class SettingsTableTests { + private IConfiguration GetConfiguration() + { + var inMemorySettings = new Dictionary { + {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"} + }; + + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + return configuration; + } [Fact] public void New() { - var client = new SettingsTable(""); + var cfg = GetConfiguration(); + var client = new SettingsTable(cfg); client.New(new SettingModel { Key = "Unit Testing", diff --git a/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs b/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs new file mode 100644 index 0000000..bb2c7bb --- /dev/null +++ b/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs @@ -0,0 +1,27 @@ + + +using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Models; + +public class SourcesTableTests +{ + [Fact] + public void NewRecordTest() + { + var client = new SourcesTable(""); + var m = new SourceModel + { + ID = Guid.NewGuid(), + Site = "Testing", + Name = "Testing", + Source = "Testing", + Type = "Testing", + Value = "Testing", + Enabled = false, + Url = "Testing", + Tags = "Unit, Testing", + Deleted = false + }; + client.New(m); + } +} \ No newline at end of file