Features/table clients (#5)

* inital table clients are almost ready

* adding new repo interfaces

* adding MS DI to the tables

* adding tables and MS DI to jobs

* updated how the config builder works and pass Iconfig to jobs

* Updated how articles interface returns a value

* updated constructors to support DI and removed static call

* added source consts and model notes

* updated sources.type to contain the value to link what job collects it

* added RssWatcherJob to hangfire

* DI and hangfire jobs wont work.  Defering to options classes

* Services updated to have options exposed over DI

* Tests have been updated.. more to come
This commit is contained in:
James Tombleson 2023-02-19 21:39:03 -08:00 committed by GitHub
parent 9f3a6323a6
commit 17e97b4e09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 842 additions and 95 deletions

View File

@ -10,6 +10,7 @@
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" /> <PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" /> <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" /> <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup> </ItemGroup>

View File

@ -8,11 +8,7 @@ var builder = WebApplication.CreateBuilder(args);
// Add services to the container. // Add services to the container.
// Build the conifg // Build the conifg
var config = new ConfigurationBuilder() var config = GetConfiguration();
.AddJsonFile("appsettings.json")
.AddEnvironmentVariables()
.Build();
var cfg = config.GetRequiredSection("Config").Get<ConfigModel>();
builder.Configuration.AddConfiguration(config); builder.Configuration.AddConfiguration(config);
builder.Services.AddHangfire(f => f.UseMemoryStorage()); builder.Services.AddHangfire(f => f.UseMemoryStorage());
@ -35,10 +31,40 @@ if (app.Environment.IsDevelopment())
app.UseHttpsRedirection(); app.UseHttpsRedirection();
app.UseHangfireDashboard(); app.UseHangfireDashboard();
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.Execute(), "0/2 * * * *"); SetupRecurringJobs(config);
app.UseAuthorization(); app.UseAuthorization();
app.MapControllers(); app.MapControllers();
app.Run(); app.Run();
static IConfiguration GetConfiguration()
{
return new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true)
.AddEnvironmentVariables()
.Build();
}
static void SetupRecurringJobs(IConfiguration configuration)
{
var databaseConnectionString = configuration.GetConnectionString("database");
if (databaseConnectionString is null)
{
databaseConnectionString = "";
}
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
{
Message = "Hello from the background!"
}), "0/2 * * * *");
//RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(config), "15 0-23 * * *");
var c = new RssWatcherJob();
BackgroundJob.Enqueue(() => c.InitAndExecute(new RssWatcherJobOptions
{
ConnectionString = databaseConnectionString
}));
}

View File

@ -7,33 +7,33 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Final Fantasy XIV Entries -- Final Fantasy XIV Entries
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); (uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'scrape', 'ffxiv', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); (uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'scrape', 'ffxiv', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); (uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'scrape', 'ffxiv', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); (uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'scrape', 'ffxiv', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); (uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'scrape', 'ffxiv', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries -- Reddit Entries
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); (uuid_generate_v4(), 'reddit', 'dadjokes', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); (uuid_generate_v4(), 'reddit', 'steamdeck', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries -- Youtube Entries
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); (uuid_generate_v4(), 'youtube', 'Game Grumps', 'feed', 'youtube', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries -- RSS Entries
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); (uuid_generate_v4(), 'steampowered', 'steam deck', 'feed', 'rss', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries -- Twitch Entries
INSERT INTO sources VALUES INSERT INTO sources VALUES
(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo'); (uuid_generate_v4(), 'twitch', 'Nintendo', 'api', 'twitch', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-- +goose StatementEnd -- +goose StatementEnd

View File

@ -6,6 +6,6 @@ ALTER TABLE sources Add COLUMN Deleted BOOLEAN;
-- +goose Down -- +goose Down
-- +goose StatementBegin -- +goose StatementBegin
SELECT 'down SQL query'; --SELECT 'down SQL query';
ALTER TABLE sources Drop Deleted Deleted BOOLEAN; ALTER TABLE sources Drop Column Deleted;
-- +goose StatementEnd -- +goose StatementEnd

View File

@ -6,6 +6,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="dapper" Version="2.0.123" /> <PackageReference Include="dapper" Version="2.0.123" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Npgsql" Version="7.0.2" /> <PackageReference Include="Npgsql" Version="7.0.2" />
</ItemGroup> </ItemGroup>

View File

@ -1,11 +1,13 @@
using System.Data; using System.Data;
using Dapper; using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Npgsql; using Npgsql;
namespace Newsbot.Collector.Database.Repositories; namespace Newsbot.Collector.Database.Repositories;
public class ArticlesTable public class ArticlesTable : IArticlesRepository
{ {
private string _connectionString; private string _connectionString;
@ -15,21 +17,31 @@ public class ArticlesTable
_connectionString = connectionString; _connectionString = connectionString;
} }
public static IDbConnection OpenConnection(string connectionString) public ArticlesTable(IConfiguration configuration)
{ {
var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; var conn = configuration.GetConnectionString("database");
var conn = new NpgsqlConnection(cs); if (conn is null)
{
conn = "";
}
_connectionString = conn;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open(); conn.Open();
return conn; return conn;
} }
public List<ArticlesModel> List(int Page = 0, int Count = 25) public List<ArticlesModel> List(int page = 0, int count = 25)
{ {
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>(@"select * from articles var res = conn.Query<ArticlesModel>(@"select * from articles
Order By PubDate Desc Order By PubDate Desc
Offset @Page Offset @Page
Fetch Next @Count Rows Only", new { Page = Page * Count, Count = Count }).ToList(); Fetch Next @Count Rows Only", new { Page = page * count, Count = count }).ToList();
return res; return res;
} }
@ -37,6 +49,10 @@ public class ArticlesTable
{ {
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID = ID }); var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID = ID });
if (res.Count() == 0)
{
return new ArticlesModel();
}
return res.First(); return res.First();
} }
@ -44,36 +60,36 @@ public class ArticlesTable
{ {
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url }); var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
if (res.Count() == 0)
{
return new ArticlesModel();
}
return res.First(); return res.First();
} }
public void New(ArticlesModel model) public ArticlesModel New(ArticlesModel model)
{ {
model.ID = Guid.NewGuid(); model.ID = Guid.NewGuid();
using var conn = OpenConnection(_connectionString); using var conn = OpenConnection(_connectionString);
var q = @"INSERT INTO Articles var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage) var res = conn.Execute(q, new
Values {
(@Id, @SourceId, @Tags, @Title, @Url, @PubDate, @Video, @VideoHeight, @VideoWidth, @Thumbnail, @Description, @AuthorName, @AuthorImage); id = Guid.NewGuid(),
"; sourceid = model.SourceID,
var res = conn.Execute(q, model); tags = model.Tags,
//new{ title = model.Title,
// Id = Guid.NewGuid(), url = model.URL,
// SourceId = model.SourceID, pubdate = model.PubDate,
// Tags = model.Tags, video = model.Video,
// Title = model.Title, videoheight = model.VideoHeight,
// Url = model.URL, videowidth = model.VideoWidth,
// PubDate = model.PubDate, thumbnail = model.Thumbnail,
// Video = model.Video, description = model.Description,
// VideoHeight = model.VideoHeight, authorname = model.AuthorName,
// VideoWidth = model.VideoWidth, authorimage = model.AuthorImage
// Thumbnail = model.Thumbnail, });
// Description = model.Description, return model;
// AuthorName = model.AuthorName,
// AuthorImage = model.AuthorImage
//});
Console.WriteLine(res);
} }
} }

View File

@ -0,0 +1,54 @@
using System.Data;
using Dapper;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class DiscordQueueTable : IDiscordQueueRepository
{
private string _connectionString;
public DiscordQueueTable(string connectionString)
{
_connectionString = connectionString;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public void New(DiscordQueueModel model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);";
conn.Execute(query, new
{
id = Guid.NewGuid(),
articleid = model.ArticleID
});
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From DiscordQueue Where ID = @id;";
conn.Execute(query, new
{
id = id
});
}
public List<DiscordQueueModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordQueue LIMIT @id;";
return conn.Query<DiscordQueueModel>(query, new {
limit = limit
}).ToList();
}
}

View File

@ -1,5 +1,6 @@
using System.Data; using System.Data;
using Dapper; using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Npgsql; using Npgsql;
@ -15,10 +16,19 @@ public class SettingsTable
_connectionString = connectionString; _connectionString = connectionString;
} }
public static IDbConnection OpenConnection(string connectionString) public SettingsTable(IConfiguration configuration)
{ {
var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; var connstr = configuration.GetConnectionString("database");
var conn = new NpgsqlConnection(cs); if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open(); conn.Open();
return conn; return conn;
} }

View File

@ -0,0 +1,161 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class SourcesTable : ISourcesRepository
{
private string _connectionString;
public SourcesTable(string connectionString)
{
_connectionString = connectionString;
}
public SourcesTable(IConfiguration configuration)
{
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public SourceModel New(SourceModel model)
{
model.ID = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var query = "Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags);";
conn.Execute(query, new
{
id = model.ID,
model.Site,
model.Name,
model.Source,
model.Type,
model.Value,
model.Enabled,
model.Url,
model.Tags
});
return model;
}
public SourceModel GetByID(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where ID = @id Limit 1;";
var res = conn.Query<SourceModel>(query, new
{
id = ID
});
if (res.Count() == 0)
{
return new SourceModel();
}
return res.First();
}
public SourceModel GetByID(string ID)
{
var uid = Guid.Parse(ID);
return GetByID(uid);
}
public SourceModel GetByName(string Name)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources where name = @name Limit 1;";
var res = conn.Query<SourceModel>(query, new
{
name = Name
});
if (res.Count() == 0)
{
return new SourceModel();
}
return res.First();
}
public SourceModel GetByNameAndSource(string name, string source)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources WHERE name = @name and source = @source;";
var res = conn.Query<SourceModel>(query, new
{
name = name,
source = source
});
if (res.Count() == 0)
{
return new SourceModel();
}
return res.First();
}
public List<SourceModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources Limit @limit;";
return conn.Query<SourceModel>(query, new
{
limit = 25
}).ToList();
}
public List<SourceModel> ListBySource(string source, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where Source = @source Limit @limit;";
return conn.Query<SourceModel>(query, new
{
source = source,
limit = limit
}).ToList();
}
public List<SourceModel> ListByType(string type, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where Type = @type Limit @limit;";
return conn.Query<SourceModel>(query, new
{
type = type,
limit = limit
}).ToList();
}
public int Disable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
public int Enable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
}

View File

@ -0,0 +1,97 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class SubscriptionsTable
{
private string _connectionString;
public SubscriptionsTable(string connectionString)
{
_connectionString = connectionString;
}
public SubscriptionsTable(IConfiguration configuration)
{
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public void New(SubscriptionModel model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);";
conn.Execute(query, new
{
id = Guid.NewGuid(),
webhookid = model.DiscordWebHookID,
sourceid = model.SourceID
});
}
public List<SubscriptionModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions Limit @limit;";
return conn.Query<SubscriptionModel>(query, new
{
limit = limit,
}).ToList();
}
// todo add paging
public List<SubscriptionModel> ListBySourceID(Guid sourceID)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions where sourceid = @sourceid";
return conn.Query<SubscriptionModel>(query, new
{
sourceid = sourceID
}).ToList();
}
public List<SubscriptionModel> GetByWebhookAndSource(Guid webhookId, Guid sourceId)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions Where discordwebhookid = @webhookid and sourceid = @sourceid;";
return conn.Query<SubscriptionModel>(query, new
{
webhookid = webhookId,
sourceid = sourceId,
}).ToList();
}
public List<SubscriptionModel> ListByWebhook(Guid webhookId)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions Where discordwebhookid = @webhookid";
return conn.Query<SubscriptionModel>(query, new
{
webhookid = webhookId,
}).ToList();
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From subscriptions Where id = @id;";
conn.Execute(query, new {
id = id
});
}
}

View File

@ -0,0 +1,121 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class WebhooksTable
{
private string _connectionString;
public WebhooksTable(string connectionString)
{
_connectionString = connectionString;
}
public WebhooksTable(IConfiguration configuration)
{
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public void New(DiscordWebHook model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);";
conn.Execute(query, new
{
id = model.ID,
url = model.Url,
server = model.Server,
channel = model.Channel,
enabled = model.Enabled
});
}
public DiscordWebHook GetByID(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;";
return conn.Query<DiscordWebHook>(query, new
{
id = ID
}).First();
}
public DiscordWebHook GetByUrl(string url)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks Where url = @url;";
return conn.QueryFirst<DiscordWebHook>(query, new
{
url = url
});
}
public List<DiscordWebHook> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks @limit;";
return conn.Query<DiscordWebHook>(query, new
{
limit = limit
}).ToList();
}
public List<DiscordWebHook> ListByServer(string server, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;";
return conn.Query<DiscordWebHook>(query, new
{
server = server,
limit = limit
}).ToList();
}
public List<DiscordWebHook> ListByServerAndChannel(string server, string channel, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;";
return conn.Query<DiscordWebHook>(query, new
{
server = server,
channel = channel,
limit = limit
}).ToList();
}
public int Disable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
public int Enable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
}

View File

@ -0,0 +1,12 @@
namespace Newsbot.Collector.Domain.Consts;
public class SourceTypes
{
public const string Reddit = "reddit";
public const string Rss = "rss";
public const string YouTube = "youtube";
public const string Twitch = "twitch";
public const string FinalFantasyXiv = "ffxiv";
public const string GitHub = "github";
}

View File

@ -0,0 +1,11 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IArticlesRepository : ITableRepository
{
List<ArticlesModel>List(int age, int count);
ArticlesModel GetById(Guid ID);
ArticlesModel GetByUrl(string url);
ArticlesModel New(ArticlesModel model);
}

View File

@ -2,7 +2,8 @@ using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces; namespace Newsbot.Collector.Domain.Interfaces;
public interface ICollector ///
public interface ICollector : IHangfireJob
{ {
List<ArticlesModel> Collect(); List<ArticlesModel> Collect();
} }

View File

@ -0,0 +1,10 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IDiscordQueueRepository
{
void New(DiscordQueueModel model);
void Delete(Guid id);
List<DiscordQueueModel> List(int limit);
}

View File

@ -0,0 +1,8 @@
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IHangfireJob
{
void InitAndExecute(IConfiguration config);
}

View File

@ -0,0 +1,18 @@
using System.Globalization;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ISourcesRepository
{
public SourceModel New(SourceModel model);
public SourceModel GetByID(Guid ID);
public SourceModel GetByID(string ID);
public SourceModel GetByName(string name);
public SourceModel GetByNameAndSource(string name, string source);
public List<SourceModel> List(int limit);
public List<SourceModel> ListBySource(string source, int limit);
public List<SourceModel> ListByType(string type, int limit = 25);
public int Disable(Guid ID);
public int Enable(Guid ID);
}

View File

@ -0,0 +1,8 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ITableRepository
{
}

View File

@ -7,10 +7,10 @@ public class ArticlesModel
public string Tags { get; set; } = ""; public string Tags { get; set; } = "";
public string Title { get; set; } = ""; public string Title { get; set; } = "";
public string URL { get; set; } = ""; public string URL { get; set; } = "";
public DateTime PubDate { get; set; } public DateTime PubDate { get; set; } = DateTime.Now;
public string Video { get; set; } = ""; public string Video { get; set; } = "";
public int VideoHeight { get; set; } public int VideoHeight { get; set; } = 0;
public int VideoWidth { get; set; } public int VideoWidth { get; set; } = 0;
public string Thumbnail { get; set; } = ""; public string Thumbnail { get; set; } = "";
public string Description { get; set; } = ""; public string Description { get; set; } = "";
public string AuthorName { get; set; } = ""; public string AuthorName { get; set; } = "";
@ -60,6 +60,8 @@ public class SourceModel
public Guid ID { get; set; } public Guid ID { get; set; }
public string Site { get; set; } = ""; public string Site { get; set; } = "";
public string Name { get; set; } = ""; public string Name { get; set; } = "";
// Source use to deinfe the worker to query with but moving to Type as it was not used really.
public string Source { get; set; } = ""; public string Source { get; set; } = "";
public string Type { get; set; } = ""; public string Type { get; set; } = "";
public string Value { get; set; } = ""; public string Value { get; set; } = "";

View File

@ -6,4 +6,8 @@
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
</ItemGroup>
</Project> </Project>

View File

@ -1,23 +1,30 @@
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Services.Jobs;
public class HelloWorldJobOptions
{
public string Message { get; set; } = "";
}
public class HelloWorldJob public class HelloWorldJob
{ {
private HelloWorldJobOptions _options;
public string _message { get; set; } public HelloWorldJob(HelloWorldJobOptions options)
public HelloWorldJob(string message)
{ {
_message = message; _options = options;
} }
public void SetMessage(string message) public void InitAndExecute(HelloWorldJobOptions options)
{ {
_message = message; _options = options;
Execute();
} }
public void Execute() private void Execute()
{ {
Console.WriteLine(_message); Console.WriteLine(_options.Message);
} }
} }

View File

@ -1,59 +1,151 @@
using System.Runtime.InteropServices;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Services.Jobs; namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJob : ICollector public class RssWatcherJobOptions
{ {
public string ConnectionString { get; set; } = "";
private string? _url;
public RssWatcherJob(string url)
{
_url = url;
} }
public List<ArticlesModel> Collect() // This class was made to work with Hangfire and it does not support constructors.
public class RssWatcherJob : IHangfireJob
{
private IArticlesRepository _articles;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
public RssWatcherJob()
{
_articles = new ArticlesTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
}
public void InitAndExecute(RssWatcherJobOptions options)
{
Console.WriteLine("Job was triggered");
Console.WriteLine("Setting up the job");
Init(options.ConnectionString);
var articles = new List<ArticlesModel>();
Console.WriteLine("Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss);
Console.WriteLine($"Got {sources.Count()} back");
foreach (var source in sources)
{
Console.WriteLine("Starting to request feed to be processed");
var results = Collect(source.Url);
articles.AddRange(results);
}
UpdateDatabase(articles);
}
public void InitAndExecute(IConfiguration config)
{
// reach out to the db and find all the rss feeds
var connectionString = config.GetConnectionString("database");
if (connectionString is null)
{
connectionString = "";
}
Init(connectionString);
var articles = new List<ArticlesModel>();
var sources = _source.ListByType(SourceTypes.Rss);
foreach (var source in sources)
{
var results = Collect(source.Url);
articles.AddRange(results);
}
UpdateDatabase(articles);
}
public void Init(string connectionString)
{
_articles = new ArticlesTable(connectionString);
_queue = new DiscordQueueTable(connectionString);
_source = new SourcesTable(connectionString);
}
public List<ArticlesModel> Collect(string url, int sleep = 3000)
{ {
var CollectedPosts = new List<ArticlesModel>(); var CollectedPosts = new List<ArticlesModel>();
if (_url is null) using var reader = XmlReader.Create(url);
{
_url = "";
}
using var reader = XmlReader.Create(_url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
var posts = feed.Items.ToList();
foreach (var post in posts) foreach (var post in feed.Items.ToList())
{ {
var url = post.Links[0].Uri.AbsoluteUri; var articleUrl = post.Links[0].Uri.AbsoluteUri;
// Check if we have seen the url before // Check if we have seen the url before
// If we have, skip and save the site bandwidth // If we have, skip and save the site bandwidth
if (IsThisUrlKnown(articleUrl) == true)
{
continue;
}
var meta = new HtmlPageReader(url); var meta = new HtmlPageReader(articleUrl);
var article = new ArticlesModel var article = new ArticlesModel
{ {
Title = post.Title.Text, Title = post.Title.Text,
Tags = FetchTags(post), Tags = FetchTags(post),
URL = post.Links[0].Uri.ToString(), URL = articleUrl,
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Meta.Image, Thumbnail = meta.Data.Header.Meta.Image,
Description = meta.Data.Header.Meta.Description, Description = meta.Data.Header.Meta.Description,
}; };
CollectedPosts.Add(article); CollectedPosts.Add(article);
// try to not be too greedy // try to not be too greedy
Thread.Sleep(3000); Thread.Sleep(sleep);
} }
return CollectedPosts; return CollectedPosts;
} }
public void UpdateDatabase(List<ArticlesModel> items)
{
foreach (var item in items)
{
if (IsThisUrlKnown(item.URL) == false)
{
continue;
}
var p = _articles.New(item);
_queue.New(new DiscordQueueModel
{
ArticleID = p.ID
});
}
}
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url)
{
return true;
}
return false;
}
private string FetchTags(SyndicationItem post) private string FetchTags(SyndicationItem post)
{ {
string result = ""; string result = "";

View File

@ -1,14 +1,41 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Tests.Jobs; namespace Newsbot.Collector.Tests.Jobs;
public class RssWatcherJobTest public class RssWatcherJobTest
{ {
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string> {
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
private string ConnectionString()
{
return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
}
[Fact] [Fact]
public void CanFindItems() public void CanFindItemsNoDb()
{ {
var url = "https://www.engadget.com/rss.xml"; var url = "https://www.engadget.com/rss.xml";
var client = new RssWatcherJob(url); var client = new RssWatcherJob();
var items = client.Collect(); var items = client.Collect(url);
}
[Fact]
public void CanAddItemsToDb()
{
var url = "https://www.engadget.com/rss.xml";
var client = new RssWatcherJob();
client.Init(ConnectionString());
client.Collect(url, 0);
} }
} }

View File

@ -3,12 +3,13 @@
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>disable</Nullable>
<IsPackable>false</IsPackable> <IsPackable>false</IsPackable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" /> <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
<PackageReference Include="xunit" Version="2.4.2" /> <PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5"> <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">

View File

@ -1,3 +1,4 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
@ -5,11 +6,23 @@ namespace Newsbot.Collector.Tests.Tables;
public class ArticlesTableTests public class ArticlesTableTests
{ {
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string> {
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
[Fact] [Fact]
public void ArticlesListTest() public void ArticlesListTest()
{ {
var client = new ArticlesTable(""); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
client.List(); client.List();
} }
@ -18,7 +31,8 @@ public class ArticlesTableTests
{ {
var uid = Guid.Parse("4ac46772-253c-4c3d-8a2c-29239abd2ad4"); var uid = Guid.Parse("4ac46772-253c-4c3d-8a2c-29239abd2ad4");
var client = new ArticlesTable(""); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var res = client.GetById(uid); var res = client.GetById(uid);
if (!res.ID.Equals(uid)) if (!res.ID.Equals(uid))
{ {
@ -29,12 +43,17 @@ public class ArticlesTableTests
[Fact] [Fact]
public void NewRecordTest() public void NewRecordTest()
{ {
var client = new ArticlesTable(""); var cfg = GetConfiguration();
client.New(new ArticlesModel var client = new ArticlesTable(cfg);
var m = new ArticlesModel
{ {
Title = "Unit Testing!", ID = Guid.NewGuid(),
SourceID = Guid.NewGuid(), SourceID = Guid.NewGuid(),
PubDate = DateTime.Now Tags = "thing, thing2",
}); Title = "Unit Testing!",
URL = "https://google.com",
PubDate = DateTime.Now.ToLocalTime(),
};
client.New(m);
} }
} }

View File

@ -1,3 +1,4 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
@ -5,10 +6,22 @@ namespace Newsbot.Collector.Tests.Tables;
public class SettingsTableTests public class SettingsTableTests
{ {
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string> {
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
[Fact] [Fact]
public void New() public void New()
{ {
var client = new SettingsTable(""); var cfg = GetConfiguration();
var client = new SettingsTable(cfg);
client.New(new SettingModel client.New(new SettingModel
{ {
Key = "Unit Testing", Key = "Unit Testing",

View File

@ -0,0 +1,27 @@
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Models;
public class SourcesTableTests
{
[Fact]
public void NewRecordTest()
{
var client = new SourcesTable("");
var m = new SourceModel
{
ID = Guid.NewGuid(),
Site = "Testing",
Name = "Testing",
Source = "Testing",
Type = "Testing",
Value = "Testing",
Enabled = false,
Url = "Testing",
Tags = "Unit, Testing",
Deleted = false
};
client.New(m);
}
}