jtom38
/
Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Features/table clients (#5) 

* inital table clients are almost ready

* adding new repo interfaces

* adding MS DI to the tables

* adding tables and MS DI to jobs

* updated how the config builder works and pass Iconfig to jobs

* Updated how articles interface returns a value

* updated constructors to support DI and removed static call

* added source consts and model notes

* updated sources.type to contain the value to link what job collects it

* added RssWatcherJob to hangfire

* DI and hangfire jobs wont work.  Defering to options classes

* Services updated to have options exposed over DI

* Tests have been updated.. more to come
This commit is contained in:
James Tombleson 2023-02-19 21:39:03 -08:00 committed by GitHub
parent 9f3a6323a6
commit 17e97b4e09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 842 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
View File

@ -10,6 +10,7 @@
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup>

38
Newsbot.Collector.Api/Program.cs
View File

@ -8,11 +8,7 @@ var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
// Build the conifg
var config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.AddEnvironmentVariables()
.Build();
var cfg = config.GetRequiredSection("Config").Get<ConfigModel>();
var config = GetConfiguration();
builder.Configuration.AddConfiguration(config);
builder.Services.AddHangfire(f => f.UseMemoryStorage());
@ -35,10 +31,40 @@ if (app.Environment.IsDevelopment())
app.UseHttpsRedirection();
app.UseHangfireDashboard();
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.Execute(), "0/2 * * * *");
SetupRecurringJobs(config);
app.UseAuthorization();
app.MapControllers();
app.Run();
static IConfiguration GetConfiguration()
{
return new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true)
.AddEnvironmentVariables()
.Build();
}
static void SetupRecurringJobs(IConfiguration configuration)
{
var databaseConnectionString = configuration.GetConnectionString("database");
if (databaseConnectionString is null)
{
databaseConnectionString = "";
}
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
{
Message = "Hello from the background!"
}), "0/2 * * * *");
//RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(config), "15 0-23 * * *");
var c = new RssWatcherJob();
BackgroundJob.Enqueue(() => c.InitAndExecute(new RssWatcherJobOptions
{
ConnectionString = databaseConnectionString
}));
}

20
Newsbot.Collector.Database/Migrations/20220529082459_seed.sql
View File

@ -7,33 +7,33 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Final Fantasy XIV Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'scrape', 'ffxiv', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'scrape', 'ffxiv', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'scrape', 'ffxiv', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'scrape', 'ffxiv', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'scrape', 'ffxiv', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
(uuid_generate_v4(), 'reddit', 'dadjokes', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
(uuid_generate_v4(), 'reddit', 'steamdeck', 'feed', 'reddit', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'feed', 'youtube', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
(uuid_generate_v4(), 'steampowered', 'steam deck', 'feed', 'rss', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
(uuid_generate_v4(), 'twitch', 'Nintendo', 'api', 'twitch', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-- +goose StatementEnd

4
Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql
View File

@ -6,6 +6,6 @@ ALTER TABLE sources Add COLUMN Deleted BOOLEAN;
-- +goose Down
-- +goose StatementBegin
SELECT 'down SQL query';
ALTER TABLE sources Drop Deleted Deleted BOOLEAN;
--SELECT 'down SQL query';
ALTER TABLE sources Drop Column Deleted;
-- +goose StatementEnd

1
Newsbot.Collector.Database/Newsbot.Collector.Database.csproj
View File

@ -6,6 +6,7 @@
<ItemGroup>
<PackageReference Include="dapper" Version="2.0.123" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Npgsql" Version="7.0.2" />
</ItemGroup>

74
Newsbot.Collector.Database/Repositories/ArticlesTable.cs
View File

@ -1,11 +1,13 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class ArticlesTable
public class ArticlesTable : IArticlesRepository
{
private string _connectionString;
@ -15,21 +17,31 @@ public class ArticlesTable
_connectionString = connectionString;
}
public static IDbConnection OpenConnection(string connectionString)
public ArticlesTable(IConfiguration configuration)
{
var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
var conn = new NpgsqlConnection(cs);
var conn = configuration.GetConnectionString("database");
if (conn is null)
{
conn = "";
}
_connectionString = conn;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public List<ArticlesModel> List(int Page = 0, int Count = 25)
public List<ArticlesModel> List(int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>(@"select * from articles
Order By PubDate Desc
Offset @Page
Fetch Next @Count Rows Only", new { Page = Page * Count, Count = Count }).ToList();
Fetch Next @Count Rows Only", new { Page = page * count, Count = count }).ToList();
return res;
}
@ -37,6 +49,10 @@ public class ArticlesTable
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID = ID });
if (res.Count() == 0)
{
return new ArticlesModel();
}
return res.First();
}
@ -44,36 +60,36 @@ public class ArticlesTable
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
if (res.Count() == 0)
{
return new ArticlesModel();
}
return res.First();
}
public void New(ArticlesModel model)
public ArticlesModel New(ArticlesModel model)
{
model.ID = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var q = @"INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
(@Id, @SourceId, @Tags, @Title, @Url, @PubDate, @Video, @VideoHeight, @VideoWidth, @Thumbnail, @Description, @AuthorName, @AuthorImage);
";
var res = conn.Execute(q, model);
//new{
// Id = Guid.NewGuid(),
// SourceId = model.SourceID,
// Tags = model.Tags,
// Title = model.Title,
// Url = model.URL,
// PubDate = model.PubDate,
// Video = model.Video,
// VideoHeight = model.VideoHeight,
// VideoWidth = model.VideoWidth,
// Thumbnail = model.Thumbnail,
// Description = model.Description,
// AuthorName = model.AuthorName,
// AuthorImage = model.AuthorImage
//});
Console.WriteLine(res);
var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
var res = conn.Execute(q, new
{
id = Guid.NewGuid(),
sourceid = model.SourceID,
tags = model.Tags,
title = model.Title,
url = model.URL,
pubdate = model.PubDate,
video = model.Video,
videoheight = model.VideoHeight,
videowidth = model.VideoWidth,
thumbnail = model.Thumbnail,
description = model.Description,
authorname = model.AuthorName,
authorimage = model.AuthorImage
});
return model;
}
}

54
Newsbot.Collector.Database/Repositories/DiscordQueue.cs Normal file
View File

@ -0,0 +1,54 @@
using System.Data;
using Dapper;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class DiscordQueueTable : IDiscordQueueRepository
{
private string _connectionString;
public DiscordQueueTable(string connectionString)
{
_connectionString = connectionString;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public void New(DiscordQueueModel model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);";
conn.Execute(query, new
{
id = Guid.NewGuid(),
articleid = model.ArticleID
});
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From DiscordQueue Where ID = @id;";
conn.Execute(query, new
{
id = id
});
}
public List<DiscordQueueModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordQueue LIMIT @id;";
return conn.Query<DiscordQueueModel>(query, new {
limit = limit
}).ToList();
}
}

16
Newsbot.Collector.Database/Repositories/SettingsTable.cs
View File

@ -1,5 +1,6 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models;
using Npgsql;
@ -15,10 +16,19 @@ public class SettingsTable
_connectionString = connectionString;
}
public static IDbConnection OpenConnection(string connectionString)
public SettingsTable(IConfiguration configuration)
{
var cs = "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
var conn = new NpgsqlConnection(cs);
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}

161
Newsbot.Collector.Database/Repositories/SourcesTable.cs Normal file
View File

@ -0,0 +1,161 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class SourcesTable : ISourcesRepository
{
private string _connectionString;
public SourcesTable(string connectionString)
{
_connectionString = connectionString;
}
public SourcesTable(IConfiguration configuration)
{
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public SourceModel New(SourceModel model)
{
model.ID = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var query = "Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags);";
conn.Execute(query, new
{
id = model.ID,
model.Site,
model.Name,
model.Source,
model.Type,
model.Value,
model.Enabled,
model.Url,
model.Tags
});
return model;
}
public SourceModel GetByID(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where ID = @id Limit 1;";
var res = conn.Query<SourceModel>(query, new
{
id = ID
});
if (res.Count() == 0)
{
return new SourceModel();
}
return res.First();
}
public SourceModel GetByID(string ID)
{
var uid = Guid.Parse(ID);
return GetByID(uid);
}
public SourceModel GetByName(string Name)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources where name = @name Limit 1;";
var res = conn.Query<SourceModel>(query, new
{
name = Name
});
if (res.Count() == 0)
{
return new SourceModel();
}
return res.First();
}
public SourceModel GetByNameAndSource(string name, string source)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources WHERE name = @name and source = @source;";
var res = conn.Query<SourceModel>(query, new
{
name = name,
source = source
});
if (res.Count() == 0)
{
return new SourceModel();
}
return res.First();
}
public List<SourceModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources Limit @limit;";
return conn.Query<SourceModel>(query, new
{
limit = 25
}).ToList();
}
public List<SourceModel> ListBySource(string source, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where Source = @source Limit @limit;";
return conn.Query<SourceModel>(query, new
{
source = source,
limit = limit
}).ToList();
}
public List<SourceModel> ListByType(string type, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where Type = @type Limit @limit;";
return conn.Query<SourceModel>(query, new
{
type = type,
limit = limit
}).ToList();
}
public int Disable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
public int Enable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
}

97
Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs Normal file
View File

@ -0,0 +1,97 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class SubscriptionsTable
{
private string _connectionString;
public SubscriptionsTable(string connectionString)
{
_connectionString = connectionString;
}
public SubscriptionsTable(IConfiguration configuration)
{
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public void New(SubscriptionModel model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);";
conn.Execute(query, new
{
id = Guid.NewGuid(),
webhookid = model.DiscordWebHookID,
sourceid = model.SourceID
});
}
public List<SubscriptionModel> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions Limit @limit;";
return conn.Query<SubscriptionModel>(query, new
{
limit = limit,
}).ToList();
}
// todo add paging
public List<SubscriptionModel> ListBySourceID(Guid sourceID)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions where sourceid = @sourceid";
return conn.Query<SubscriptionModel>(query, new
{
sourceid = sourceID
}).ToList();
}
public List<SubscriptionModel> GetByWebhookAndSource(Guid webhookId, Guid sourceId)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions Where discordwebhookid = @webhookid and sourceid = @sourceid;";
return conn.Query<SubscriptionModel>(query, new
{
webhookid = webhookId,
sourceid = sourceId,
}).ToList();
}
public List<SubscriptionModel> ListByWebhook(Guid webhookId)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From subscriptions Where discordwebhookid = @webhookid";
return conn.Query<SubscriptionModel>(query, new
{
webhookid = webhookId,
}).ToList();
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From subscriptions Where id = @id;";
conn.Execute(query, new {
id = id
});
}
}

121
Newsbot.Collector.Database/Repositories/WebhooksTable.cs Normal file
View File

@ -0,0 +1,121 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
public class WebhooksTable
{
private string _connectionString;
public WebhooksTable(string connectionString)
{
_connectionString = connectionString;
}
public WebhooksTable(IConfiguration configuration)
{
var connstr = configuration.GetConnectionString("database");
if (connstr is null)
{
connstr = "";
}
_connectionString = connstr;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public void New(DiscordWebHook model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);";
conn.Execute(query, new
{
id = model.ID,
url = model.Url,
server = model.Server,
channel = model.Channel,
enabled = model.Enabled
});
}
public DiscordWebHook GetByID(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;";
return conn.Query<DiscordWebHook>(query, new
{
id = ID
}).First();
}
public DiscordWebHook GetByUrl(string url)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks Where url = @url;";
return conn.QueryFirst<DiscordWebHook>(query, new
{
url = url
});
}
public List<DiscordWebHook> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks @limit;";
return conn.Query<DiscordWebHook>(query, new
{
limit = limit
}).ToList();
}
public List<DiscordWebHook> ListByServer(string server, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;";
return conn.Query<DiscordWebHook>(query, new
{
server = server,
limit = limit
}).ToList();
}
public List<DiscordWebHook> ListByServerAndChannel(string server, string channel, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;";
return conn.Query<DiscordWebHook>(query, new
{
server = server,
channel = channel,
limit = limit
}).ToList();
}
public int Disable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
public int Enable(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
{
id = ID
});
}
}

12
Newsbot.Collector.Domain/Consts/SourcesConst.cs Normal file
View File

@ -0,0 +1,12 @@
namespace Newsbot.Collector.Domain.Consts;
public class SourceTypes
{
public const string Reddit = "reddit";
public const string Rss = "rss";
public const string YouTube = "youtube";
public const string Twitch = "twitch";
public const string FinalFantasyXiv = "ffxiv";
public const string GitHub = "github";
}

11
Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs Normal file
View File

@ -0,0 +1,11 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IArticlesRepository : ITableRepository
{
List<ArticlesModel>List(int age, int count);
ArticlesModel GetById(Guid ID);
ArticlesModel GetByUrl(string url);
ArticlesModel New(ArticlesModel model);
}

3
Newsbot.Collector.Domain/Interfaces/ICollector.cs
View File

@ -2,7 +2,8 @@ using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ICollector
///
public interface ICollector : IHangfireJob
{
List<ArticlesModel> Collect();
}

10
Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs Normal file
View File

@ -0,0 +1,10 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IDiscordQueueRepository
{
void New(DiscordQueueModel model);
void Delete(Guid id);
List<DiscordQueueModel> List(int limit);
}

8
Newsbot.Collector.Domain/Interfaces/IHangfireJob.cs Normal file
View File

@ -0,0 +1,8 @@
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IHangfireJob
{
void InitAndExecute(IConfiguration config);
}

18
Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs Normal file
View File

@ -0,0 +1,18 @@
using System.Globalization;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ISourcesRepository
{
public SourceModel New(SourceModel model);
public SourceModel GetByID(Guid ID);
public SourceModel GetByID(string ID);
public SourceModel GetByName(string name);
public SourceModel GetByNameAndSource(string name, string source);
public List<SourceModel> List(int limit);
public List<SourceModel> ListBySource(string source, int limit);
public List<SourceModel> ListByType(string type, int limit = 25);
public int Disable(Guid ID);
public int Enable(Guid ID);
}

8
Newsbot.Collector.Domain/Interfaces/ITableRepository.cs Normal file
View File

@ -0,0 +1,8 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ITableRepository
{
}

8
Newsbot.Collector.Domain/Models/DatabaseModel.cs
View File

@ -7,10 +7,10 @@ public class ArticlesModel
public string Tags { get; set; } = "";
public string Title { get; set; } = "";
public string URL { get; set; } = "";
public DateTime PubDate { get; set; }
public DateTime PubDate { get; set; } = DateTime.Now;
public string Video { get; set; } = "";
public int VideoHeight { get; set; }
public int VideoWidth { get; set; }
public int VideoHeight { get; set; } = 0;
public int VideoWidth { get; set; } = 0;
public string Thumbnail { get; set; } = "";
public string Description { get; set; } = "";
public string AuthorName { get; set; } = "";
@ -60,6 +60,8 @@ public class SourceModel
public Guid ID { get; set; }
public string Site { get; set; } = "";
public string Name { get; set; } = "";
// Source use to deinfe the worker to query with but moving to Type as it was not used really.
public string Source { get; set; } = "";
public string Type { get; set; } = "";
public string Value { get; set; } = "";

4
Newsbot.Collector.Domain/Newsbot.Collector.Domain.csproj
View File

@ -6,4 +6,8 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
</ItemGroup>
</Project>

25
Newsbot.Collector.Services/Jobs/HelloWorldJob.cs
View File

@ -1,23 +1,30 @@
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Services.Jobs;
public class HelloWorldJobOptions
{
public string Message { get; set; } = "";
}
public class HelloWorldJob
{
public string _message { get; set; }
public HelloWorldJob(string message)
private HelloWorldJobOptions _options;
public HelloWorldJob(HelloWorldJobOptions options)
{
_message = message;
_options = options;
}
public void SetMessage(string message)
public void InitAndExecute(HelloWorldJobOptions options)
{
_message = message;
_options = options;
Execute();
}
public void Execute()
private void Execute()
{
Console.WriteLine(_message);
Console.WriteLine(_options.Message);
}
}

126
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -1,59 +1,151 @@
using System.Runtime.InteropServices;
using System.ServiceModel.Syndication;
using System.Xml;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJob : ICollector
public class RssWatcherJobOptions
{
public string ConnectionString { get; set; } = "";
}
// This class was made to work with Hangfire and it does not support constructors.
public class RssWatcherJob : IHangfireJob
{
private string? _url;
private IArticlesRepository _articles;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
public RssWatcherJob(string url)
public RssWatcherJob()
{
_url = url;
_articles = new ArticlesTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
}
public List<ArticlesModel> Collect()
public void InitAndExecute(RssWatcherJobOptions options)
{
Console.WriteLine("Job was triggered");
Console.WriteLine("Setting up the job");
Init(options.ConnectionString);
var articles = new List<ArticlesModel>();
Console.WriteLine("Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss);
Console.WriteLine($"Got {sources.Count()} back");
foreach (var source in sources)
{
Console.WriteLine("Starting to request feed to be processed");
var results = Collect(source.Url);
articles.AddRange(results);
}
UpdateDatabase(articles);
}
public void InitAndExecute(IConfiguration config)
{
// reach out to the db and find all the rss feeds
var connectionString = config.GetConnectionString("database");
if (connectionString is null)
{
connectionString = "";
}
Init(connectionString);
var articles = new List<ArticlesModel>();
var sources = _source.ListByType(SourceTypes.Rss);
foreach (var source in sources)
{
var results = Collect(source.Url);
articles.AddRange(results);
}
UpdateDatabase(articles);
}
public void Init(string connectionString)
{
_articles = new ArticlesTable(connectionString);
_queue = new DiscordQueueTable(connectionString);
_source = new SourcesTable(connectionString);
}
public List<ArticlesModel> Collect(string url, int sleep = 3000)
{
var CollectedPosts = new List<ArticlesModel>();
if (_url is null)
{
_url = "";
}
using var reader = XmlReader.Create(_url);
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
var posts = feed.Items.ToList();
foreach (var post in posts)
foreach (var post in feed.Items.ToList())
{
var url = post.Links[0].Uri.AbsoluteUri;
var articleUrl = post.Links[0].Uri.AbsoluteUri;
// Check if we have seen the url before
// If we have, skip and save the site bandwidth
if (IsThisUrlKnown(articleUrl) == true)
{
continue;
}
var meta = new HtmlPageReader(url);
var meta = new HtmlPageReader(articleUrl);
var article = new ArticlesModel
{
Title = post.Title.Text,
Tags = FetchTags(post),
URL = post.Links[0].Uri.ToString(),
URL = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Meta.Image,
Description = meta.Data.Header.Meta.Description,
};
CollectedPosts.Add(article);
// try to not be too greedy
Thread.Sleep(3000);
Thread.Sleep(sleep);
}
return CollectedPosts;
}
public void UpdateDatabase(List<ArticlesModel> items)
{
foreach (var item in items)
{
if (IsThisUrlKnown(item.URL) == false)
{
continue;
}
var p = _articles.New(item);
_queue.New(new DiscordQueueModel
{
ArticleID = p.ID
});
}
}
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url)
{
return true;
}
return false;
}
private string FetchTags(SyndicationItem post)
{
string result = "";

33
Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs
View File

@ -1,14 +1,41 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Tests.Jobs;
public class RssWatcherJobTest
{
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string> {
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
private string ConnectionString()
{
return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
}
[Fact]
public void CanFindItems()
public void CanFindItemsNoDb()
{
var url = "https://www.engadget.com/rss.xml";
var client = new RssWatcherJob(url);
var items = client.Collect();
var client = new RssWatcherJob();
var items = client.Collect(url);
}
[Fact]
public void CanAddItemsToDb()
{
var url = "https://www.engadget.com/rss.xml";
var client = new RssWatcherJob();
client.Init(ConnectionString());
client.Collect(url, 0);
}
}

9
Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj
View File

@ -3,12 +3,13 @@
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Nullable>disable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
<PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
@ -21,9 +22,9 @@
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Services\Newsbot.Collector.Services.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" />
<ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Services\Newsbot.Collector.Services.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" />
</ItemGroup>
</Project>

33
Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs
View File

@ -1,3 +1,4 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Models;
@ -5,11 +6,23 @@ namespace Newsbot.Collector.Tests.Tables;
public class ArticlesTableTests
{
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string> {
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
[Fact]
public void ArticlesListTest()
{
var client = new ArticlesTable("");
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
client.List();
}
@ -18,7 +31,8 @@ public class ArticlesTableTests
{
var uid = Guid.Parse("4ac46772-253c-4c3d-8a2c-29239abd2ad4");
var client = new ArticlesTable("");
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var res = client.GetById(uid);
if (!res.ID.Equals(uid))
{
@ -29,12 +43,17 @@ public class ArticlesTableTests
[Fact]
public void NewRecordTest()
{
var client = new ArticlesTable("");
client.New(new ArticlesModel
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var m = new ArticlesModel
{
Title = "Unit Testing!",
ID = Guid.NewGuid(),
SourceID = Guid.NewGuid(),
PubDate = DateTime.Now
});
Tags = "thing, thing2",
Title = "Unit Testing!",
URL = "https://google.com",
PubDate = DateTime.Now.ToLocalTime(),
};
client.New(m);
}
}

15
Newsbot.Collector.Tests/Tables/SettingsTableTests.cs
View File

@ -1,3 +1,4 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Models;
@ -5,10 +6,22 @@ namespace Newsbot.Collector.Tests.Tables;
public class SettingsTableTests
{
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string> {
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
[Fact]
public void New()
{
var client = new SettingsTable("");
var cfg = GetConfiguration();
var client = new SettingsTable(cfg);
client.New(new SettingModel
{
Key = "Unit Testing",

27
Newsbot.Collector.Tests/Tables/SourcesTableTests.cs Normal file
View File

@ -0,0 +1,27 @@
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Models;
public class SourcesTableTests
{
[Fact]
public void NewRecordTest()
{
var client = new SourcesTable("");
var m = new SourceModel
{
ID = Guid.NewGuid(),
Site = "Testing",
Name = "Testing",
Source = "Testing",
Type = "Testing",
Value = "Testing",
Enabled = false,
Url = "Testing",
Tags = "Unit, Testing",
Deleted = false
};
client.New(m);
}
}