jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

features/cutover-to-ef #9

Merged
jtom38 merged 27 commits from features/cutover-to-ef into main 2023-06-25 21:15:58 -07:00
Conversation 0 Commits 27 Files Changed 64 +61 -54
7 changed files with 61 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 3e3da9027f - Show all commits

9
Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs
View File

@ -1,10 +1,11 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto; namespace Newsbot.Collector.Domain.Dto;
public class ArticleDetailsDto public class ArticleDetailsDto
{ {
public Guid ID { get; set; } public Guid Id { get; set; }
public string[]? Tags { get; set; } public string[]? Tags { get; set; }
public string? Title { get; set; } public string? Title { get; set; }
public string? Url { get; set; } public string? Url { get; set; }
@ -19,14 +20,14 @@ public class ArticleDetailsDto
public SourceDto? Source { get; set; } public SourceDto? Source { get; set; }
public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source) public static ArticleDetailsDto Convert(ArticlesEntity article, SourceModel source)
{ {
return new ArticleDetailsDto return new ArticleDetailsDto
{ {
ID = article.ID, Id = article.Id,
Tags = article.Tags.Split(','), Tags = article.Tags.Split(','),
Title = article.Title, Title = article.Title,
Url = article.URL, Url = article.Url,
PubDate = article.PubDate, PubDate = article.PubDate,
Video = article.Video, Video = article.Video,
VideoHeight = article.VideoHeight, VideoHeight = article.VideoHeight,

25
Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs
View File

@ -3,6 +3,7 @@ using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Domain.Models.Config;
@ -63,7 +64,7 @@ public class CodeProjectWatcherJob
var sources = _source.ListByType(SourceTypes.CodeProject); var sources = _source.ListByType(SourceTypes.CodeProject);
// query sources for things to pull // query sources for things to pull
var items = new List<ArticlesModel>(); var items = new List<ArticlesEntity>();
foreach (var source in sources) foreach (var source in sources)
{ {
@ -76,12 +77,12 @@ public class CodeProjectWatcherJob
_articles.New(item); _articles.New(item);
_queue.New(new DiscordQueueModel _queue.New(new DiscordQueueModel
{ {
ArticleID = item.ID ArticleID = item.Id
}); });
} }
} }
public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source) public IEnumerable<ArticlesEntity> CheckForReleases(SourceModel source)
{ {
var url = new Uri(source.Url); var url = new Uri(source.Url);
var links = new List<string> var links = new List<string>
@ -110,10 +111,10 @@ public class CodeProjectWatcherJob
url.AbsoluteUri); url.AbsoluteUri);
} }
return new List<ArticlesModel>(); return new List<ArticlesEntity>();
} }
public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source) public IEnumerable<ArticlesEntity> CheckForCommits(SourceModel source)
{ {
var url = new Uri(source.Url); var url = new Uri(source.Url);
var links = new List<string> var links = new List<string>
@ -136,19 +137,19 @@ public class CodeProjectWatcherJob
url.AbsoluteUri); url.AbsoluteUri);
} }
return new List<ArticlesModel>(); return new List<ArticlesEntity>();
} }
private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source, private IEnumerable<ArticlesEntity> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
bool isRelease, bool isCommit) bool isRelease, bool isCommit)
{ {
var items = new List<ArticlesModel>(); var items = new List<ArticlesEntity>();
foreach (var item in feed) foreach (var item in feed)
{ {
var itemUrl = item.Links[0].Uri.AbsoluteUri; var itemUrl = item.Links[0].Uri.AbsoluteUri;
var exits = _articles.GetByUrl(itemUrl); var exits = _articles.GetByUrl(itemUrl);
if (exits.ID != Guid.Empty) continue; if (exits.Id != Guid.Empty) continue;
var parser = new HtmlPageReader(new HtmlPageReaderOptions var parser = new HtmlPageReader(new HtmlPageReaderOptions
{ {
@ -156,12 +157,12 @@ public class CodeProjectWatcherJob
}); });
parser.Parse(); parser.Parse();
var a = new ArticlesModel var a = new ArticlesEntity
{ {
SourceID = source.ID, SourceId = source.ID,
Tags = source.Tags, Tags = source.Tags,
Title = item.Title.Text, Title = item.Title.Text,
URL = itemUrl, Url = itemUrl,
PubDate = item.LastUpdatedTime.DateTime, PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image, Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text, Description = item.Title.Text,

15
Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs
View File

@ -1,4 +1,5 @@
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
@ -90,11 +91,11 @@ public class DiscordNotificationJob
var articleDetails = _article.GetById(request.ArticleID); var articleDetails = _article.GetById(request.ArticleID);
// Get the details of the source // Get the details of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID); var sourceDetails = _sources.GetByID(articleDetails.SourceId);
if (sourceDetails.ID == Guid.Empty) if (sourceDetails.ID == Guid.Empty)
{ {
_logger.Error( _logger.Error(
$"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue."); $"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.ID); _queue.Delete(request.ID);
return; return;
} }
@ -120,7 +121,7 @@ public class DiscordNotificationJob
_queue.Delete(request.ID); _queue.Delete(request.ID);
} }
public void SendSubscriptionNotification(Guid requestId, ArticlesModel articleDetails, SourceModel sourceDetails, public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceModel sourceDetails,
IconModel sourceIcon, SubscriptionModel sub) IconModel sourceIcon, SubscriptionModel sub)
{ {
// Check if the subscription code flags // Check if the subscription code flags
@ -143,7 +144,7 @@ public class DiscordNotificationJob
{ {
_logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message); _logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message);
_logger.Debug("Queue Record: {RequestId}", requestId); _logger.Debug("Queue Record: {RequestId}", requestId);
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.ID); _logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id);
_logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID); _logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID);
_logger.Debug("Subscription: {SubId}", sub.Id); _logger.Debug("Subscription: {SubId}", sub.Id);
} }
@ -151,7 +152,7 @@ public class DiscordNotificationJob
Thread.Sleep(3000); Thread.Sleep(3000);
} }
public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesModel article, IconModel icon) public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesEntity article, IconModel icon)
{ {
var embed = new DiscordMessageEmbed var embed = new DiscordMessageEmbed
{ {
@ -172,13 +173,13 @@ public class DiscordNotificationJob
new() new()
{ {
Name = "Link", Name = "Link",
Value = article.URL, Value = article.Url,
Inline = false Inline = false
} }
} }
}; };
if (article.URL is not null && article.URL != "") embed.Url = article.URL; if (article.Url is not null && article.Url != "") embed.Url = article.Url;
if (article.Thumbnail != "") if (article.Thumbnail != "")
embed.Image = new DiscordMessageEmbedImage embed.Image = new DiscordMessageEmbedImage

27
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Domain.Models.Config;
@ -56,15 +57,15 @@ public class RssWatcherJob
public void Execute() public void Execute()
{ {
var articles = new List<ArticlesModel>(); var articles = new List<ArticlesEntity>();
_logger.Information($"{JobName} - Requesting sources"); _logger.Information($"{JobName} - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss); var sources = _source.ListByType(SourceTypes.Rss);
_logger.Information($"{JobName} - Got {sources.Count} back"); _logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count);
foreach (var source in sources) foreach (var source in sources)
{ {
_logger.Information($"{JobName} - Starting to process '{source.Name}'"); _logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name);
_logger.Information($"{JobName} - Starting to request feed to be processed"); _logger.Information($"{JobName} - Starting to request feed to be processed");
var results = Collect(source.Url, source.ID); var results = Collect(source.Url, source.ID);
@ -78,9 +79,9 @@ public class RssWatcherJob
_logger.Information($"{JobName} - Done!"); _logger.Information($"{JobName} - Done!");
} }
public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000) public List<ArticlesEntity> Collect(string url, Guid sourceId, int sleep = 3000)
{ {
var collectedPosts = new List<ArticlesModel>(); var collectedPosts = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url); using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
@ -99,15 +100,15 @@ public class RssWatcherJob
}); });
meta.Parse(); meta.Parse();
var article = new ArticlesModel var article = new ArticlesEntity
{ {
Title = post.Title.Text, Title = post.Title.Text,
Tags = FetchTags(post), Tags = FetchTags(post),
URL = articleUrl, Url = articleUrl,
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image, Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description, Description = meta.Data.Header.Description,
SourceID = sourceId SourceId = sourceId
}; };
collectedPosts.Add(article); collectedPosts.Add(article);
@ -119,20 +120,20 @@ public class RssWatcherJob
return collectedPosts; return collectedPosts;
} }
public void UpdateDatabase(List<ArticlesModel> items) public void UpdateDatabase(List<ArticlesEntity> items)
{ {
foreach (var item in items) foreach (var item in items)
{ {
if (item.URL is null) if (item.Url is null)
{ {
Log.Warning("RSS Watcher collected a blank url and was skipped."); Log.Warning("RSS Watcher collected a blank url and was skipped");
continue; continue;
} }
var p = _articles.New(item); var p = _articles.New(item);
_queue.New(new DiscordQueueModel _queue.New(new DiscordQueueModel
{ {
ArticleID = p.ID ArticleID = p.Id
}); });
} }
} }
@ -140,7 +141,7 @@ public class RssWatcherJob
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)
{ {
var isKnown = _articles.GetByUrl(url); var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) return true; if (isKnown.Url == url) return true;
return false; return false;
} }

15
Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs
View File

@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.HtmlParser; using Newsbot.Collector.Services.HtmlParser;
@ -84,7 +85,7 @@ public class YoutubeWatcherJob
_articles.New(video); _articles.New(video);
_queue.New(new DiscordQueueModel _queue.New(new DiscordQueueModel
{ {
ArticleID = video.ID ArticleID = video.Id
}); });
} }
} }
@ -109,9 +110,9 @@ public class YoutubeWatcherJob
return id; return id;
} }
private List<ArticlesModel> CheckFeed(string url, SourceModel source) private List<ArticlesEntity> CheckFeed(string url, SourceModel source)
{ {
var videos = new List<ArticlesModel>(); var videos = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url); using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
@ -126,17 +127,17 @@ public class YoutubeWatcherJob
}); });
videoDetails.Parse(); videoDetails.Parse();
var article = new ArticlesModel var article = new ArticlesEntity
{ {
//Todo add the icon //Todo add the icon
AuthorName = post.Authors[0].Name, AuthorName = post.Authors[0].Name,
Title = post.Title.Text, Title = post.Title.Text,
Tags = FetchTags(post), Tags = FetchTags(post),
URL = articleUrl, Url = articleUrl,
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image, Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description, Description = videoDetails.Data.Header.Description,
SourceID = source.ID, SourceId = source.ID,
Video = "true" Video = "true"
}; };
@ -151,7 +152,7 @@ public class YoutubeWatcherJob
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)
{ {
var isKnown = _articles.GetByUrl(url); var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) return true; if (isKnown.Url == url) return true;
return false; return false;
} }

9
Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs
View File

@ -1,3 +1,4 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Jobs;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
@ -24,11 +25,11 @@ public class DiscordNotificationJobTest
Url = "https://github.com", Url = "https://github.com",
Tags = "Unit, Testing" Tags = "Unit, Testing"
}, },
new ArticlesModel new ArticlesEntity
{ {
Tags = "more,unit,testing", Tags = "more,unit,testing",
Title = "Nope not real", Title = "Nope not real",
URL = "https://github.com/jtom38", Url = "https://github.com/jtom38",
PubDate = DateTime.Now, PubDate = DateTime.Now,
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
Description = "Please work", Description = "Please work",
@ -50,11 +51,11 @@ public class DiscordNotificationJobTest
{ {
client.SendSubscriptionNotification( client.SendSubscriptionNotification(
new Guid(), new Guid(),
new ArticlesModel new ArticlesEntity
{ {
Tags = "more,unit,testing", Tags = "more,unit,testing",
Title = "Nope not real", Title = "Nope not real",
URL = "https://github.com/jtom38", Url = "https://github.com/jtom38",
PubDate = DateTime.Now, PubDate = DateTime.Now,
Thumbnail = Thumbnail =
"https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",

15
Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs
View File

@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Tests.Tables; namespace Newsbot.Collector.Tests.Tables;
@ -23,7 +24,7 @@ public class ArticlesTableTests
{ {
var cfg = GetConfiguration(); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg); var client = new ArticlesTable(cfg);
client.List(); client.List(0, 25);
} }
[Fact] [Fact]
@ -34,7 +35,7 @@ public class ArticlesTableTests
var cfg = GetConfiguration(); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg); var client = new ArticlesTable(cfg);
var res = client.GetById(uid); var res = client.GetById(uid);
if (!res.ID.Equals(uid)) if (!res.Id.Equals(uid))
{ {
Assert.Fail("Incorrect record or not found"); Assert.Fail("Incorrect record or not found");
} }
@ -45,14 +46,14 @@ public class ArticlesTableTests
{ {
var cfg = GetConfiguration(); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg); var client = new ArticlesTable(cfg);
var m = new ArticlesModel var m = new ArticlesEntity
{ {
ID = Guid.NewGuid(), Id = Guid.NewGuid(),
SourceID = Guid.NewGuid(), SourceId = Guid.NewGuid(),
Tags = "thing, thing2", Tags = "thing, thing2",
Title = "Unit Testing!", Title = "Unit Testing!",
URL = "https://google.com", Url = "https://google.com",
PubDate = DateTime.Now.ToLocalTime(), PubDate = DateTime.Now.ToUniversalTime(),
}; };
client.New(m); client.New(m);
} }