jtom38
/
Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

refs updated for articlesEntity

This commit is contained in:
James Tombleson 2023-06-23 20:09:57 -07:00
parent 4f46247d0c
commit 3e3da9027f
7 changed files with 61 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs
View File

@ -1,10 +1,11 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto;
public class ArticleDetailsDto
{
public Guid ID { get; set; }
public Guid Id { get; set; }
public string[]? Tags { get; set; }
public string? Title { get; set; }
public string? Url { get; set; }
@ -19,14 +20,14 @@ public class ArticleDetailsDto
public SourceDto? Source { get; set; }
public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source)
public static ArticleDetailsDto Convert(ArticlesEntity article, SourceModel source)
{
return new ArticleDetailsDto
{
ID = article.ID,
Id = article.Id,
Tags = article.Tags.Split(','),
Title = article.Title,
Url = article.URL,
Url = article.Url,
PubDate = article.PubDate,
Video = article.Video,
VideoHeight = article.VideoHeight,

25
Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs
View File

@ -3,6 +3,7 @@ using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
@ -63,7 +64,7 @@ public class CodeProjectWatcherJob
var sources = _source.ListByType(SourceTypes.CodeProject);
// query sources for things to pull
var items = new List<ArticlesModel>();
var items = new List<ArticlesEntity>();
foreach (var source in sources)
{
@ -76,12 +77,12 @@ public class CodeProjectWatcherJob
_articles.New(item);
_queue.New(new DiscordQueueModel
{
ArticleID = item.ID
ArticleID = item.Id
});
}
}
public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source)
public IEnumerable<ArticlesEntity> CheckForReleases(SourceModel source)
{
var url = new Uri(source.Url);
var links = new List<string>
@ -110,10 +111,10 @@ public class CodeProjectWatcherJob
url.AbsoluteUri);
}
return new List<ArticlesModel>();
return new List<ArticlesEntity>();
}
public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source)
public IEnumerable<ArticlesEntity> CheckForCommits(SourceModel source)
{
var url = new Uri(source.Url);
var links = new List<string>
@ -136,19 +137,19 @@ public class CodeProjectWatcherJob
url.AbsoluteUri);
}
return new List<ArticlesModel>();
return new List<ArticlesEntity>();
}
private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
private IEnumerable<ArticlesEntity> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
bool isRelease, bool isCommit)
{
var items = new List<ArticlesModel>();
var items = new List<ArticlesEntity>();
foreach (var item in feed)
{
var itemUrl = item.Links[0].Uri.AbsoluteUri;
var exits = _articles.GetByUrl(itemUrl);
if (exits.ID != Guid.Empty) continue;
if (exits.Id != Guid.Empty) continue;
var parser = new HtmlPageReader(new HtmlPageReaderOptions
{
@ -156,12 +157,12 @@ public class CodeProjectWatcherJob
});
parser.Parse();
var a = new ArticlesModel
var a = new ArticlesEntity
{
SourceID = source.ID,
SourceId = source.ID,
Tags = source.Tags,
Title = item.Title.Text,
URL = itemUrl,
Url = itemUrl,
PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text,

15
Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs
View File

@ -1,4 +1,5 @@
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Notifications.Discord;
@ -90,11 +91,11 @@ public class DiscordNotificationJob
var articleDetails = _article.GetById(request.ArticleID);
// Get the details of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID);
var sourceDetails = _sources.GetByID(articleDetails.SourceId);
if (sourceDetails.ID == Guid.Empty)
{
_logger.Error(
$"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
$"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.ID);
return;
}
@ -120,7 +121,7 @@ public class DiscordNotificationJob
_queue.Delete(request.ID);
}
public void SendSubscriptionNotification(Guid requestId, ArticlesModel articleDetails, SourceModel sourceDetails,
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceModel sourceDetails,
IconModel sourceIcon, SubscriptionModel sub)
{
// Check if the subscription code flags
@ -143,7 +144,7 @@ public class DiscordNotificationJob
{
_logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message);
_logger.Debug("Queue Record: {RequestId}", requestId);
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.ID);
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id);
_logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID);
_logger.Debug("Subscription: {SubId}", sub.Id);
}
@ -151,7 +152,7 @@ public class DiscordNotificationJob
Thread.Sleep(3000);
}
public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesModel article, IconModel icon)
public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesEntity article, IconModel icon)
{
var embed = new DiscordMessageEmbed
{
@ -172,13 +173,13 @@ public class DiscordNotificationJob
new()
{
Name = "Link",
Value = article.URL,
Value = article.Url,
Inline = false
}
}
};
if (article.URL is not null && article.URL != "") embed.Url = article.URL;
if (article.Url is not null && article.Url != "") embed.Url = article.Url;
if (article.Thumbnail != "")
embed.Image = new DiscordMessageEmbedImage

27
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
@ -56,15 +57,15 @@ public class RssWatcherJob
public void Execute()
{
var articles = new List<ArticlesModel>();
var articles = new List<ArticlesEntity>();
_logger.Information($"{JobName} - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss);
_logger.Information($"{JobName} - Got {sources.Count} back");
_logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count);
foreach (var source in sources)
{
_logger.Information($"{JobName} - Starting to process '{source.Name}'");
_logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name);
_logger.Information($"{JobName} - Starting to request feed to be processed");
var results = Collect(source.Url, source.ID);
@ -78,9 +79,9 @@ public class RssWatcherJob
_logger.Information($"{JobName} - Done!");
}
public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
public List<ArticlesEntity> Collect(string url, Guid sourceId, int sleep = 3000)
{
var collectedPosts = new List<ArticlesModel>();
var collectedPosts = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
@ -99,15 +100,15 @@ public class RssWatcherJob
});
meta.Parse();
var article = new ArticlesModel
var article = new ArticlesEntity
{
Title = post.Title.Text,
Tags = FetchTags(post),
URL = articleUrl,
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description,
SourceID = sourceId
SourceId = sourceId
};
collectedPosts.Add(article);
@ -119,20 +120,20 @@ public class RssWatcherJob
return collectedPosts;
}
public void UpdateDatabase(List<ArticlesModel> items)
public void UpdateDatabase(List<ArticlesEntity> items)
{
foreach (var item in items)
{
if (item.URL is null)
if (item.Url is null)
{
Log.Warning("RSS Watcher collected a blank url and was skipped.");
Log.Warning("RSS Watcher collected a blank url and was skipped");
continue;
}
var p = _articles.New(item);
_queue.New(new DiscordQueueModel
{
ArticleID = p.ID
ArticleID = p.Id
});
}
}
@ -140,7 +141,7 @@ public class RssWatcherJob
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) return true;
if (isKnown.Url == url) return true;
return false;
}

15
Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs
View File

@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.HtmlParser;
@ -84,7 +85,7 @@ public class YoutubeWatcherJob
_articles.New(video);
_queue.New(new DiscordQueueModel
{
ArticleID = video.ID
ArticleID = video.Id
});
}
}
@ -109,9 +110,9 @@ public class YoutubeWatcherJob
return id;
}
private List<ArticlesModel> CheckFeed(string url, SourceModel source)
private List<ArticlesEntity> CheckFeed(string url, SourceModel source)
{
var videos = new List<ArticlesModel>();
var videos = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
@ -126,17 +127,17 @@ public class YoutubeWatcherJob
});
videoDetails.Parse();
var article = new ArticlesModel
var article = new ArticlesEntity
{
//Todo add the icon
AuthorName = post.Authors[0].Name,
Title = post.Title.Text,
Tags = FetchTags(post),
URL = articleUrl,
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description,
SourceID = source.ID,
SourceId = source.ID,
Video = "true"
};
@ -151,7 +152,7 @@ public class YoutubeWatcherJob
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) return true;
if (isKnown.Url == url) return true;
return false;
}

9
Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs
View File

@ -1,3 +1,4 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs;
using Newsbot.Collector.Services.Notifications.Discord;
@ -24,11 +25,11 @@ public class DiscordNotificationJobTest
Url = "https://github.com",
Tags = "Unit, Testing"
},
new ArticlesModel
new ArticlesEntity
{
Tags = "more,unit,testing",
Title = "Nope not real",
URL = "https://github.com/jtom38",
Url = "https://github.com/jtom38",
PubDate = DateTime.Now,
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
Description = "Please work",
@ -50,11 +51,11 @@ public class DiscordNotificationJobTest
{
client.SendSubscriptionNotification(
new Guid(),
new ArticlesModel
new ArticlesEntity
{
Tags = "more,unit,testing",
Title = "Nope not real",
URL = "https://github.com/jtom38",
Url = "https://github.com/jtom38",
PubDate = DateTime.Now,
Thumbnail =
"https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",

15
Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs
View File

@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Tests.Tables;
@ -23,7 +24,7 @@ public class ArticlesTableTests
{
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
client.List();
client.List(0, 25);
}
[Fact]
@ -34,7 +35,7 @@ public class ArticlesTableTests
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var res = client.GetById(uid);
if (!res.ID.Equals(uid))
if (!res.Id.Equals(uid))
{
Assert.Fail("Incorrect record or not found");
}
@ -45,14 +46,14 @@ public class ArticlesTableTests
{
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var m = new ArticlesModel
var m = new ArticlesEntity
{
ID = Guid.NewGuid(),
SourceID = Guid.NewGuid(),
Id = Guid.NewGuid(),
SourceId = Guid.NewGuid(),
Tags = "thing, thing2",
Title = "Unit Testing!",
URL = "https://google.com",
PubDate = DateTime.Now.ToLocalTime(),
Url = "https://google.com",
PubDate = DateTime.Now.ToUniversalTime(),
};
client.New(m);
}