refs updated for articlesEntity
This commit is contained in:
parent
4f46247d0c
commit
3e3da9027f
@ -1,10 +1,11 @@
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class ArticleDetailsDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public Guid Id { get; set; }
|
||||
public string[]? Tags { get; set; }
|
||||
public string? Title { get; set; }
|
||||
public string? Url { get; set; }
|
||||
@ -19,14 +20,14 @@ public class ArticleDetailsDto
|
||||
|
||||
public SourceDto? Source { get; set; }
|
||||
|
||||
public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source)
|
||||
public static ArticleDetailsDto Convert(ArticlesEntity article, SourceModel source)
|
||||
{
|
||||
return new ArticleDetailsDto
|
||||
{
|
||||
ID = article.ID,
|
||||
Id = article.Id,
|
||||
Tags = article.Tags.Split(','),
|
||||
Title = article.Title,
|
||||
Url = article.URL,
|
||||
Url = article.Url,
|
||||
PubDate = article.PubDate,
|
||||
Video = article.Video,
|
||||
VideoHeight = article.VideoHeight,
|
||||
|
@ -3,6 +3,7 @@ using System.ServiceModel.Syndication;
|
||||
using System.Xml;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Consts;
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Domain.Models.Config;
|
||||
@ -63,7 +64,7 @@ public class CodeProjectWatcherJob
|
||||
var sources = _source.ListByType(SourceTypes.CodeProject);
|
||||
|
||||
// query sources for things to pull
|
||||
var items = new List<ArticlesModel>();
|
||||
var items = new List<ArticlesEntity>();
|
||||
|
||||
foreach (var source in sources)
|
||||
{
|
||||
@ -76,12 +77,12 @@ public class CodeProjectWatcherJob
|
||||
_articles.New(item);
|
||||
_queue.New(new DiscordQueueModel
|
||||
{
|
||||
ArticleID = item.ID
|
||||
ArticleID = item.Id
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source)
|
||||
public IEnumerable<ArticlesEntity> CheckForReleases(SourceModel source)
|
||||
{
|
||||
var url = new Uri(source.Url);
|
||||
var links = new List<string>
|
||||
@ -110,10 +111,10 @@ public class CodeProjectWatcherJob
|
||||
url.AbsoluteUri);
|
||||
}
|
||||
|
||||
return new List<ArticlesModel>();
|
||||
return new List<ArticlesEntity>();
|
||||
}
|
||||
|
||||
public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source)
|
||||
public IEnumerable<ArticlesEntity> CheckForCommits(SourceModel source)
|
||||
{
|
||||
var url = new Uri(source.Url);
|
||||
var links = new List<string>
|
||||
@ -136,19 +137,19 @@ public class CodeProjectWatcherJob
|
||||
url.AbsoluteUri);
|
||||
}
|
||||
|
||||
return new List<ArticlesModel>();
|
||||
return new List<ArticlesEntity>();
|
||||
}
|
||||
|
||||
private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
|
||||
private IEnumerable<ArticlesEntity> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
|
||||
bool isRelease, bool isCommit)
|
||||
{
|
||||
var items = new List<ArticlesModel>();
|
||||
var items = new List<ArticlesEntity>();
|
||||
|
||||
foreach (var item in feed)
|
||||
{
|
||||
var itemUrl = item.Links[0].Uri.AbsoluteUri;
|
||||
var exits = _articles.GetByUrl(itemUrl);
|
||||
if (exits.ID != Guid.Empty) continue;
|
||||
if (exits.Id != Guid.Empty) continue;
|
||||
|
||||
var parser = new HtmlPageReader(new HtmlPageReaderOptions
|
||||
{
|
||||
@ -156,12 +157,12 @@ public class CodeProjectWatcherJob
|
||||
});
|
||||
parser.Parse();
|
||||
|
||||
var a = new ArticlesModel
|
||||
var a = new ArticlesEntity
|
||||
{
|
||||
SourceID = source.ID,
|
||||
SourceId = source.ID,
|
||||
Tags = source.Tags,
|
||||
Title = item.Title.Text,
|
||||
URL = itemUrl,
|
||||
Url = itemUrl,
|
||||
PubDate = item.LastUpdatedTime.DateTime,
|
||||
Thumbnail = parser.Data.Header.Image,
|
||||
Description = item.Title.Text,
|
||||
|
@ -1,4 +1,5 @@
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Services.Notifications.Discord;
|
||||
@ -90,11 +91,11 @@ public class DiscordNotificationJob
|
||||
var articleDetails = _article.GetById(request.ArticleID);
|
||||
|
||||
// Get the details of the source
|
||||
var sourceDetails = _sources.GetByID(articleDetails.SourceID);
|
||||
var sourceDetails = _sources.GetByID(articleDetails.SourceId);
|
||||
if (sourceDetails.ID == Guid.Empty)
|
||||
{
|
||||
_logger.Error(
|
||||
$"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
|
||||
$"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue.");
|
||||
_queue.Delete(request.ID);
|
||||
return;
|
||||
}
|
||||
@ -120,7 +121,7 @@ public class DiscordNotificationJob
|
||||
_queue.Delete(request.ID);
|
||||
}
|
||||
|
||||
public void SendSubscriptionNotification(Guid requestId, ArticlesModel articleDetails, SourceModel sourceDetails,
|
||||
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceModel sourceDetails,
|
||||
IconModel sourceIcon, SubscriptionModel sub)
|
||||
{
|
||||
// Check if the subscription code flags
|
||||
@ -143,7 +144,7 @@ public class DiscordNotificationJob
|
||||
{
|
||||
_logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message);
|
||||
_logger.Debug("Queue Record: {RequestId}", requestId);
|
||||
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.ID);
|
||||
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id);
|
||||
_logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID);
|
||||
_logger.Debug("Subscription: {SubId}", sub.Id);
|
||||
}
|
||||
@ -151,7 +152,7 @@ public class DiscordNotificationJob
|
||||
Thread.Sleep(3000);
|
||||
}
|
||||
|
||||
public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesModel article, IconModel icon)
|
||||
public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesEntity article, IconModel icon)
|
||||
{
|
||||
var embed = new DiscordMessageEmbed
|
||||
{
|
||||
@ -172,13 +173,13 @@ public class DiscordNotificationJob
|
||||
new()
|
||||
{
|
||||
Name = "Link",
|
||||
Value = article.URL,
|
||||
Value = article.Url,
|
||||
Inline = false
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (article.URL is not null && article.URL != "") embed.Url = article.URL;
|
||||
if (article.Url is not null && article.Url != "") embed.Url = article.Url;
|
||||
|
||||
if (article.Thumbnail != "")
|
||||
embed.Image = new DiscordMessageEmbedImage
|
||||
|
@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
|
||||
using System.Xml;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Consts;
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Domain.Models.Config;
|
||||
@ -56,15 +57,15 @@ public class RssWatcherJob
|
||||
|
||||
public void Execute()
|
||||
{
|
||||
var articles = new List<ArticlesModel>();
|
||||
var articles = new List<ArticlesEntity>();
|
||||
|
||||
_logger.Information($"{JobName} - Requesting sources");
|
||||
var sources = _source.ListByType(SourceTypes.Rss);
|
||||
_logger.Information($"{JobName} - Got {sources.Count} back");
|
||||
_logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count);
|
||||
|
||||
foreach (var source in sources)
|
||||
{
|
||||
_logger.Information($"{JobName} - Starting to process '{source.Name}'");
|
||||
_logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name);
|
||||
_logger.Information($"{JobName} - Starting to request feed to be processed");
|
||||
var results = Collect(source.Url, source.ID);
|
||||
|
||||
@ -78,9 +79,9 @@ public class RssWatcherJob
|
||||
_logger.Information($"{JobName} - Done!");
|
||||
}
|
||||
|
||||
public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
|
||||
public List<ArticlesEntity> Collect(string url, Guid sourceId, int sleep = 3000)
|
||||
{
|
||||
var collectedPosts = new List<ArticlesModel>();
|
||||
var collectedPosts = new List<ArticlesEntity>();
|
||||
|
||||
using var reader = XmlReader.Create(url);
|
||||
var feed = SyndicationFeed.Load(reader);
|
||||
@ -99,15 +100,15 @@ public class RssWatcherJob
|
||||
});
|
||||
meta.Parse();
|
||||
|
||||
var article = new ArticlesModel
|
||||
var article = new ArticlesEntity
|
||||
{
|
||||
Title = post.Title.Text,
|
||||
Tags = FetchTags(post),
|
||||
URL = articleUrl,
|
||||
Url = articleUrl,
|
||||
PubDate = post.PublishDate.DateTime,
|
||||
Thumbnail = meta.Data.Header.Image,
|
||||
Description = meta.Data.Header.Description,
|
||||
SourceID = sourceId
|
||||
SourceId = sourceId
|
||||
};
|
||||
|
||||
collectedPosts.Add(article);
|
||||
@ -119,20 +120,20 @@ public class RssWatcherJob
|
||||
return collectedPosts;
|
||||
}
|
||||
|
||||
public void UpdateDatabase(List<ArticlesModel> items)
|
||||
public void UpdateDatabase(List<ArticlesEntity> items)
|
||||
{
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (item.URL is null)
|
||||
if (item.Url is null)
|
||||
{
|
||||
Log.Warning("RSS Watcher collected a blank url and was skipped.");
|
||||
Log.Warning("RSS Watcher collected a blank url and was skipped");
|
||||
continue;
|
||||
}
|
||||
|
||||
var p = _articles.New(item);
|
||||
_queue.New(new DiscordQueueModel
|
||||
{
|
||||
ArticleID = p.ID
|
||||
ArticleID = p.Id
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -140,7 +141,7 @@ public class RssWatcherJob
|
||||
private bool IsThisUrlKnown(string url)
|
||||
{
|
||||
var isKnown = _articles.GetByUrl(url);
|
||||
if (isKnown.URL == url) return true;
|
||||
if (isKnown.Url == url) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
|
||||
using System.Xml;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Consts;
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Services.HtmlParser;
|
||||
@ -84,7 +85,7 @@ public class YoutubeWatcherJob
|
||||
_articles.New(video);
|
||||
_queue.New(new DiscordQueueModel
|
||||
{
|
||||
ArticleID = video.ID
|
||||
ArticleID = video.Id
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -109,9 +110,9 @@ public class YoutubeWatcherJob
|
||||
return id;
|
||||
}
|
||||
|
||||
private List<ArticlesModel> CheckFeed(string url, SourceModel source)
|
||||
private List<ArticlesEntity> CheckFeed(string url, SourceModel source)
|
||||
{
|
||||
var videos = new List<ArticlesModel>();
|
||||
var videos = new List<ArticlesEntity>();
|
||||
|
||||
using var reader = XmlReader.Create(url);
|
||||
var feed = SyndicationFeed.Load(reader);
|
||||
@ -126,17 +127,17 @@ public class YoutubeWatcherJob
|
||||
});
|
||||
videoDetails.Parse();
|
||||
|
||||
var article = new ArticlesModel
|
||||
var article = new ArticlesEntity
|
||||
{
|
||||
//Todo add the icon
|
||||
AuthorName = post.Authors[0].Name,
|
||||
Title = post.Title.Text,
|
||||
Tags = FetchTags(post),
|
||||
URL = articleUrl,
|
||||
Url = articleUrl,
|
||||
PubDate = post.PublishDate.DateTime,
|
||||
Thumbnail = videoDetails.Data.Header.Image,
|
||||
Description = videoDetails.Data.Header.Description,
|
||||
SourceID = source.ID,
|
||||
SourceId = source.ID,
|
||||
Video = "true"
|
||||
};
|
||||
|
||||
@ -151,7 +152,7 @@ public class YoutubeWatcherJob
|
||||
private bool IsThisUrlKnown(string url)
|
||||
{
|
||||
var isKnown = _articles.GetByUrl(url);
|
||||
if (isKnown.URL == url) return true;
|
||||
if (isKnown.Url == url) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -1,3 +1,4 @@
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Services.Jobs;
|
||||
using Newsbot.Collector.Services.Notifications.Discord;
|
||||
@ -24,11 +25,11 @@ public class DiscordNotificationJobTest
|
||||
Url = "https://github.com",
|
||||
Tags = "Unit, Testing"
|
||||
},
|
||||
new ArticlesModel
|
||||
new ArticlesEntity
|
||||
{
|
||||
Tags = "more,unit,testing",
|
||||
Title = "Nope not real",
|
||||
URL = "https://github.com/jtom38",
|
||||
Url = "https://github.com/jtom38",
|
||||
PubDate = DateTime.Now,
|
||||
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
|
||||
Description = "Please work",
|
||||
@ -50,11 +51,11 @@ public class DiscordNotificationJobTest
|
||||
{
|
||||
client.SendSubscriptionNotification(
|
||||
new Guid(),
|
||||
new ArticlesModel
|
||||
new ArticlesEntity
|
||||
{
|
||||
Tags = "more,unit,testing",
|
||||
Title = "Nope not real",
|
||||
URL = "https://github.com/jtom38",
|
||||
Url = "https://github.com/jtom38",
|
||||
PubDate = DateTime.Now,
|
||||
Thumbnail =
|
||||
"https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
|
||||
|
@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Entities;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Tests.Tables;
|
||||
@ -23,7 +24,7 @@ public class ArticlesTableTests
|
||||
{
|
||||
var cfg = GetConfiguration();
|
||||
var client = new ArticlesTable(cfg);
|
||||
client.List();
|
||||
client.List(0, 25);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@ -34,7 +35,7 @@ public class ArticlesTableTests
|
||||
var cfg = GetConfiguration();
|
||||
var client = new ArticlesTable(cfg);
|
||||
var res = client.GetById(uid);
|
||||
if (!res.ID.Equals(uid))
|
||||
if (!res.Id.Equals(uid))
|
||||
{
|
||||
Assert.Fail("Incorrect record or not found");
|
||||
}
|
||||
@ -45,14 +46,14 @@ public class ArticlesTableTests
|
||||
{
|
||||
var cfg = GetConfiguration();
|
||||
var client = new ArticlesTable(cfg);
|
||||
var m = new ArticlesModel
|
||||
var m = new ArticlesEntity
|
||||
{
|
||||
ID = Guid.NewGuid(),
|
||||
SourceID = Guid.NewGuid(),
|
||||
Id = Guid.NewGuid(),
|
||||
SourceId = Guid.NewGuid(),
|
||||
Tags = "thing, thing2",
|
||||
Title = "Unit Testing!",
|
||||
URL = "https://google.com",
|
||||
PubDate = DateTime.Now.ToLocalTime(),
|
||||
Url = "https://google.com",
|
||||
PubDate = DateTime.Now.ToUniversalTime(),
|
||||
};
|
||||
client.New(m);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user