Collector jobs have been updated to handle authors

This commit is contained in:
James Tombleson 2023-08-04 23:04:32 -07:00
parent 56a655e4f1
commit 0d751cc571
3 changed files with 85 additions and 39 deletions

View File

@ -30,6 +30,7 @@ public class CodeProjectWatcherJob
private ILogger _logger; private ILogger _logger;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
private ISourcesRepository _source; private ISourcesRepository _source;
private IAuthorTable _author;
public CodeProjectWatcherJob() public CodeProjectWatcherJob()
{ {
@ -37,12 +38,14 @@ public class CodeProjectWatcherJob
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_source = new SourcesTable(""); _source = new SourcesTable("");
_logger = JobLogger.GetLogger("", JobName); _logger = JobLogger.GetLogger("", JobName);
_author = new AuthorsTable("");
} }
public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options) public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@ -52,6 +55,7 @@ public class CodeProjectWatcherJob
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@ -157,17 +161,29 @@ public class CodeProjectWatcherJob
}); });
parser.Parse(); parser.Parse();
if (item.Authors[0].Name is null)
{
_logger.Warning("Author was missing from the record and will continue with a missing author");
}
var authorExists = _author.CreateIfMissingAsync(new AuthorEntity
{
Name = item.Authors[0].Name,
SourceId = source.Id,
Image = "",
});
authorExists.Wait();
var a = new ArticlesEntity var a = new ArticlesEntity
{ {
SourceId = source.Id, SourceId = source.Id,
AuthorId = authorExists.Result.Id,
Tags = source.Tags, Tags = source.Tags,
Title = item.Title.Text, Title = item.Title.Text,
Url = itemUrl, Url = itemUrl,
PubDate = item.LastUpdatedTime.DateTime, PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image, Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text, Description = item.Title.Text,
AuthorName = item.Authors[0].Name ?? "",
AuthorImage = item.Authors[0].Uri ?? "",
CodeIsRelease = isRelease, CodeIsRelease = isRelease,
CodeIsCommit = isCommit, CodeIsCommit = isCommit,
}; };

View File

@ -36,6 +36,7 @@ public class DiscordNotificationJob
private ILogger _logger; private ILogger _logger;
//private DatabaseContext _databaseContext; //private DatabaseContext _databaseContext;
private IArticlesRepository _article; private IArticlesRepository _article;
private IAuthorTable _author;
private IIconsRepository _icons; private IIconsRepository _icons;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
private ISourcesRepository _sources; private ISourcesRepository _sources;
@ -46,6 +47,7 @@ public class DiscordNotificationJob
{ {
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_article = new ArticlesTable(""); _article = new ArticlesTable("");
_author = new AuthorsTable("");
_webhook = new DiscordWebhooksTable(""); _webhook = new DiscordWebhooksTable("");
_sources = new SourcesTable(""); _sources = new SourcesTable("");
_subs = new DiscordNotificationTable(""); _subs = new DiscordNotificationTable("");
@ -92,7 +94,6 @@ public class DiscordNotificationJob
_logger.Debug($"{JobName} - Processing {request.Id}"); _logger.Debug($"{JobName} - Processing {request.Id}");
// Get all details on the article in the queue // Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleId); var articleDetails = _article.GetById(request.ArticleId);
// Get the details of the source // Get the details of the source
var sourceDetails = _sources.GetById(articleDetails.SourceId); var sourceDetails = _sources.GetById(articleDetails.SourceId);
if (sourceDetails.Id == Guid.Empty) if (sourceDetails.Id == Guid.Empty)
@ -103,6 +104,9 @@ public class DiscordNotificationJob
return; return;
} }
var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name);
author.Wait();
var sourceIcon = new IconEntity(); var sourceIcon = new IconEntity();
try try
{ {
@ -118,13 +122,13 @@ public class DiscordNotificationJob
var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id); var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
foreach (var sub in allSubscriptions) foreach (var sub in allSubscriptions)
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity());
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id); _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id);
_queue.Delete(request.Id); _queue.Delete(request.Id);
} }
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub) public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity)
{ {
// Check if the subscription code flags // Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip. // If the article is a code commit and the subscription does not want them, skip.
@ -140,7 +144,7 @@ public class DiscordNotificationJob
var client = new DiscordClient(discordDetails.Url); var client = new DiscordClient(discordDetails.Url);
try try
{ {
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon)); client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity));
} }
catch (Exception e) catch (Exception e)
{ {
@ -154,7 +158,7 @@ public class DiscordNotificationJob
Thread.Sleep(3000); Thread.Sleep(3000);
} }
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon) public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author)
{ {
var embed = new DiscordMessageEmbed var embed = new DiscordMessageEmbed
{ {
@ -163,7 +167,7 @@ public class DiscordNotificationJob
Description = MessageValidation.ConvertHtmlCodes(article.Description), Description = MessageValidation.ConvertHtmlCodes(article.Description),
Author = new DiscordMessageEmbedAuthor Author = new DiscordMessageEmbedAuthor
{ {
Name = article.AuthorName, Name = author.Name,
IconUrl = icon.FileName IconUrl = icon.FileName
}, },
Footer = new DiscordMessageEmbedFooter Footer = new DiscordMessageEmbedFooter
@ -189,7 +193,7 @@ public class DiscordNotificationJob
Url = article.Thumbnail Url = article.Thumbnail
}; };
if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage; embed.Author.IconUrl = author.Image ?? "";
return new DiscordMessage return new DiscordMessage
{ {

View File

@ -25,6 +25,7 @@ public class YoutubeWatcherJob
private readonly YoutubeWatcherJobOptions _options; private readonly YoutubeWatcherJobOptions _options;
private IArticlesRepository _articles; private IArticlesRepository _articles;
private IAuthorTable _author;
private IIconsRepository _icons; private IIconsRepository _icons;
private ILogger _logger; private ILogger _logger;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
@ -34,6 +35,7 @@ public class YoutubeWatcherJob
{ {
_options = new YoutubeWatcherJobOptions(); _options = new YoutubeWatcherJobOptions();
_articles = new ArticlesTable(""); _articles = new ArticlesTable("");
_author = new AuthorsTable("");
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_source = new SourcesTable(""); _source = new SourcesTable("");
_icons = new IconsTable(""); _icons = new IconsTable("");
@ -43,6 +45,7 @@ public class YoutubeWatcherJob
public void InitAndExecute(YoutubeWatcherJobOptions options) public void InitAndExecute(YoutubeWatcherJobOptions options)
{ {
_articles = new ArticlesTable(options.DatabaseConnectionString ?? ""); _articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
_author = new AuthorsTable(options.DatabaseConnectionString ?? "");
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? ""); _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_source = new SourcesTable(options.DatabaseConnectionString ?? ""); _source = new SourcesTable(options.DatabaseConnectionString ?? "");
_icons = new IconsTable(options.DatabaseConnectionString ?? ""); _icons = new IconsTable(options.DatabaseConnectionString ?? "");
@ -77,11 +80,23 @@ public class YoutubeWatcherJob
_logger.Information($"{JobName} - Checking '{source.Name}'"); _logger.Information($"{JobName} - Checking '{source.Name}'");
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}"; var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = CheckFeed(url, source); var newVideos = FindMissingPosts(url, source);
_logger.Debug($"{JobName} - Collected {newVideos.Count} new videos"); _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
foreach (var video in newVideos) foreach (var video in newVideos)
{ {
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found"); var author = _author.GetById(video.AuthorId);
author.Wait();
if (author.Result is null)
{
_logger.Warning("Missing author record for article id {VideoId}", video.Id);
}
else
{
_logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title);
}
_articles.New(video); _articles.New(video);
_queue.New(new DiscordQueueEntity _queue.New(new DiscordQueueEntity
{ {
@ -104,13 +119,12 @@ public class YoutubeWatcherJob
var id = pageReader.Data.Header.YoutubeChannelID ?? ""; var id = pageReader.Data.Header.YoutubeChannelID ?? "";
if (id == "") if (id == "")
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."), _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), "");
url);
return id; return id;
} }
private List<ArticlesEntity> CheckFeed(string url, SourceEntity source) private List<ArticlesEntity> FindMissingPosts(string url, SourceEntity source)
{ {
var videos = new List<ArticlesEntity>(); var videos = new List<ArticlesEntity>();
@ -118,37 +132,49 @@ public class YoutubeWatcherJob
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
foreach (var post in feed.Items.ToList()) foreach (var post in feed.Items.ToList())
{ {
var articleUrl = post.Links[0].Uri.AbsoluteUri; var article = CheckFeedItem(post, source.Id);
if (IsThisUrlKnown(articleUrl)) continue; if (article is null) continue;
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
{
Url = articleUrl
});
videoDetails.Parse();
var article = new ArticlesEntity
{
//Todo add the icon
AuthorName = post.Authors[0].Name,
Title = post.Title.Text,
Tags = FetchTags(post),
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description,
SourceId = source.Id,
Video = "true"
};
videos.Add(article); videos.Add(article);
Thread.Sleep(_options.SleepTimer);
} }
return videos; return videos;
} }
private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId)
{
var articleUrl = post.Links[0].Uri.AbsoluteUri;
if (IsThisUrlKnown(articleUrl)) return null;
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
{
Url = articleUrl
});
videoDetails.Parse();
var author = _author.CreateIfMissingAsync(new AuthorEntity
{
Image = post.Authors[0].Uri,
Name = post.Authors[0].Name
});
author.Wait();
var article = new ArticlesEntity
{
//Todo add the icon
AuthorId = author.Result.Id,
Title = post.Title.Text,
Tags = FetchTags(post),
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description,
SourceId = sourceId,
Video = "true"
};
return article;
}
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)
{ {
var isKnown = _articles.GetByUrl(url); var isKnown = _articles.GetByUrl(url);