jtom38
/
Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Collector jobs have been updated to handle authors

This commit is contained in:
James Tombleson 2023-08-04 23:04:32 -07:00
parent 56a655e4f1
commit 0d751cc571
3 changed files with 85 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs
View File

@ -30,6 +30,7 @@ public class CodeProjectWatcherJob
private ILogger _logger;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
private IAuthorTable _author;
public CodeProjectWatcherJob()
{
@ -37,12 +38,14 @@ public class CodeProjectWatcherJob
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
_logger = JobLogger.GetLogger("", JobName);
_author = new AuthorsTable("");
}
public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@ -52,6 +55,7 @@ public class CodeProjectWatcherJob
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@ -157,17 +161,29 @@ public class CodeProjectWatcherJob
});
parser.Parse();
if (item.Authors[0].Name is null)
{
_logger.Warning("Author was missing from the record and will continue with a missing author");
}
var authorExists = _author.CreateIfMissingAsync(new AuthorEntity
{
Name = item.Authors[0].Name,
SourceId = source.Id,
Image = "",
});
authorExists.Wait();
var a = new ArticlesEntity
{
SourceId = source.Id,
AuthorId = authorExists.Result.Id,
Tags = source.Tags,
Title = item.Title.Text,
Url = itemUrl,
PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text,
AuthorName = item.Authors[0].Name ?? "",
AuthorImage = item.Authors[0].Uri ?? "",
CodeIsRelease = isRelease,
CodeIsCommit = isCommit,
};

18
Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs
View File

@ -36,6 +36,7 @@ public class DiscordNotificationJob
private ILogger _logger;
//private DatabaseContext _databaseContext;
private IArticlesRepository _article;
private IAuthorTable _author;
private IIconsRepository _icons;
private IDiscordQueueRepository _queue;
private ISourcesRepository _sources;
@ -46,6 +47,7 @@ public class DiscordNotificationJob
{
_queue = new DiscordQueueTable("");
_article = new ArticlesTable("");
_author = new AuthorsTable("");
_webhook = new DiscordWebhooksTable("");
_sources = new SourcesTable("");
_subs = new DiscordNotificationTable("");
@ -92,7 +94,6 @@ public class DiscordNotificationJob
_logger.Debug($"{JobName} - Processing {request.Id}");
// Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleId);
// Get the details of the source
var sourceDetails = _sources.GetById(articleDetails.SourceId);
if (sourceDetails.Id == Guid.Empty)
@ -103,6 +104,9 @@ public class DiscordNotificationJob
return;
}
var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name);
author.Wait();
var sourceIcon = new IconEntity();
try
{
@ -118,13 +122,13 @@ public class DiscordNotificationJob
var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
foreach (var sub in allSubscriptions)
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub);
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity());
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id);
_queue.Delete(request.Id);
}
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub)
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity)
{
// Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip.
@ -140,7 +144,7 @@ public class DiscordNotificationJob
var client = new DiscordClient(discordDetails.Url);
try
{
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon));
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity));
}
catch (Exception e)
{
@ -154,7 +158,7 @@ public class DiscordNotificationJob
Thread.Sleep(3000);
}
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon)
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author)
{
var embed = new DiscordMessageEmbed
{
@ -163,7 +167,7 @@ public class DiscordNotificationJob
Description = MessageValidation.ConvertHtmlCodes(article.Description),
Author = new DiscordMessageEmbedAuthor
{
Name = article.AuthorName,
Name = author.Name,
IconUrl = icon.FileName
},
Footer = new DiscordMessageEmbedFooter
@ -189,7 +193,7 @@ public class DiscordNotificationJob
Url = article.Thumbnail
};
if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage;
embed.Author.IconUrl = author.Image ?? "";
return new DiscordMessage
{

86
Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs
View File

@ -25,6 +25,7 @@ public class YoutubeWatcherJob
private readonly YoutubeWatcherJobOptions _options;
private IArticlesRepository _articles;
private IAuthorTable _author;
private IIconsRepository _icons;
private ILogger _logger;
private IDiscordQueueRepository _queue;
@ -34,6 +35,7 @@ public class YoutubeWatcherJob
{
_options = new YoutubeWatcherJobOptions();
_articles = new ArticlesTable("");
_author = new AuthorsTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
_icons = new IconsTable("");
@ -43,6 +45,7 @@ public class YoutubeWatcherJob
public void InitAndExecute(YoutubeWatcherJobOptions options)
{
_articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
_author = new AuthorsTable(options.DatabaseConnectionString ?? "");
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_source = new SourcesTable(options.DatabaseConnectionString ?? "");
_icons = new IconsTable(options.DatabaseConnectionString ?? "");
@ -77,11 +80,23 @@ public class YoutubeWatcherJob
_logger.Information($"{JobName} - Checking '{source.Name}'");
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = CheckFeed(url, source);
var newVideos = FindMissingPosts(url, source);
_logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
foreach (var video in newVideos)
{
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
var author = _author.GetById(video.AuthorId);
author.Wait();
if (author.Result is null)
{
_logger.Warning("Missing author record for article id {VideoId}", video.Id);
}
else
{
_logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title);
}
_articles.New(video);
_queue.New(new DiscordQueueEntity
{
@ -104,13 +119,12 @@ public class YoutubeWatcherJob
var id = pageReader.Data.Header.YoutubeChannelID ?? "";
if (id == "")
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
url);
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), "");
return id;
}
private List<ArticlesEntity> CheckFeed(string url, SourceEntity source)
private List<ArticlesEntity> FindMissingPosts(string url, SourceEntity source)
{
var videos = new List<ArticlesEntity>();
@ -118,37 +132,49 @@ public class YoutubeWatcherJob
var feed = SyndicationFeed.Load(reader);
foreach (var post in feed.Items.ToList())
{
var articleUrl = post.Links[0].Uri.AbsoluteUri;
if (IsThisUrlKnown(articleUrl)) continue;
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
{
Url = articleUrl
});
videoDetails.Parse();
var article = new ArticlesEntity
{
//Todo add the icon
AuthorName = post.Authors[0].Name,
Title = post.Title.Text,
Tags = FetchTags(post),
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description,
SourceId = source.Id,
Video = "true"
};
var article = CheckFeedItem(post, source.Id);
if (article is null) continue;
videos.Add(article);
Thread.Sleep(_options.SleepTimer);
}
return videos;
}
private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId)
{
var articleUrl = post.Links[0].Uri.AbsoluteUri;
if (IsThisUrlKnown(articleUrl)) return null;
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
{
Url = articleUrl
});
videoDetails.Parse();
var author = _author.CreateIfMissingAsync(new AuthorEntity
{
Image = post.Authors[0].Uri,
Name = post.Authors[0].Name
});
author.Wait();
var article = new ArticlesEntity
{
//Todo add the icon
AuthorId = author.Result.Id,
Title = post.Title.Text,
Tags = FetchTags(post),
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description,
SourceId = sourceId,
Video = "true"
};
return article;
}
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);