Collector jobs have been updated to handle authors
This commit is contained in:
parent
56a655e4f1
commit
0d751cc571
@ -30,6 +30,7 @@ public class CodeProjectWatcherJob
|
||||
private ILogger _logger;
|
||||
private IDiscordQueueRepository _queue;
|
||||
private ISourcesRepository _source;
|
||||
private IAuthorTable _author;
|
||||
|
||||
public CodeProjectWatcherJob()
|
||||
{
|
||||
@ -37,12 +38,14 @@ public class CodeProjectWatcherJob
|
||||
_queue = new DiscordQueueTable("");
|
||||
_source = new SourcesTable("");
|
||||
_logger = JobLogger.GetLogger("", JobName);
|
||||
_author = new AuthorsTable("");
|
||||
}
|
||||
|
||||
public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
|
||||
{
|
||||
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
|
||||
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
|
||||
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
|
||||
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
|
||||
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
|
||||
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
|
||||
@ -52,6 +55,7 @@ public class CodeProjectWatcherJob
|
||||
{
|
||||
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
|
||||
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
|
||||
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
|
||||
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
|
||||
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
|
||||
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
|
||||
@ -157,17 +161,29 @@ public class CodeProjectWatcherJob
|
||||
});
|
||||
parser.Parse();
|
||||
|
||||
if (item.Authors[0].Name is null)
|
||||
{
|
||||
_logger.Warning("Author was missing from the record and will continue with a missing author");
|
||||
}
|
||||
|
||||
var authorExists = _author.CreateIfMissingAsync(new AuthorEntity
|
||||
{
|
||||
Name = item.Authors[0].Name,
|
||||
SourceId = source.Id,
|
||||
Image = "",
|
||||
});
|
||||
authorExists.Wait();
|
||||
|
||||
var a = new ArticlesEntity
|
||||
{
|
||||
SourceId = source.Id,
|
||||
AuthorId = authorExists.Result.Id,
|
||||
Tags = source.Tags,
|
||||
Title = item.Title.Text,
|
||||
Url = itemUrl,
|
||||
PubDate = item.LastUpdatedTime.DateTime,
|
||||
Thumbnail = parser.Data.Header.Image,
|
||||
Description = item.Title.Text,
|
||||
AuthorName = item.Authors[0].Name ?? "",
|
||||
AuthorImage = item.Authors[0].Uri ?? "",
|
||||
CodeIsRelease = isRelease,
|
||||
CodeIsCommit = isCommit,
|
||||
};
|
||||
|
@ -36,6 +36,7 @@ public class DiscordNotificationJob
|
||||
private ILogger _logger;
|
||||
//private DatabaseContext _databaseContext;
|
||||
private IArticlesRepository _article;
|
||||
private IAuthorTable _author;
|
||||
private IIconsRepository _icons;
|
||||
private IDiscordQueueRepository _queue;
|
||||
private ISourcesRepository _sources;
|
||||
@ -46,6 +47,7 @@ public class DiscordNotificationJob
|
||||
{
|
||||
_queue = new DiscordQueueTable("");
|
||||
_article = new ArticlesTable("");
|
||||
_author = new AuthorsTable("");
|
||||
_webhook = new DiscordWebhooksTable("");
|
||||
_sources = new SourcesTable("");
|
||||
_subs = new DiscordNotificationTable("");
|
||||
@ -92,7 +94,6 @@ public class DiscordNotificationJob
|
||||
_logger.Debug($"{JobName} - Processing {request.Id}");
|
||||
// Get all details on the article in the queue
|
||||
var articleDetails = _article.GetById(request.ArticleId);
|
||||
|
||||
// Get the details of the source
|
||||
var sourceDetails = _sources.GetById(articleDetails.SourceId);
|
||||
if (sourceDetails.Id == Guid.Empty)
|
||||
@ -103,6 +104,9 @@ public class DiscordNotificationJob
|
||||
return;
|
||||
}
|
||||
|
||||
var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name);
|
||||
author.Wait();
|
||||
|
||||
var sourceIcon = new IconEntity();
|
||||
try
|
||||
{
|
||||
@ -118,13 +122,13 @@ public class DiscordNotificationJob
|
||||
var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
|
||||
|
||||
foreach (var sub in allSubscriptions)
|
||||
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub);
|
||||
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity());
|
||||
|
||||
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id);
|
||||
_queue.Delete(request.Id);
|
||||
}
|
||||
|
||||
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub)
|
||||
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity)
|
||||
{
|
||||
// Check if the subscription code flags
|
||||
// If the article is a code commit and the subscription does not want them, skip.
|
||||
@ -140,7 +144,7 @@ public class DiscordNotificationJob
|
||||
var client = new DiscordClient(discordDetails.Url);
|
||||
try
|
||||
{
|
||||
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon));
|
||||
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
@ -154,7 +158,7 @@ public class DiscordNotificationJob
|
||||
Thread.Sleep(3000);
|
||||
}
|
||||
|
||||
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon)
|
||||
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author)
|
||||
{
|
||||
var embed = new DiscordMessageEmbed
|
||||
{
|
||||
@ -163,7 +167,7 @@ public class DiscordNotificationJob
|
||||
Description = MessageValidation.ConvertHtmlCodes(article.Description),
|
||||
Author = new DiscordMessageEmbedAuthor
|
||||
{
|
||||
Name = article.AuthorName,
|
||||
Name = author.Name,
|
||||
IconUrl = icon.FileName
|
||||
},
|
||||
Footer = new DiscordMessageEmbedFooter
|
||||
@ -189,7 +193,7 @@ public class DiscordNotificationJob
|
||||
Url = article.Thumbnail
|
||||
};
|
||||
|
||||
if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage;
|
||||
embed.Author.IconUrl = author.Image ?? "";
|
||||
|
||||
return new DiscordMessage
|
||||
{
|
||||
|
@ -25,6 +25,7 @@ public class YoutubeWatcherJob
|
||||
|
||||
private readonly YoutubeWatcherJobOptions _options;
|
||||
private IArticlesRepository _articles;
|
||||
private IAuthorTable _author;
|
||||
private IIconsRepository _icons;
|
||||
private ILogger _logger;
|
||||
private IDiscordQueueRepository _queue;
|
||||
@ -34,6 +35,7 @@ public class YoutubeWatcherJob
|
||||
{
|
||||
_options = new YoutubeWatcherJobOptions();
|
||||
_articles = new ArticlesTable("");
|
||||
_author = new AuthorsTable("");
|
||||
_queue = new DiscordQueueTable("");
|
||||
_source = new SourcesTable("");
|
||||
_icons = new IconsTable("");
|
||||
@ -43,6 +45,7 @@ public class YoutubeWatcherJob
|
||||
public void InitAndExecute(YoutubeWatcherJobOptions options)
|
||||
{
|
||||
_articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
|
||||
_author = new AuthorsTable(options.DatabaseConnectionString ?? "");
|
||||
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
|
||||
_source = new SourcesTable(options.DatabaseConnectionString ?? "");
|
||||
_icons = new IconsTable(options.DatabaseConnectionString ?? "");
|
||||
@ -77,11 +80,23 @@ public class YoutubeWatcherJob
|
||||
_logger.Information($"{JobName} - Checking '{source.Name}'");
|
||||
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
|
||||
|
||||
var newVideos = CheckFeed(url, source);
|
||||
var newVideos = FindMissingPosts(url, source);
|
||||
_logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
|
||||
|
||||
foreach (var video in newVideos)
|
||||
{
|
||||
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
|
||||
var author = _author.GetById(video.AuthorId);
|
||||
author.Wait();
|
||||
|
||||
if (author.Result is null)
|
||||
{
|
||||
_logger.Warning("Missing author record for article id {VideoId}", video.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title);
|
||||
}
|
||||
|
||||
_articles.New(video);
|
||||
_queue.New(new DiscordQueueEntity
|
||||
{
|
||||
@ -104,13 +119,12 @@ public class YoutubeWatcherJob
|
||||
|
||||
var id = pageReader.Data.Header.YoutubeChannelID ?? "";
|
||||
if (id == "")
|
||||
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
|
||||
url);
|
||||
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), "");
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
private List<ArticlesEntity> CheckFeed(string url, SourceEntity source)
|
||||
private List<ArticlesEntity> FindMissingPosts(string url, SourceEntity source)
|
||||
{
|
||||
var videos = new List<ArticlesEntity>();
|
||||
|
||||
@ -118,37 +132,49 @@ public class YoutubeWatcherJob
|
||||
var feed = SyndicationFeed.Load(reader);
|
||||
foreach (var post in feed.Items.ToList())
|
||||
{
|
||||
var articleUrl = post.Links[0].Uri.AbsoluteUri;
|
||||
if (IsThisUrlKnown(articleUrl)) continue;
|
||||
|
||||
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
|
||||
{
|
||||
Url = articleUrl
|
||||
});
|
||||
videoDetails.Parse();
|
||||
|
||||
var article = new ArticlesEntity
|
||||
{
|
||||
//Todo add the icon
|
||||
AuthorName = post.Authors[0].Name,
|
||||
Title = post.Title.Text,
|
||||
Tags = FetchTags(post),
|
||||
Url = articleUrl,
|
||||
PubDate = post.PublishDate.DateTime,
|
||||
Thumbnail = videoDetails.Data.Header.Image,
|
||||
Description = videoDetails.Data.Header.Description,
|
||||
SourceId = source.Id,
|
||||
Video = "true"
|
||||
};
|
||||
|
||||
var article = CheckFeedItem(post, source.Id);
|
||||
if (article is null) continue;
|
||||
videos.Add(article);
|
||||
|
||||
Thread.Sleep(_options.SleepTimer);
|
||||
}
|
||||
|
||||
return videos;
|
||||
}
|
||||
|
||||
private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId)
|
||||
{
|
||||
var articleUrl = post.Links[0].Uri.AbsoluteUri;
|
||||
if (IsThisUrlKnown(articleUrl)) return null;
|
||||
|
||||
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
|
||||
{
|
||||
Url = articleUrl
|
||||
});
|
||||
videoDetails.Parse();
|
||||
|
||||
var author = _author.CreateIfMissingAsync(new AuthorEntity
|
||||
{
|
||||
Image = post.Authors[0].Uri,
|
||||
Name = post.Authors[0].Name
|
||||
});
|
||||
author.Wait();
|
||||
|
||||
var article = new ArticlesEntity
|
||||
{
|
||||
//Todo add the icon
|
||||
AuthorId = author.Result.Id,
|
||||
Title = post.Title.Text,
|
||||
Tags = FetchTags(post),
|
||||
Url = articleUrl,
|
||||
PubDate = post.PublishDate.DateTime,
|
||||
Thumbnail = videoDetails.Data.Header.Image,
|
||||
Description = videoDetails.Data.Header.Description,
|
||||
SourceId = sourceId,
|
||||
Video = "true"
|
||||
};
|
||||
|
||||
return article;
|
||||
}
|
||||
|
||||
private bool IsThisUrlKnown(string url)
|
||||
{
|
||||
var isKnown = _articles.GetByUrl(url);
|
||||
|
Loading…
Reference in New Issue
Block a user