From 56a655e4f13560b15215cd66271cb070ff11dd82 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 4 Aug 2023 23:04:09 -0700 Subject: [PATCH 1/5] author table was added --- ...0230805060324_add author table.Designer.cs | 575 ++++++++++++++++++ .../20230805060324_add author table.cs | 51 ++ .../DatabaseContextModelSnapshot.cs | 8 +- .../Repositories/AuthorsTable.cs | 40 +- .../Entities/ArticlesEntity.cs | 3 +- 5 files changed, 659 insertions(+), 18 deletions(-) create mode 100644 Newsbot.Collector.Database/Migrations/20230805060324_add author table.Designer.cs create mode 100644 Newsbot.Collector.Database/Migrations/20230805060324_add author table.cs diff --git a/Newsbot.Collector.Database/Migrations/20230805060324_add author table.Designer.cs b/Newsbot.Collector.Database/Migrations/20230805060324_add author table.Designer.cs new file mode 100644 index 0000000..5d45814 --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230805060324_add author table.Designer.cs @@ -0,0 +1,575 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Newsbot.Collector.Database; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Newsbot.Collector.Database.Migrations +{ + [DbContext(typeof(DatabaseContext))] + [Migration("20230805060324_add author table")] + partial class addauthortable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("AspNetRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUser", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("AccessFailedCount") + .HasColumnType("integer"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("EmailConfirmed") + .HasColumnType("boolean"); + + b.Property("LockoutEnabled") + .HasColumnType("boolean"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("PasswordHash") + .HasColumnType("text"); + + b.Property("PhoneNumber") + .HasColumnType("text"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("boolean"); + + b.Property("SecurityStamp") + .HasColumnType("text"); + + b.Property("TwoFactorEnabled") + .HasColumnType("boolean"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("AspNetUsers", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("ProviderKey") + .HasColumnType("text"); + + b.Property("ProviderDisplayName") + .HasColumnType("text"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("text"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("text"); + + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Value") + .HasColumnType("text"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens", (string)null); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.ArticlesEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AuthorId") + .HasColumnType("uuid"); + + b.Property("CodeIsCommit") + .HasColumnType("boolean"); + + b.Property("CodeIsRelease") + .HasColumnType("boolean"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("PubDate") + .HasColumnType("timestamp with time zone"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text"); + + b.Property("Thumbnail") + .IsRequired() + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .HasColumnType("text"); + + b.Property("Video") + .IsRequired() + .HasColumnType("text"); + + b.Property("VideoHeight") + .HasColumnType("integer"); + + b.Property("VideoWidth") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Articles"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.AuthorEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Image") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Authors"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordNotificationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CodeAllowCommits") + .HasColumnType("boolean"); + + b.Property("CodeAllowReleases") + .HasColumnType("boolean"); + + b.Property("DiscordWebHookId") + .HasColumnType("uuid"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DiscordNotification"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordQueueEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ArticleId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("DiscordQueue"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Channel") + .IsRequired() + .HasColumnType("text"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("Server") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("DiscordWebhooks"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.IconEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("Site") + .IsRequired() + .HasColumnType("text"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Icons"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.RefreshTokenEntity", b => + { + b.Property("Token") + .HasColumnType("text"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiryDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Invalidated") + .HasColumnType("boolean"); + + b.Property("JwtId") + .HasColumnType("text"); + + b.Property("Used") + .HasColumnType("boolean"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Token"); + + b.HasIndex("UserId"); + + b.ToTable("RefreshTokens"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SourceEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Deleted") + .HasColumnType("boolean"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Site") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("YoutubeId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Sources"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.UserSourceSubscriptionEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DateAdded") + .HasColumnType("timestamp with time zone"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("UserSourceSubscription"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", "User") + .WithMany() + .HasForeignKey("UserId"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.RefreshTokenEntity", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", "User") + .WithMany() + .HasForeignKey("UserId"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.UserSourceSubscriptionEntity", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", "User") + .WithMany() + .HasForeignKey("UserId"); + + b.Navigation("User"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Newsbot.Collector.Database/Migrations/20230805060324_add author table.cs b/Newsbot.Collector.Database/Migrations/20230805060324_add author table.cs new file mode 100644 index 0000000..cc98ec6 --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230805060324_add author table.cs @@ -0,0 +1,51 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Newsbot.Collector.Database.Migrations +{ + /// + public partial class addauthortable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "AuthorImage", + table: "Articles"); + + migrationBuilder.DropColumn( + name: "AuthorName", + table: "Articles"); + + migrationBuilder.AddColumn( + name: "AuthorId", + table: "Articles", + type: "uuid", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "AuthorId", + table: "Articles"); + + migrationBuilder.AddColumn( + name: "AuthorImage", + table: "Articles", + type: "text", + nullable: true); + + migrationBuilder.AddColumn( + name: "AuthorName", + table: "Articles", + type: "text", + nullable: false, + defaultValue: ""); + } + } +} diff --git a/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs b/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs index 8677532..f349ad0 100644 --- a/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs +++ b/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs @@ -224,12 +224,8 @@ namespace Newsbot.Collector.Database.Migrations .ValueGeneratedOnAdd() .HasColumnType("uuid"); - b.Property("AuthorImage") - .HasColumnType("text"); - - b.Property("AuthorName") - .IsRequired() - .HasColumnType("text"); + b.Property("AuthorId") + .HasColumnType("uuid"); b.Property("CodeIsCommit") .HasColumnType("boolean"); diff --git a/Newsbot.Collector.Database/Repositories/AuthorsTable.cs b/Newsbot.Collector.Database/Repositories/AuthorsTable.cs index 0d24a3a..ecd093f 100644 --- a/Newsbot.Collector.Database/Repositories/AuthorsTable.cs +++ b/Newsbot.Collector.Database/Repositories/AuthorsTable.cs @@ -1,3 +1,4 @@ +using Microsoft.EntityFrameworkCore; using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; @@ -18,30 +19,49 @@ public class AuthorsTable : IAuthorTable _context = context; } - public AuthorEntity New(AuthorEntity entity) + public async Task NewAsync(AuthorEntity entity) { entity.Id = Guid.NewGuid(); _context.Authors.Add(entity); - _context.SaveChanges(); + await _context.SaveChangesAsync(); return entity; } - public List ListBySourceId(Guid sourceId) + public async Task CreateIfMissingAsync(AuthorEntity entity) { - return _context.Authors - .Where(s => s.SourceId.Equals(sourceId)).ToList(); + var res = await GetBySourceIdAndNameAsync(entity.SourceId, entity.Name); + if (res is null) + { + entity.Id = Guid.NewGuid(); + _context.Authors.Add(entity); + await _context.SaveChangesAsync(); + return entity; + } + + return res; } - public int TotalPosts(Guid id) + public async Task> ListBySourceIdAsync(Guid sourceId) { - return _context.Authors.Count(x => x.Id.Equals(id)); + return await _context.Authors + .Where(s => s.SourceId.Equals(sourceId)).ToListAsync(); } - public AuthorEntity? GetBySourceIdAndName(Guid sourceId, string name) + public async Task TotalPostsAsync(Guid id) { - return _context.Authors + return await _context.Authors.CountAsync(x => x.Id.Equals(id)); + } + + public async Task GetBySourceIdAndNameAsync(Guid sourceId, string name) + { + return await _context.Authors .Where(s => s.SourceId.Equals(sourceId)) - .FirstOrDefault(n => n.Name.Equals(name)); + .FirstOrDefaultAsync(n => n.Name.Equals(name)); } + + public async Task GetById(Guid id) + { + return await _context.Authors.FirstOrDefaultAsync(q => q.Id.Equals(id)); + } } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs b/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs index 626e2c1..47ad792 100644 --- a/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs +++ b/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs @@ -4,6 +4,7 @@ public class ArticlesEntity { public Guid Id { get; set; } public Guid SourceId { get; set; } + public Guid AuthorId { get; set; } public string Tags { get; set; } = ""; public string Title { get; set; } = ""; public string? Url { get; set; } @@ -13,8 +14,6 @@ public class ArticlesEntity public int VideoWidth { get; set; } = 0; public string Thumbnail { get; set; } = ""; public string Description { get; set; } = ""; - public string AuthorName { get; set; } = ""; - public string? AuthorImage { get; set; } public bool CodeIsRelease { get; set; } public bool CodeIsCommit { get; set; } } \ No newline at end of file -- 2.45.2 From 0d751cc571830c441684ed7705be18125d3732d9 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 4 Aug 2023 23:04:32 -0700 Subject: [PATCH 2/5] Collector jobs have been updated to handle authors --- .../Jobs/CodeProjectWatcherJob.cs | 20 ++++- .../Jobs/DiscordNotificationJob.cs | 18 ++-- .../Jobs/YoutubeWatcherJob.cs | 86 ++++++++++++------- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs index 1414a08..1551cbd 100644 --- a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs @@ -30,6 +30,7 @@ public class CodeProjectWatcherJob private ILogger _logger; private IDiscordQueueRepository _queue; private ISourcesRepository _source; + private IAuthorTable _author; public CodeProjectWatcherJob() { @@ -37,12 +38,14 @@ public class CodeProjectWatcherJob _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); _logger = JobLogger.GetLogger("", JobName); + _author = new AuthorsTable(""); } public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options) { options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); + _author = new AuthorsTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); @@ -52,6 +55,7 @@ public class CodeProjectWatcherJob { options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); + _author = new AuthorsTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); @@ -157,17 +161,29 @@ public class CodeProjectWatcherJob }); parser.Parse(); + if (item.Authors[0].Name is null) + { + _logger.Warning("Author was missing from the record and will continue with a missing author"); + } + + var authorExists = _author.CreateIfMissingAsync(new AuthorEntity + { + Name = item.Authors[0].Name, + SourceId = source.Id, + Image = "", + }); + authorExists.Wait(); + var a = new ArticlesEntity { SourceId = source.Id, + AuthorId = authorExists.Result.Id, Tags = source.Tags, Title = item.Title.Text, Url = itemUrl, PubDate = item.LastUpdatedTime.DateTime, Thumbnail = parser.Data.Header.Image, Description = item.Title.Text, - AuthorName = item.Authors[0].Name ?? "", - AuthorImage = item.Authors[0].Uri ?? "", CodeIsRelease = isRelease, CodeIsCommit = isCommit, }; diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index 9544c63..c2a80fa 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -36,6 +36,7 @@ public class DiscordNotificationJob private ILogger _logger; //private DatabaseContext _databaseContext; private IArticlesRepository _article; + private IAuthorTable _author; private IIconsRepository _icons; private IDiscordQueueRepository _queue; private ISourcesRepository _sources; @@ -46,6 +47,7 @@ public class DiscordNotificationJob { _queue = new DiscordQueueTable(""); _article = new ArticlesTable(""); + _author = new AuthorsTable(""); _webhook = new DiscordWebhooksTable(""); _sources = new SourcesTable(""); _subs = new DiscordNotificationTable(""); @@ -92,7 +94,6 @@ public class DiscordNotificationJob _logger.Debug($"{JobName} - Processing {request.Id}"); // Get all details on the article in the queue var articleDetails = _article.GetById(request.ArticleId); - // Get the details of the source var sourceDetails = _sources.GetById(articleDetails.SourceId); if (sourceDetails.Id == Guid.Empty) @@ -103,6 +104,9 @@ public class DiscordNotificationJob return; } + var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name); + author.Wait(); + var sourceIcon = new IconEntity(); try { @@ -118,13 +122,13 @@ public class DiscordNotificationJob var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id); foreach (var sub in allSubscriptions) - SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); + SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity()); _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id); _queue.Delete(request.Id); } - public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub) + public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity) { // Check if the subscription code flags // If the article is a code commit and the subscription does not want them, skip. @@ -140,7 +144,7 @@ public class DiscordNotificationJob var client = new DiscordClient(discordDetails.Url); try { - client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon)); + client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity)); } catch (Exception e) { @@ -154,7 +158,7 @@ public class DiscordNotificationJob Thread.Sleep(3000); } - public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon) + public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author) { var embed = new DiscordMessageEmbed { @@ -163,7 +167,7 @@ public class DiscordNotificationJob Description = MessageValidation.ConvertHtmlCodes(article.Description), Author = new DiscordMessageEmbedAuthor { - Name = article.AuthorName, + Name = author.Name, IconUrl = icon.FileName }, Footer = new DiscordMessageEmbedFooter @@ -189,7 +193,7 @@ public class DiscordNotificationJob Url = article.Thumbnail }; - if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage; + embed.Author.IconUrl = author.Image ?? ""; return new DiscordMessage { diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index 2374fe5..325847c 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -25,6 +25,7 @@ public class YoutubeWatcherJob private readonly YoutubeWatcherJobOptions _options; private IArticlesRepository _articles; + private IAuthorTable _author; private IIconsRepository _icons; private ILogger _logger; private IDiscordQueueRepository _queue; @@ -34,6 +35,7 @@ public class YoutubeWatcherJob { _options = new YoutubeWatcherJobOptions(); _articles = new ArticlesTable(""); + _author = new AuthorsTable(""); _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); _icons = new IconsTable(""); @@ -43,6 +45,7 @@ public class YoutubeWatcherJob public void InitAndExecute(YoutubeWatcherJobOptions options) { _articles = new ArticlesTable(options.DatabaseConnectionString ?? ""); + _author = new AuthorsTable(options.DatabaseConnectionString ?? ""); _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? ""); _source = new SourcesTable(options.DatabaseConnectionString ?? ""); _icons = new IconsTable(options.DatabaseConnectionString ?? ""); @@ -77,11 +80,23 @@ public class YoutubeWatcherJob _logger.Information($"{JobName} - Checking '{source.Name}'"); var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}"; - var newVideos = CheckFeed(url, source); + var newVideos = FindMissingPosts(url, source); _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos"); + foreach (var video in newVideos) { - _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found"); + var author = _author.GetById(video.AuthorId); + author.Wait(); + + if (author.Result is null) + { + _logger.Warning("Missing author record for article id {VideoId}", video.Id); + } + else + { + _logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title); + } + _articles.New(video); _queue.New(new DiscordQueueEntity { @@ -104,13 +119,12 @@ public class YoutubeWatcherJob var id = pageReader.Data.Header.YoutubeChannelID ?? ""; if (id == "") - _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."), - url); + _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), ""); return id; } - private List CheckFeed(string url, SourceEntity source) + private List FindMissingPosts(string url, SourceEntity source) { var videos = new List(); @@ -118,37 +132,49 @@ public class YoutubeWatcherJob var feed = SyndicationFeed.Load(reader); foreach (var post in feed.Items.ToList()) { - var articleUrl = post.Links[0].Uri.AbsoluteUri; - if (IsThisUrlKnown(articleUrl)) continue; - - var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions - { - Url = articleUrl - }); - videoDetails.Parse(); - - var article = new ArticlesEntity - { - //Todo add the icon - AuthorName = post.Authors[0].Name, - Title = post.Title.Text, - Tags = FetchTags(post), - Url = articleUrl, - PubDate = post.PublishDate.DateTime, - Thumbnail = videoDetails.Data.Header.Image, - Description = videoDetails.Data.Header.Description, - SourceId = source.Id, - Video = "true" - }; - + var article = CheckFeedItem(post, source.Id); + if (article is null) continue; videos.Add(article); - - Thread.Sleep(_options.SleepTimer); } return videos; } + private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId) + { + var articleUrl = post.Links[0].Uri.AbsoluteUri; + if (IsThisUrlKnown(articleUrl)) return null; + + var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions + { + Url = articleUrl + }); + videoDetails.Parse(); + + var author = _author.CreateIfMissingAsync(new AuthorEntity + { + Image = post.Authors[0].Uri, + Name = post.Authors[0].Name + }); + author.Wait(); + + var article = new ArticlesEntity + { + //Todo add the icon + AuthorId = author.Result.Id, + Title = post.Title.Text, + Tags = FetchTags(post), + Url = articleUrl, + PubDate = post.PublishDate.DateTime, + Thumbnail = videoDetails.Data.Header.Image, + Description = videoDetails.Data.Header.Description, + SourceId = sourceId, + Video = "true" + }; + + return article; + } + private bool IsThisUrlKnown(string url) { var isKnown = _articles.GetByUrl(url); -- 2.45.2 From 2aa14548aa1caf5901203b8801b7c662cb12aaaf Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 4 Aug 2023 23:05:26 -0700 Subject: [PATCH 3/5] DTO was updated to reflect author changes --- Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs | 8 ++++---- Newsbot.Collector.Domain/Dto/ArticleDto.cs | 10 +++------- Newsbot.Collector.Domain/Dto/AuthorDto.cs | 9 +++++++++ 3 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 Newsbot.Collector.Domain/Dto/AuthorDto.cs diff --git a/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs b/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs index ca1fb6d..9813288 100644 --- a/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs +++ b/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs @@ -19,8 +19,9 @@ public class ArticleDetailsDto public string? AuthorImage { get; set; } public SourceDto? Source { get; set; } + public AuthorEntity? Author { get; set; } - public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source) + public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source, AuthorEntity? author) { return new ArticleDetailsDto { @@ -34,9 +35,8 @@ public class ArticleDetailsDto VideoWidth = article.VideoWidth, Thumbnail = article.Thumbnail, Description = article.Description, - AuthorName = article.AuthorName, - AuthorImage = article.AuthorImage, - Source = SourceDto.Convert(source) + Source = SourceDto.Convert(source), + }; } } diff --git a/Newsbot.Collector.Domain/Dto/ArticleDto.cs b/Newsbot.Collector.Domain/Dto/ArticleDto.cs index 7a55872..dbd27d1 100644 --- a/Newsbot.Collector.Domain/Dto/ArticleDto.cs +++ b/Newsbot.Collector.Domain/Dto/ArticleDto.cs @@ -7,6 +7,7 @@ public class ArticleDto { public Guid ID { get; set; } public Guid SourceID { get; set; } + public Guid AuthorId { get; set; } public string[]? Tags { get; set; } public string? Title { get; set; } public string? Url { get; set; } @@ -16,8 +17,6 @@ public class ArticleDto public int VideoWidth { get; set; } public string? Thumbnail { get; set; } public string? Description { get; set; } - public string? AuthorName { get; set; } - public string? AuthorImage { get; set; } public static ArticleDto Convert(ArticlesModel article) { return new ArticleDto @@ -33,8 +32,6 @@ public class ArticleDto VideoWidth = article.VideoWidth, Thumbnail = article.Thumbnail, Description = article.Description, - AuthorName = article.AuthorName, - AuthorImage = article.AuthorImage, }; } @@ -44,6 +41,7 @@ public class ArticleDto { ID = article.Id, SourceID = article.SourceId, + AuthorId = article.AuthorId, Tags = article.Tags.Split(','), Title = article.Title, Url = article.Url, @@ -52,9 +50,7 @@ public class ArticleDto VideoHeight = article.VideoHeight, VideoWidth = article.VideoWidth, Thumbnail = article.Thumbnail, - Description = article.Description, - AuthorName = article.AuthorName, - AuthorImage = article.AuthorImage, + Description = article.Description }; } } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Dto/AuthorDto.cs b/Newsbot.Collector.Domain/Dto/AuthorDto.cs new file mode 100644 index 0000000..b56090b --- /dev/null +++ b/Newsbot.Collector.Domain/Dto/AuthorDto.cs @@ -0,0 +1,9 @@ +namespace Newsbot.Collector.Domain.Dto; + +public class AuthorDto +{ + public Guid Id { get; set; } + public Guid SourceId { get; set; } + public string? Name { get; set; } + public string? Image { get; set; } +} \ No newline at end of file -- 2.45.2 From 751bbd3a7b20fd7379e65c25b13b262c87b16e00 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 4 Aug 2023 23:05:49 -0700 Subject: [PATCH 4/5] IAuthorTable.cs was added --- Newsbot.Collector.Domain/Interfaces/IAuthorTable.cs | 11 +++++++---- Newsbot.Collector.Domain/Interfaces/ICollector.cs | 9 --------- 2 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 Newsbot.Collector.Domain/Interfaces/ICollector.cs diff --git a/Newsbot.Collector.Domain/Interfaces/IAuthorTable.cs b/Newsbot.Collector.Domain/Interfaces/IAuthorTable.cs index 6c777a2..5071a54 100644 --- a/Newsbot.Collector.Domain/Interfaces/IAuthorTable.cs +++ b/Newsbot.Collector.Domain/Interfaces/IAuthorTable.cs @@ -4,8 +4,11 @@ namespace Newsbot.Collector.Domain.Interfaces; public interface IAuthorTable { - AuthorEntity New(AuthorEntity entity); - List ListBySourceId(Guid sourceId); - int TotalPosts(Guid id); - AuthorEntity? GetBySourceIdAndName(Guid sourceId, string name); + Task NewAsync(AuthorEntity entity); + + Task CreateIfMissingAsync(AuthorEntity entity); + Task> ListBySourceIdAsync(Guid sourceId); + Task TotalPostsAsync(Guid id); + Task GetBySourceIdAndNameAsync(Guid sourceId, string name); + Task GetById(Guid id); } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/ICollector.cs b/Newsbot.Collector.Domain/Interfaces/ICollector.cs deleted file mode 100644 index ac2b55b..0000000 --- a/Newsbot.Collector.Domain/Interfaces/ICollector.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Newsbot.Collector.Domain.Models; - -namespace Newsbot.Collector.Domain.Interfaces; - -/// -public interface ICollector : IHangfireJob -{ - List Collect(); -} \ No newline at end of file -- 2.45.2 From a3085bb27fcaffebb4f12b299a3c79bbdbe00342 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 4 Aug 2023 23:07:04 -0700 Subject: [PATCH 5/5] Article controller was updated and tests updated --- .../Controllers/v1/ArticlesController.cs | 9 +++++++-- .../Jobs/DiscordNotificationJobTest.cs | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Newsbot.Collector.Api/Controllers/v1/ArticlesController.cs b/Newsbot.Collector.Api/Controllers/v1/ArticlesController.cs index b8ed6d5..81f1c59 100644 --- a/Newsbot.Collector.Api/Controllers/v1/ArticlesController.cs +++ b/Newsbot.Collector.Api/Controllers/v1/ArticlesController.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Newsbot.Collector.Domain.Dto; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Results; @@ -15,11 +16,13 @@ public class ArticlesController : ControllerBase //private readonly ILogger _logger; private readonly IArticlesRepository _articles; private readonly ISourcesRepository _sources; + private readonly IAuthorTable _author; - public ArticlesController(IArticlesRepository articles, ISourcesRepository sources) + public ArticlesController(IArticlesRepository articles, ISourcesRepository sources, IAuthorTable author) { _articles = articles; _sources = sources; + _author = author; } [HttpGet(Name = "GetArticles")] @@ -59,11 +62,13 @@ public class ArticlesController : ControllerBase { var item = _articles.GetById(id); var sourceItem = _sources.GetById(item.SourceId); + var author = _author.GetBySourceIdAndNameAsync(sourceItem.Id, sourceItem.Name); + author.Wait(); return new OkObjectResult(new ArticleDetailsResult { IsSuccessful = true, - Item = ArticleDetailsDto.Convert(item, sourceItem) + Item = ArticleDetailsDto.Convert(item, sourceItem, author.Result) }); } diff --git a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs index ed50aae..01c1597 100644 --- a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs @@ -32,12 +32,15 @@ public class DiscordNotificationJobTest PubDate = DateTime.Now, Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", Description = "Please work", - AuthorName = "No one knows" }, new IconEntity() { Id = Guid.NewGuid(), FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" + }, new AuthorEntity + { + Id = Guid.NewGuid(), + Name = "test" }); webhookClient.SendMessage(msg); } @@ -59,7 +62,6 @@ public class DiscordNotificationJobTest Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", Description = "Please work", - AuthorName = "No one knows", CodeIsCommit = true }, new SourceEntity @@ -82,6 +84,9 @@ public class DiscordNotificationJobTest { CodeAllowCommits = false, CodeAllowReleases = true + }, new AuthorEntity + { + Name = "a" }); Assert.Fail("Expected a error to come back."); } -- 2.45.2