Compare commits

..

No commits in common. "a3085bb27fcaffebb4f12b299a3c79bbdbe00342" and "2ea971af541372eb8626b2fef7b3fc84a627579a" have entirely different histories.

15 changed files with 85 additions and 781 deletions

View File

@ -2,7 +2,6 @@ using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Newsbot.Collector.Domain.Dto; using Newsbot.Collector.Domain.Dto;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Results; using Newsbot.Collector.Domain.Results;
@ -16,13 +15,11 @@ public class ArticlesController : ControllerBase
//private readonly ILogger<ArticlesController> _logger; //private readonly ILogger<ArticlesController> _logger;
private readonly IArticlesRepository _articles; private readonly IArticlesRepository _articles;
private readonly ISourcesRepository _sources; private readonly ISourcesRepository _sources;
private readonly IAuthorTable _author;
public ArticlesController(IArticlesRepository articles, ISourcesRepository sources, IAuthorTable author) public ArticlesController(IArticlesRepository articles, ISourcesRepository sources)
{ {
_articles = articles; _articles = articles;
_sources = sources; _sources = sources;
_author = author;
} }
[HttpGet(Name = "GetArticles")] [HttpGet(Name = "GetArticles")]
@ -62,13 +59,11 @@ public class ArticlesController : ControllerBase
{ {
var item = _articles.GetById(id); var item = _articles.GetById(id);
var sourceItem = _sources.GetById(item.SourceId); var sourceItem = _sources.GetById(item.SourceId);
var author = _author.GetBySourceIdAndNameAsync(sourceItem.Id, sourceItem.Name);
author.Wait();
return new OkObjectResult(new ArticleDetailsResult return new OkObjectResult(new ArticleDetailsResult
{ {
IsSuccessful = true, IsSuccessful = true,
Item = ArticleDetailsDto.Convert(item, sourceItem, author.Result) Item = ArticleDetailsDto.Convert(item, sourceItem)
}); });
} }

View File

@ -1,575 +0,0 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Newsbot.Collector.Database;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace Newsbot.Collector.Database.Migrations
{
[DbContext(typeof(DatabaseContext))]
[Migration("20230805060324_add author table")]
partial class addauthortable
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b =>
{
b.Property<string>("Id")
.HasColumnType("text");
b.Property<string>("ConcurrencyStamp")
.IsConcurrencyToken()
.HasColumnType("text");
b.Property<string>("Name")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("NormalizedName")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.HasKey("Id");
b.HasIndex("NormalizedName")
.IsUnique()
.HasDatabaseName("RoleNameIndex");
b.ToTable("AspNetRoles", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("ClaimType")
.HasColumnType("text");
b.Property<string>("ClaimValue")
.HasColumnType("text");
b.Property<string>("RoleId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("RoleId");
b.ToTable("AspNetRoleClaims", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUser", b =>
{
b.Property<string>("Id")
.HasColumnType("text");
b.Property<int>("AccessFailedCount")
.HasColumnType("integer");
b.Property<string>("ConcurrencyStamp")
.IsConcurrencyToken()
.HasColumnType("text");
b.Property<string>("Email")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<bool>("EmailConfirmed")
.HasColumnType("boolean");
b.Property<bool>("LockoutEnabled")
.HasColumnType("boolean");
b.Property<DateTimeOffset?>("LockoutEnd")
.HasColumnType("timestamp with time zone");
b.Property<string>("NormalizedEmail")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("NormalizedUserName")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("PasswordHash")
.HasColumnType("text");
b.Property<string>("PhoneNumber")
.HasColumnType("text");
b.Property<bool>("PhoneNumberConfirmed")
.HasColumnType("boolean");
b.Property<string>("SecurityStamp")
.HasColumnType("text");
b.Property<bool>("TwoFactorEnabled")
.HasColumnType("boolean");
b.Property<string>("UserName")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.HasKey("Id");
b.HasIndex("NormalizedEmail")
.HasDatabaseName("EmailIndex");
b.HasIndex("NormalizedUserName")
.IsUnique()
.HasDatabaseName("UserNameIndex");
b.ToTable("AspNetUsers", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
b.Property<string>("ClaimType")
.HasColumnType("text");
b.Property<string>("ClaimValue")
.HasColumnType("text");
b.Property<string>("UserId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("UserId");
b.ToTable("AspNetUserClaims", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
{
b.Property<string>("LoginProvider")
.HasColumnType("text");
b.Property<string>("ProviderKey")
.HasColumnType("text");
b.Property<string>("ProviderDisplayName")
.HasColumnType("text");
b.Property<string>("UserId")
.IsRequired()
.HasColumnType("text");
b.HasKey("LoginProvider", "ProviderKey");
b.HasIndex("UserId");
b.ToTable("AspNetUserLogins", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
{
b.Property<string>("UserId")
.HasColumnType("text");
b.Property<string>("RoleId")
.HasColumnType("text");
b.HasKey("UserId", "RoleId");
b.HasIndex("RoleId");
b.ToTable("AspNetUserRoles", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
{
b.Property<string>("UserId")
.HasColumnType("text");
b.Property<string>("LoginProvider")
.HasColumnType("text");
b.Property<string>("Name")
.HasColumnType("text");
b.Property<string>("Value")
.HasColumnType("text");
b.HasKey("UserId", "LoginProvider", "Name");
b.ToTable("AspNetUserTokens", (string)null);
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.ArticlesEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Guid>("AuthorId")
.HasColumnType("uuid");
b.Property<bool>("CodeIsCommit")
.HasColumnType("boolean");
b.Property<bool>("CodeIsRelease")
.HasColumnType("boolean");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("PubDate")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.Property<string>("Tags")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Thumbnail")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.HasColumnType("text");
b.Property<string>("Video")
.IsRequired()
.HasColumnType("text");
b.Property<int>("VideoHeight")
.HasColumnType("integer");
b.Property<int>("VideoWidth")
.HasColumnType("integer");
b.HasKey("Id");
b.ToTable("Articles");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.AuthorEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Image")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("Authors");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordNotificationEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("CodeAllowCommits")
.HasColumnType("boolean");
b.Property<bool>("CodeAllowReleases")
.HasColumnType("boolean");
b.Property<Guid>("DiscordWebHookId")
.HasColumnType("uuid");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.Property<string>("UserId")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("DiscordNotification");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordQueueEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Guid>("ArticleId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("DiscordQueue");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Channel")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("Server")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<string>("UserId")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("UserId");
b.ToTable("DiscordWebhooks");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.IconEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("FileName")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Site")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("Icons");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.RefreshTokenEntity", b =>
{
b.Property<string>("Token")
.HasColumnType("text");
b.Property<DateTime>("CreatedDate")
.HasColumnType("timestamp with time zone");
b.Property<DateTime>("ExpiryDate")
.HasColumnType("timestamp with time zone");
b.Property<bool>("Invalidated")
.HasColumnType("boolean");
b.Property<string>("JwtId")
.HasColumnType("text");
b.Property<bool>("Used")
.HasColumnType("boolean");
b.Property<string>("UserId")
.HasColumnType("text");
b.HasKey("Token");
b.HasIndex("UserId");
b.ToTable("RefreshTokens");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SourceEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("Deleted")
.HasColumnType("boolean");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Site")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Tags")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Value")
.IsRequired()
.HasColumnType("text");
b.Property<string>("YoutubeId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("Sources");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.UserSourceSubscriptionEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<DateTimeOffset>("DateAdded")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.Property<string>("UserId")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("UserId");
b.ToTable("UserSourceSubscription");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
.WithMany()
.HasForeignKey("RoleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null)
.WithMany()
.HasForeignKey("RoleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<string>", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null)
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", "User")
.WithMany()
.HasForeignKey("UserId");
b.Navigation("User");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.RefreshTokenEntity", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", "User")
.WithMany()
.HasForeignKey("UserId");
b.Navigation("User");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.UserSourceSubscriptionEntity", b =>
{
b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", "User")
.WithMany()
.HasForeignKey("UserId");
b.Navigation("User");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -1,51 +0,0 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Newsbot.Collector.Database.Migrations
{
/// <inheritdoc />
public partial class addauthortable : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "AuthorImage",
table: "Articles");
migrationBuilder.DropColumn(
name: "AuthorName",
table: "Articles");
migrationBuilder.AddColumn<Guid>(
name: "AuthorId",
table: "Articles",
type: "uuid",
nullable: false,
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "AuthorId",
table: "Articles");
migrationBuilder.AddColumn<string>(
name: "AuthorImage",
table: "Articles",
type: "text",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "AuthorName",
table: "Articles",
type: "text",
nullable: false,
defaultValue: "");
}
}
}

View File

@ -224,8 +224,12 @@ namespace Newsbot.Collector.Database.Migrations
.ValueGeneratedOnAdd() .ValueGeneratedOnAdd()
.HasColumnType("uuid"); .HasColumnType("uuid");
b.Property<Guid>("AuthorId") b.Property<string>("AuthorImage")
.HasColumnType("uuid"); .HasColumnType("text");
b.Property<string>("AuthorName")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("CodeIsCommit") b.Property<bool>("CodeIsCommit")
.HasColumnType("boolean"); .HasColumnType("boolean");

View File

@ -1,4 +1,3 @@
using Microsoft.EntityFrameworkCore;
using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
@ -19,49 +18,30 @@ public class AuthorsTable : IAuthorTable
_context = context; _context = context;
} }
public async Task<AuthorEntity> NewAsync(AuthorEntity entity) public AuthorEntity New(AuthorEntity entity)
{ {
entity.Id = Guid.NewGuid(); entity.Id = Guid.NewGuid();
_context.Authors.Add(entity); _context.Authors.Add(entity);
await _context.SaveChangesAsync(); _context.SaveChanges();
return entity; return entity;
} }
public async Task<AuthorEntity> CreateIfMissingAsync(AuthorEntity entity) public List<AuthorEntity> ListBySourceId(Guid sourceId)
{ {
var res = await GetBySourceIdAndNameAsync(entity.SourceId, entity.Name); return _context.Authors
if (res is null) .Where(s => s.SourceId.Equals(sourceId)).ToList();
{
entity.Id = Guid.NewGuid();
_context.Authors.Add(entity);
await _context.SaveChangesAsync();
return entity;
} }
return res; public int TotalPosts(Guid id)
{
return _context.Authors.Count(x => x.Id.Equals(id));
} }
public async Task<List<AuthorEntity>> ListBySourceIdAsync(Guid sourceId) public AuthorEntity? GetBySourceIdAndName(Guid sourceId, string name)
{ {
return await _context.Authors return _context.Authors
.Where(s => s.SourceId.Equals(sourceId)).ToListAsync();
}
public async Task<int> TotalPostsAsync(Guid id)
{
return await _context.Authors.CountAsync(x => x.Id.Equals(id));
}
public async Task<AuthorEntity?> GetBySourceIdAndNameAsync(Guid sourceId, string name)
{
return await _context.Authors
.Where(s => s.SourceId.Equals(sourceId)) .Where(s => s.SourceId.Equals(sourceId))
.FirstOrDefaultAsync(n => n.Name.Equals(name)); .FirstOrDefault(n => n.Name.Equals(name));
} }
public async Task<AuthorEntity?> GetById(Guid id)
{
return await _context.Authors.FirstOrDefaultAsync(q => q.Id.Equals(id));
}
} }

View File

@ -19,9 +19,8 @@ public class ArticleDetailsDto
public string? AuthorImage { get; set; } public string? AuthorImage { get; set; }
public SourceDto? Source { get; set; } public SourceDto? Source { get; set; }
public AuthorEntity? Author { get; set; }
public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source, AuthorEntity? author) public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source)
{ {
return new ArticleDetailsDto return new ArticleDetailsDto
{ {
@ -35,8 +34,9 @@ public class ArticleDetailsDto
VideoWidth = article.VideoWidth, VideoWidth = article.VideoWidth,
Thumbnail = article.Thumbnail, Thumbnail = article.Thumbnail,
Description = article.Description, Description = article.Description,
Source = SourceDto.Convert(source), AuthorName = article.AuthorName,
AuthorImage = article.AuthorImage,
Source = SourceDto.Convert(source)
}; };
} }
} }

View File

@ -7,7 +7,6 @@ public class ArticleDto
{ {
public Guid ID { get; set; } public Guid ID { get; set; }
public Guid SourceID { get; set; } public Guid SourceID { get; set; }
public Guid AuthorId { get; set; }
public string[]? Tags { get; set; } public string[]? Tags { get; set; }
public string? Title { get; set; } public string? Title { get; set; }
public string? Url { get; set; } public string? Url { get; set; }
@ -17,6 +16,8 @@ public class ArticleDto
public int VideoWidth { get; set; } public int VideoWidth { get; set; }
public string? Thumbnail { get; set; } public string? Thumbnail { get; set; }
public string? Description { get; set; } public string? Description { get; set; }
public string? AuthorName { get; set; }
public string? AuthorImage { get; set; }
public static ArticleDto Convert(ArticlesModel article) public static ArticleDto Convert(ArticlesModel article)
{ {
return new ArticleDto return new ArticleDto
@ -32,6 +33,8 @@ public class ArticleDto
VideoWidth = article.VideoWidth, VideoWidth = article.VideoWidth,
Thumbnail = article.Thumbnail, Thumbnail = article.Thumbnail,
Description = article.Description, Description = article.Description,
AuthorName = article.AuthorName,
AuthorImage = article.AuthorImage,
}; };
} }
@ -41,7 +44,6 @@ public class ArticleDto
{ {
ID = article.Id, ID = article.Id,
SourceID = article.SourceId, SourceID = article.SourceId,
AuthorId = article.AuthorId,
Tags = article.Tags.Split(','), Tags = article.Tags.Split(','),
Title = article.Title, Title = article.Title,
Url = article.Url, Url = article.Url,
@ -50,7 +52,9 @@ public class ArticleDto
VideoHeight = article.VideoHeight, VideoHeight = article.VideoHeight,
VideoWidth = article.VideoWidth, VideoWidth = article.VideoWidth,
Thumbnail = article.Thumbnail, Thumbnail = article.Thumbnail,
Description = article.Description Description = article.Description,
AuthorName = article.AuthorName,
AuthorImage = article.AuthorImage,
}; };
} }
} }

View File

@ -1,9 +0,0 @@
namespace Newsbot.Collector.Domain.Dto;
public class AuthorDto
{
public Guid Id { get; set; }
public Guid SourceId { get; set; }
public string? Name { get; set; }
public string? Image { get; set; }
}

View File

@ -4,7 +4,6 @@ public class ArticlesEntity
{ {
public Guid Id { get; set; } public Guid Id { get; set; }
public Guid SourceId { get; set; } public Guid SourceId { get; set; }
public Guid AuthorId { get; set; }
public string Tags { get; set; } = ""; public string Tags { get; set; } = "";
public string Title { get; set; } = ""; public string Title { get; set; } = "";
public string? Url { get; set; } public string? Url { get; set; }
@ -14,6 +13,8 @@ public class ArticlesEntity
public int VideoWidth { get; set; } = 0; public int VideoWidth { get; set; } = 0;
public string Thumbnail { get; set; } = ""; public string Thumbnail { get; set; } = "";
public string Description { get; set; } = ""; public string Description { get; set; } = "";
public string AuthorName { get; set; } = "";
public string? AuthorImage { get; set; }
public bool CodeIsRelease { get; set; } public bool CodeIsRelease { get; set; }
public bool CodeIsCommit { get; set; } public bool CodeIsCommit { get; set; }
} }

View File

@ -4,11 +4,8 @@ namespace Newsbot.Collector.Domain.Interfaces;
public interface IAuthorTable public interface IAuthorTable
{ {
Task<AuthorEntity> NewAsync(AuthorEntity entity); AuthorEntity New(AuthorEntity entity);
List<AuthorEntity> ListBySourceId(Guid sourceId);
Task<AuthorEntity> CreateIfMissingAsync(AuthorEntity entity); int TotalPosts(Guid id);
Task<List<AuthorEntity>> ListBySourceIdAsync(Guid sourceId); AuthorEntity? GetBySourceIdAndName(Guid sourceId, string name);
Task<int> TotalPostsAsync(Guid id);
Task<AuthorEntity?> GetBySourceIdAndNameAsync(Guid sourceId, string name);
Task<AuthorEntity?> GetById(Guid id);
} }

View File

@ -0,0 +1,9 @@
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
///
public interface ICollector : IHangfireJob
{
List<ArticlesModel> Collect();
}

View File

@ -30,7 +30,6 @@ public class CodeProjectWatcherJob
private ILogger _logger; private ILogger _logger;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
private ISourcesRepository _source; private ISourcesRepository _source;
private IAuthorTable _author;
public CodeProjectWatcherJob() public CodeProjectWatcherJob()
{ {
@ -38,14 +37,12 @@ public class CodeProjectWatcherJob
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_source = new SourcesTable(""); _source = new SourcesTable("");
_logger = JobLogger.GetLogger("", JobName); _logger = JobLogger.GetLogger("", JobName);
_author = new AuthorsTable("");
} }
public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options) public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@ -55,7 +52,6 @@ public class CodeProjectWatcherJob
{ {
options.ConnectionStrings ??= new ConfigSectionConnectionStrings(); options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? ""); _articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_author = new AuthorsTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? ""); _queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? ""); _source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName); _logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
@ -161,29 +157,17 @@ public class CodeProjectWatcherJob
}); });
parser.Parse(); parser.Parse();
if (item.Authors[0].Name is null)
{
_logger.Warning("Author was missing from the record and will continue with a missing author");
}
var authorExists = _author.CreateIfMissingAsync(new AuthorEntity
{
Name = item.Authors[0].Name,
SourceId = source.Id,
Image = "",
});
authorExists.Wait();
var a = new ArticlesEntity var a = new ArticlesEntity
{ {
SourceId = source.Id, SourceId = source.Id,
AuthorId = authorExists.Result.Id,
Tags = source.Tags, Tags = source.Tags,
Title = item.Title.Text, Title = item.Title.Text,
Url = itemUrl, Url = itemUrl,
PubDate = item.LastUpdatedTime.DateTime, PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image, Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text, Description = item.Title.Text,
AuthorName = item.Authors[0].Name ?? "",
AuthorImage = item.Authors[0].Uri ?? "",
CodeIsRelease = isRelease, CodeIsRelease = isRelease,
CodeIsCommit = isCommit, CodeIsCommit = isCommit,
}; };

View File

@ -36,7 +36,6 @@ public class DiscordNotificationJob
private ILogger _logger; private ILogger _logger;
//private DatabaseContext _databaseContext; //private DatabaseContext _databaseContext;
private IArticlesRepository _article; private IArticlesRepository _article;
private IAuthorTable _author;
private IIconsRepository _icons; private IIconsRepository _icons;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
private ISourcesRepository _sources; private ISourcesRepository _sources;
@ -47,7 +46,6 @@ public class DiscordNotificationJob
{ {
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_article = new ArticlesTable(""); _article = new ArticlesTable("");
_author = new AuthorsTable("");
_webhook = new DiscordWebhooksTable(""); _webhook = new DiscordWebhooksTable("");
_sources = new SourcesTable(""); _sources = new SourcesTable("");
_subs = new DiscordNotificationTable(""); _subs = new DiscordNotificationTable("");
@ -94,6 +92,7 @@ public class DiscordNotificationJob
_logger.Debug($"{JobName} - Processing {request.Id}"); _logger.Debug($"{JobName} - Processing {request.Id}");
// Get all details on the article in the queue // Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleId); var articleDetails = _article.GetById(request.ArticleId);
// Get the details of the source // Get the details of the source
var sourceDetails = _sources.GetById(articleDetails.SourceId); var sourceDetails = _sources.GetById(articleDetails.SourceId);
if (sourceDetails.Id == Guid.Empty) if (sourceDetails.Id == Guid.Empty)
@ -104,9 +103,6 @@ public class DiscordNotificationJob
return; return;
} }
var author = _author.GetBySourceIdAndNameAsync(sourceDetails.Id, sourceDetails.Name);
author.Wait();
var sourceIcon = new IconEntity(); var sourceIcon = new IconEntity();
try try
{ {
@ -122,13 +118,13 @@ public class DiscordNotificationJob
var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id); var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
foreach (var sub in allSubscriptions) foreach (var sub in allSubscriptions)
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub, author.Result ?? new AuthorEntity()); SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub);
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id); _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id);
_queue.Delete(request.Id); _queue.Delete(request.Id);
} }
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub, AuthorEntity authorEntity) public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, DiscordNotificationEntity sub)
{ {
// Check if the subscription code flags // Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip. // If the article is a code commit and the subscription does not want them, skip.
@ -144,7 +140,7 @@ public class DiscordNotificationJob
var client = new DiscordClient(discordDetails.Url); var client = new DiscordClient(discordDetails.Url);
try try
{ {
client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon, authorEntity)); client.SendMessage(GenerateDiscordMessage(sourceDetails, articleDetails, sourceIcon));
} }
catch (Exception e) catch (Exception e)
{ {
@ -158,7 +154,7 @@ public class DiscordNotificationJob
Thread.Sleep(3000); Thread.Sleep(3000);
} }
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon, AuthorEntity author) public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon)
{ {
var embed = new DiscordMessageEmbed var embed = new DiscordMessageEmbed
{ {
@ -167,7 +163,7 @@ public class DiscordNotificationJob
Description = MessageValidation.ConvertHtmlCodes(article.Description), Description = MessageValidation.ConvertHtmlCodes(article.Description),
Author = new DiscordMessageEmbedAuthor Author = new DiscordMessageEmbedAuthor
{ {
Name = author.Name, Name = article.AuthorName,
IconUrl = icon.FileName IconUrl = icon.FileName
}, },
Footer = new DiscordMessageEmbedFooter Footer = new DiscordMessageEmbedFooter
@ -193,7 +189,7 @@ public class DiscordNotificationJob
Url = article.Thumbnail Url = article.Thumbnail
}; };
embed.Author.IconUrl = author.Image ?? ""; if (article.AuthorImage is not null && article.AuthorImage != "") embed.Author.IconUrl = article.AuthorImage;
return new DiscordMessage return new DiscordMessage
{ {

View File

@ -25,7 +25,6 @@ public class YoutubeWatcherJob
private readonly YoutubeWatcherJobOptions _options; private readonly YoutubeWatcherJobOptions _options;
private IArticlesRepository _articles; private IArticlesRepository _articles;
private IAuthorTable _author;
private IIconsRepository _icons; private IIconsRepository _icons;
private ILogger _logger; private ILogger _logger;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
@ -35,7 +34,6 @@ public class YoutubeWatcherJob
{ {
_options = new YoutubeWatcherJobOptions(); _options = new YoutubeWatcherJobOptions();
_articles = new ArticlesTable(""); _articles = new ArticlesTable("");
_author = new AuthorsTable("");
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_source = new SourcesTable(""); _source = new SourcesTable("");
_icons = new IconsTable(""); _icons = new IconsTable("");
@ -45,7 +43,6 @@ public class YoutubeWatcherJob
public void InitAndExecute(YoutubeWatcherJobOptions options) public void InitAndExecute(YoutubeWatcherJobOptions options)
{ {
_articles = new ArticlesTable(options.DatabaseConnectionString ?? ""); _articles = new ArticlesTable(options.DatabaseConnectionString ?? "");
_author = new AuthorsTable(options.DatabaseConnectionString ?? "");
_queue = new DiscordQueueTable(options.DatabaseConnectionString ?? ""); _queue = new DiscordQueueTable(options.DatabaseConnectionString ?? "");
_source = new SourcesTable(options.DatabaseConnectionString ?? ""); _source = new SourcesTable(options.DatabaseConnectionString ?? "");
_icons = new IconsTable(options.DatabaseConnectionString ?? ""); _icons = new IconsTable(options.DatabaseConnectionString ?? "");
@ -80,23 +77,11 @@ public class YoutubeWatcherJob
_logger.Information($"{JobName} - Checking '{source.Name}'"); _logger.Information($"{JobName} - Checking '{source.Name}'");
var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}"; var url = $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
var newVideos = FindMissingPosts(url, source); var newVideos = CheckFeed(url, source);
_logger.Debug($"{JobName} - Collected {newVideos.Count} new videos"); _logger.Debug($"{JobName} - Collected {newVideos.Count} new videos");
foreach (var video in newVideos) foreach (var video in newVideos)
{ {
var author = _author.GetById(video.AuthorId); _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
author.Wait();
if (author.Result is null)
{
_logger.Warning("Missing author record for article id {VideoId}", video.Id);
}
else
{
_logger.Debug("{JobName} - {ResultName} \'{VideoTitle}\' was found", JobName, author.Result.Name, video.Title);
}
_articles.New(video); _articles.New(video);
_queue.New(new DiscordQueueEntity _queue.New(new DiscordQueueEntity
{ {
@ -119,31 +104,22 @@ public class YoutubeWatcherJob
var id = pageReader.Data.Header.YoutubeChannelID ?? ""; var id = pageReader.Data.Header.YoutubeChannelID ?? "";
if (id == "") if (id == "")
_logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url"), ""); _logger.Error(new Exception($"{JobName} - Unable to find the Youtube Channel ID for the requested url."),
url);
return id; return id;
} }
private List<ArticlesEntity> FindMissingPosts(string url, SourceEntity source) private List<ArticlesEntity> CheckFeed(string url, SourceEntity source)
{ {
var videos = new List<ArticlesEntity>(); var videos = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url); using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
foreach (var post in feed.Items.ToList()) foreach (var post in feed.Items.ToList())
{
var article = CheckFeedItem(post, source.Id);
if (article is null) continue;
videos.Add(article);
}
return videos;
}
private ArticlesEntity? CheckFeedItem(SyndicationItem post, Guid sourceId)
{ {
var articleUrl = post.Links[0].Uri.AbsoluteUri; var articleUrl = post.Links[0].Uri.AbsoluteUri;
if (IsThisUrlKnown(articleUrl)) return null; if (IsThisUrlKnown(articleUrl)) continue;
var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions var videoDetails = new HtmlPageReader(new HtmlPageReaderOptions
{ {
@ -151,28 +127,26 @@ public class YoutubeWatcherJob
}); });
videoDetails.Parse(); videoDetails.Parse();
var author = _author.CreateIfMissingAsync(new AuthorEntity
{
Image = post.Authors[0].Uri,
Name = post.Authors[0].Name
});
author.Wait();
var article = new ArticlesEntity var article = new ArticlesEntity
{ {
//Todo add the icon //Todo add the icon
AuthorId = author.Result.Id, AuthorName = post.Authors[0].Name,
Title = post.Title.Text, Title = post.Title.Text,
Tags = FetchTags(post), Tags = FetchTags(post),
Url = articleUrl, Url = articleUrl,
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image, Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description, Description = videoDetails.Data.Header.Description,
SourceId = sourceId, SourceId = source.Id,
Video = "true" Video = "true"
}; };
return article; videos.Add(article);
Thread.Sleep(_options.SleepTimer);
}
return videos;
} }
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)

View File

@ -32,15 +32,12 @@ public class DiscordNotificationJobTest
PubDate = DateTime.Now, PubDate = DateTime.Now,
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
Description = "Please work", Description = "Please work",
AuthorName = "No one knows"
}, },
new IconEntity() new IconEntity()
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png"
}, new AuthorEntity
{
Id = Guid.NewGuid(),
Name = "test"
}); });
webhookClient.SendMessage(msg); webhookClient.SendMessage(msg);
} }
@ -62,6 +59,7 @@ public class DiscordNotificationJobTest
Thumbnail = Thumbnail =
"https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
Description = "Please work", Description = "Please work",
AuthorName = "No one knows",
CodeIsCommit = true CodeIsCommit = true
}, },
new SourceEntity new SourceEntity
@ -84,9 +82,6 @@ public class DiscordNotificationJobTest
{ {
CodeAllowCommits = false, CodeAllowCommits = false,
CodeAllowReleases = true CodeAllowReleases = true
}, new AuthorEntity
{
Name = "a"
}); });
Assert.Fail("Expected a error to come back."); Assert.Fail("Expected a error to come back.");
} }