From 8149ce21517974315083f6f498a4476d03dff7ed Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 19 Jun 2023 15:36:43 -0700 Subject: [PATCH 01/27] ef was added to the project --- Newsbot.Collector.Database/DatabaseContext.cs | 39 +++++++++++++++++++ .../DesignTimeContext.cs | 15 +++++++ .../Newsbot.Collector.Database.csproj | 8 +++- 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 Newsbot.Collector.Database/DatabaseContext.cs create mode 100644 Newsbot.Collector.Database/DesignTimeContext.cs diff --git a/Newsbot.Collector.Database/DatabaseContext.cs b/Newsbot.Collector.Database/DatabaseContext.cs new file mode 100644 index 0000000..46e1cf9 --- /dev/null +++ b/Newsbot.Collector.Database/DatabaseContext.cs @@ -0,0 +1,39 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Entities; + +namespace Newsbot.Collector.Database; + +public class DatabaseContext : DbContext +{ + public DbSet Articles { get; set; } = null!; + public DbSet DiscordQueue { get; set; } = null!; + public DbSet DiscordWebhooks { get; set; } = null!; + public DbSet Icons { get; set; } = null!; + public DbSet Sources { get; set; } = null!; + public DbSet Subscriptions { get; set; } = null!; + + private string ConnectionString { get; set; } + + public DatabaseContext(IConfiguration appsettings) + { + var connString = appsettings.GetConnectionString(ConfigConnectionStringConst.Database); + ConnectionString = connString ?? ""; + } + + public DatabaseContext(string connectionString) + { + ConnectionString = connectionString; + } + + protected override void OnConfiguring(DbContextOptionsBuilder options) + { + options.UseNpgsql(ConnectionString); + } + + public DatabaseContext(DbContextOptions options) + : base(options) + { + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Database/DesignTimeContext.cs b/Newsbot.Collector.Database/DesignTimeContext.cs new file mode 100644 index 0000000..ab717c9 --- /dev/null +++ b/Newsbot.Collector.Database/DesignTimeContext.cs @@ -0,0 +1,15 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; + +namespace Newsbot.Collector.Database; + +public class DesignTimeContext :IDesignTimeDbContextFactory +{ + public DatabaseContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder(); + // pass your design time connection string here + optionsBuilder.UseNpgsql(""); + return new DatabaseContext(optionsBuilder.Options); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj b/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj index 8f50477..c191f7f 100644 --- a/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj +++ b/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj @@ -6,8 +6,14 @@ + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + - + + From d2a5d2ed8108e7a140fe8c5994b469f0fa9e2d21 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 19 Jun 2023 15:37:05 -0700 Subject: [PATCH 02/27] Migrations have been moved and new ef migrations made --- ...30619043102_MigrationFromGoose.Designer.cs | 223 ++++++++++++++++++ .../20230619043102_MigrationFromGoose.cs | 140 +++++++++++ .../DatabaseContextModelSnapshot.cs | 220 +++++++++++++++++ .../20220522083756_init.sql | 0 .../20220529082459_seed.sql | 0 .../20220619085634_subscriptions.sql | 0 .../20221207213427_source_delete.sql | 0 .../20230316185340_sources_youtube_id.sql | 0 .../20230326223545_icon_sourceid.sql | 0 .../20230411071020_article_code_columns.sql | 0 ...230413203006_subscription_code_options.sql | 0 .../20230413212643_icon_sourceid.sql | 0 12 files changed, 583 insertions(+) create mode 100644 Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.Designer.cs create mode 100644 Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.cs create mode 100644 Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20220522083756_init.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20220529082459_seed.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20220619085634_subscriptions.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20221207213427_source_delete.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20230316185340_sources_youtube_id.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20230326223545_icon_sourceid.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20230411071020_article_code_columns.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20230413203006_subscription_code_options.sql (100%) rename Newsbot.Collector.Database/{Migrations => OldMigrations}/20230413212643_icon_sourceid.sql (100%) diff --git a/Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.Designer.cs b/Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.Designer.cs new file mode 100644 index 0000000..b31a819 --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.Designer.cs @@ -0,0 +1,223 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Newsbot.Collector.Database; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Newsbot.Collector.Database.Migrations +{ + [DbContext(typeof(DatabaseContext))] + [Migration("20230619043102_MigrationFromGoose")] + partial class MigrationFromGoose + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.ArticlesEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AuthorImage") + .HasColumnType("text"); + + b.Property("AuthorName") + .IsRequired() + .HasColumnType("text"); + + b.Property("CodeIsCommit") + .HasColumnType("boolean"); + + b.Property("CodeIsRelease") + .HasColumnType("boolean"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("PubDate") + .HasColumnType("timestamp with time zone"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text"); + + b.Property("Thumbnail") + .IsRequired() + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .HasColumnType("text"); + + b.Property("Video") + .IsRequired() + .HasColumnType("text"); + + b.Property("VideoHeight") + .HasColumnType("integer"); + + b.Property("VideoWidth") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Articles"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordQueueEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ArticleId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("DiscordQueue"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Channel") + .IsRequired() + .HasColumnType("text"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("Server") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DiscordWebhooks"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.IconEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("Site") + .IsRequired() + .HasColumnType("text"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Icons"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SourceEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Deleted") + .HasColumnType("boolean"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Site") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("YoutubeId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Sources"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SubscriptionEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CodeAllowCommits") + .HasColumnType("boolean"); + + b.Property("CodeAllowReleases") + .HasColumnType("boolean"); + + b.Property("DiscordWebHookId") + .HasColumnType("uuid"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Subscriptions"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.cs b/Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.cs new file mode 100644 index 0000000..122daeb --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.cs @@ -0,0 +1,140 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.Configuration; + +#nullable disable + +namespace Newsbot.Collector.Database.Migrations +{ + /// + public partial class MigrationFromGoose : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "Articles", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + SourceId = table.Column(type: "uuid", nullable: false), + Tags = table.Column(type: "text", nullable: false), + Title = table.Column(type: "text", nullable: false), + Url = table.Column(type: "text", nullable: true), + PubDate = table.Column(type: "timestamp with time zone", nullable: false), + Video = table.Column(type: "text", nullable: false), + VideoHeight = table.Column(type: "integer", nullable: false), + VideoWidth = table.Column(type: "integer", nullable: false), + Thumbnail = table.Column(type: "text", nullable: false), + Description = table.Column(type: "text", nullable: false), + AuthorName = table.Column(type: "text", nullable: false), + AuthorImage = table.Column(type: "text", nullable: true), + CodeIsRelease = table.Column(type: "boolean", nullable: false), + CodeIsCommit = table.Column(type: "boolean", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Articles", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "DiscordQueue", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + ArticleId = table.Column(type: "uuid", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_DiscordQueue", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "DiscordWebhooks", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Url = table.Column(type: "text", nullable: false), + Server = table.Column(type: "text", nullable: false), + Channel = table.Column(type: "text", nullable: false), + Enabled = table.Column(type: "boolean", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_DiscordWebhooks", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Icons", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + FileName = table.Column(type: "text", nullable: false), + Site = table.Column(type: "text", nullable: false), + SourceId = table.Column(type: "uuid", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Icons", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Sources", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Site = table.Column(type: "text", nullable: false), + Name = table.Column(type: "text", nullable: false), + Source = table.Column(type: "text", nullable: false), + Type = table.Column(type: "text", nullable: false), + Value = table.Column(type: "text", nullable: false), + Enabled = table.Column(type: "boolean", nullable: false), + Url = table.Column(type: "text", nullable: false), + Tags = table.Column(type: "text", nullable: false), + Deleted = table.Column(type: "boolean", nullable: false), + YoutubeId = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Sources", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Subscriptions", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + CodeAllowReleases = table.Column(type: "boolean", nullable: false), + CodeAllowCommits = table.Column(type: "boolean", nullable: false), + SourceId = table.Column(type: "uuid", nullable: false), + DiscordWebHookId = table.Column(type: "uuid", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Subscriptions", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "Articles"); + + migrationBuilder.DropTable( + name: "DiscordQueue"); + + migrationBuilder.DropTable( + name: "DiscordWebhooks"); + + migrationBuilder.DropTable( + name: "Icons"); + + migrationBuilder.DropTable( + name: "Sources"); + + migrationBuilder.DropTable( + name: "Subscriptions"); + } + } +} diff --git a/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs b/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs new file mode 100644 index 0000000..a2c6ecb --- /dev/null +++ b/Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs @@ -0,0 +1,220 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Newsbot.Collector.Database; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Newsbot.Collector.Database.Migrations +{ + [DbContext(typeof(DatabaseContext))] + partial class DatabaseContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.ArticlesEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AuthorImage") + .HasColumnType("text"); + + b.Property("AuthorName") + .IsRequired() + .HasColumnType("text"); + + b.Property("CodeIsCommit") + .HasColumnType("boolean"); + + b.Property("CodeIsRelease") + .HasColumnType("boolean"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("PubDate") + .HasColumnType("timestamp with time zone"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text"); + + b.Property("Thumbnail") + .IsRequired() + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .HasColumnType("text"); + + b.Property("Video") + .IsRequired() + .HasColumnType("text"); + + b.Property("VideoHeight") + .HasColumnType("integer"); + + b.Property("VideoWidth") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Articles"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordQueueEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ArticleId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("DiscordQueue"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Channel") + .IsRequired() + .HasColumnType("text"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("Server") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DiscordWebhooks"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.IconEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("Site") + .IsRequired() + .HasColumnType("text"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Icons"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SourceEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Deleted") + .HasColumnType("boolean"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Site") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Tags") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("YoutubeId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Sources"); + }); + + modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SubscriptionEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CodeAllowCommits") + .HasColumnType("boolean"); + + b.Property("CodeAllowReleases") + .HasColumnType("boolean"); + + b.Property("DiscordWebHookId") + .HasColumnType("uuid"); + + b.Property("SourceId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Subscriptions"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Newsbot.Collector.Database/Migrations/20220522083756_init.sql b/Newsbot.Collector.Database/OldMigrations/20220522083756_init.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20220522083756_init.sql rename to Newsbot.Collector.Database/OldMigrations/20220522083756_init.sql diff --git a/Newsbot.Collector.Database/Migrations/20220529082459_seed.sql b/Newsbot.Collector.Database/OldMigrations/20220529082459_seed.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20220529082459_seed.sql rename to Newsbot.Collector.Database/OldMigrations/20220529082459_seed.sql diff --git a/Newsbot.Collector.Database/Migrations/20220619085634_subscriptions.sql b/Newsbot.Collector.Database/OldMigrations/20220619085634_subscriptions.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20220619085634_subscriptions.sql rename to Newsbot.Collector.Database/OldMigrations/20220619085634_subscriptions.sql diff --git a/Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql b/Newsbot.Collector.Database/OldMigrations/20221207213427_source_delete.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql rename to Newsbot.Collector.Database/OldMigrations/20221207213427_source_delete.sql diff --git a/Newsbot.Collector.Database/Migrations/20230316185340_sources_youtube_id.sql b/Newsbot.Collector.Database/OldMigrations/20230316185340_sources_youtube_id.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20230316185340_sources_youtube_id.sql rename to Newsbot.Collector.Database/OldMigrations/20230316185340_sources_youtube_id.sql diff --git a/Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql b/Newsbot.Collector.Database/OldMigrations/20230326223545_icon_sourceid.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql rename to Newsbot.Collector.Database/OldMigrations/20230326223545_icon_sourceid.sql diff --git a/Newsbot.Collector.Database/Migrations/20230411071020_article_code_columns.sql b/Newsbot.Collector.Database/OldMigrations/20230411071020_article_code_columns.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20230411071020_article_code_columns.sql rename to Newsbot.Collector.Database/OldMigrations/20230411071020_article_code_columns.sql diff --git a/Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql b/Newsbot.Collector.Database/OldMigrations/20230413203006_subscription_code_options.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql rename to Newsbot.Collector.Database/OldMigrations/20230413203006_subscription_code_options.sql diff --git a/Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql b/Newsbot.Collector.Database/OldMigrations/20230413212643_icon_sourceid.sql similarity index 100% rename from Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql rename to Newsbot.Collector.Database/OldMigrations/20230413212643_icon_sourceid.sql From eeb6327046f3784dbdc68bcbacc82b9b7b1d90b7 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 19 Jun 2023 15:37:21 -0700 Subject: [PATCH 03/27] docker compose was updated --- docker-compose.dev.yaml | 18 ++++++++++++++++++ ...compose.yaml => docker-compose.example.yaml | 8 +------- 2 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 docker-compose.dev.yaml rename docker-compose.yaml => docker-compose.example.yaml (93%) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml new file mode 100644 index 0000000..b0c094d --- /dev/null +++ b/docker-compose.dev.yaml @@ -0,0 +1,18 @@ +# docker compose -f docker-compose.dev.yaml up -d + +version: "3" + +volumes: + db: + +services: + db: + image: postgres:latest + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - "5432:5432" + volumes: + - db:/var/lib/postgresql/data \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.example.yaml similarity index 93% rename from docker-compose.yaml rename to docker-compose.example.yaml index 372c939..0ae311c 100644 --- a/docker-compose.yaml +++ b/docker-compose.example.yaml @@ -10,7 +10,7 @@ services: db: image: postgres:latest environment: - POSTGRES_USER: ${PostgresUser} + POSTGRES_USER: ${PostgresUser}] POSTGRES_PASSWORD: ${PostgresPassword} POSTGRES_DB: ${PostgresDatabaseName} ports: @@ -25,12 +25,6 @@ services: timeout: "30s" retries: 5 - adminer: - image: adminer - restart: always - ports: - - "8080:8080" - api: image: newsbot.collector:latest environment: From 79c4da89e4da7b195a8bf4f0ab34e59b348ffe38 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 19 Jun 2023 15:38:17 -0700 Subject: [PATCH 04/27] Entities have been made to isolate db models --- .../Entities/ArticlesEntity.cs | 20 +++++++++++++++++++ .../Entities/AuthorEntity.cs | 9 +++++++++ .../Entities/DiscordQueueEntity.cs | 7 +++++++ .../Entities/DiscordWebhookEntity.cs | 10 ++++++++++ .../Entities/IconEntity.cs | 10 ++++++++++ .../Entities/SourceEntity.cs | 18 +++++++++++++++++ .../Entities/SubscriptionEntity.cs | 11 ++++++++++ 7 files changed, 85 insertions(+) create mode 100644 Newsbot.Collector.Domain/Entities/ArticlesEntity.cs create mode 100644 Newsbot.Collector.Domain/Entities/AuthorEntity.cs create mode 100644 Newsbot.Collector.Domain/Entities/DiscordQueueEntity.cs create mode 100644 Newsbot.Collector.Domain/Entities/DiscordWebhookEntity.cs create mode 100644 Newsbot.Collector.Domain/Entities/IconEntity.cs create mode 100644 Newsbot.Collector.Domain/Entities/SourceEntity.cs create mode 100644 Newsbot.Collector.Domain/Entities/SubscriptionEntity.cs diff --git a/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs b/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs new file mode 100644 index 0000000..626e2c1 --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/ArticlesEntity.cs @@ -0,0 +1,20 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class ArticlesEntity +{ + public Guid Id { get; set; } + public Guid SourceId { get; set; } + public string Tags { get; set; } = ""; + public string Title { get; set; } = ""; + public string? Url { get; set; } + public DateTime PubDate { get; set; } = DateTime.Now; + public string Video { get; set; } = ""; + public int VideoHeight { get; set; } = 0; + public int VideoWidth { get; set; } = 0; + public string Thumbnail { get; set; } = ""; + public string Description { get; set; } = ""; + public string AuthorName { get; set; } = ""; + public string? AuthorImage { get; set; } + public bool CodeIsRelease { get; set; } + public bool CodeIsCommit { get; set; } +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/AuthorEntity.cs b/Newsbot.Collector.Domain/Entities/AuthorEntity.cs new file mode 100644 index 0000000..36d06a8 --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/AuthorEntity.cs @@ -0,0 +1,9 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class AuthorEntity +{ + public Guid Id { get; set; } + public Guid SourceId { get; set; } + public string Name { get; set; } = ""; + public string Image { get; set; } = ""; +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/DiscordQueueEntity.cs b/Newsbot.Collector.Domain/Entities/DiscordQueueEntity.cs new file mode 100644 index 0000000..9e29842 --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/DiscordQueueEntity.cs @@ -0,0 +1,7 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class DiscordQueueEntity +{ + public Guid Id { get; set; } + public Guid ArticleId { get; set; } +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/DiscordWebhookEntity.cs b/Newsbot.Collector.Domain/Entities/DiscordWebhookEntity.cs new file mode 100644 index 0000000..7e7ec40 --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/DiscordWebhookEntity.cs @@ -0,0 +1,10 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class DiscordWebhookEntity +{ + public Guid Id { get; set; } + public string Url { get; set; } = ""; + public string Server { get; set; } = ""; + public string Channel { get; set; } = ""; + public bool Enabled { get; set; } +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/IconEntity.cs b/Newsbot.Collector.Domain/Entities/IconEntity.cs new file mode 100644 index 0000000..c4f17b6 --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/IconEntity.cs @@ -0,0 +1,10 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class IconEntity +{ + public Guid Id { get; set; } + public string FileName { get; set; } = ""; + public string Site { get; set; } = ""; + + public Guid SourceId { get; set; } +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/SourceEntity.cs b/Newsbot.Collector.Domain/Entities/SourceEntity.cs new file mode 100644 index 0000000..e7bc247 --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/SourceEntity.cs @@ -0,0 +1,18 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class SourceEntity +{ + public Guid Id { get; set; } + public string Site { get; set; } = ""; + public string Name { get; set; } = ""; + + // Source use to define the worker to query with but moving to Type as it was not used really. + public string Source { get; set; } = ""; + public string Type { get; set; } = ""; + public string Value { get; set; } = ""; + public bool Enabled { get; set; } + public string Url { get; set; } = ""; + public string Tags { get; set; } = ""; + public bool Deleted { get; set; } + public string YoutubeId { get; set; } = ""; +} \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Entities/SubscriptionEntity.cs b/Newsbot.Collector.Domain/Entities/SubscriptionEntity.cs new file mode 100644 index 0000000..ae405ed --- /dev/null +++ b/Newsbot.Collector.Domain/Entities/SubscriptionEntity.cs @@ -0,0 +1,11 @@ +namespace Newsbot.Collector.Domain.Entities; + +public class SubscriptionEntity +{ + public Guid Id { get; set; } + public bool CodeAllowReleases { get; set; } + public bool CodeAllowCommits { get; set; } + + public Guid SourceId { get; set; } + public Guid DiscordWebHookId { get; set; } +} \ No newline at end of file From bb8b3c4b74fffc5cece571c84535135c11dc762c Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 19 Jun 2023 15:38:33 -0700 Subject: [PATCH 05/27] removed goose from Dockerfile --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 153e250..ab3e3ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM golang:latest as goose +#FROM golang:latest as goose -RUN go install github.com/pressly/goose/v3/cmd/goose@latest +#RUN go install github.com/pressly/goose/v3/cmd/goose@latest FROM mcr.microsoft.com/dotnet/sdk:7.0.103 as build @@ -26,9 +26,9 @@ ENV DOTNET_URLS=http://*:5000 WORKDIR /app -RUN mkdir /migrations +#RUN mkdir /migrations COPY --from=publish /app/build /app -COPY --from=build ./app/Newsbot.Collector.Database/Migrations /app/migrations -COPY --from=goose /go/bin/goose /app +#COPY --from=build ./app/Newsbot.Collector.Database/Migrations /app/migrations +#COPY --from=goose /go/bin/goose /app CMD [ "dotnet", "Newsbot.Collector.Api.dll" ] \ No newline at end of file From ae8b0266e5f60ee7343d4eda81ff1553ebd5d6c6 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 19 Jun 2023 15:38:55 -0700 Subject: [PATCH 06/27] added command to generate migration bundle to makefile --- .gitignore | 2 ++ makefile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 845d631..0394bff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore seed.secrects.json +efbundle +efbundle.exe out/ #Docker Compose Secrets diff --git a/makefile b/makefile index faab6d1..2341cb6 100644 --- a/makefile +++ b/makefile @@ -29,3 +29,6 @@ migrate-dev-down: ## revert sql migrations to dev db migrate-refresh: ## Rolls back all migrations goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset + +ef-bundle: ## Create a migration bundle + dotnet ef migrations bundle --project Newsbot.Collector.Database \ No newline at end of file From a343fa2d7f8c27432d42c48f48370fc7b28a13e9 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 22 Jun 2023 08:04:18 -0700 Subject: [PATCH 07/27] started to update articlesTable with entities --- Newsbot.Collector.Domain/Dto/ArticleDto.cs | 23 ++++++++++++++++++- .../Interfaces/IArticlesRepository.cs | 5 ++-- makefile | 15 ++++-------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Newsbot.Collector.Domain/Dto/ArticleDto.cs b/Newsbot.Collector.Domain/Dto/ArticleDto.cs index 0532f32..7a55872 100644 --- a/Newsbot.Collector.Domain/Dto/ArticleDto.cs +++ b/Newsbot.Collector.Domain/Dto/ArticleDto.cs @@ -1,3 +1,4 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Dto; @@ -17,7 +18,7 @@ public class ArticleDto public string? Description { get; set; } public string? AuthorName { get; set; } public string? AuthorImage { get; set; } - public static ArticleDto Convert(ArticlesModel article) + public static ArticleDto Convert(ArticlesModel article) { return new ArticleDto { @@ -36,4 +37,24 @@ public class ArticleDto AuthorImage = article.AuthorImage, }; } + + public static ArticleDto Convert(ArticlesEntity article) + { + return new ArticleDto + { + ID = article.Id, + SourceID = article.SourceId, + Tags = article.Tags.Split(','), + Title = article.Title, + Url = article.Url, + PubDate = article.PubDate, + Video = article.Video, + VideoHeight = article.VideoHeight, + VideoWidth = article.VideoWidth, + Thumbnail = article.Thumbnail, + Description = article.Description, + AuthorName = article.AuthorName, + AuthorImage = article.AuthorImage, + }; + } } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs index c17e62c..d5b619f 100644 --- a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs @@ -1,13 +1,14 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Interfaces; public interface IArticlesRepository : ITableRepository { - List List(int page, int count); + List List(int page, int count); List ListBySourceId(Guid id, int page = 0, int count = 25); ArticlesModel GetById(Guid ID); ArticlesModel GetByUrl(string url); - ArticlesModel New(ArticlesModel model); + ArticlesEntity New(ArticlesEntity model); void DeleteAllBySourceId(Guid sourceId); } \ No newline at end of file diff --git a/makefile b/makefile index 2341cb6..858ec2b 100644 --- a/makefile +++ b/makefile @@ -21,14 +21,9 @@ docker-run: ## Runs the docker compose docker-migrate: ## Runs the migrations stored in the Docker image docker run -it --env-file .env ghcr.io/jtom38/newsbot.collector:master /app/goose --dir "/app/migrations" up -migrate-dev: ## Apply sql migrations to dev db - goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" up +ef-build: ## Builds migration artifact + dotnet ef migrations bundle --project "Newsbot.Collector.Database" -migrate-dev-down: ## revert sql migrations to dev db - goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down - -migrate-refresh: ## Rolls back all migrations - goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset - -ef-bundle: ## Create a migration bundle - dotnet ef migrations bundle --project Newsbot.Collector.Database \ No newline at end of file +ef-migrate: ## Runs migrations based on the newest artifact + dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force + ./efbundle --connection "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable" From 731b2cf9b7debbd6cd96bf7ed8908fa43faafc5d Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 22 Jun 2023 08:09:18 -0700 Subject: [PATCH 08/27] moving some commands over to ef --- .../Repositories/ArticlesTable.cs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs index 168ce32..de7c117 100644 --- a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs +++ b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs @@ -1,6 +1,8 @@ using System.Data; using Dapper; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Npgsql; @@ -11,9 +13,12 @@ public class ArticlesTable : IArticlesRepository { private readonly string _connectionString; + private DatabaseContext _context; + public ArticlesTable(string connectionString) { _connectionString = connectionString; + _context = new DatabaseContext(connectionString); } public ArticlesTable(IConfiguration configuration) @@ -22,12 +27,24 @@ public class ArticlesTable : IArticlesRepository if (conn is null) conn = ""; _connectionString = conn; + _context = new DatabaseContext(conn); } - public List List(int page = 0, int count = 25) + public async Task> ListAsync(int page = 0, int count = 25) + { + await using var context = new DatabaseContext(_connectionString); + var query = context.Articles + .OrderBy(d => d.PubDate) + .Take(25); + Console.WriteLine(query.ToQueryString()); + await query.ToListAsync(); + return await query.ToListAsync(); + } + + public List List(int page = 0, int count = 25) { using var conn = OpenConnection(_connectionString); - var res = conn.Query(@"select * from articles + var res = conn.Query(@"select * from Articles Order By PubDate Desc Offset @Page Fetch Next @Count Rows Only", new @@ -70,7 +87,16 @@ public class ArticlesTable : IArticlesRepository }).ToList(); } - public ArticlesModel New(ArticlesModel model) + public ArticlesEntity New(ArticlesEntity model) + { + using var context = new DatabaseContext(_connectionString); + model.Id = new Guid(); + var query = context.Articles.Add(model); + context.SaveChanges(); + return model; + } + + public ArticlesModel NewDapper(ArticlesModel model) { model.ID = Guid.NewGuid(); From 4f46247d0cf534daa0ac0d757407698555c8b08a Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 20:09:15 -0700 Subject: [PATCH 09/27] articles table was updated for the entity --- .../Repositories/ArticlesTable.cs | 83 ++++++++----------- .../Interfaces/IArticlesRepository.cs | 6 +- 2 files changed, 37 insertions(+), 52 deletions(-) diff --git a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs index de7c117..d75ecf5 100644 --- a/Newsbot.Collector.Database/Repositories/ArticlesTable.cs +++ b/Newsbot.Collector.Database/Repositories/ArticlesTable.cs @@ -30,61 +30,41 @@ public class ArticlesTable : IArticlesRepository _context = new DatabaseContext(conn); } - public async Task> ListAsync(int page = 0, int count = 25) + public List List(int page = 0, int count = 25) { - await using var context = new DatabaseContext(_connectionString); + using var context = new DatabaseContext(_connectionString); var query = context.Articles + .Skip(page * count) .OrderBy(d => d.PubDate) .Take(25); Console.WriteLine(query.ToQueryString()); - await query.ToListAsync(); - return await query.ToListAsync(); + return query.ToList(); } - - public List List(int page = 0, int count = 25) + + public ArticlesEntity GetById(Guid id) { - using var conn = OpenConnection(_connectionString); - var res = conn.Query(@"select * from Articles - Order By PubDate Desc - Offset @Page - Fetch Next @Count Rows Only", new - { - Page = page * count, - Count = count - }) - .ToList(); + using var context = new DatabaseContext(_connectionString); + var query = context.Articles + .FirstOrDefault(d => d.Id.Equals(id)); + query ??= new ArticlesEntity(); + return query; + } + + public ArticlesEntity GetByUrl(string url) + { + using var context = new DatabaseContext(_connectionString); + var res = context.Articles.FirstOrDefault(d => d.Url!.Equals(url)); + res ??= new ArticlesEntity(); return res; } - public ArticlesModel GetById(Guid ID) + public List ListBySourceId(Guid id, int page, int count) { - using var conn = OpenConnection(_connectionString); - var res = conn.Query("select * from articles where ID = @ID", new { ID }); - if (res.Count() == 0) return new ArticlesModel(); - return res.First(); - } - - public ArticlesModel GetByUrl(string url) - { - using var conn = OpenConnection(_connectionString); - var res = conn.Query("select * from articles where Url = @Url Limit 1", new { Url = url }); - if (res.Count() == 0) return new ArticlesModel(); - return res.First(); - } - - public List ListBySourceId(Guid id, int page, int count) - { - using var conn = OpenConnection(_connectionString); - var query = @"Select * from articles - where sourceid = @sourceid - Offset @page - Fetch next @count rows only"; - return conn.Query(query, new - { - sourceid = id, - page = page * count, - count - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.Articles + .Skip(page * count) + .Where(d => d.SourceId.Equals(id)); + return res.ToList(); } public ArticlesEntity New(ArticlesEntity model) @@ -126,12 +106,17 @@ public class ArticlesTable : IArticlesRepository public void DeleteAllBySourceId(Guid sourceId) { - using var conn = OpenConnection(_connectionString); - var res = conn.Execute("Delete from articles where sourceid = '@id'", new + using var context = new DatabaseContext(_connectionString); + var res = context.Articles + .Where(d => d.SourceId.Equals(sourceId)) + .ToList(); + + foreach (var item in res) { - sourceId - }); - if (res == 0) throw new Exception($"No records where deleted that linked to SourceId = '{sourceId}'"); + context.Articles.Remove(item); + } + + context.SaveChanges(); } private IDbConnection OpenConnection(string connectionString) diff --git a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs index d5b619f..4cabce7 100644 --- a/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs @@ -6,9 +6,9 @@ namespace Newsbot.Collector.Domain.Interfaces; public interface IArticlesRepository : ITableRepository { List List(int page, int count); - List ListBySourceId(Guid id, int page = 0, int count = 25); - ArticlesModel GetById(Guid ID); - ArticlesModel GetByUrl(string url); + List ListBySourceId(Guid id, int page = 0, int count = 25); + ArticlesEntity GetById(Guid id); + ArticlesEntity GetByUrl(string url); ArticlesEntity New(ArticlesEntity model); void DeleteAllBySourceId(Guid sourceId); } \ No newline at end of file From 3e3da9027f77d02ade6cf2bf8d99a52cf831bc2b Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 20:09:57 -0700 Subject: [PATCH 10/27] refs updated for articlesEntity --- .../Dto/ArticleDetailsDto.cs | 9 ++++--- .../Jobs/CodeProjectWatcherJob.cs | 25 ++++++++--------- .../Jobs/DiscordNotificationJob.cs | 15 ++++++----- .../Jobs/RssWatcherJob.cs | 27 ++++++++++--------- .../Jobs/YoutubeWatcherJob.cs | 15 ++++++----- .../Jobs/DiscordNotificationJobTest.cs | 9 ++++--- .../Tables/ArticlesTableTests.cs | 15 ++++++----- 7 files changed, 61 insertions(+), 54 deletions(-) diff --git a/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs b/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs index 1624a58..e89280c 100644 --- a/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs +++ b/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs @@ -1,10 +1,11 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Dto; public class ArticleDetailsDto { - public Guid ID { get; set; } + public Guid Id { get; set; } public string[]? Tags { get; set; } public string? Title { get; set; } public string? Url { get; set; } @@ -19,14 +20,14 @@ public class ArticleDetailsDto public SourceDto? Source { get; set; } - public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source) + public static ArticleDetailsDto Convert(ArticlesEntity article, SourceModel source) { return new ArticleDetailsDto { - ID = article.ID, + Id = article.Id, Tags = article.Tags.Split(','), Title = article.Title, - Url = article.URL, + Url = article.Url, PubDate = article.PubDate, Video = article.Video, VideoHeight = article.VideoHeight, diff --git a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs index 22891d4..9904ab9 100644 --- a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs @@ -3,6 +3,7 @@ using System.ServiceModel.Syndication; using System.Xml; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models.Config; @@ -63,7 +64,7 @@ public class CodeProjectWatcherJob var sources = _source.ListByType(SourceTypes.CodeProject); // query sources for things to pull - var items = new List(); + var items = new List(); foreach (var source in sources) { @@ -76,12 +77,12 @@ public class CodeProjectWatcherJob _articles.New(item); _queue.New(new DiscordQueueModel { - ArticleID = item.ID + ArticleID = item.Id }); } } - public IEnumerable CheckForReleases(SourceModel source) + public IEnumerable CheckForReleases(SourceModel source) { var url = new Uri(source.Url); var links = new List @@ -110,10 +111,10 @@ public class CodeProjectWatcherJob url.AbsoluteUri); } - return new List(); + return new List(); } - public IEnumerable CheckForCommits(SourceModel source) + public IEnumerable CheckForCommits(SourceModel source) { var url = new Uri(source.Url); var links = new List @@ -136,19 +137,19 @@ public class CodeProjectWatcherJob url.AbsoluteUri); } - return new List(); + return new List(); } - private IEnumerable ProcessFeed(IEnumerable feed, SourceModel source, + private IEnumerable ProcessFeed(IEnumerable feed, SourceModel source, bool isRelease, bool isCommit) { - var items = new List(); + var items = new List(); foreach (var item in feed) { var itemUrl = item.Links[0].Uri.AbsoluteUri; var exits = _articles.GetByUrl(itemUrl); - if (exits.ID != Guid.Empty) continue; + if (exits.Id != Guid.Empty) continue; var parser = new HtmlPageReader(new HtmlPageReaderOptions { @@ -156,12 +157,12 @@ public class CodeProjectWatcherJob }); parser.Parse(); - var a = new ArticlesModel + var a = new ArticlesEntity { - SourceID = source.ID, + SourceId = source.ID, Tags = source.Tags, Title = item.Title.Text, - URL = itemUrl, + Url = itemUrl, PubDate = item.LastUpdatedTime.DateTime, Thumbnail = parser.Data.Header.Image, Description = item.Title.Text, diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index 57af19a..a933191 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -1,4 +1,5 @@ using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.Notifications.Discord; @@ -90,11 +91,11 @@ public class DiscordNotificationJob var articleDetails = _article.GetById(request.ArticleID); // Get the details of the source - var sourceDetails = _sources.GetByID(articleDetails.SourceID); + var sourceDetails = _sources.GetByID(articleDetails.SourceId); if (sourceDetails.ID == Guid.Empty) { _logger.Error( - $"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue."); + $"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue."); _queue.Delete(request.ID); return; } @@ -120,7 +121,7 @@ public class DiscordNotificationJob _queue.Delete(request.ID); } - public void SendSubscriptionNotification(Guid requestId, ArticlesModel articleDetails, SourceModel sourceDetails, + public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceModel sourceDetails, IconModel sourceIcon, SubscriptionModel sub) { // Check if the subscription code flags @@ -143,7 +144,7 @@ public class DiscordNotificationJob { _logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message); _logger.Debug("Queue Record: {RequestId}", requestId); - _logger.Debug("Article: {ArticleDetailsId}", articleDetails.ID); + _logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id); _logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID); _logger.Debug("Subscription: {SubId}", sub.Id); } @@ -151,7 +152,7 @@ public class DiscordNotificationJob Thread.Sleep(3000); } - public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesModel article, IconModel icon) + public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesEntity article, IconModel icon) { var embed = new DiscordMessageEmbed { @@ -172,13 +173,13 @@ public class DiscordNotificationJob new() { Name = "Link", - Value = article.URL, + Value = article.Url, Inline = false } } }; - if (article.URL is not null && article.URL != "") embed.Url = article.URL; + if (article.Url is not null && article.Url != "") embed.Url = article.Url; if (article.Thumbnail != "") embed.Image = new DiscordMessageEmbedImage diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index 0e7e512..3641345 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -2,6 +2,7 @@ using System.ServiceModel.Syndication; using System.Xml; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models.Config; @@ -56,15 +57,15 @@ public class RssWatcherJob public void Execute() { - var articles = new List(); + var articles = new List(); _logger.Information($"{JobName} - Requesting sources"); var sources = _source.ListByType(SourceTypes.Rss); - _logger.Information($"{JobName} - Got {sources.Count} back"); + _logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count); foreach (var source in sources) { - _logger.Information($"{JobName} - Starting to process '{source.Name}'"); + _logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name); _logger.Information($"{JobName} - Starting to request feed to be processed"); var results = Collect(source.Url, source.ID); @@ -78,9 +79,9 @@ public class RssWatcherJob _logger.Information($"{JobName} - Done!"); } - public List Collect(string url, Guid sourceId, int sleep = 3000) + public List Collect(string url, Guid sourceId, int sleep = 3000) { - var collectedPosts = new List(); + var collectedPosts = new List(); using var reader = XmlReader.Create(url); var feed = SyndicationFeed.Load(reader); @@ -99,15 +100,15 @@ public class RssWatcherJob }); meta.Parse(); - var article = new ArticlesModel + var article = new ArticlesEntity { Title = post.Title.Text, Tags = FetchTags(post), - URL = articleUrl, + Url = articleUrl, PubDate = post.PublishDate.DateTime, Thumbnail = meta.Data.Header.Image, Description = meta.Data.Header.Description, - SourceID = sourceId + SourceId = sourceId }; collectedPosts.Add(article); @@ -119,20 +120,20 @@ public class RssWatcherJob return collectedPosts; } - public void UpdateDatabase(List items) + public void UpdateDatabase(List items) { foreach (var item in items) { - if (item.URL is null) + if (item.Url is null) { - Log.Warning("RSS Watcher collected a blank url and was skipped."); + Log.Warning("RSS Watcher collected a blank url and was skipped"); continue; } var p = _articles.New(item); _queue.New(new DiscordQueueModel { - ArticleID = p.ID + ArticleID = p.Id }); } } @@ -140,7 +141,7 @@ public class RssWatcherJob private bool IsThisUrlKnown(string url) { var isKnown = _articles.GetByUrl(url); - if (isKnown.URL == url) return true; + if (isKnown.Url == url) return true; return false; } diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index 3e742e5..e055ad0 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -2,6 +2,7 @@ using System.ServiceModel.Syndication; using System.Xml; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.HtmlParser; @@ -84,7 +85,7 @@ public class YoutubeWatcherJob _articles.New(video); _queue.New(new DiscordQueueModel { - ArticleID = video.ID + ArticleID = video.Id }); } } @@ -109,9 +110,9 @@ public class YoutubeWatcherJob return id; } - private List CheckFeed(string url, SourceModel source) + private List CheckFeed(string url, SourceModel source) { - var videos = new List(); + var videos = new List(); using var reader = XmlReader.Create(url); var feed = SyndicationFeed.Load(reader); @@ -126,17 +127,17 @@ public class YoutubeWatcherJob }); videoDetails.Parse(); - var article = new ArticlesModel + var article = new ArticlesEntity { //Todo add the icon AuthorName = post.Authors[0].Name, Title = post.Title.Text, Tags = FetchTags(post), - URL = articleUrl, + Url = articleUrl, PubDate = post.PublishDate.DateTime, Thumbnail = videoDetails.Data.Header.Image, Description = videoDetails.Data.Header.Description, - SourceID = source.ID, + SourceId = source.ID, Video = "true" }; @@ -151,7 +152,7 @@ public class YoutubeWatcherJob private bool IsThisUrlKnown(string url) { var isKnown = _articles.GetByUrl(url); - if (isKnown.URL == url) return true; + if (isKnown.Url == url) return true; return false; } diff --git a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs index 41f3fde..249133e 100644 --- a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs @@ -1,3 +1,4 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Notifications.Discord; @@ -24,11 +25,11 @@ public class DiscordNotificationJobTest Url = "https://github.com", Tags = "Unit, Testing" }, - new ArticlesModel + new ArticlesEntity { Tags = "more,unit,testing", Title = "Nope not real", - URL = "https://github.com/jtom38", + Url = "https://github.com/jtom38", PubDate = DateTime.Now, Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", Description = "Please work", @@ -50,11 +51,11 @@ public class DiscordNotificationJobTest { client.SendSubscriptionNotification( new Guid(), - new ArticlesModel + new ArticlesEntity { Tags = "more,unit,testing", Title = "Nope not real", - URL = "https://github.com/jtom38", + Url = "https://github.com/jtom38", PubDate = DateTime.Now, Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", diff --git a/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs b/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs index 2884600..e2d247e 100644 --- a/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs +++ b/Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Configuration; using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Tests.Tables; @@ -23,7 +24,7 @@ public class ArticlesTableTests { var cfg = GetConfiguration(); var client = new ArticlesTable(cfg); - client.List(); + client.List(0, 25); } [Fact] @@ -34,7 +35,7 @@ public class ArticlesTableTests var cfg = GetConfiguration(); var client = new ArticlesTable(cfg); var res = client.GetById(uid); - if (!res.ID.Equals(uid)) + if (!res.Id.Equals(uid)) { Assert.Fail("Incorrect record or not found"); } @@ -45,14 +46,14 @@ public class ArticlesTableTests { var cfg = GetConfiguration(); var client = new ArticlesTable(cfg); - var m = new ArticlesModel + var m = new ArticlesEntity { - ID = Guid.NewGuid(), - SourceID = Guid.NewGuid(), + Id = Guid.NewGuid(), + SourceId = Guid.NewGuid(), Tags = "thing, thing2", Title = "Unit Testing!", - URL = "https://google.com", - PubDate = DateTime.Now.ToLocalTime(), + Url = "https://google.com", + PubDate = DateTime.Now.ToUniversalTime(), }; client.New(m); } From 7d176cb6435aaf850651c9812314323323957e84 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 20:11:09 -0700 Subject: [PATCH 11/27] controller updated for entity --- Newsbot.Collector.Api/Controllers/ArticlesController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Newsbot.Collector.Api/Controllers/ArticlesController.cs b/Newsbot.Collector.Api/Controllers/ArticlesController.cs index a8962c9..f5408ee 100644 --- a/Newsbot.Collector.Api/Controllers/ArticlesController.cs +++ b/Newsbot.Collector.Api/Controllers/ArticlesController.cs @@ -43,7 +43,7 @@ public class ArticlesController : ControllerBase public ArticleDetailsDto GetDetailsById(Guid id) { var item = _articles.GetById(id); - var sourceItem = _sources.GetByID(item.SourceID); + var sourceItem = _sources.GetByID(item.SourceId); return ArticleDetailsDto.Convert(item, sourceItem); } From a6ea89be62a6f86993d61d9c5eef58af68d5eb62 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 20:19:09 -0700 Subject: [PATCH 12/27] Discord Queue Entity updated --- .../Repositories/DiscordQueue.cs | 36 +++++++++---------- .../Interfaces/IDiscordQueueRepository.cs | 6 ++-- .../Jobs/CodeProjectWatcherJob.cs | 4 +-- .../Jobs/DiscordNotificationJob.cs | 14 ++++---- .../Jobs/RssWatcherJob.cs | 4 +-- .../Jobs/YoutubeWatcherJob.cs | 4 +-- 6 files changed, 32 insertions(+), 36 deletions(-) diff --git a/Newsbot.Collector.Database/Repositories/DiscordQueue.cs b/Newsbot.Collector.Database/Repositories/DiscordQueue.cs index 233d8ac..ad77070 100644 --- a/Newsbot.Collector.Database/Repositories/DiscordQueue.cs +++ b/Newsbot.Collector.Database/Repositories/DiscordQueue.cs @@ -1,5 +1,6 @@ using System.Data; using Dapper; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Npgsql; @@ -22,33 +23,28 @@ public class DiscordQueueTable : IDiscordQueueRepository return conn; } - public void New(DiscordQueueModel model) + public void New(DiscordQueueEntity model) { - using var conn = OpenConnection(_connectionString); - var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);"; - conn.Execute(query, new - { - id = Guid.NewGuid(), - articleid = model.ArticleID - }); + model.Id = new Guid(); + + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordQueue.Add(model); + context.SaveChanges(); } public void Delete(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Delete From DiscordQueue Where ID = @id;"; - conn.Execute(query, new - { - id = id - }); + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordQueue.FirstOrDefault(d => d.Id.Equals(id)); + res ??= new DiscordQueueEntity(); + context.DiscordQueue.Remove(res); + context.SaveChanges(); } - public List List(int limit = 25) + public List List(int limit = 25) { - using var conn = OpenConnection(_connectionString); - var query = "Select * from DiscordQueue LIMIT @limit;"; - return conn.Query(query, new { - limit = limit - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordQueue.Take(limit).ToList(); + return res; } } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs b/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs index 2af0c6d..4cba109 100644 --- a/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs @@ -1,10 +1,10 @@ -using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Entities; namespace Newsbot.Collector.Domain.Interfaces; public interface IDiscordQueueRepository { - void New(DiscordQueueModel model); + void New(DiscordQueueEntity model); void Delete(Guid id); - List List(int limit); + List List(int limit); } \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs index 9904ab9..8b58df4 100644 --- a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs @@ -75,9 +75,9 @@ public class CodeProjectWatcherJob foreach (var item in items) { _articles.New(item); - _queue.New(new DiscordQueueModel + _queue.New(new DiscordQueueEntity() { - ArticleID = item.Id + ArticleId = item.Id }); } } diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index a933191..de090f8 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -84,11 +84,11 @@ public class DiscordNotificationJob _logger.Information($"{JobName} - Loop has been completed."); } - public void ProcessQueueItem(DiscordQueueModel request) + public void ProcessQueueItem(DiscordQueueEntity request) { - _logger.Debug($"{JobName} - Processing {request.ID}"); + _logger.Debug($"{JobName} - Processing {request.Id}"); // Get all details on the article in the queue - var articleDetails = _article.GetById(request.ArticleID); + var articleDetails = _article.GetById(request.ArticleId); // Get the details of the source var sourceDetails = _sources.GetByID(articleDetails.SourceId); @@ -96,7 +96,7 @@ public class DiscordNotificationJob { _logger.Error( $"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue."); - _queue.Delete(request.ID); + _queue.Delete(request.Id); return; } @@ -115,10 +115,10 @@ public class DiscordNotificationJob var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID); foreach (var sub in allSubscriptions) - SendSubscriptionNotification(request.ID, articleDetails, sourceDetails, sourceIcon, sub); + SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); - _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.ID); - _queue.Delete(request.ID); + _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id); + _queue.Delete(request.Id); } public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceModel sourceDetails, diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index 3641345..8548006 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -131,9 +131,9 @@ public class RssWatcherJob } var p = _articles.New(item); - _queue.New(new DiscordQueueModel + _queue.New(new DiscordQueueEntity { - ArticleID = p.Id + ArticleId = p.Id }); } } diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index e055ad0..8297684 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -83,9 +83,9 @@ public class YoutubeWatcherJob { _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found"); _articles.New(video); - _queue.New(new DiscordQueueModel + _queue.New(new DiscordQueueEntity { - ArticleID = video.Id + ArticleId = video.Id }); } } From f2995ff1569560e7732b8d95cbaea4a5ec6fe09b Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 20:19:31 -0700 Subject: [PATCH 13/27] docker will now build the migration bundle --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index ab3e3ce..7427653 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,21 +14,20 @@ FROM build AS publish RUN dotnet publish -c Release -o /app/publish RUN dotnet publish -o build -#--self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true RUN ls build +FROM build as ef +RUN dotnet tool install dotnet-ef --tool-path /usr/bin +RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database" + FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app ENV ASPNETCORE_URLS=http://*:5000 ENV DOTNET_URLS=http://*:5000 -#RUN apt-get install chromium -y - WORKDIR /app -#RUN mkdir /migrations COPY --from=publish /app/build /app -#COPY --from=build ./app/Newsbot.Collector.Database/Migrations /app/migrations -#COPY --from=goose /go/bin/goose /app +COPY --from=ef /app/efbundle /app CMD [ "dotnet", "Newsbot.Collector.Api.dll" ] \ No newline at end of file From 0b682611813515294da51bd3aade65a78b437dc3 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 21:55:56 -0700 Subject: [PATCH 14/27] discord web hooks updated with ef --- .../Controllers/DiscordWebHooksController.cs | 7 +- .../Repositories/DiscordWebhooksTable.cs | 143 +++++++++--------- .../Dto/DiscordWebHookDto.cs | 6 +- .../Interfaces/IDiscordWebHooksRepository.cs | 15 +- .../Jobs/DiscordNotificationJob.cs | 2 +- 5 files changed, 87 insertions(+), 86 deletions(-) diff --git a/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs b/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs index 5706972..194c6ff 100644 --- a/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs +++ b/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs @@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Dto; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; @@ -40,12 +41,12 @@ public class DiscordWebHookController : ControllerBase { var exists = _webhooks.GetByUrl(url); // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - if (exists.ID != Guid.Empty) + if (exists.Id != Guid.Empty) { return DiscordWebHookDto.Convert(exists); } - var res = _webhooks.New(new DiscordWebHookModel + var res = _webhooks.New(new DiscordWebhookEntity { Url = url, Server = server, @@ -72,7 +73,7 @@ public class DiscordWebHookController : ControllerBase [HttpGet("{id}")] public DiscordWebHookDto GetById(Guid id) { - var res = _webhooks.GetByID(id); + var res = _webhooks.GetById(id); return DiscordWebHookDto.Convert(res); } diff --git a/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs b/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs index f41517c..58aac8a 100644 --- a/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs +++ b/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs @@ -2,7 +2,7 @@ using System.Data; using Dapper; using Microsoft.Extensions.Configuration; using Newsbot.Collector.Domain.Interfaces; -using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Entities; using Npgsql; namespace Newsbot.Collector.Database.Repositories; @@ -22,103 +22,104 @@ public class DiscordWebhooksTable : IDiscordWebHooksRepository _connectionString = connstr; } - public DiscordWebHookModel New(DiscordWebHookModel model) + public DiscordWebhookEntity New(DiscordWebhookEntity model) { - var uid = Guid.NewGuid(); - using var conn = OpenConnection(_connectionString); - var query = - "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);"; - conn.Execute(query, new - { - id = uid, - url = model.Url, - server = model.Server, - channel = model.Channel, - enabled = model.Enabled - }); - model.ID = uid; + model.Id = new Guid(); + + using var context = new DatabaseContext(_connectionString); + context.DiscordWebhooks.Add(model); + context.SaveChanges(); return model; } - public DiscordWebHookModel GetByID(Guid id) + public DiscordWebhookEntity GetById(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;"; - return conn.Query(query, new - { - id - }).First(); + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordWebhooks.FirstOrDefault(d => d.Id.Equals(id)); + res ??= new DiscordWebhookEntity(); + return res; } - public DiscordWebHookModel GetByUrl(string url) + public DiscordWebhookEntity GetByUrl(string url) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From DiscordWebHooks Where url = @url;"; - try - { - var res = conn.QueryFirst(query, new - { - url - }); - return res; - } - catch - { - return new DiscordWebHookModel(); - } + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordWebhooks.FirstOrDefault(d => d.Url.Equals(url)); + res ??= new DiscordWebhookEntity(); + return res; } - public List List(int page, int count = 25) + public List List(int page, int count = 25) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From DiscordWebHooks - Offset @offset Fetch Next @count Rows Only;"; - return conn.Query(query, new - { - offset = page * count, count - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordWebhooks + .Skip(page * count) + .Take(count) + .ToList(); + res ??= new List(); + return res; } - public List ListByServer(string server, int limit = 25) + public List ListByServer(string server, int limit = 25) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;"; - return conn.Query(query, new - { - server, limit - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordWebhooks + .Where(d => d.Server.Equals(server)) + .Take(limit) + .ToList(); + res ??= new List(); + return res; } - public List ListByServerAndChannel(string server, string channel, int limit = 25) + public List ListByServerAndChannel(string server, string channel, int limit = 25) { - using var conn = OpenConnection(_connectionString); - var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;"; - return conn.Query(query, new - { - server, - channel, - limit - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.DiscordWebhooks + .Where(s => s.Server.Equals(server)) + .Where(c => c.Channel.Equals(channel)) + .Take(limit) + .ToList(); + res ??= new List(); + return res; } public int Disable(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;"; - return conn.Execute(query, new + var res = GetById(id); + using var context = new DatabaseContext(_connectionString); + + res.Enabled = true; + + context.DiscordWebhooks.Update(res); + try { - id - }); + context.SaveChanges(); + return 1; + } + catch(Exception ex) + { + Console.WriteLine($"Failed to update DiscordWebhook ID = {id}. {ex.Message}"); + return 0; + } } public int Enable(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;"; - return conn.Execute(query, new + var res = GetById(id); + using var context = new DatabaseContext(_connectionString); + + res.Enabled = false; + + context.DiscordWebhooks.Update(res); + try { - id - }); + context.SaveChanges(); + return 1; + } + catch(Exception ex) + { + Console.WriteLine($"Failed to update DiscordWebhook ID = {id}. {ex.Message}"); + return 0; + } } private IDbConnection OpenConnection(string connectionString) diff --git a/Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs b/Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs index 0eeea4d..d8a93b7 100644 --- a/Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs +++ b/Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs @@ -1,4 +1,4 @@ -using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Entities; namespace Newsbot.Collector.Domain.Dto; @@ -10,11 +10,11 @@ public class DiscordWebHookDto public string? Channel { get; set; } public bool Enabled { get; set; } - public static DiscordWebHookDto Convert(DiscordWebHookModel model) + public static DiscordWebHookDto Convert(DiscordWebhookEntity model) { return new DiscordWebHookDto { - ID = model.ID, + ID = model.Id, Url = model.Url, Server = model.Server, Channel = model.Channel, diff --git a/Newsbot.Collector.Domain/Interfaces/IDiscordWebHooksRepository.cs b/Newsbot.Collector.Domain/Interfaces/IDiscordWebHooksRepository.cs index cffbaca..2192371 100644 --- a/Newsbot.Collector.Domain/Interfaces/IDiscordWebHooksRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/IDiscordWebHooksRepository.cs @@ -1,18 +1,17 @@ -using Microsoft.VisualBasic; -using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Entities; namespace Newsbot.Collector.Domain.Interfaces; public interface IDiscordWebHooksRepository { - DiscordWebHookModel New(DiscordWebHookModel model); + DiscordWebhookEntity New(DiscordWebhookEntity model); - DiscordWebHookModel GetByID(Guid id); - DiscordWebHookModel GetByUrl(string url); + DiscordWebhookEntity GetById(Guid id); + DiscordWebhookEntity GetByUrl(string url); - List List(int page, int count = 25); - List ListByServer(string server, int limit); - List ListByServerAndChannel(string server, string channel, int limit); + List List(int page, int count = 25); + List ListByServer(string server, int limit); + List ListByServerAndChannel(string server, string channel, int limit); int Disable(Guid id); int Enable(Guid id); diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index de090f8..ed30ba2 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -132,7 +132,7 @@ public class DiscordNotificationJob if (articleDetails.CodeIsRelease && !sub.CodeAllowReleases) throw new MessageTypeNotRequestedException("Message was a code release and was not requested by the subscription"); // find the discord webhooks we need to post to - var discordDetails = _webhook.GetByID(sub.DiscordWebHookId); + var discordDetails = _webhook.GetById(sub.DiscordWebHookId); if (discordDetails.Enabled == false) return; var client = new DiscordWebhookClient(discordDetails.Url); From 2c886d22ad806a13bd09ff63b0a62f88989f9c9f Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 21:56:15 -0700 Subject: [PATCH 15/27] Icons updated with ef --- .../Repositories/IconsTable.cs | 37 ++++++++----------- .../Interfaces/IIconsRepository.cs | 8 ++-- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/Newsbot.Collector.Database/Repositories/IconsTable.cs b/Newsbot.Collector.Database/Repositories/IconsTable.cs index 3a12682..1ef0c42 100644 --- a/Newsbot.Collector.Database/Repositories/IconsTable.cs +++ b/Newsbot.Collector.Database/Repositories/IconsTable.cs @@ -1,6 +1,7 @@ using System.Data; using Dapper; using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Npgsql; @@ -23,37 +24,29 @@ public class IconsTable : IIconsRepository _connectionString = connstr; } - public void New(IconModel model) + public void New(IconEntity model) { + using var context = new DatabaseContext(_connectionString); model.Id = Guid.NewGuid(); - using var conn = OpenConnection(_connectionString); - var q = @"Insert Into icons (id, filename, site, sourceid) values (@Id,@FileName, @Site, @SourceId)"; - conn.Execute(q, model); + context.Icons.Add(model); + context.SaveChanges(); } - public IconModel GetById(Guid id) + public IconEntity GetById(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From icons where ID = @id Limit 1;"; - var res = conn.Query(query, new - { - id - }); - if (!res.Any()) return new IconModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Icons.FirstOrDefault(f => f.Id.Equals(id)); + res ??= new IconEntity(); + return res; } - public IconModel GetBySourceId(Guid id) + public IconEntity GetBySourceId(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From icons where sourceid = @id Limit 1;"; - var res = conn.Query(query, new - { - id - }); - if (!res.Any()) return new IconModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Icons.FirstOrDefault(f => f.SourceId.Equals(id)); + res ??= new IconEntity(); + return res; } private IDbConnection OpenConnection(string connectionString) diff --git a/Newsbot.Collector.Domain/Interfaces/IIconsRepository.cs b/Newsbot.Collector.Domain/Interfaces/IIconsRepository.cs index c502f48..750048b 100644 --- a/Newsbot.Collector.Domain/Interfaces/IIconsRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/IIconsRepository.cs @@ -1,11 +1,11 @@ -using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Domain.Entities; namespace Newsbot.Collector.Domain.Interfaces; public interface IIconsRepository { - public void New(IconModel model); + public void New(IconEntity model); - public IconModel GetById(Guid id); - public IconModel GetBySourceId(Guid id); + public IconEntity GetById(Guid id); + public IconEntity GetBySourceId(Guid id); } \ No newline at end of file From 0797fb85029f2fd42e31a8e846d23365fe10222a Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Fri, 23 Jun 2023 23:01:40 -0700 Subject: [PATCH 16/27] Souces has been moved to ef and with entites --- .../Controllers/ArticlesController.cs | 2 +- .../Controllers/SourcesController.cs | 39 ++-- .../Controllers/SubscriptionsController.cs | 20 +- .../Repositories/SourcesTable.cs | 207 +++++++++--------- .../Dto/ArticleDetailsDto.cs | 2 +- Newsbot.Collector.Domain/Dto/SourceDto.cs | 5 +- .../Dto/SubscriptionDetailsDto.cs | 5 +- .../Interfaces/ISourcesRepository.cs | 19 +- .../Jobs/CodeProjectWatcherJob.cs | 10 +- .../Jobs/DiscordNotificationJob.cs | 19 +- .../Jobs/RssWatcherJob.cs | 4 +- .../Jobs/YoutubeWatcherJob.cs | 8 +- .../Jobs/CodeProjectWatcherJobTests.cs | 13 +- .../Jobs/DiscordNotificationJobTest.cs | 12 +- .../Tables/SourcesTableTests.cs | 5 +- 15 files changed, 186 insertions(+), 184 deletions(-) diff --git a/Newsbot.Collector.Api/Controllers/ArticlesController.cs b/Newsbot.Collector.Api/Controllers/ArticlesController.cs index f5408ee..502d04a 100644 --- a/Newsbot.Collector.Api/Controllers/ArticlesController.cs +++ b/Newsbot.Collector.Api/Controllers/ArticlesController.cs @@ -43,7 +43,7 @@ public class ArticlesController : ControllerBase public ArticleDetailsDto GetDetailsById(Guid id) { var item = _articles.GetById(id); - var sourceItem = _sources.GetByID(item.SourceId); + var sourceItem = _sources.GetById(item.SourceId); return ArticleDetailsDto.Convert(item, sourceItem); } diff --git a/Newsbot.Collector.Api/Controllers/SourcesController.cs b/Newsbot.Collector.Api/Controllers/SourcesController.cs index 57e244d..41bd859 100644 --- a/Newsbot.Collector.Api/Controllers/SourcesController.cs +++ b/Newsbot.Collector.Api/Controllers/SourcesController.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Options; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Dto; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.HtmlParser; @@ -39,10 +40,10 @@ public class SourcesController : ControllerBase } [HttpGet("by/type")] - public IEnumerable GetByType(string type) + public IEnumerable GetByType(string type, int page) { var res = new List(); - var temp = _sources.ListByType(type); + var temp = _sources.ListByType(type, page); foreach (var item in temp) res.Add(SourceDto.Convert(item)); return res; } @@ -51,7 +52,7 @@ public class SourcesController : ControllerBase public SourceDto NewReddit(string name) { var res = _sources.GetByNameAndType(name, SourceTypes.Reddit); - if (res.ID != Guid.Empty) return SourceDto.Convert(res); + if (res.Id != Guid.Empty) return SourceDto.Convert(res); var uri = new Uri($"https://reddit.com/r/{name}"); @@ -61,7 +62,7 @@ public class SourcesController : ControllerBase }); pageReader.Parse(); - var item = _sources.New(new SourceModel + var item = _sources.New(new SourceEntity { Site = SourceTypes.Reddit, Name = name, @@ -74,11 +75,11 @@ public class SourcesController : ControllerBase // Not all subreddits have an Icon, so we only want to add a record when it has one. if (pageReader.Data.Header.Image != "") - _icons.New(new IconModel + _icons.New(new IconEntity() { Id = Guid.NewGuid(), FileName = pageReader.Data.Header.Image, - SourceId = item.ID + SourceId = item.Id }); return SourceDto.Convert(item); } @@ -87,9 +88,9 @@ public class SourcesController : ControllerBase public SourceDto NewRss(string name, string url) { var res = _sources.GetByNameAndType(name, SourceTypes.Rss); - if (res.ID != Guid.Empty) return SourceDto.Convert(res); + if (res.Id != Guid.Empty) return SourceDto.Convert(res); - var m = new SourceModel + var m = new SourceEntity { Site = SourceTypes.Rss, Name = name, @@ -107,7 +108,7 @@ public class SourcesController : ControllerBase public SourceDto NewYoutube(string url) { var res = _sources.GetByUrl(url); - if (res.ID != Guid.Empty) return SourceDto.Convert(res); + if (res.Id != Guid.Empty) return SourceDto.Convert(res); var htmlClient = new HtmlPageReader(new HtmlPageReaderOptions { @@ -115,7 +116,7 @@ public class SourcesController : ControllerBase }); htmlClient.Parse(); - var item = _sources.New(new SourceModel + var item = _sources.New(new SourceEntity { Site = SourceTypes.YouTube, Type = SourceTypes.YouTube, @@ -127,11 +128,11 @@ public class SourcesController : ControllerBase YoutubeId = htmlClient.Data.Header.YoutubeChannelID ?? "" }); - _icons.New(new IconModel + _icons.New(new IconEntity() { Id = Guid.NewGuid(), FileName = htmlClient.Data.Header.Image, - SourceId = item.ID + SourceId = item.Id }); return SourceDto.Convert(item); @@ -141,9 +142,9 @@ public class SourcesController : ControllerBase public SourceDto NewTwitch(string name) { var res = _sources.GetByNameAndType(name, SourceTypes.Twitch); - if (res.ID != Guid.Empty) return SourceDto.Convert(res); + if (res.Id != Guid.Empty) return SourceDto.Convert(res); - var item = _sources.New(new SourceModel + var item = _sources.New(new SourceEntity { Site = SourceTypes.Twitch, Type = SourceTypes.Twitch, @@ -162,7 +163,7 @@ public class SourcesController : ControllerBase //if (!url.Contains("github.com")) return new SourceDto(); var res = _sources.GetByUrl(url); - if (res.ID != Guid.Empty) return SourceDto.Convert(res); + if (res.Id != Guid.Empty) return SourceDto.Convert(res); var slice = url.Split('/'); @@ -172,7 +173,7 @@ public class SourcesController : ControllerBase }); pageReader.Parse(); - var item = _sources.New(new SourceModel + var item = _sources.New(new SourceEntity { Site = SourceTypes.CodeProject, Type = SourceTypes.CodeProject, @@ -183,11 +184,11 @@ public class SourcesController : ControllerBase Tags = $"{slice[2]},{slice[3]},{slice[4]}" }); - _icons.New(new IconModel + _icons.New(new IconEntity() { Id = Guid.NewGuid(), FileName = pageReader.Data.Header.Image, - SourceId = item.ID + SourceId = item.Id }); return SourceDto.Convert(item); @@ -196,7 +197,7 @@ public class SourcesController : ControllerBase [HttpGet("{id}")] public SourceDto GetById(Guid id) { - var item = _sources.GetByID(id); + var item = _sources.GetById(id); return SourceDto.Convert(item); } diff --git a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs index 624547b..bd6a953 100644 --- a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs +++ b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs @@ -43,8 +43,8 @@ public class SubscriptionsController : ControllerBase public SubscriptionDetailsDto GetDetailsById(Guid id) { var sub = _subscription.GetById(id); - var webhook = _discord.GetByID(sub.DiscordWebHookId); - var source = _sources.GetByID(sub.SourceId); + var webhook = _discord.GetById(sub.DiscordWebHookId); + var source = _sources.GetById(sub.SourceId); return SubscriptionDetailsDto.Convert(sub, source, webhook); } @@ -82,11 +82,11 @@ public class SubscriptionsController : ControllerBase var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists); - var discord = _discord.GetByID(discordId); - if (discord.ID == Guid.Empty) return new BadRequestResult(); + var discord = _discord.GetById(discordId); + if (discord.Id == Guid.Empty) return new BadRequestResult(); - var source = _sources.GetByID(sourceId); - if (source.ID == Guid.Empty) return new BadRequestResult(); + var source = _sources.GetById(sourceId); + if (source.Id == Guid.Empty) return new BadRequestResult(); var item = _subscription.New(new SubscriptionModel { @@ -110,11 +110,11 @@ public class SubscriptionsController : ControllerBase var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists); - var discord = _discord.GetByID(discordId); - if (discord.ID == Guid.Empty) return new BadRequestResult(); + var discord = _discord.GetById(discordId); + if (discord.Id == Guid.Empty) return new BadRequestResult(); - var source = _sources.GetByID(sourceId); - if (source.ID == Guid.Empty) return new BadRequestResult(); + var source = _sources.GetById(sourceId); + if (source.Id == Guid.Empty) return new BadRequestResult(); var sub = _subscription.New(new SubscriptionModel { diff --git a/Newsbot.Collector.Database/Repositories/SourcesTable.cs b/Newsbot.Collector.Database/Repositories/SourcesTable.cs index 61348cf..1ea6520 100644 --- a/Newsbot.Collector.Database/Repositories/SourcesTable.cs +++ b/Newsbot.Collector.Database/Repositories/SourcesTable.cs @@ -1,6 +1,7 @@ using System.Data; using Dapper; using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Npgsql; @@ -23,157 +24,153 @@ public class SourcesTable : ISourcesRepository _connectionString = connstr; } - public SourceModel New(SourceModel model) + public SourceEntity New(SourceEntity model) { - model.ID = Guid.NewGuid(); - using var conn = OpenConnection(_connectionString); - var query = - "Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags, YoutubeId) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags,@youtubeid);"; - conn.Execute(query, new + model.Id = Guid.NewGuid(); + using var context = new DatabaseContext(_connectionString); + context.Sources.Add(model); + try { - id = model.ID, - model.Site, - model.Name, - model.Source, - model.Type, - model.Value, - model.Enabled, - model.Url, - model.Tags, - model.YoutubeId - }); + context.SaveChanges(); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to save "); + } + return model; } - public SourceModel GetByID(Guid ID) + public SourceEntity GetById(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From Sources where ID = @id Limit 1;"; - var res = conn.Query(query, new - { - id = ID - }); - if (res.Count() == 0) return new SourceModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources.FirstOrDefault(f => f.Id.Equals(id)); + res ??= new SourceEntity(); + return res; } - public SourceModel GetByID(string ID) + public SourceEntity GetById(string id) { - var uid = Guid.Parse(ID); - return GetByID(uid); + var uid = Guid.Parse(id); + return GetById(uid); } - public SourceModel GetByName(string Name) + public SourceEntity GetByName(string name) { - using var conn = OpenConnection(_connectionString); - var query = "Select * from Sources where name = @name Limit 1;"; - var res = conn.Query(query, new - { - name = Name - }); - - if (res.Count() == 0) return new SourceModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources.FirstOrDefault(f => f.Name.Equals(name)); + res ??= new SourceEntity(); + return res; } - public SourceModel GetByNameAndType(string name, string type) + public SourceEntity GetByNameAndType(string name, string type) { - using var conn = OpenConnection(_connectionString); - var query = "Select * from Sources WHERE name = @name and type = @type;"; - var res = conn.Query(query, new - { - name, type - }); - - if (res.Count() == 0) return new SourceModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources + .Where(f => f.Name.Equals(name)) + .FirstOrDefault(f => f.Type.Equals(type)); + res ??= new SourceEntity(); + return res; } - public SourceModel GetByUrl(string url) + public SourceEntity GetByUrl(string url) { - using var conn = OpenConnection(_connectionString); - var query = "Select * from Sources WHERE url = @url;"; - var res = conn.Query(query, new - { - url - }); - - if (res.ToList().Count == 0) return new SourceModel(); - - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources + .FirstOrDefault(f => f.Url.Equals(url)); + res ??= new SourceEntity(); + return res; } - public List List(int page = 0, int count = 25) + public List List(int page = 0, int count = 100) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From Sources - Offset @page - Fetch Next @count Rows Only;"; - return conn.Query(query, new - { - page = page * count, count - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources + .Skip(page * count) + .Take(count) + .ToList(); + return res; } - public List ListBySource(string source, int limit = 25) + public List ListBySource(string source, int page = 0, int limit = 25) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From Sources where Source = @source Limit @limit;"; - return conn.Query(query, new - { - source, limit - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources + .Where(f => f.Source.Equals(source)) + .Skip(page * limit) + .Take(limit) + .ToList(); + return res; } - public List ListByType(string type, int limit = 25) + public List ListByType(string type,int page = 0, int limit = 25) { - using var conn = OpenConnection(_connectionString); - var query = "Select * From Sources where Type = @type Limit @limit;"; - return conn.Query(query, new - { - type, limit - }).ToList(); + using var context = new DatabaseContext(_connectionString); + var res = context.Sources + .Where(f => f.Type.Equals(type)) + .Skip(page * limit) + .Take(limit) + .ToList(); + return res; } public int Disable(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Update Sources Set Enabled = FALSE where ID = @id;"; - return conn.Execute(query, new + using var context = new DatabaseContext(_connectionString); + var res = GetById(id); + res.Enabled = false; + context.Sources.Update(res); + try { - id - }); + context.SaveChanges(); + return 1; + } + catch + { + return 0; + } } public int Enable(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Update Sources Set Enabled = TRUE where ID = @id;"; - return conn.Execute(query, new + using var context = new DatabaseContext(_connectionString); + var res = GetById(id); + res.Enabled = true; + context.Sources.Update(res); + try { - id - }); + context.SaveChanges(); + return 1; + } + catch + { + return 0; + } } public void Delete(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Delete From sources where id = @id;"; - var res = conn.Execute(query, new - { - id - }); - if (res == 0) throw new Exception("Nothing was deleted"); + using var context = new DatabaseContext(_connectionString); + var res = GetById(id); + context.Sources.Remove(res); + context.SaveChanges(); } public int UpdateYoutubeId(Guid id, string youtubeId) { - using var conn = OpenConnection(_connectionString); - var query = "Update Sources Set youtubeid = @youtubeId where ID = @id;"; - return conn.Execute(query, new + using var context = new DatabaseContext(_connectionString); + var res = GetById(id); + res.YoutubeId = youtubeId; + context.Sources.Update(res); + try { - id, youtubeId - }); + context.SaveChanges(); + return 1; + } + catch + { + return 0; + } } private IDbConnection OpenConnection(string connectionString) diff --git a/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs b/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs index e89280c..ca1fb6d 100644 --- a/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs +++ b/Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs @@ -20,7 +20,7 @@ public class ArticleDetailsDto public SourceDto? Source { get; set; } - public static ArticleDetailsDto Convert(ArticlesEntity article, SourceModel source) + public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source) { return new ArticleDetailsDto { diff --git a/Newsbot.Collector.Domain/Dto/SourceDto.cs b/Newsbot.Collector.Domain/Dto/SourceDto.cs index 8a9e051..bd3f7a1 100644 --- a/Newsbot.Collector.Domain/Dto/SourceDto.cs +++ b/Newsbot.Collector.Domain/Dto/SourceDto.cs @@ -1,4 +1,5 @@ using System.Net.NetworkInformation; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Dto; @@ -16,10 +17,10 @@ public class SourceDto public string[]? Tags { get; set; } public bool Deleted { get; set; } - public static SourceDto Convert(SourceModel model) { + public static SourceDto Convert(SourceEntity model) { return new SourceDto { - ID = model.ID, + ID = model.Id, Site = model.Site, Name = model.Name, Source = model.Source, diff --git a/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs b/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs index bc9c9d7..a73c9cd 100644 --- a/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs +++ b/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs @@ -1,3 +1,4 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Dto; @@ -10,8 +11,8 @@ public class SubscriptionDetailsDto public SourceDto? Source { get; set; } public DiscordWebHookDto? DiscordWebHook { get; set; } - public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source, - DiscordWebHookModel discord) + public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceEntity source, + DiscordWebhookEntity discord) { return new SubscriptionDetailsDto { diff --git a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs index c528230..9c4c843 100644 --- a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs @@ -1,18 +1,19 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Interfaces; public interface ISourcesRepository { - public SourceModel New(SourceModel model); - public SourceModel GetByID(Guid ID); - public SourceModel GetByID(string ID); - public SourceModel GetByName(string name); - public SourceModel GetByNameAndType(string name, string type); - SourceModel GetByUrl(string url); - public List List(int page, int count); - public List ListBySource(string source, int limit); - public List ListByType(string type, int limit = 25); + public SourceEntity New(SourceEntity model); + public SourceEntity GetById(Guid id); + public SourceEntity GetById(string id); + public SourceEntity GetByName(string name); + public SourceEntity GetByNameAndType(string name, string type); + SourceEntity GetByUrl(string url); + public List List(int page, int count); + public List ListBySource(string source,int page, int limit); + public List ListByType(string type,int page, int limit = 25); public int Disable(Guid id); public int Enable(Guid id); public void Delete(Guid id); diff --git a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs index 8b58df4..1414a08 100644 --- a/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs @@ -61,7 +61,7 @@ public class CodeProjectWatcherJob private void Execute() { - var sources = _source.ListByType(SourceTypes.CodeProject); + var sources = _source.ListByType(SourceTypes.CodeProject, 0, 100); // query sources for things to pull var items = new List(); @@ -82,7 +82,7 @@ public class CodeProjectWatcherJob } } - public IEnumerable CheckForReleases(SourceModel source) + public IEnumerable CheckForReleases(SourceEntity source) { var url = new Uri(source.Url); var links = new List @@ -114,7 +114,7 @@ public class CodeProjectWatcherJob return new List(); } - public IEnumerable CheckForCommits(SourceModel source) + public IEnumerable CheckForCommits(SourceEntity source) { var url = new Uri(source.Url); var links = new List @@ -140,7 +140,7 @@ public class CodeProjectWatcherJob return new List(); } - private IEnumerable ProcessFeed(IEnumerable feed, SourceModel source, + private IEnumerable ProcessFeed(IEnumerable feed, SourceEntity source, bool isRelease, bool isCommit) { var items = new List(); @@ -159,7 +159,7 @@ public class CodeProjectWatcherJob var a = new ArticlesEntity { - SourceId = source.ID, + SourceId = source.Id, Tags = source.Tags, Title = item.Title.Text, Url = itemUrl, diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index ed30ba2..be60738 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -91,8 +91,8 @@ public class DiscordNotificationJob var articleDetails = _article.GetById(request.ArticleId); // Get the details of the source - var sourceDetails = _sources.GetByID(articleDetails.SourceId); - if (sourceDetails.ID == Guid.Empty) + var sourceDetails = _sources.GetById(articleDetails.SourceId); + if (sourceDetails.Id == Guid.Empty) { _logger.Error( $"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue."); @@ -100,19 +100,19 @@ public class DiscordNotificationJob return; } - var sourceIcon = new IconModel(); + var sourceIcon = new IconEntity(); try { - sourceIcon = _icons.GetBySourceId(sourceDetails.ID); + sourceIcon = _icons.GetBySourceId(sourceDetails.Id); } catch { _logger.Warning("{JobName} - Source ID \'{SourceDetailsId}\' is missing an icon", JobName, - sourceDetails.ID); + sourceDetails.Id); } // Find all the subscriptions for that source - var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID); + var allSubscriptions = _subs.ListBySourceID(sourceDetails.Id); foreach (var sub in allSubscriptions) SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); @@ -121,8 +121,7 @@ public class DiscordNotificationJob _queue.Delete(request.Id); } - public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceModel sourceDetails, - IconModel sourceIcon, SubscriptionModel sub) + public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionModel sub) { // Check if the subscription code flags // If the article is a code commit and the subscription does not want them, skip. @@ -145,14 +144,14 @@ public class DiscordNotificationJob _logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message); _logger.Debug("Queue Record: {RequestId}", requestId); _logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id); - _logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID); + _logger.Debug("Source: {SourceDetailsId}", sourceDetails.Id); _logger.Debug("Subscription: {SubId}", sub.Id); } Thread.Sleep(3000); } - public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesEntity article, IconModel icon) + public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon) { var embed = new DiscordMessageEmbed { diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index 8548006..be35362 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -60,14 +60,14 @@ public class RssWatcherJob var articles = new List(); _logger.Information($"{JobName} - Requesting sources"); - var sources = _source.ListByType(SourceTypes.Rss); + var sources = _source.ListByType(SourceTypes.Rss, 0, 100); _logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count); foreach (var source in sources) { _logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name); _logger.Information($"{JobName} - Starting to request feed to be processed"); - var results = Collect(source.Url, source.ID); + var results = Collect(source.Url, source.Id); _logger.Information($"{JobName} - Collected {results.Count} posts"); articles.AddRange(results); diff --git a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs index 8297684..2374fe5 100644 --- a/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs @@ -67,11 +67,11 @@ public class YoutubeWatcherJob if (channelId == "") { channelId = GetChannelId(source.Url); - _source.UpdateYoutubeId(source.ID, channelId); + _source.UpdateYoutubeId(source.Id, channelId); } // Make sure we have a Icon for the channel - var icon = _icons.GetBySourceId(source.ID); + var icon = _icons.GetBySourceId(source.Id); if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V"); _logger.Information($"{JobName} - Checking '{source.Name}'"); @@ -110,7 +110,7 @@ public class YoutubeWatcherJob return id; } - private List CheckFeed(string url, SourceModel source) + private List CheckFeed(string url, SourceEntity source) { var videos = new List(); @@ -137,7 +137,7 @@ public class YoutubeWatcherJob PubDate = post.PublishDate.DateTime, Thumbnail = videoDetails.Data.Header.Image, Description = videoDetails.Data.Header.Description, - SourceId = source.ID, + SourceId = source.Id, Video = "true" }; diff --git a/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs b/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs index 36ecb04..101b918 100644 --- a/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs +++ b/Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Configuration; using Newsbot.Collector.Domain.Consts; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Services.Jobs; @@ -18,9 +19,9 @@ public class CodeProjectWatcherJobTests FeaturePullCommits = true, FeaturePullReleases = true }); - var results = client.CheckForReleases(new SourceModel + var results = client.CheckForReleases(new SourceEntity { - ID = Guid.NewGuid(), + Id = Guid.NewGuid(), Url = "https://github.com/jtom38/dvb", Type = SourceTypes.CodeProject, Site = SourceTypes.CodeProject, @@ -44,9 +45,9 @@ public class CodeProjectWatcherJobTests FeaturePullCommits = true, FeaturePullReleases = true }); - var results = client.CheckForReleases(new SourceModel + var results = client.CheckForReleases(new SourceEntity { - ID = Guid.NewGuid(), + Id = Guid.NewGuid(), Url = "https://github.com/python/cpython", Type = SourceTypes.CodeProject, Site = SourceTypes.CodeProject, @@ -70,9 +71,9 @@ public class CodeProjectWatcherJobTests FeaturePullCommits = true, FeaturePullReleases = true }); - var results = client.CheckForCommits(new SourceModel + var results = client.CheckForCommits(new SourceEntity { - ID = Guid.NewGuid(), + Id = Guid.NewGuid(), Url = "https://github.com/jtom38/dvb", Type = SourceTypes.CodeProject, Site = SourceTypes.CodeProject, diff --git a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs index 249133e..46192b2 100644 --- a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs @@ -14,9 +14,9 @@ public class DiscordNotificationJobTest var webhookClient = new DiscordWebhookClient(uri); var client = new DiscordNotificationJob(); - var msg = client.GenerateDiscordMessage(new SourceModel + var msg = client.GenerateDiscordMessage(new SourceEntity { - ID = Guid.NewGuid(), + Id = Guid.NewGuid(), Site = "Unit Test", Source = "placeholder", Type = "a", @@ -35,7 +35,7 @@ public class DiscordNotificationJobTest Description = "Please work", AuthorName = "No one knows" }, - new IconModel + new IconEntity() { Id = Guid.NewGuid(), FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" @@ -63,9 +63,9 @@ public class DiscordNotificationJobTest AuthorName = "No one knows", CodeIsCommit = true }, - new SourceModel + new SourceEntity { - ID = Guid.NewGuid(), + Id = Guid.NewGuid(), Site = "Unit Test", Source = "placeholder", Type = "a", @@ -74,7 +74,7 @@ public class DiscordNotificationJobTest Url = "https://github.com", Tags = "Unit, Testing" }, - new IconModel + new IconEntity() { Id = Guid.NewGuid(), FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" diff --git a/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs b/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs index c514cba..9903747 100644 --- a/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs +++ b/Newsbot.Collector.Tests/Tables/SourcesTableTests.cs @@ -1,4 +1,5 @@ using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Tests.Tables; @@ -9,9 +10,9 @@ public class SourcesTableTests public void NewRecordTest() { var client = new SourcesTable(""); - var m = new SourceModel + var m = new SourceEntity { - ID = Guid.NewGuid(), + Id = Guid.NewGuid(), Site = "Testing", Name = "Testing", Source = "Testing", From c2b0f47d1c6cce5349290a8dde8fcdb5f7f2593f Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 20:59:53 -0700 Subject: [PATCH 17/27] minor change to SourcesTable.cs --- Newsbot.Collector.Database/Repositories/SourcesTable.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Newsbot.Collector.Database/Repositories/SourcesTable.cs b/Newsbot.Collector.Database/Repositories/SourcesTable.cs index 1ea6520..09e364b 100644 --- a/Newsbot.Collector.Database/Repositories/SourcesTable.cs +++ b/Newsbot.Collector.Database/Repositories/SourcesTable.cs @@ -35,7 +35,7 @@ public class SourcesTable : ISourcesRepository } catch (Exception ex) { - Console.WriteLine($"Failed to save "); + Console.WriteLine($"Failed to save source to db. {model.ToString()} {ex.Message} "); } return model; From 49342a8c39c0ad2756fcff3fc263222f8b94bfd8 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:02:26 -0700 Subject: [PATCH 18/27] subscriptions updated for ef --- .../Repositories/SubscriptionsTable.cs | 115 ++++++------------ .../Dto/SubscriptionDetailsDto.cs | 2 +- .../Dto/SubscriptionDto.cs | 3 +- .../Interfaces/ISubscriptionsRepository.cs | 13 +- .../Jobs/DiscordNotificationJob.cs | 4 +- .../Jobs/DiscordNotificationJobTest.cs | 4 +- 6 files changed, 52 insertions(+), 89 deletions(-) diff --git a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs index 61b30a6..a4c13ad 100644 --- a/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs +++ b/Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs @@ -1,6 +1,7 @@ using System.Data; using Dapper; using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; using Npgsql; @@ -23,104 +24,64 @@ public class SubscriptionsTable : ISubscriptionRepository _connectionString = connstr; } - public SubscriptionModel New(SubscriptionModel model) + public SubscriptionEntity New(SubscriptionEntity model) { - model.Id = Guid.NewGuid(); - using var conn = OpenConnection(_connectionString); - var query = - "Insert Into subscriptions (ID, DiscordWebHookId, SourceId, codeAllowCommits, codeAllowReleases) Values (@id, @webhookId, @sourceId, @codeAllowCommits, @codeAllowReleases);"; - try - { - conn.Execute(query, new - { - id = model.Id, - webhookId = model.DiscordWebHookId, - sourceId = model.SourceId, - codeAllowCommits = model.CodeAllowCommits, - codeAllowReleases = model.CodeAllowReleases - }); - return model; - } - catch - { - return new SubscriptionModel(); - } + model.Id = new Guid(); + using var context = new DatabaseContext(_connectionString); + context.Subscriptions.Add(model); + context.SaveChanges(); + return model; } - public List List(int page = 0, int count = 25) + public List List(int page = 0, int count = 25) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From subscriptions - Offset @page Fetch Next @count Rows Only;"; - return conn.Query(query, new - { - page = page * count, count - }).ToList(); + using var context = new DatabaseContext(_connectionString); + return context.Subscriptions.Skip(page * count).Take(count).ToList(); } - public List ListBySourceID(Guid id, int page = 0, int count = 25) + public List ListBySourceId(Guid id, int page = 0, int count = 25) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From subscriptions - Where sourceid = @sourceid - Offset @page Fetch Next @count Rows Only"; - return conn.Query(query, new - { - page = page * count, - count, - sourceid = id - }).ToList(); + using var context = new DatabaseContext(_connectionString); + return context.Subscriptions.Where(f => f.SourceId.Equals(id)) + .Skip(page * count) + .ToList(); } - public List ListByWebhook(Guid id, int page = 0, int count = 25) + public List ListByWebhook(Guid id, int page = 0, int count = 25) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From subscriptions - Where discordwebhookid = @webhookid - Offset @page Fetch Next @count Rows Only"; - return conn.Query(query, new - { - page = page * count, - count, - webhookid = id - }).ToList(); + using var context = new DatabaseContext(_connectionString); + return context.Subscriptions.Where(f => f.DiscordWebHookId.Equals(id)).Skip(page * count).ToList(); } - public SubscriptionModel GetById(Guid id) + public SubscriptionEntity GetById(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From subscriptions Where id = @id;"; - var res = conn.Query(query, new - { - id - }); - if (res.Count() == 0) return new SubscriptionModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Subscriptions + .FirstOrDefault(f => f.Id.Equals(id)); + return res ??= new SubscriptionEntity(); } - public SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId) + public SubscriptionEntity GetByWebhookAndSource(Guid webhookId, Guid sourceId) { - using var conn = OpenConnection(_connectionString); - var query = @"Select * From subscriptions - Where discordwebhookid = @webhookid - and sourceid = @sourceid;"; - var res = conn.Query(query, new - { - webhookid = webhookId, - sourceid = sourceId - }); - if (res.Count() == 0) return new SubscriptionModel(); - return res.First(); + using var context = new DatabaseContext(_connectionString); + var res = context.Subscriptions + .Where(f => f.DiscordWebHookId.Equals(webhookId)) + .FirstOrDefault(f => f.SourceId.Equals(sourceId)); + return res ??= new SubscriptionEntity(); } public void Delete(Guid id) { - using var conn = OpenConnection(_connectionString); - var query = "Delete From subscriptions Where id = @id;"; - conn.Execute(query, new + using var context = new DatabaseContext(_connectionString); + var res = context.Subscriptions.FirstOrDefault(f => f.Id.Equals(id)); + if (res is null) { - id - }); + return; + } + + context.Subscriptions.Remove(res); + context.SaveChanges(); + } private IDbConnection OpenConnection(string connectionString) diff --git a/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs b/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs index a73c9cd..90ae0af 100644 --- a/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs +++ b/Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs @@ -11,7 +11,7 @@ public class SubscriptionDetailsDto public SourceDto? Source { get; set; } public DiscordWebHookDto? DiscordWebHook { get; set; } - public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceEntity source, + public static SubscriptionDetailsDto Convert(SubscriptionEntity subscription, SourceEntity source, DiscordWebhookEntity discord) { return new SubscriptionDetailsDto diff --git a/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs b/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs index 1b6f1c2..56b7efb 100644 --- a/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs +++ b/Newsbot.Collector.Domain/Dto/SubscriptionDto.cs @@ -1,3 +1,4 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Dto; @@ -10,7 +11,7 @@ public class SubscriptionDto public bool CodeAllowReleases { get; set; } public bool CodeAllowCommits { get; set; } - public static SubscriptionDto Convert(SubscriptionModel model) + public static SubscriptionDto Convert(SubscriptionEntity model) { return new SubscriptionDto { diff --git a/Newsbot.Collector.Domain/Interfaces/ISubscriptionsRepository.cs b/Newsbot.Collector.Domain/Interfaces/ISubscriptionsRepository.cs index ecfe54b..c955bbf 100644 --- a/Newsbot.Collector.Domain/Interfaces/ISubscriptionsRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/ISubscriptionsRepository.cs @@ -1,17 +1,18 @@ +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models; namespace Newsbot.Collector.Domain.Interfaces; public interface ISubscriptionRepository { - SubscriptionModel New(SubscriptionModel model); + SubscriptionEntity New(SubscriptionEntity model); - List List(int page = 0, int count = 25); - List ListBySourceID(Guid id, int page = 0, int count = 25); - List ListByWebhook(Guid id, int page = 0, int count = 25); + List List(int page = 0, int count = 25); + List ListBySourceId(Guid id, int page = 0, int count = 25); + List ListByWebhook(Guid id, int page = 0, int count = 25); - SubscriptionModel GetById(Guid id); - SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId); + SubscriptionEntity GetById(Guid id); + SubscriptionEntity GetByWebhookAndSource(Guid webhookId, Guid sourceId); void Delete(Guid id); } \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs index be60738..6ae432e 100644 --- a/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs +++ b/Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs @@ -112,7 +112,7 @@ public class DiscordNotificationJob } // Find all the subscriptions for that source - var allSubscriptions = _subs.ListBySourceID(sourceDetails.Id); + var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id); foreach (var sub in allSubscriptions) SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); @@ -121,7 +121,7 @@ public class DiscordNotificationJob _queue.Delete(request.Id); } - public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionModel sub) + public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionEntity sub) { // Check if the subscription code flags // If the article is a code commit and the subscription does not want them, skip. diff --git a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs index 46192b2..a3b3c71 100644 --- a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs @@ -1,5 +1,4 @@ using Newsbot.Collector.Domain.Entities; -using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Notifications.Discord; @@ -79,7 +78,7 @@ public class DiscordNotificationJobTest Id = Guid.NewGuid(), FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" }, - new SubscriptionModel + new SubscriptionEntity { CodeAllowCommits = false, CodeAllowReleases = true @@ -88,6 +87,7 @@ public class DiscordNotificationJobTest } catch (MessageTypeNotRequestedException) { + Console.Write($"Message did not send as expected"); } From d286c4fd9a5fc80ce3fe60fd32641546913d2ed3 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:03:27 -0700 Subject: [PATCH 19/27] controller updated for entity --- .../Controllers/SubscriptionsController.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs index bd6a953..981a6b2 100644 --- a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs +++ b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Dto; +using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; @@ -68,7 +69,7 @@ public class SubscriptionsController : ControllerBase public IEnumerable GetBySourceId(Guid id) { var res = new List(); - var items = _subscription.ListBySourceID(id); + var items = _subscription.ListBySourceId(id); foreach (var item in items) res.Add(SubscriptionDto.Convert(item)); return res; } @@ -88,7 +89,7 @@ public class SubscriptionsController : ControllerBase var source = _sources.GetById(sourceId); if (source.Id == Guid.Empty) return new BadRequestResult(); - var item = _subscription.New(new SubscriptionModel + var item = _subscription.New(new SubscriptionEntity { Id = Guid.NewGuid(), SourceId = sourceId, @@ -116,7 +117,7 @@ public class SubscriptionsController : ControllerBase var source = _sources.GetById(sourceId); if (source.Id == Guid.Empty) return new BadRequestResult(); - var sub = _subscription.New(new SubscriptionModel + var sub = _subscription.New(new SubscriptionEntity { DiscordWebHookId = discordId, SourceId = sourceId, From 4720c23915c1e58c76f1149931259aa4ddbfb92d Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:03:58 -0700 Subject: [PATCH 20/27] projects updated to sync ef requirements --- Newsbot.Collector.Database/Newsbot.Collector.Database.csproj | 4 ++-- Newsbot.Collector.Services/Newsbot.Collector.Services.csproj | 4 ++++ Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj b/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj index c191f7f..efbaba5 100644 --- a/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj +++ b/Newsbot.Collector.Database/Newsbot.Collector.Database.csproj @@ -6,8 +6,8 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj b/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj index 98a3e25..132e6a8 100644 --- a/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj +++ b/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj @@ -7,6 +7,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj b/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj index 3fde038..967b7e9 100644 --- a/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj +++ b/Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj @@ -9,6 +9,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + From aa5296d084054b88f81ce667734f78d2ff15526c Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:04:31 -0700 Subject: [PATCH 21/27] ef migrations was updated with the requirements of DesignTimeContext.cs --- Newsbot.Collector.Database/DatabaseContext.cs | 15 +++++++++++++-- Newsbot.Collector.Database/DesignTimeContext.cs | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Newsbot.Collector.Database/DatabaseContext.cs b/Newsbot.Collector.Database/DatabaseContext.cs index 46e1cf9..d48c067 100644 --- a/Newsbot.Collector.Database/DatabaseContext.cs +++ b/Newsbot.Collector.Database/DatabaseContext.cs @@ -16,7 +16,7 @@ public class DatabaseContext : DbContext private string ConnectionString { get; set; } - public DatabaseContext(IConfiguration appsettings) + public DatabaseContext(IConfiguration appsettings, string connectionString) { var connString = appsettings.GetConnectionString(ConfigConnectionStringConst.Database); ConnectionString = connString ?? ""; @@ -26,14 +26,25 @@ public class DatabaseContext : DbContext { ConnectionString = connectionString; } + + public DatabaseContext(DbContextOptions connectionString) + { + ConnectionString = ""; + } + + public DatabaseContext() + { + ConnectionString = ""; + } protected override void OnConfiguring(DbContextOptionsBuilder options) { options.UseNpgsql(ConnectionString); } - public DatabaseContext(DbContextOptions options) + public DatabaseContext(DbContextOptions options, string connectionString) : base(options) { + ConnectionString = connectionString; } } \ No newline at end of file diff --git a/Newsbot.Collector.Database/DesignTimeContext.cs b/Newsbot.Collector.Database/DesignTimeContext.cs index ab717c9..f471a0b 100644 --- a/Newsbot.Collector.Database/DesignTimeContext.cs +++ b/Newsbot.Collector.Database/DesignTimeContext.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Configuration; namespace Newsbot.Collector.Database; From b4b09b0e2794cccadd45ac2f992ea4760e79279f Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:04:49 -0700 Subject: [PATCH 22/27] commented a line that was not getting used in a test --- Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs b/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs index b0575bc..92f3fb3 100644 --- a/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs +++ b/Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs @@ -61,7 +61,7 @@ public class RssWatcherJobTest [Fact] public void CanReadHtmlDrivenFeedPage() { - var url = "https://www.howtogeek.com/feed/"; + //var url = "https://www.howtogeek.com/feed/"; var client = new RssWatcherJob(); client.InitAndExecute(new RssWatcherJobOptions { From d6e65270914066e4d9b5bc59b2ec99cd5417fcef Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:05:12 -0700 Subject: [PATCH 23/27] api adds ef packages and adds db context to start up --- Newsbot.Collector.Api/Newsbot.Collector.Api.csproj | 5 +++++ Newsbot.Collector.Api/Program.cs | 3 +++ 2 files changed, 8 insertions(+) diff --git a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj index c4fb0d5..74cc3ae 100644 --- a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj +++ b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj @@ -14,6 +14,11 @@ + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Newsbot.Collector.Api/Program.cs b/Newsbot.Collector.Api/Program.cs index 534230a..9dad4cf 100644 --- a/Newsbot.Collector.Api/Program.cs +++ b/Newsbot.Collector.Api/Program.cs @@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.OpenApi.Models; using Newsbot.Collector.Api; using Newsbot.Collector.Api.Authentication; +using Newsbot.Collector.Database; using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models.Config; @@ -71,6 +72,8 @@ builder.Services.AddSwaggerGen(cfg => cfg.AddSecurityRequirement(requirement); }); +builder.Services.AddDbContext(); + var app = builder.Build(); // Configure the HTTP request pipeline. From 3c302d2f0eea601f459b9799e574916d8c416aa6 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:05:27 -0700 Subject: [PATCH 24/27] pinned the ef version in Dockerfile --- .DockerIgnore | 5 +++-- Dockerfile | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.DockerIgnore b/.DockerIgnore index 07c6bb4..eb22bc9 100644 --- a/.DockerIgnore +++ b/.DockerIgnore @@ -1,3 +1,4 @@ appsettings.json -**/bin/ -**/obj/ \ No newline at end of file +**/bin/** +**/obj/** +**/efbundle \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7427653..2a5abbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN dotnet publish -o build RUN ls build FROM build as ef -RUN dotnet tool install dotnet-ef --tool-path /usr/bin +RUN dotnet tool install dotnet-ef --tool-path /usr/bin --version 7.0.8 RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database" FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app From 54155bfc3f7808f542efe31a755a5e00e4a0f189 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:05:50 -0700 Subject: [PATCH 25/27] makefile will now overwrite any local migration file --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 858ec2b..1007228 100644 --- a/makefile +++ b/makefile @@ -22,7 +22,7 @@ docker-migrate: ## Runs the migrations stored in the Docker image docker run -it --env-file .env ghcr.io/jtom38/newsbot.collector:master /app/goose --dir "/app/migrations" up ef-build: ## Builds migration artifact - dotnet ef migrations bundle --project "Newsbot.Collector.Database" + dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force ef-migrate: ## Runs migrations based on the newest artifact dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force From 52e86c01cea407f9c28bd7e74512b94572e86308 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:06:05 -0700 Subject: [PATCH 26/27] starting to update the README.md --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 93d20c4..91ffc75 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,26 @@ # Newsbot.Collector -C# API Server to collect news from around the world + +C# API Server to collect news from around the world. + +## Features + +- YouTube +- Github + - Should support other source control services but untested +- RSS + +## Notifications + +- Discord Webhooks + +## Legacy Features No longer support + +- Twitter + - API Changes +- Reddit + - API Changes + - Could come back as long as the json dump is still available +- Instagram + - No plans to attempt to get this working again. + + From f92480e717d6b2bcedc4f53525c789a4bd016054 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 25 Jun 2023 21:14:34 -0700 Subject: [PATCH 27/27] added --force to ensure a migration file is always made. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2a5abbc..32dc8e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN ls build FROM build as ef RUN dotnet tool install dotnet-ef --tool-path /usr/bin --version 7.0.8 -RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database" +RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app