starting to update the README.md

.DockerIgnore

@@ -1,3 +1,4 @@
 appsettings.json
-**/bin/
-**/obj/
+**/bin/**
+**/obj/**
+**/efbundle

Dockerfile

@@ -17,7 +17,7 @@ RUN dotnet publish -o build
 RUN ls build
 
 FROM build as ef
-RUN dotnet tool install dotnet-ef --tool-path /usr/bin
+RUN dotnet tool install dotnet-ef --tool-path /usr/bin --version 7.0.8
 RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database"
 
 FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app

Newsbot.Collector.Api/Controllers/SubscriptionsController.cs

@@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Dto;
+using Newsbot.Collector.Domain.Entities;
 using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
 
@@ -68,7 +69,7 @@ public class SubscriptionsController : ControllerBase
     public IEnumerable<SubscriptionDto> GetBySourceId(Guid id)
     {
         var res = new List<SubscriptionDto>();
-        var items = _subscription.ListBySourceID(id);
+        var items = _subscription.ListBySourceId(id);
         foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
         return res;
     }
@@ -88,7 +89,7 @@ public class SubscriptionsController : ControllerBase
         var source = _sources.GetById(sourceId);
         if (source.Id == Guid.Empty) return new BadRequestResult();
 
-        var item = _subscription.New(new SubscriptionModel
+        var item = _subscription.New(new SubscriptionEntity
         {
             Id = Guid.NewGuid(),
             SourceId = sourceId,
@@ -116,7 +117,7 @@ public class SubscriptionsController : ControllerBase
         var source = _sources.GetById(sourceId);
         if (source.Id == Guid.Empty) return new BadRequestResult();
 
-        var sub = _subscription.New(new SubscriptionModel
+        var sub = _subscription.New(new SubscriptionEntity
         {
             DiscordWebHookId = discordId,
             SourceId = sourceId,

Newsbot.Collector.Api/Newsbot.Collector.Api.csproj

@@ -14,6 +14,11 @@
         <PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
         <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
         <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.8" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
         <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
         <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.4.0" />
         <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.4.0" />

Newsbot.Collector.Api/Program.cs

@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Diagnostics.HealthChecks;
 using Microsoft.OpenApi.Models;
 using Newsbot.Collector.Api;
 using Newsbot.Collector.Api.Authentication;
+using Newsbot.Collector.Database;
 using Newsbot.Collector.Domain.Consts;
 using Newsbot.Collector.Domain.Models;
 using Newsbot.Collector.Domain.Models.Config;
@@ -71,6 +72,8 @@ builder.Services.AddSwaggerGen(cfg =>
     cfg.AddSecurityRequirement(requirement);
 });
 
+builder.Services.AddDbContext<DatabaseContext>();
+
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.

Newsbot.Collector.Database/DatabaseContext.cs

@@ -16,7 +16,7 @@ public class DatabaseContext : DbContext
     
     private string ConnectionString { get; set; }
 
-    public DatabaseContext(IConfiguration appsettings)
+    public DatabaseContext(IConfiguration appsettings, string connectionString)
     {
         var connString = appsettings.GetConnectionString(ConfigConnectionStringConst.Database);
         ConnectionString = connString ?? "";
@@ -26,14 +26,25 @@ public class DatabaseContext : DbContext
     {
         ConnectionString = connectionString;
     }
+    
+    public DatabaseContext(DbContextOptions<DatabaseContext> connectionString)
+    {
+        ConnectionString = "";
+    }
+
+    public DatabaseContext()
+    {
+        ConnectionString = "";
+    }
 
     protected override void OnConfiguring(DbContextOptionsBuilder options)
     {
         options.UseNpgsql(ConnectionString);
     }
     
-    public DatabaseContext(DbContextOptions<DatabaseContext> options)
+    public DatabaseContext(DbContextOptions<DatabaseContext> options, string connectionString)
         : base(options)
     {
+        ConnectionString = connectionString;
     }
 }

Newsbot.Collector.Database/DesignTimeContext.cs

@@ -1,5 +1,6 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
+using Microsoft.Extensions.Configuration;
 
 namespace Newsbot.Collector.Database;
 

Newsbot.Collector.Database/Newsbot.Collector.Database.csproj

@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="dapper" Version="2.0.123" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.7" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.7">
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.8" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>

Newsbot.Collector.Database/Repositories/SourcesTable.cs

@@ -35,7 +35,7 @@ public class SourcesTable : ISourcesRepository
         }
         catch (Exception ex)
         {
-            Console.WriteLine($"Failed to save ");
+            Console.WriteLine($"Failed to save source to db. {model.ToString()} {ex.Message} ");
         }
 
         return model;

Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs

@@ -1,6 +1,7 @@
 using System.Data;
 using Dapper;
 using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Domain.Entities;
 using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
 using Npgsql;
@@ -23,104 +24,64 @@ public class SubscriptionsTable : ISubscriptionRepository
         _connectionString = connstr;
     }
 
-    public SubscriptionModel New(SubscriptionModel model)
+    public SubscriptionEntity New(SubscriptionEntity model)
     {
-        model.Id = Guid.NewGuid();
-        using var conn = OpenConnection(_connectionString);
-        var query =
-            "Insert Into subscriptions (ID, DiscordWebHookId, SourceId, codeAllowCommits, codeAllowReleases) Values (@id, @webhookId, @sourceId, @codeAllowCommits, @codeAllowReleases);";
-        try
-        {
-            conn.Execute(query, new
-            {
-                id = model.Id,
-                webhookId = model.DiscordWebHookId,
-                sourceId = model.SourceId,
-                codeAllowCommits = model.CodeAllowCommits,
-                codeAllowReleases = model.CodeAllowReleases
-            });
-            return model;
-        }
-        catch
-        {
-            return new SubscriptionModel();
-        }
+        model.Id = new Guid();
+        using var context = new DatabaseContext(_connectionString);
+        context.Subscriptions.Add(model);
+        context.SaveChanges();
+        return model;
     }
 
-    public List<SubscriptionModel> List(int page = 0, int count = 25)
+    public List<SubscriptionEntity> List(int page = 0, int count = 25)
     {
-        using var conn = OpenConnection(_connectionString);
-        var query = @"Select * From subscriptions
-        Offset @page Fetch Next @count Rows Only;";
-        return conn.Query<SubscriptionModel>(query, new
-        {
-            page = page * count, count
-        }).ToList();
+        using var context = new DatabaseContext(_connectionString);
+        return context.Subscriptions.Skip(page * count).Take(count).ToList();
     }
 
-    public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25)
+    public List<SubscriptionEntity> ListBySourceId(Guid id, int page = 0, int count = 25)
     {
-        using var conn = OpenConnection(_connectionString);
-        var query = @"Select * From subscriptions 
-        Where sourceid = @sourceid
-        Offset @page Fetch Next @count Rows Only";
-        return conn.Query<SubscriptionModel>(query, new
-        {
-            page = page * count,
-            count,
-            sourceid = id
-        }).ToList();
+        using var context = new DatabaseContext(_connectionString);
+        return context.Subscriptions.Where(f => f.SourceId.Equals(id))
+            .Skip(page * count)
+            .ToList();
     }
 
-    public List<SubscriptionModel> ListByWebhook(Guid id, int page = 0, int count = 25)
+    public List<SubscriptionEntity> ListByWebhook(Guid id, int page = 0, int count = 25)
     {
-        using var conn = OpenConnection(_connectionString);
-        var query = @"Select * From subscriptions 
-        Where discordwebhookid = @webhookid
-        Offset @page Fetch Next @count Rows Only";
-        return conn.Query<SubscriptionModel>(query, new
-        {
-            page = page * count,
-            count,
-            webhookid = id
-        }).ToList();
+        using var context = new DatabaseContext(_connectionString);
+        return context.Subscriptions.Where(f => f.DiscordWebHookId.Equals(id)).Skip(page * count).ToList();
     }
 
-    public SubscriptionModel GetById(Guid id)
+    public SubscriptionEntity GetById(Guid id)
     {
-        using var conn = OpenConnection(_connectionString);
-        var query = @"Select * From subscriptions Where id = @id;";
-        var res = conn.Query<SubscriptionModel>(query, new
-        {
-            id
-        });
-        if (res.Count() == 0) return new SubscriptionModel();
-        return res.First();
+        using var context = new DatabaseContext(_connectionString);
+        var res = context.Subscriptions
+            .FirstOrDefault(f => f.Id.Equals(id));
+        return res ??= new SubscriptionEntity();
     }
 
-    public SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId)
+    public SubscriptionEntity GetByWebhookAndSource(Guid webhookId, Guid sourceId)
     {
-        using var conn = OpenConnection(_connectionString);
-        var query = @"Select * From subscriptions 
-        Where discordwebhookid = @webhookid 
-        and sourceid = @sourceid;";
-        var res = conn.Query<SubscriptionModel>(query, new
-        {
-            webhookid = webhookId,
-            sourceid = sourceId
-        });
-        if (res.Count() == 0) return new SubscriptionModel();
-        return res.First();
+        using var context = new DatabaseContext(_connectionString);
+        var res = context.Subscriptions
+            .Where(f => f.DiscordWebHookId.Equals(webhookId))
+            .FirstOrDefault(f => f.SourceId.Equals(sourceId));
+        return res ??= new SubscriptionEntity();
     }
 
     public void Delete(Guid id)
     {
-        using var conn = OpenConnection(_connectionString);
-        var query = "Delete From subscriptions Where id = @id;";
-        conn.Execute(query, new
+        using var context = new DatabaseContext(_connectionString);
+        var res = context.Subscriptions.FirstOrDefault(f => f.Id.Equals(id));
+        if (res is null)
         {
-            id
-        });
+            return;
+        }
+
+        context.Subscriptions.Remove(res);
+        context.SaveChanges();
+
     }
 
     private IDbConnection OpenConnection(string connectionString)

Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs

@@ -11,7 +11,7 @@ public class SubscriptionDetailsDto
     public SourceDto? Source { get; set; }
     public DiscordWebHookDto? DiscordWebHook { get; set; }
 
-    public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceEntity source,
+    public static SubscriptionDetailsDto Convert(SubscriptionEntity subscription, SourceEntity source,
         DiscordWebhookEntity discord)
     {
         return new SubscriptionDetailsDto

Newsbot.Collector.Domain/Dto/SubscriptionDto.cs

@@ -1,3 +1,4 @@
+using Newsbot.Collector.Domain.Entities;
 using Newsbot.Collector.Domain.Models;
 
 namespace Newsbot.Collector.Domain.Dto;
@@ -10,7 +11,7 @@ public class SubscriptionDto
     public bool CodeAllowReleases { get; set; }
     public bool CodeAllowCommits { get; set; }
 
-    public static SubscriptionDto Convert(SubscriptionModel model)
+    public static SubscriptionDto Convert(SubscriptionEntity model)
     {
         return new SubscriptionDto
         {

Newsbot.Collector.Domain/Interfaces/ISubscriptionsRepository.cs

@@ -1,17 +1,18 @@
+using Newsbot.Collector.Domain.Entities;
 using Newsbot.Collector.Domain.Models;
 
 namespace Newsbot.Collector.Domain.Interfaces;
 
 public interface ISubscriptionRepository
 {
-    SubscriptionModel New(SubscriptionModel model);
+    SubscriptionEntity New(SubscriptionEntity model);
 
-    List<SubscriptionModel> List(int page = 0, int count = 25);
-    List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25);
-    List<SubscriptionModel> ListByWebhook(Guid id, int page = 0, int count = 25);
+    List<SubscriptionEntity> List(int page = 0, int count = 25);
+    List<SubscriptionEntity> ListBySourceId(Guid id, int page = 0, int count = 25);
+    List<SubscriptionEntity> ListByWebhook(Guid id, int page = 0, int count = 25);
 
-    SubscriptionModel GetById(Guid id);
-    SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId);
+    SubscriptionEntity GetById(Guid id);
+    SubscriptionEntity GetByWebhookAndSource(Guid webhookId, Guid sourceId);
 
     void Delete(Guid id);
 }

Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs

@@ -112,7 +112,7 @@ public class DiscordNotificationJob
         }
 
         // Find all the subscriptions for that source
-        var allSubscriptions = _subs.ListBySourceID(sourceDetails.Id);
+        var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
 
         foreach (var sub in allSubscriptions)
             SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub);
@@ -121,7 +121,7 @@ public class DiscordNotificationJob
         _queue.Delete(request.Id);
     }
 
-    public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionModel sub)
+    public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionEntity sub)
     {
         // Check if the subscription code flags
         // If the article is a code commit and the subscription does not want them, skip.

Newsbot.Collector.Services/Newsbot.Collector.Services.csproj

@@ -7,6 +7,10 @@
 
     <ItemGroup>
         <PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
         <PackageReference Include="Selenium.WebDriver" Version="4.8.1" />
         <PackageReference Include="Selenium.WebDriver.GeckoDriver" Version="0.32.2" />
         <PackageReference Include="Serilog" Version="2.12.0" />

Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs

@@ -1,5 +1,4 @@
 using Newsbot.Collector.Domain.Entities;
-using Newsbot.Collector.Domain.Models;
 using Newsbot.Collector.Services.Jobs;
 using Newsbot.Collector.Services.Notifications.Discord;
 
@@ -79,7 +78,7 @@ public class DiscordNotificationJobTest
                     Id = Guid.NewGuid(),
                     FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png"
                 },
-                new SubscriptionModel
+                new SubscriptionEntity
                 {
                     CodeAllowCommits = false,
                     CodeAllowReleases = true
@@ -88,6 +87,7 @@ public class DiscordNotificationJobTest
         }
         catch (MessageTypeNotRequestedException)
         {
+            
             Console.Write($"Message did not send as expected");
         }
 

Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs

@@ -61,7 +61,7 @@ public class RssWatcherJobTest
     [Fact]
     public void CanReadHtmlDrivenFeedPage()
     {
-        var url = "https://www.howtogeek.com/feed/";
+        //var url = "https://www.howtogeek.com/feed/";
         var client = new RssWatcherJob();
         client.InitAndExecute(new RssWatcherJobOptions
         {

Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj

@@ -9,6 +9,10 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />

README.md

@@ -1,2 +1,26 @@
 # Newsbot.Collector
-C# API Server to collect news from around the world
+
+C# API Server to collect news from around the world.
+
+## Features
+
+- YouTube
+- Github
+  - Should support other source control services but untested
+- RSS
+
+## Notifications
+
+- Discord Webhooks
+
+## Legacy Features No longer support
+
+- Twitter
+  - API Changes
+- Reddit
+  - API Changes
+  - Could come back as long as the json dump is still available
+- Instagram
+  - No plans to attempt to get this working again.
+
+

makefile

@@ -22,7 +22,7 @@ docker-migrate: ## Runs the migrations stored in the Docker image
     docker run -it --env-file .env ghcr.io/jtom38/newsbot.collector:master /app/goose --dir "/app/migrations" up
 
 ef-build: ## Builds migration artifact
-	dotnet ef migrations bundle --project "Newsbot.Collector.Database"
+	dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force
 
 ef-migrate: ## Runs migrations based on the newest artifact
 	dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force