jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Merge pull request 'features/cutover-to-ef' (#9) from features/cutover-to-ef into main

Browse Source 
Reviewed-on: #9
This commit is contained in:
jtom38 2023-06-25 21:15:56 -07:00
commit 7931a4fe6a
64 changed files with 1326 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.DockerIgnore
View File

@ -1,3 +1,4 @@
appsettings.json
**/bin/
**/obj/
**/bin/**
**/obj/**
**/efbundle

2
.gitignore vendored
View File

@ -4,6 +4,8 @@
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
seed.secrects.json
efbundle
efbundle.exe
out/
#Docker Compose Secrets

15
Dockerfile
View File

@ -1,6 +1,6 @@
FROM golang:latest as goose
#FROM golang:latest as goose
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
#RUN go install github.com/pressly/goose/v3/cmd/goose@latest
FROM mcr.microsoft.com/dotnet/sdk:7.0.103 as build
@ -14,21 +14,20 @@ FROM build AS publish
RUN dotnet publish -c Release -o /app/publish
RUN dotnet publish -o build
#--self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true
RUN ls build
FROM build as ef
RUN dotnet tool install dotnet-ef --tool-path /usr/bin --version 7.0.8
RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force
FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app
ENV ASPNETCORE_URLS=http://*:5000
ENV DOTNET_URLS=http://*:5000
#RUN apt-get install chromium -y
WORKDIR /app
RUN mkdir /migrations
COPY --from=publish /app/build /app
COPY --from=build ./app/Newsbot.Collector.Database/Migrations /app/migrations
COPY --from=goose /go/bin/goose /app
COPY --from=ef /app/efbundle /app
CMD [ "dotnet", "Newsbot.Collector.Api.dll" ]

2
Newsbot.Collector.Api/Controllers/ArticlesController.cs
View File

@ -43,7 +43,7 @@ public class ArticlesController : ControllerBase
public ArticleDetailsDto GetDetailsById(Guid id)
{
var item = _articles.GetById(id);
var sourceItem = _sources.GetByID(item.SourceID);
var sourceItem = _sources.GetById(item.SourceId);
return ArticleDetailsDto.Convert(item, sourceItem);
}

7
Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs
View File

@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Dto;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
@ -40,12 +41,12 @@ public class DiscordWebHookController : ControllerBase
{
var exists = _webhooks.GetByUrl(url);
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (exists.ID != Guid.Empty)
if (exists.Id != Guid.Empty)
{
return DiscordWebHookDto.Convert(exists);
}
var res = _webhooks.New(new DiscordWebHookModel
var res = _webhooks.New(new DiscordWebhookEntity
{
Url = url,
Server = server,
@ -72,7 +73,7 @@ public class DiscordWebHookController : ControllerBase
[HttpGet("{id}")]
public DiscordWebHookDto GetById(Guid id)
{
var res = _webhooks.GetByID(id);
var res = _webhooks.GetById(id);
return DiscordWebHookDto.Convert(res);
}

39
Newsbot.Collector.Api/Controllers/SourcesController.cs
View File

@ -3,6 +3,7 @@ using Microsoft.Extensions.Options;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Dto;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.HtmlParser;
@ -39,10 +40,10 @@ public class SourcesController : ControllerBase
}
[HttpGet("by/type")]
public IEnumerable<SourceDto> GetByType(string type)
public IEnumerable<SourceDto> GetByType(string type, int page)
{
var res = new List<SourceDto>();
var temp = _sources.ListByType(type);
var temp = _sources.ListByType(type, page);
foreach (var item in temp) res.Add(SourceDto.Convert(item));
return res;
}
@ -51,7 +52,7 @@ public class SourcesController : ControllerBase
public SourceDto NewReddit(string name)
{
var res = _sources.GetByNameAndType(name, SourceTypes.Reddit);
if (res.ID != Guid.Empty) return SourceDto.Convert(res);
if (res.Id != Guid.Empty) return SourceDto.Convert(res);
var uri = new Uri($"https://reddit.com/r/{name}");
@ -61,7 +62,7 @@ public class SourcesController : ControllerBase
});
pageReader.Parse();
var item = _sources.New(new SourceModel
var item = _sources.New(new SourceEntity
{
Site = SourceTypes.Reddit,
Name = name,
@ -74,11 +75,11 @@ public class SourcesController : ControllerBase
// Not all subreddits have an Icon, so we only want to add a record when it has one.
if (pageReader.Data.Header.Image != "")
_icons.New(new IconModel
_icons.New(new IconEntity()
{
Id = Guid.NewGuid(),
FileName = pageReader.Data.Header.Image,
SourceId = item.ID
SourceId = item.Id
});
return SourceDto.Convert(item);
}
@ -87,9 +88,9 @@ public class SourcesController : ControllerBase
public SourceDto NewRss(string name, string url)
{
var res = _sources.GetByNameAndType(name, SourceTypes.Rss);
if (res.ID != Guid.Empty) return SourceDto.Convert(res);
if (res.Id != Guid.Empty) return SourceDto.Convert(res);
var m = new SourceModel
var m = new SourceEntity
{
Site = SourceTypes.Rss,
Name = name,
@ -107,7 +108,7 @@ public class SourcesController : ControllerBase
public SourceDto NewYoutube(string url)
{
var res = _sources.GetByUrl(url);
if (res.ID != Guid.Empty) return SourceDto.Convert(res);
if (res.Id != Guid.Empty) return SourceDto.Convert(res);
var htmlClient = new HtmlPageReader(new HtmlPageReaderOptions
{
@ -115,7 +116,7 @@ public class SourcesController : ControllerBase
});
htmlClient.Parse();
var item = _sources.New(new SourceModel
var item = _sources.New(new SourceEntity
{
Site = SourceTypes.YouTube,
Type = SourceTypes.YouTube,
@ -127,11 +128,11 @@ public class SourcesController : ControllerBase
YoutubeId = htmlClient.Data.Header.YoutubeChannelID ?? ""
});
_icons.New(new IconModel
_icons.New(new IconEntity()
{
Id = Guid.NewGuid(),
FileName = htmlClient.Data.Header.Image,
SourceId = item.ID
SourceId = item.Id
});
return SourceDto.Convert(item);
@ -141,9 +142,9 @@ public class SourcesController : ControllerBase
public SourceDto NewTwitch(string name)
{
var res = _sources.GetByNameAndType(name, SourceTypes.Twitch);
if (res.ID != Guid.Empty) return SourceDto.Convert(res);
if (res.Id != Guid.Empty) return SourceDto.Convert(res);
var item = _sources.New(new SourceModel
var item = _sources.New(new SourceEntity
{
Site = SourceTypes.Twitch,
Type = SourceTypes.Twitch,
@ -162,7 +163,7 @@ public class SourcesController : ControllerBase
//if (!url.Contains("github.com")) return new SourceDto();
var res = _sources.GetByUrl(url);
if (res.ID != Guid.Empty) return SourceDto.Convert(res);
if (res.Id != Guid.Empty) return SourceDto.Convert(res);
var slice = url.Split('/');
@ -172,7 +173,7 @@ public class SourcesController : ControllerBase
});
pageReader.Parse();
var item = _sources.New(new SourceModel
var item = _sources.New(new SourceEntity
{
Site = SourceTypes.CodeProject,
Type = SourceTypes.CodeProject,
@ -183,11 +184,11 @@ public class SourcesController : ControllerBase
Tags = $"{slice[2]},{slice[3]},{slice[4]}"
});
_icons.New(new IconModel
_icons.New(new IconEntity()
{
Id = Guid.NewGuid(),
FileName = pageReader.Data.Header.Image,
SourceId = item.ID
SourceId = item.Id
});
return SourceDto.Convert(item);
@ -196,7 +197,7 @@ public class SourcesController : ControllerBase
[HttpGet("{id}")]
public SourceDto GetById(Guid id)
{
var item = _sources.GetByID(id);
var item = _sources.GetById(id);
return SourceDto.Convert(item);
}

27
Newsbot.Collector.Api/Controllers/SubscriptionsController.cs
View File

@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Dto;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
@ -43,8 +44,8 @@ public class SubscriptionsController : ControllerBase
public SubscriptionDetailsDto GetDetailsById(Guid id)
{
var sub = _subscription.GetById(id);
var webhook = _discord.GetByID(sub.DiscordWebHookId);
var source = _sources.GetByID(sub.SourceId);
var webhook = _discord.GetById(sub.DiscordWebHookId);
var source = _sources.GetById(sub.SourceId);
return SubscriptionDetailsDto.Convert(sub, source, webhook);
}
@ -68,7 +69,7 @@ public class SubscriptionsController : ControllerBase
public IEnumerable<SubscriptionDto> GetBySourceId(Guid id)
{
var res = new List<SubscriptionDto>();
var items = _subscription.ListBySourceID(id);
var items = _subscription.ListBySourceId(id);
foreach (var item in items) res.Add(SubscriptionDto.Convert(item));
return res;
}
@ -82,13 +83,13 @@ public class SubscriptionsController : ControllerBase
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetByID(discordId);
if (discord.ID == Guid.Empty) return new BadRequestResult();
var discord = _discord.GetById(discordId);
if (discord.Id == Guid.Empty) return new BadRequestResult();
var source = _sources.GetByID(sourceId);
if (source.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetById(sourceId);
if (source.Id == Guid.Empty) return new BadRequestResult();
var item = _subscription.New(new SubscriptionModel
var item = _subscription.New(new SubscriptionEntity
{
Id = Guid.NewGuid(),
SourceId = sourceId,
@ -110,13 +111,13 @@ public class SubscriptionsController : ControllerBase
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetByID(discordId);
if (discord.ID == Guid.Empty) return new BadRequestResult();
var discord = _discord.GetById(discordId);
if (discord.Id == Guid.Empty) return new BadRequestResult();
var source = _sources.GetByID(sourceId);
if (source.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetById(sourceId);
if (source.Id == Guid.Empty) return new BadRequestResult();
var sub = _subscription.New(new SubscriptionModel
var sub = _subscription.New(new SubscriptionEntity
{
DiscordWebHookId = discordId,
SourceId = sourceId,

5
Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
View File

@ -14,6 +14,11 @@
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.4.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.4.0" />

3
Newsbot.Collector.Api/Program.cs
View File

@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Diagnostics.HealthChecks;
using Microsoft.OpenApi.Models;
using Newsbot.Collector.Api;
using Newsbot.Collector.Api.Authentication;
using Newsbot.Collector.Database;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
@ -71,6 +72,8 @@ builder.Services.AddSwaggerGen(cfg =>
cfg.AddSecurityRequirement(requirement);
});
builder.Services.AddDbContext<DatabaseContext>();
var app = builder.Build();
// Configure the HTTP request pipeline.

50
Newsbot.Collector.Database/DatabaseContext.cs Normal file
View File

@ -0,0 +1,50 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
namespace Newsbot.Collector.Database;
public class DatabaseContext : DbContext
{
public DbSet<ArticlesEntity> Articles { get; set; } = null!;
public DbSet<DiscordQueueEntity> DiscordQueue { get; set; } = null!;
public DbSet<DiscordWebhookEntity> DiscordWebhooks { get; set; } = null!;
public DbSet<IconEntity> Icons { get; set; } = null!;
public DbSet<SourceEntity> Sources { get; set; } = null!;
public DbSet<SubscriptionEntity> Subscriptions { get; set; } = null!;
private string ConnectionString { get; set; }
public DatabaseContext(IConfiguration appsettings, string connectionString)
{
var connString = appsettings.GetConnectionString(ConfigConnectionStringConst.Database);
ConnectionString = connString ?? "";
}
public DatabaseContext(string connectionString)
{
ConnectionString = connectionString;
}
public DatabaseContext(DbContextOptions<DatabaseContext> connectionString)
{
ConnectionString = "";
}
public DatabaseContext()
{
ConnectionString = "";
}
protected override void OnConfiguring(DbContextOptionsBuilder options)
{
options.UseNpgsql(ConnectionString);
}
public DatabaseContext(DbContextOptions<DatabaseContext> options, string connectionString)
: base(options)
{
ConnectionString = connectionString;
}
}

16
Newsbot.Collector.Database/DesignTimeContext.cs Normal file
View File

@ -0,0 +1,16 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
using Microsoft.Extensions.Configuration;
namespace Newsbot.Collector.Database;
public class DesignTimeContext :IDesignTimeDbContextFactory<DatabaseContext>
{
public DatabaseContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<DatabaseContext>();
// pass your design time connection string here
optionsBuilder.UseNpgsql("<connection_string>");
return new DatabaseContext(optionsBuilder.Options);
}
}

223
Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.Designer.cs generated Normal file
View File

@ -0,0 +1,223 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Newsbot.Collector.Database;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace Newsbot.Collector.Database.Migrations
{
[DbContext(typeof(DatabaseContext))]
[Migration("20230619043102_MigrationFromGoose")]
partial class MigrationFromGoose
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.ArticlesEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AuthorImage")
.HasColumnType("text");
b.Property<string>("AuthorName")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("CodeIsCommit")
.HasColumnType("boolean");
b.Property<bool>("CodeIsRelease")
.HasColumnType("boolean");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("PubDate")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.Property<string>("Tags")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Thumbnail")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.HasColumnType("text");
b.Property<string>("Video")
.IsRequired()
.HasColumnType("text");
b.Property<int>("VideoHeight")
.HasColumnType("integer");
b.Property<int>("VideoWidth")
.HasColumnType("integer");
b.HasKey("Id");
b.ToTable("Articles");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordQueueEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Guid>("ArticleId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("DiscordQueue");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Channel")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("Server")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("DiscordWebhooks");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.IconEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("FileName")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Site")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("Icons");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SourceEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("Deleted")
.HasColumnType("boolean");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Site")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Tags")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Value")
.IsRequired()
.HasColumnType("text");
b.Property<string>("YoutubeId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("Sources");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SubscriptionEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("CodeAllowCommits")
.HasColumnType("boolean");
b.Property<bool>("CodeAllowReleases")
.HasColumnType("boolean");
b.Property<Guid>("DiscordWebHookId")
.HasColumnType("uuid");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("Subscriptions");
});
#pragma warning restore 612, 618
}
}
}

140
Newsbot.Collector.Database/Migrations/20230619043102_MigrationFromGoose.cs Normal file
View File

@ -0,0 +1,140 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.Extensions.Configuration;
#nullable disable
namespace Newsbot.Collector.Database.Migrations
{
/// <inheritdoc />
public partial class MigrationFromGoose : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "Articles",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
SourceId = table.Column<Guid>(type: "uuid", nullable: false),
Tags = table.Column<string>(type: "text", nullable: false),
Title = table.Column<string>(type: "text", nullable: false),
Url = table.Column<string>(type: "text", nullable: true),
PubDate = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
Video = table.Column<string>(type: "text", nullable: false),
VideoHeight = table.Column<int>(type: "integer", nullable: false),
VideoWidth = table.Column<int>(type: "integer", nullable: false),
Thumbnail = table.Column<string>(type: "text", nullable: false),
Description = table.Column<string>(type: "text", nullable: false),
AuthorName = table.Column<string>(type: "text", nullable: false),
AuthorImage = table.Column<string>(type: "text", nullable: true),
CodeIsRelease = table.Column<bool>(type: "boolean", nullable: false),
CodeIsCommit = table.Column<bool>(type: "boolean", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_Articles", x => x.Id);
});
migrationBuilder.CreateTable(
name: "DiscordQueue",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
ArticleId = table.Column<Guid>(type: "uuid", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DiscordQueue", x => x.Id);
});
migrationBuilder.CreateTable(
name: "DiscordWebhooks",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Url = table.Column<string>(type: "text", nullable: false),
Server = table.Column<string>(type: "text", nullable: false),
Channel = table.Column<string>(type: "text", nullable: false),
Enabled = table.Column<bool>(type: "boolean", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DiscordWebhooks", x => x.Id);
});
migrationBuilder.CreateTable(
name: "Icons",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
FileName = table.Column<string>(type: "text", nullable: false),
Site = table.Column<string>(type: "text", nullable: false),
SourceId = table.Column<Guid>(type: "uuid", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_Icons", x => x.Id);
});
migrationBuilder.CreateTable(
name: "Sources",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Site = table.Column<string>(type: "text", nullable: false),
Name = table.Column<string>(type: "text", nullable: false),
Source = table.Column<string>(type: "text", nullable: false),
Type = table.Column<string>(type: "text", nullable: false),
Value = table.Column<string>(type: "text", nullable: false),
Enabled = table.Column<bool>(type: "boolean", nullable: false),
Url = table.Column<string>(type: "text", nullable: false),
Tags = table.Column<string>(type: "text", nullable: false),
Deleted = table.Column<bool>(type: "boolean", nullable: false),
YoutubeId = table.Column<string>(type: "text", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_Sources", x => x.Id);
});
migrationBuilder.CreateTable(
name: "Subscriptions",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
CodeAllowReleases = table.Column<bool>(type: "boolean", nullable: false),
CodeAllowCommits = table.Column<bool>(type: "boolean", nullable: false),
SourceId = table.Column<Guid>(type: "uuid", nullable: false),
DiscordWebHookId = table.Column<Guid>(type: "uuid", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_Subscriptions", x => x.Id);
});
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "Articles");
migrationBuilder.DropTable(
name: "DiscordQueue");
migrationBuilder.DropTable(
name: "DiscordWebhooks");
migrationBuilder.DropTable(
name: "Icons");
migrationBuilder.DropTable(
name: "Sources");
migrationBuilder.DropTable(
name: "Subscriptions");
}
}
}

220
Newsbot.Collector.Database/Migrations/DatabaseContextModelSnapshot.cs Normal file
View File

@ -0,0 +1,220 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Newsbot.Collector.Database;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace Newsbot.Collector.Database.Migrations
{
[DbContext(typeof(DatabaseContext))]
partial class DatabaseContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.ArticlesEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AuthorImage")
.HasColumnType("text");
b.Property<string>("AuthorName")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("CodeIsCommit")
.HasColumnType("boolean");
b.Property<bool>("CodeIsRelease")
.HasColumnType("boolean");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("PubDate")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.Property<string>("Tags")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Thumbnail")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.HasColumnType("text");
b.Property<string>("Video")
.IsRequired()
.HasColumnType("text");
b.Property<int>("VideoHeight")
.HasColumnType("integer");
b.Property<int>("VideoWidth")
.HasColumnType("integer");
b.HasKey("Id");
b.ToTable("Articles");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordQueueEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<Guid>("ArticleId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("DiscordQueue");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.DiscordWebhookEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Channel")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("Server")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("DiscordWebhooks");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.IconEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("FileName")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Site")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("Icons");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SourceEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("Deleted")
.HasColumnType("boolean");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Site")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Tags")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Value")
.IsRequired()
.HasColumnType("text");
b.Property<string>("YoutubeId")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("Sources");
});
modelBuilder.Entity("Newsbot.Collector.Domain.Entities.SubscriptionEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("CodeAllowCommits")
.HasColumnType("boolean");
b.Property<bool>("CodeAllowReleases")
.HasColumnType("boolean");
b.Property<Guid>("DiscordWebHookId")
.HasColumnType("uuid");
b.Property<Guid>("SourceId")
.HasColumnType("uuid");
b.HasKey("Id");
b.ToTable("Subscriptions");
});
#pragma warning restore 612, 618
}
}
}

8
Newsbot.Collector.Database/Newsbot.Collector.Database.csproj
View File

@ -6,8 +6,14 @@
<ItemGroup>
<PackageReference Include="dapper" Version="2.0.123" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Npgsql" Version="7.0.2" />
<PackageReference Include="Npgsql" Version="7.0.4" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.4" />
</ItemGroup>
<PropertyGroup>

0
Newsbot.Collector.Database/Migrations/20220522083756_init.sql → Newsbot.Collector.Database/OldMigrations/20220522083756_init.sql
View File

0
Newsbot.Collector.Database/Migrations/20220529082459_seed.sql → Newsbot.Collector.Database/OldMigrations/20220529082459_seed.sql
View File

0
Newsbot.Collector.Database/Migrations/20220619085634_subscriptions.sql → Newsbot.Collector.Database/OldMigrations/20220619085634_subscriptions.sql
View File

0
Newsbot.Collector.Database/Migrations/20221207213427_source_delete.sql → Newsbot.Collector.Database/OldMigrations/20221207213427_source_delete.sql
View File

0
Newsbot.Collector.Database/Migrations/20230316185340_sources_youtube_id.sql → Newsbot.Collector.Database/OldMigrations/20230316185340_sources_youtube_id.sql
View File

0
Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql → Newsbot.Collector.Database/OldMigrations/20230326223545_icon_sourceid.sql
View File

0
Newsbot.Collector.Database/Migrations/20230411071020_article_code_columns.sql → Newsbot.Collector.Database/OldMigrations/20230411071020_article_code_columns.sql
View File

0
Newsbot.Collector.Database/Migrations/20230413203006_subscription_code_options.sql → Newsbot.Collector.Database/OldMigrations/20230413203006_subscription_code_options.sql
View File

0
Newsbot.Collector.Database/Migrations/20230413212643_icon_sourceid.sql → Newsbot.Collector.Database/OldMigrations/20230413212643_icon_sourceid.sql
View File

97
Newsbot.Collector.Database/Repositories/ArticlesTable.cs
View File

@ -1,6 +1,8 @@
using System.Data;
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
@ -11,9 +13,12 @@ public class ArticlesTable : IArticlesRepository
{
private readonly string _connectionString;
private DatabaseContext _context;
public ArticlesTable(string connectionString)
{
_connectionString = connectionString;
_context = new DatabaseContext(connectionString);
}
public ArticlesTable(IConfiguration configuration)
@ -22,55 +27,56 @@ public class ArticlesTable : IArticlesRepository
if (conn is null) conn = "";
_connectionString = conn;
_context = new DatabaseContext(conn);
}
public List<ArticlesModel> List(int page = 0, int count = 25)
public List<ArticlesEntity> List(int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>(@"select * from articles
Order By PubDate Desc
Offset @Page
Fetch Next @Count Rows Only", new
{
Page = page * count,
Count = count
})
.ToList();
using var context = new DatabaseContext(_connectionString);
var query = context.Articles
.Skip(page * count)
.OrderBy(d => d.PubDate)
.Take(25);
Console.WriteLine(query.ToQueryString());
return query.ToList();
}
public ArticlesEntity GetById(Guid id)
{
using var context = new DatabaseContext(_connectionString);
var query = context.Articles
.FirstOrDefault(d => d.Id.Equals(id));
query ??= new ArticlesEntity();
return query;
}
public ArticlesEntity GetByUrl(string url)
{
using var context = new DatabaseContext(_connectionString);
var res = context.Articles.FirstOrDefault(d => d.Url!.Equals(url));
res ??= new ArticlesEntity();
return res;
}
public ArticlesModel GetById(Guid ID)
public List<ArticlesEntity> ListBySourceId(Guid id, int page, int count)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID });
if (res.Count() == 0) return new ArticlesModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Articles
.Skip(page * count)
.Where(d => d.SourceId.Equals(id));
return res.ToList();
}
public ArticlesModel GetByUrl(string url)
public ArticlesEntity New(ArticlesEntity model)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
if (res.Count() == 0) return new ArticlesModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
model.Id = new Guid();
var query = context.Articles.Add(model);
context.SaveChanges();
return model;
}
public List<ArticlesModel> ListBySourceId(Guid id, int page, int count)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * from articles
where sourceid = @sourceid
Offset @page
Fetch next @count rows only";
return conn.Query<ArticlesModel>(query, new
{
sourceid = id,
page = page * count,
count
}).ToList();
}
public ArticlesModel New(ArticlesModel model)
public ArticlesModel NewDapper(ArticlesModel model)
{
model.ID = Guid.NewGuid();
@ -100,12 +106,17 @@ public class ArticlesTable : IArticlesRepository
public void DeleteAllBySourceId(Guid sourceId)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Execute("Delete from articles where sourceid = '@id'", new
using var context = new DatabaseContext(_connectionString);
var res = context.Articles
.Where(d => d.SourceId.Equals(sourceId))
.ToList();
foreach (var item in res)
{
sourceId
});
if (res == 0) throw new Exception($"No records where deleted that linked to SourceId = '{sourceId}'");
context.Articles.Remove(item);
}
context.SaveChanges();
}
private IDbConnection OpenConnection(string connectionString)

36
Newsbot.Collector.Database/Repositories/DiscordQueue.cs
View File

@ -1,5 +1,6 @@
using System.Data;
using Dapper;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
@ -22,33 +23,28 @@ public class DiscordQueueTable : IDiscordQueueRepository
return conn;
}
public void New(DiscordQueueModel model)
public void New(DiscordQueueEntity model)
{
using var conn = OpenConnection(_connectionString);
var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);";
conn.Execute(query, new
{
id = Guid.NewGuid(),
articleid = model.ArticleID
});
model.Id = new Guid();
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordQueue.Add(model);
context.SaveChanges();
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From DiscordQueue Where ID = @id;";
conn.Execute(query, new
{
id = id
});
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordQueue.FirstOrDefault(d => d.Id.Equals(id));
res ??= new DiscordQueueEntity();
context.DiscordQueue.Remove(res);
context.SaveChanges();
}
public List<DiscordQueueModel> List(int limit = 25)
public List<DiscordQueueEntity> List(int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordQueue LIMIT @limit;";
return conn.Query<DiscordQueueModel>(query, new {
limit = limit
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordQueue.Take(limit).ToList();
return res;
}
}

143
Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs
View File

@ -2,7 +2,7 @@ using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Entities;
using Npgsql;
namespace Newsbot.Collector.Database.Repositories;
@ -22,103 +22,104 @@ public class DiscordWebhooksTable : IDiscordWebHooksRepository
_connectionString = connstr;
}
public DiscordWebHookModel New(DiscordWebHookModel model)
public DiscordWebhookEntity New(DiscordWebhookEntity model)
{
var uid = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var query =
"Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);";
conn.Execute(query, new
{
id = uid,
url = model.Url,
server = model.Server,
channel = model.Channel,
enabled = model.Enabled
});
model.ID = uid;
model.Id = new Guid();
using var context = new DatabaseContext(_connectionString);
context.DiscordWebhooks.Add(model);
context.SaveChanges();
return model;
}
public DiscordWebHookModel GetByID(Guid id)
public DiscordWebhookEntity GetById(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;";
return conn.Query<DiscordWebHookModel>(query, new
{
id
}).First();
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordWebhooks.FirstOrDefault(d => d.Id.Equals(id));
res ??= new DiscordWebhookEntity();
return res;
}
public DiscordWebHookModel GetByUrl(string url)
public DiscordWebhookEntity GetByUrl(string url)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks Where url = @url;";
try
{
var res = conn.QueryFirst<DiscordWebHookModel>(query, new
{
url
});
return res;
}
catch
{
return new DiscordWebHookModel();
}
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordWebhooks.FirstOrDefault(d => d.Url.Equals(url));
res ??= new DiscordWebhookEntity();
return res;
}
public List<DiscordWebHookModel> List(int page, int count = 25)
public List<DiscordWebhookEntity> List(int page, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From DiscordWebHooks
Offset @offset Fetch Next @count Rows Only;";
return conn.Query<DiscordWebHookModel>(query, new
{
offset = page * count, count
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordWebhooks
.Skip(page * count)
.Take(count)
.ToList();
res ??= new List<DiscordWebhookEntity>();
return res;
}
public List<DiscordWebHookModel> ListByServer(string server, int limit = 25)
public List<DiscordWebhookEntity> ListByServer(string server, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;";
return conn.Query<DiscordWebHookModel>(query, new
{
server, limit
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordWebhooks
.Where(d => d.Server.Equals(server))
.Take(limit)
.ToList();
res ??= new List<DiscordWebhookEntity>();
return res;
}
public List<DiscordWebHookModel> ListByServerAndChannel(string server, string channel, int limit = 25)
public List<DiscordWebhookEntity> ListByServerAndChannel(string server, string channel, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;";
return conn.Query<DiscordWebHookModel>(query, new
{
server,
channel,
limit
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.DiscordWebhooks
.Where(s => s.Server.Equals(server))
.Where(c => c.Channel.Equals(channel))
.Take(limit)
.ToList();
res ??= new List<DiscordWebhookEntity>();
return res;
}
public int Disable(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
var res = GetById(id);
using var context = new DatabaseContext(_connectionString);
res.Enabled = true;
context.DiscordWebhooks.Update(res);
try
{
id
});
context.SaveChanges();
return 1;
}
catch(Exception ex)
{
Console.WriteLine($"Failed to update DiscordWebhook ID = {id}. {ex.Message}");
return 0;
}
}
public int Enable(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
var res = GetById(id);
using var context = new DatabaseContext(_connectionString);
res.Enabled = false;
context.DiscordWebhooks.Update(res);
try
{
id
});
context.SaveChanges();
return 1;
}
catch(Exception ex)
{
Console.WriteLine($"Failed to update DiscordWebhook ID = {id}. {ex.Message}");
return 0;
}
}
private IDbConnection OpenConnection(string connectionString)

37
Newsbot.Collector.Database/Repositories/IconsTable.cs
View File

@ -1,6 +1,7 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
@ -23,37 +24,29 @@ public class IconsTable : IIconsRepository
_connectionString = connstr;
}
public void New(IconModel model)
public void New(IconEntity model)
{
using var context = new DatabaseContext(_connectionString);
model.Id = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var q = @"Insert Into icons (id, filename, site, sourceid) values (@Id,@FileName, @Site, @SourceId)";
conn.Execute(q, model);
context.Icons.Add(model);
context.SaveChanges();
}
public IconModel GetById(Guid id)
public IconEntity GetById(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From icons where ID = @id Limit 1;";
var res = conn.Query<IconModel>(query, new
{
id
});
if (!res.Any()) return new IconModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Icons.FirstOrDefault(f => f.Id.Equals(id));
res ??= new IconEntity();
return res;
}
public IconModel GetBySourceId(Guid id)
public IconEntity GetBySourceId(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From icons where sourceid = @id Limit 1;";
var res = conn.Query<IconModel>(query, new
{
id
});
if (!res.Any()) return new IconModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Icons.FirstOrDefault(f => f.SourceId.Equals(id));
res ??= new IconEntity();
return res;
}
private IDbConnection OpenConnection(string connectionString)

207
Newsbot.Collector.Database/Repositories/SourcesTable.cs
View File

@ -1,6 +1,7 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
@ -23,157 +24,153 @@ public class SourcesTable : ISourcesRepository
_connectionString = connstr;
}
public SourceModel New(SourceModel model)
public SourceEntity New(SourceEntity model)
{
model.ID = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var query =
"Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags, YoutubeId) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags,@youtubeid);";
conn.Execute(query, new
model.Id = Guid.NewGuid();
using var context = new DatabaseContext(_connectionString);
context.Sources.Add(model);
try
{
id = model.ID,
model.Site,
model.Name,
model.Source,
model.Type,
model.Value,
model.Enabled,
model.Url,
model.Tags,
model.YoutubeId
});
context.SaveChanges();
}
catch (Exception ex)
{
Console.WriteLine($"Failed to save source to db. {model.ToString()} {ex.Message} ");
}
return model;
}
public SourceModel GetByID(Guid ID)
public SourceEntity GetById(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where ID = @id Limit 1;";
var res = conn.Query<SourceModel>(query, new
{
id = ID
});
if (res.Count() == 0) return new SourceModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources.FirstOrDefault(f => f.Id.Equals(id));
res ??= new SourceEntity();
return res;
}
public SourceModel GetByID(string ID)
public SourceEntity GetById(string id)
{
var uid = Guid.Parse(ID);
return GetByID(uid);
var uid = Guid.Parse(id);
return GetById(uid);
}
public SourceModel GetByName(string Name)
public SourceEntity GetByName(string name)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources where name = @name Limit 1;";
var res = conn.Query<SourceModel>(query, new
{
name = Name
});
if (res.Count() == 0) return new SourceModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources.FirstOrDefault(f => f.Name.Equals(name));
res ??= new SourceEntity();
return res;
}
public SourceModel GetByNameAndType(string name, string type)
public SourceEntity GetByNameAndType(string name, string type)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources WHERE name = @name and type = @type;";
var res = conn.Query<SourceModel>(query, new
{
name, type
});
if (res.Count() == 0) return new SourceModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources
.Where(f => f.Name.Equals(name))
.FirstOrDefault(f => f.Type.Equals(type));
res ??= new SourceEntity();
return res;
}
public SourceModel GetByUrl(string url)
public SourceEntity GetByUrl(string url)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * from Sources WHERE url = @url;";
var res = conn.Query<SourceModel>(query, new
{
url
});
if (res.ToList().Count == 0) return new SourceModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources
.FirstOrDefault(f => f.Url.Equals(url));
res ??= new SourceEntity();
return res;
}
public List<SourceModel> List(int page = 0, int count = 25)
public List<SourceEntity> List(int page = 0, int count = 100)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From Sources
Offset @page
Fetch Next @count Rows Only;";
return conn.Query<SourceModel>(query, new
{
page = page * count, count
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources
.Skip(page * count)
.Take(count)
.ToList();
return res;
}
public List<SourceModel> ListBySource(string source, int limit = 25)
public List<SourceEntity> ListBySource(string source, int page = 0, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where Source = @source Limit @limit;";
return conn.Query<SourceModel>(query, new
{
source, limit
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources
.Where(f => f.Source.Equals(source))
.Skip(page * limit)
.Take(limit)
.ToList();
return res;
}
public List<SourceModel> ListByType(string type, int limit = 25)
public List<SourceEntity> ListByType(string type,int page = 0, int limit = 25)
{
using var conn = OpenConnection(_connectionString);
var query = "Select * From Sources where Type = @type Limit @limit;";
return conn.Query<SourceModel>(query, new
{
type, limit
}).ToList();
using var context = new DatabaseContext(_connectionString);
var res = context.Sources
.Where(f => f.Type.Equals(type))
.Skip(page * limit)
.Take(limit)
.ToList();
return res;
}
public int Disable(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
using var context = new DatabaseContext(_connectionString);
var res = GetById(id);
res.Enabled = false;
context.Sources.Update(res);
try
{
id
});
context.SaveChanges();
return 1;
}
catch
{
return 0;
}
}
public int Enable(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
using var context = new DatabaseContext(_connectionString);
var res = GetById(id);
res.Enabled = true;
context.Sources.Update(res);
try
{
id
});
context.SaveChanges();
return 1;
}
catch
{
return 0;
}
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From sources where id = @id;";
var res = conn.Execute(query, new
{
id
});
if (res == 0) throw new Exception("Nothing was deleted");
using var context = new DatabaseContext(_connectionString);
var res = GetById(id);
context.Sources.Remove(res);
context.SaveChanges();
}
public int UpdateYoutubeId(Guid id, string youtubeId)
{
using var conn = OpenConnection(_connectionString);
var query = "Update Sources Set youtubeid = @youtubeId where ID = @id;";
return conn.Execute(query, new
using var context = new DatabaseContext(_connectionString);
var res = GetById(id);
res.YoutubeId = youtubeId;
context.Sources.Update(res);
try
{
id, youtubeId
});
context.SaveChanges();
return 1;
}
catch
{
return 0;
}
}
private IDbConnection OpenConnection(string connectionString)

115
Newsbot.Collector.Database/Repositories/SubscriptionsTable.cs
View File

@ -1,6 +1,7 @@
using System.Data;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Npgsql;
@ -23,104 +24,64 @@ public class SubscriptionsTable : ISubscriptionRepository
_connectionString = connstr;
}
public SubscriptionModel New(SubscriptionModel model)
public SubscriptionEntity New(SubscriptionEntity model)
{
model.Id = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var query =
"Insert Into subscriptions (ID, DiscordWebHookId, SourceId, codeAllowCommits, codeAllowReleases) Values (@id, @webhookId, @sourceId, @codeAllowCommits, @codeAllowReleases);";
try
{
conn.Execute(query, new
{
id = model.Id,
webhookId = model.DiscordWebHookId,
sourceId = model.SourceId,
codeAllowCommits = model.CodeAllowCommits,
codeAllowReleases = model.CodeAllowReleases
});
return model;
}
catch
{
return new SubscriptionModel();
}
model.Id = new Guid();
using var context = new DatabaseContext(_connectionString);
context.Subscriptions.Add(model);
context.SaveChanges();
return model;
}
public List<SubscriptionModel> List(int page = 0, int count = 25)
public List<SubscriptionEntity> List(int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions
Offset @page Fetch Next @count Rows Only;";
return conn.Query<SubscriptionModel>(query, new
{
page = page * count, count
}).ToList();
using var context = new DatabaseContext(_connectionString);
return context.Subscriptions.Skip(page * count).Take(count).ToList();
}
public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25)
public List<SubscriptionEntity> ListBySourceId(Guid id, int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions
Where sourceid = @sourceid
Offset @page Fetch Next @count Rows Only";
return conn.Query<SubscriptionModel>(query, new
{
page = page * count,
count,
sourceid = id
}).ToList();
using var context = new DatabaseContext(_connectionString);
return context.Subscriptions.Where(f => f.SourceId.Equals(id))
.Skip(page * count)
.ToList();
}
public List<SubscriptionModel> ListByWebhook(Guid id, int page = 0, int count = 25)
public List<SubscriptionEntity> ListByWebhook(Guid id, int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions
Where discordwebhookid = @webhookid
Offset @page Fetch Next @count Rows Only";
return conn.Query<SubscriptionModel>(query, new
{
page = page * count,
count,
webhookid = id
}).ToList();
using var context = new DatabaseContext(_connectionString);
return context.Subscriptions.Where(f => f.DiscordWebHookId.Equals(id)).Skip(page * count).ToList();
}
public SubscriptionModel GetById(Guid id)
public SubscriptionEntity GetById(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions Where id = @id;";
var res = conn.Query<SubscriptionModel>(query, new
{
id
});
if (res.Count() == 0) return new SubscriptionModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Subscriptions
.FirstOrDefault(f => f.Id.Equals(id));
return res ??= new SubscriptionEntity();
}
public SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId)
public SubscriptionEntity GetByWebhookAndSource(Guid webhookId, Guid sourceId)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * From subscriptions
Where discordwebhookid = @webhookid
and sourceid = @sourceid;";
var res = conn.Query<SubscriptionModel>(query, new
{
webhookid = webhookId,
sourceid = sourceId
});
if (res.Count() == 0) return new SubscriptionModel();
return res.First();
using var context = new DatabaseContext(_connectionString);
var res = context.Subscriptions
.Where(f => f.DiscordWebHookId.Equals(webhookId))
.FirstOrDefault(f => f.SourceId.Equals(sourceId));
return res ??= new SubscriptionEntity();
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From subscriptions Where id = @id;";
conn.Execute(query, new
using var context = new DatabaseContext(_connectionString);
var res = context.Subscriptions.FirstOrDefault(f => f.Id.Equals(id));
if (res is null)
{
id
});
return;
}
context.Subscriptions.Remove(res);
context.SaveChanges();
}
private IDbConnection OpenConnection(string connectionString)

9
Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs
View File

@ -1,10 +1,11 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto;
public class ArticleDetailsDto
{
public Guid ID { get; set; }
public Guid Id { get; set; }
public string[]? Tags { get; set; }
public string? Title { get; set; }
public string? Url { get; set; }
@ -19,14 +20,14 @@ public class ArticleDetailsDto
public SourceDto? Source { get; set; }
public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source)
public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source)
{
return new ArticleDetailsDto
{
ID = article.ID,
Id = article.Id,
Tags = article.Tags.Split(','),
Title = article.Title,
Url = article.URL,
Url = article.Url,
PubDate = article.PubDate,
Video = article.Video,
VideoHeight = article.VideoHeight,

23
Newsbot.Collector.Domain/Dto/ArticleDto.cs
View File

@ -1,3 +1,4 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto;
@ -17,7 +18,7 @@ public class ArticleDto
public string? Description { get; set; }
public string? AuthorName { get; set; }
public string? AuthorImage { get; set; }
public static ArticleDto Convert(ArticlesModel article)
public static ArticleDto Convert(ArticlesModel article)
{
return new ArticleDto
{
@ -36,4 +37,24 @@ public class ArticleDto
AuthorImage = article.AuthorImage,
};
}
public static ArticleDto Convert(ArticlesEntity article)
{
return new ArticleDto
{
ID = article.Id,
SourceID = article.SourceId,
Tags = article.Tags.Split(','),
Title = article.Title,
Url = article.Url,
PubDate = article.PubDate,
Video = article.Video,
VideoHeight = article.VideoHeight,
VideoWidth = article.VideoWidth,
Thumbnail = article.Thumbnail,
Description = article.Description,
AuthorName = article.AuthorName,
AuthorImage = article.AuthorImage,
};
}
}

6
Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs
View File

@ -1,4 +1,4 @@
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Entities;
namespace Newsbot.Collector.Domain.Dto;
@ -10,11 +10,11 @@ public class DiscordWebHookDto
public string? Channel { get; set; }
public bool Enabled { get; set; }
public static DiscordWebHookDto Convert(DiscordWebHookModel model)
public static DiscordWebHookDto Convert(DiscordWebhookEntity model)
{
return new DiscordWebHookDto
{
ID = model.ID,
ID = model.Id,
Url = model.Url,
Server = model.Server,
Channel = model.Channel,

5
Newsbot.Collector.Domain/Dto/SourceDto.cs
View File

@ -1,4 +1,5 @@
using System.Net.NetworkInformation;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto;
@ -16,10 +17,10 @@ public class SourceDto
public string[]? Tags { get; set; }
public bool Deleted { get; set; }
public static SourceDto Convert(SourceModel model) {
public static SourceDto Convert(SourceEntity model) {
return new SourceDto
{
ID = model.ID,
ID = model.Id,
Site = model.Site,
Name = model.Name,
Source = model.Source,

5
Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs
View File

@ -1,3 +1,4 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto;
@ -10,8 +11,8 @@ public class SubscriptionDetailsDto
public SourceDto? Source { get; set; }
public DiscordWebHookDto? DiscordWebHook { get; set; }
public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source,
DiscordWebHookModel discord)
public static SubscriptionDetailsDto Convert(SubscriptionEntity subscription, SourceEntity source,
DiscordWebhookEntity discord)
{
return new SubscriptionDetailsDto
{

3
Newsbot.Collector.Domain/Dto/SubscriptionDto.cs
View File

@ -1,3 +1,4 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto;
@ -10,7 +11,7 @@ public class SubscriptionDto
public bool CodeAllowReleases { get; set; }
public bool CodeAllowCommits { get; set; }
public static SubscriptionDto Convert(SubscriptionModel model)
public static SubscriptionDto Convert(SubscriptionEntity model)
{
return new SubscriptionDto
{

20
Newsbot.Collector.Domain/Entities/ArticlesEntity.cs Normal file
View File

@ -0,0 +1,20 @@
namespace Newsbot.Collector.Domain.Entities;
public class ArticlesEntity
{
public Guid Id { get; set; }
public Guid SourceId { get; set; }
public string Tags { get; set; } = "";
public string Title { get; set; } = "";
public string? Url { get; set; }
public DateTime PubDate { get; set; } = DateTime.Now;
public string Video { get; set; } = "";
public int VideoHeight { get; set; } = 0;
public int VideoWidth { get; set; } = 0;
public string Thumbnail { get; set; } = "";
public string Description { get; set; } = "";
public string AuthorName { get; set; } = "";
public string? AuthorImage { get; set; }
public bool CodeIsRelease { get; set; }
public bool CodeIsCommit { get; set; }
}

9
Newsbot.Collector.Domain/Entities/AuthorEntity.cs Normal file
View File

@ -0,0 +1,9 @@
namespace Newsbot.Collector.Domain.Entities;
public class AuthorEntity
{
public Guid Id { get; set; }
public Guid SourceId { get; set; }
public string Name { get; set; } = "";
public string Image { get; set; } = "";
}

7
Newsbot.Collector.Domain/Entities/DiscordQueueEntity.cs Normal file
View File

@ -0,0 +1,7 @@
namespace Newsbot.Collector.Domain.Entities;
public class DiscordQueueEntity
{
public Guid Id { get; set; }
public Guid ArticleId { get; set; }
}

10
Newsbot.Collector.Domain/Entities/DiscordWebhookEntity.cs Normal file
View File

@ -0,0 +1,10 @@
namespace Newsbot.Collector.Domain.Entities;
public class DiscordWebhookEntity
{
public Guid Id { get; set; }
public string Url { get; set; } = "";
public string Server { get; set; } = "";
public string Channel { get; set; } = "";
public bool Enabled { get; set; }
}

10
Newsbot.Collector.Domain/Entities/IconEntity.cs Normal file
View File

@ -0,0 +1,10 @@
namespace Newsbot.Collector.Domain.Entities;
public class IconEntity
{
public Guid Id { get; set; }
public string FileName { get; set; } = "";
public string Site { get; set; } = "";
public Guid SourceId { get; set; }
}

18
Newsbot.Collector.Domain/Entities/SourceEntity.cs Normal file
View File

@ -0,0 +1,18 @@
namespace Newsbot.Collector.Domain.Entities;
public class SourceEntity
{
public Guid Id { get; set; }
public string Site { get; set; } = "";
public string Name { get; set; } = "";
// Source use to define the worker to query with but moving to Type as it was not used really.
public string Source { get; set; } = "";
public string Type { get; set; } = "";
public string Value { get; set; } = "";
public bool Enabled { get; set; }
public string Url { get; set; } = "";
public string Tags { get; set; } = "";
public bool Deleted { get; set; }
public string YoutubeId { get; set; } = "";
}

11
Newsbot.Collector.Domain/Entities/SubscriptionEntity.cs Normal file
View File

@ -0,0 +1,11 @@
namespace Newsbot.Collector.Domain.Entities;
public class SubscriptionEntity
{
public Guid Id { get; set; }
public bool CodeAllowReleases { get; set; }
public bool CodeAllowCommits { get; set; }
public Guid SourceId { get; set; }
public Guid DiscordWebHookId { get; set; }
}

11
Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs
View File

@ -1,13 +1,14 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IArticlesRepository : ITableRepository
{
List<ArticlesModel> List(int page, int count);
List<ArticlesModel> ListBySourceId(Guid id, int page = 0, int count = 25);
ArticlesModel GetById(Guid ID);
ArticlesModel GetByUrl(string url);
ArticlesModel New(ArticlesModel model);
List<ArticlesEntity> List(int page, int count);
List<ArticlesEntity> ListBySourceId(Guid id, int page = 0, int count = 25);
ArticlesEntity GetById(Guid id);
ArticlesEntity GetByUrl(string url);
ArticlesEntity New(ArticlesEntity model);
void DeleteAllBySourceId(Guid sourceId);
}

6
Newsbot.Collector.Domain/Interfaces/IDiscordQueueRepository.cs
View File

@ -1,10 +1,10 @@
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Entities;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IDiscordQueueRepository
{
void New(DiscordQueueModel model);
void New(DiscordQueueEntity model);
void Delete(Guid id);
List<DiscordQueueModel> List(int limit);
List<DiscordQueueEntity> List(int limit);
}

15
Newsbot.Collector.Domain/Interfaces/IDiscordWebHooksRepository.cs
View File

@ -1,18 +1,17 @@
using Microsoft.VisualBasic;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Entities;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IDiscordWebHooksRepository
{
DiscordWebHookModel New(DiscordWebHookModel model);
DiscordWebhookEntity New(DiscordWebhookEntity model);
DiscordWebHookModel GetByID(Guid id);
DiscordWebHookModel GetByUrl(string url);
DiscordWebhookEntity GetById(Guid id);
DiscordWebhookEntity GetByUrl(string url);
List<DiscordWebHookModel> List(int page, int count = 25);
List<DiscordWebHookModel> ListByServer(string server, int limit);
List<DiscordWebHookModel> ListByServerAndChannel(string server, string channel, int limit);
List<DiscordWebhookEntity> List(int page, int count = 25);
List<DiscordWebhookEntity> ListByServer(string server, int limit);
List<DiscordWebhookEntity> ListByServerAndChannel(string server, string channel, int limit);
int Disable(Guid id);
int Enable(Guid id);

8
Newsbot.Collector.Domain/Interfaces/IIconsRepository.cs
View File

@ -1,11 +1,11 @@
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Entities;
namespace Newsbot.Collector.Domain.Interfaces;
public interface IIconsRepository
{
public void New(IconModel model);
public void New(IconEntity model);
public IconModel GetById(Guid id);
public IconModel GetBySourceId(Guid id);
public IconEntity GetById(Guid id);
public IconEntity GetBySourceId(Guid id);
}

19
Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs
View File

@ -1,18 +1,19 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ISourcesRepository
{
public SourceModel New(SourceModel model);
public SourceModel GetByID(Guid ID);
public SourceModel GetByID(string ID);
public SourceModel GetByName(string name);
public SourceModel GetByNameAndType(string name, string type);
SourceModel GetByUrl(string url);
public List<SourceModel> List(int page, int count);
public List<SourceModel> ListBySource(string source, int limit);
public List<SourceModel> ListByType(string type, int limit = 25);
public SourceEntity New(SourceEntity model);
public SourceEntity GetById(Guid id);
public SourceEntity GetById(string id);
public SourceEntity GetByName(string name);
public SourceEntity GetByNameAndType(string name, string type);
SourceEntity GetByUrl(string url);
public List<SourceEntity> List(int page, int count);
public List<SourceEntity> ListBySource(string source,int page, int limit);
public List<SourceEntity> ListByType(string type,int page, int limit = 25);
public int Disable(Guid id);
public int Enable(Guid id);
public void Delete(Guid id);

13
Newsbot.Collector.Domain/Interfaces/ISubscriptionsRepository.cs
View File

@ -1,17 +1,18 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces;
public interface ISubscriptionRepository
{
SubscriptionModel New(SubscriptionModel model);
SubscriptionEntity New(SubscriptionEntity model);
List<SubscriptionModel> List(int page = 0, int count = 25);
List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25);
List<SubscriptionModel> ListByWebhook(Guid id, int page = 0, int count = 25);
List<SubscriptionEntity> List(int page = 0, int count = 25);
List<SubscriptionEntity> ListBySourceId(Guid id, int page = 0, int count = 25);
List<SubscriptionEntity> ListByWebhook(Guid id, int page = 0, int count = 25);
SubscriptionModel GetById(Guid id);
SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId);
SubscriptionEntity GetById(Guid id);
SubscriptionEntity GetByWebhookAndSource(Guid webhookId, Guid sourceId);
void Delete(Guid id);
}

29
Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs
View File

@ -3,6 +3,7 @@ using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
@ -60,10 +61,10 @@ public class CodeProjectWatcherJob
private void Execute()
{
var sources = _source.ListByType(SourceTypes.CodeProject);
var sources = _source.ListByType(SourceTypes.CodeProject, 0, 100);
// query sources for things to pull
var items = new List<ArticlesModel>();
var items = new List<ArticlesEntity>();
foreach (var source in sources)
{
@ -74,14 +75,14 @@ public class CodeProjectWatcherJob
foreach (var item in items)
{
_articles.New(item);
_queue.New(new DiscordQueueModel
_queue.New(new DiscordQueueEntity()
{
ArticleID = item.ID
ArticleId = item.Id
});
}
}
public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source)
public IEnumerable<ArticlesEntity> CheckForReleases(SourceEntity source)
{
var url = new Uri(source.Url);
var links = new List<string>
@ -110,10 +111,10 @@ public class CodeProjectWatcherJob
url.AbsoluteUri);
}
return new List<ArticlesModel>();
return new List<ArticlesEntity>();
}
public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source)
public IEnumerable<ArticlesEntity> CheckForCommits(SourceEntity source)
{
var url = new Uri(source.Url);
var links = new List<string>
@ -136,19 +137,19 @@ public class CodeProjectWatcherJob
url.AbsoluteUri);
}
return new List<ArticlesModel>();
return new List<ArticlesEntity>();
}
private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
private IEnumerable<ArticlesEntity> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceEntity source,
bool isRelease, bool isCommit)
{
var items = new List<ArticlesModel>();
var items = new List<ArticlesEntity>();
foreach (var item in feed)
{
var itemUrl = item.Links[0].Uri.AbsoluteUri;
var exits = _articles.GetByUrl(itemUrl);
if (exits.ID != Guid.Empty) continue;
if (exits.Id != Guid.Empty) continue;
var parser = new HtmlPageReader(new HtmlPageReaderOptions
{
@ -156,12 +157,12 @@ public class CodeProjectWatcherJob
});
parser.Parse();
var a = new ArticlesModel
var a = new ArticlesEntity
{
SourceID = source.ID,
SourceId = source.Id,
Tags = source.Tags,
Title = item.Title.Text,
URL = itemUrl,
Url = itemUrl,
PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text,

44
Newsbot.Collector.Services/Jobs/DiscordNotificationJob.cs
View File

@ -1,4 +1,5 @@
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Notifications.Discord;
@ -83,45 +84,44 @@ public class DiscordNotificationJob
_logger.Information($"{JobName} - Loop has been completed.");
}
public void ProcessQueueItem(DiscordQueueModel request)
public void ProcessQueueItem(DiscordQueueEntity request)
{
_logger.Debug($"{JobName} - Processing {request.ID}");
_logger.Debug($"{JobName} - Processing {request.Id}");
// Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleID);
var articleDetails = _article.GetById(request.ArticleId);
// Get the details of the source
var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.ID == Guid.Empty)
var sourceDetails = _sources.GetById(articleDetails.SourceId);
if (sourceDetails.Id == Guid.Empty)
{
_logger.Error(
$"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.ID);
$"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.Id);
return;
}
var sourceIcon = new IconModel();
var sourceIcon = new IconEntity();
try
{
sourceIcon = _icons.GetBySourceId(sourceDetails.ID);
sourceIcon = _icons.GetBySourceId(sourceDetails.Id);
}
catch
{
_logger.Warning("{JobName} - Source ID \'{SourceDetailsId}\' is missing an icon", JobName,
sourceDetails.ID);
sourceDetails.Id);
}
// Find all the subscriptions for that source
var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID);
var allSubscriptions = _subs.ListBySourceId(sourceDetails.Id);
foreach (var sub in allSubscriptions)
SendSubscriptionNotification(request.ID, articleDetails, sourceDetails, sourceIcon, sub);
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub);
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.ID);
_queue.Delete(request.ID);
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id);
_queue.Delete(request.Id);
}
public void SendSubscriptionNotification(Guid requestId, ArticlesModel articleDetails, SourceModel sourceDetails,
IconModel sourceIcon, SubscriptionModel sub)
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionEntity sub)
{
// Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip.
@ -131,7 +131,7 @@ public class DiscordNotificationJob
if (articleDetails.CodeIsRelease && !sub.CodeAllowReleases) throw new MessageTypeNotRequestedException("Message was a code release and was not requested by the subscription");
// find the discord webhooks we need to post to
var discordDetails = _webhook.GetByID(sub.DiscordWebHookId);
var discordDetails = _webhook.GetById(sub.DiscordWebHookId);
if (discordDetails.Enabled == false) return;
var client = new DiscordWebhookClient(discordDetails.Url);
@ -143,15 +143,15 @@ public class DiscordNotificationJob
{
_logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message);
_logger.Debug("Queue Record: {RequestId}", requestId);
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.ID);
_logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID);
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id);
_logger.Debug("Source: {SourceDetailsId}", sourceDetails.Id);
_logger.Debug("Subscription: {SubId}", sub.Id);
}
Thread.Sleep(3000);
}
public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesModel article, IconModel icon)
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon)
{
var embed = new DiscordMessageEmbed
{
@ -172,13 +172,13 @@ public class DiscordNotificationJob
new()
{
Name = "Link",
Value = article.URL,
Value = article.Url,
Inline = false
}
}
};
if (article.URL is not null && article.URL != "") embed.Url = article.URL;
if (article.Url is not null && article.Url != "") embed.Url = article.Url;
if (article.Thumbnail != "")
embed.Image = new DiscordMessageEmbedImage

33
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
@ -56,17 +57,17 @@ public class RssWatcherJob
public void Execute()
{
var articles = new List<ArticlesModel>();
var articles = new List<ArticlesEntity>();
_logger.Information($"{JobName} - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss);
_logger.Information($"{JobName} - Got {sources.Count} back");
var sources = _source.ListByType(SourceTypes.Rss, 0, 100);
_logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count);
foreach (var source in sources)
{
_logger.Information($"{JobName} - Starting to process '{source.Name}'");
_logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name);
_logger.Information($"{JobName} - Starting to request feed to be processed");
var results = Collect(source.Url, source.ID);
var results = Collect(source.Url, source.Id);
_logger.Information($"{JobName} - Collected {results.Count} posts");
articles.AddRange(results);
@ -78,9 +79,9 @@ public class RssWatcherJob
_logger.Information($"{JobName} - Done!");
}
public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
public List<ArticlesEntity> Collect(string url, Guid sourceId, int sleep = 3000)
{
var collectedPosts = new List<ArticlesModel>();
var collectedPosts = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
@ -99,15 +100,15 @@ public class RssWatcherJob
});
meta.Parse();
var article = new ArticlesModel
var article = new ArticlesEntity
{
Title = post.Title.Text,
Tags = FetchTags(post),
URL = articleUrl,
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description,
SourceID = sourceId
SourceId = sourceId
};
collectedPosts.Add(article);
@ -119,20 +120,20 @@ public class RssWatcherJob
return collectedPosts;
}
public void UpdateDatabase(List<ArticlesModel> items)
public void UpdateDatabase(List<ArticlesEntity> items)
{
foreach (var item in items)
{
if (item.URL is null)
if (item.Url is null)
{
Log.Warning("RSS Watcher collected a blank url and was skipped.");
Log.Warning("RSS Watcher collected a blank url and was skipped");
continue;
}
var p = _articles.New(item);
_queue.New(new DiscordQueueModel
_queue.New(new DiscordQueueEntity
{
ArticleID = p.ID
ArticleId = p.Id
});
}
}
@ -140,7 +141,7 @@ public class RssWatcherJob
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) return true;
if (isKnown.Url == url) return true;
return false;
}

21
Newsbot.Collector.Services/Jobs/YoutubeWatcherJob.cs
View File

@ -2,6 +2,7 @@ using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.HtmlParser;
@ -66,11 +67,11 @@ public class YoutubeWatcherJob
if (channelId == "")
{
channelId = GetChannelId(source.Url);
_source.UpdateYoutubeId(source.ID, channelId);
_source.UpdateYoutubeId(source.Id, channelId);
}
// Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.ID);
var icon = _icons.GetBySourceId(source.Id);
if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");
_logger.Information($"{JobName} - Checking '{source.Name}'");
@ -82,9 +83,9 @@ public class YoutubeWatcherJob
{
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
_articles.New(video);
_queue.New(new DiscordQueueModel
_queue.New(new DiscordQueueEntity
{
ArticleID = video.ID
ArticleId = video.Id
});
}
}
@ -109,9 +110,9 @@ public class YoutubeWatcherJob
return id;
}
private List<ArticlesModel> CheckFeed(string url, SourceModel source)
private List<ArticlesEntity> CheckFeed(string url, SourceEntity source)
{
var videos = new List<ArticlesModel>();
var videos = new List<ArticlesEntity>();
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
@ -126,17 +127,17 @@ public class YoutubeWatcherJob
});
videoDetails.Parse();
var article = new ArticlesModel
var article = new ArticlesEntity
{
//Todo add the icon
AuthorName = post.Authors[0].Name,
Title = post.Title.Text,
Tags = FetchTags(post),
URL = articleUrl,
Url = articleUrl,
PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description,
SourceID = source.ID,
SourceId = source.Id,
Video = "true"
};
@ -151,7 +152,7 @@ public class YoutubeWatcherJob
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) return true;
if (isKnown.Url == url) return true;
return false;
}

4
Newsbot.Collector.Services/Newsbot.Collector.Services.csproj
View File

@ -7,6 +7,10 @@
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Selenium.WebDriver" Version="4.8.1" />
<PackageReference Include="Selenium.WebDriver.GeckoDriver" Version="0.32.2" />
<PackageReference Include="Serilog" Version="2.12.0" />

13
Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs
View File

@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
@ -18,9 +19,9 @@ public class CodeProjectWatcherJobTests
FeaturePullCommits = true,
FeaturePullReleases = true
});
var results = client.CheckForReleases(new SourceModel
var results = client.CheckForReleases(new SourceEntity
{
ID = Guid.NewGuid(),
Id = Guid.NewGuid(),
Url = "https://github.com/jtom38/dvb",
Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject,
@ -44,9 +45,9 @@ public class CodeProjectWatcherJobTests
FeaturePullCommits = true,
FeaturePullReleases = true
});
var results = client.CheckForReleases(new SourceModel
var results = client.CheckForReleases(new SourceEntity
{
ID = Guid.NewGuid(),
Id = Guid.NewGuid(),
Url = "https://github.com/python/cpython",
Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject,
@ -70,9 +71,9 @@ public class CodeProjectWatcherJobTests
FeaturePullCommits = true,
FeaturePullReleases = true
});
var results = client.CheckForCommits(new SourceModel
var results = client.CheckForCommits(new SourceEntity
{
ID = Guid.NewGuid(),
Id = Guid.NewGuid(),
Url = "https://github.com/jtom38/dvb",
Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject,

25
Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs
View File

@ -1,4 +1,4 @@
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Services.Jobs;
using Newsbot.Collector.Services.Notifications.Discord;
@ -13,9 +13,9 @@ public class DiscordNotificationJobTest
var webhookClient = new DiscordWebhookClient(uri);
var client = new DiscordNotificationJob();
var msg = client.GenerateDiscordMessage(new SourceModel
var msg = client.GenerateDiscordMessage(new SourceEntity
{
ID = Guid.NewGuid(),
Id = Guid.NewGuid(),
Site = "Unit Test",
Source = "placeholder",
Type = "a",
@ -24,17 +24,17 @@ public class DiscordNotificationJobTest
Url = "https://github.com",
Tags = "Unit, Testing"
},
new ArticlesModel
new ArticlesEntity
{
Tags = "more,unit,testing",
Title = "Nope not real",
URL = "https://github.com/jtom38",
Url = "https://github.com/jtom38",
PubDate = DateTime.Now,
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
Description = "Please work",
AuthorName = "No one knows"
},
new IconModel
new IconEntity()
{
Id = Guid.NewGuid(),
FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png"
@ -50,11 +50,11 @@ public class DiscordNotificationJobTest
{
client.SendSubscriptionNotification(
new Guid(),
new ArticlesModel
new ArticlesEntity
{
Tags = "more,unit,testing",
Title = "Nope not real",
URL = "https://github.com/jtom38",
Url = "https://github.com/jtom38",
PubDate = DateTime.Now,
Thumbnail =
"https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
@ -62,9 +62,9 @@ public class DiscordNotificationJobTest
AuthorName = "No one knows",
CodeIsCommit = true
},
new SourceModel
new SourceEntity
{
ID = Guid.NewGuid(),
Id = Guid.NewGuid(),
Site = "Unit Test",
Source = "placeholder",
Type = "a",
@ -73,12 +73,12 @@ public class DiscordNotificationJobTest
Url = "https://github.com",
Tags = "Unit, Testing"
},
new IconModel
new IconEntity()
{
Id = Guid.NewGuid(),
FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png"
},
new SubscriptionModel
new SubscriptionEntity
{
CodeAllowCommits = false,
CodeAllowReleases = true
@ -87,6 +87,7 @@ public class DiscordNotificationJobTest
}
catch (MessageTypeNotRequestedException)
{
Console.Write($"Message did not send as expected");
}

2
Newsbot.Collector.Tests/Jobs/RssWatcherJobTest.cs
View File

@ -61,7 +61,7 @@ public class RssWatcherJobTest
[Fact]
public void CanReadHtmlDrivenFeedPage()
{
var url = "https://www.howtogeek.com/feed/";
//var url = "https://www.howtogeek.com/feed/";
var client = new RssWatcherJob();
client.InitAndExecute(new RssWatcherJobOptions
{

4
Newsbot.Collector.Tests/Newsbot.Collector.Tests.csproj
View File

@ -9,6 +9,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />

15
Newsbot.Collector.Tests/Tables/ArticlesTableTests.cs
View File

@ -1,5 +1,6 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Tests.Tables;
@ -23,7 +24,7 @@ public class ArticlesTableTests
{
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
client.List();
client.List(0, 25);
}
[Fact]
@ -34,7 +35,7 @@ public class ArticlesTableTests
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var res = client.GetById(uid);
if (!res.ID.Equals(uid))
if (!res.Id.Equals(uid))
{
Assert.Fail("Incorrect record or not found");
}
@ -45,14 +46,14 @@ public class ArticlesTableTests
{
var cfg = GetConfiguration();
var client = new ArticlesTable(cfg);
var m = new ArticlesModel
var m = new ArticlesEntity
{
ID = Guid.NewGuid(),
SourceID = Guid.NewGuid(),
Id = Guid.NewGuid(),
SourceId = Guid.NewGuid(),
Tags = "thing, thing2",
Title = "Unit Testing!",
URL = "https://google.com",
PubDate = DateTime.Now.ToLocalTime(),
Url = "https://google.com",
PubDate = DateTime.Now.ToUniversalTime(),
};
client.New(m);
}

5
Newsbot.Collector.Tests/Tables/SourcesTableTests.cs
View File

@ -1,4 +1,5 @@
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Tests.Tables;
@ -9,9 +10,9 @@ public class SourcesTableTests
public void NewRecordTest()
{
var client = new SourcesTable("");
var m = new SourceModel
var m = new SourceEntity
{
ID = Guid.NewGuid(),
Id = Guid.NewGuid(),
Site = "Testing",
Name = "Testing",
Source = "Testing",

26
README.md
View File

@ -1,2 +1,26 @@
# Newsbot.Collector
C# API Server to collect news from around the world
C# API Server to collect news from around the world.
## Features
- YouTube
- Github
- Should support other source control services but untested
- RSS
## Notifications
- Discord Webhooks
## Legacy Features No longer support
- Twitter
- API Changes
- Reddit
- API Changes
- Could come back as long as the json dump is still available
- Instagram
- No plans to attempt to get this working again.

18
docker-compose.dev.yaml Normal file
View File

@ -0,0 +1,18 @@
# docker compose -f docker-compose.dev.yaml up -d
version: "3"
volumes:
db:
services:
db:
image: postgres:latest
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- "5432:5432"
volumes:
- db:/var/lib/postgresql/data

8
docker-compose.yaml → docker-compose.example.yaml
View File

@ -10,7 +10,7 @@ services:
db:
image: postgres:latest
environment:
POSTGRES_USER: ${PostgresUser}
POSTGRES_USER: ${PostgresUser}]
POSTGRES_PASSWORD: ${PostgresPassword}
POSTGRES_DB: ${PostgresDatabaseName}
ports:
@ -25,12 +25,6 @@ services:
timeout: "30s"
retries: 5
adminer:
image: adminer
restart: always
ports:
- "8080:8080"
api:
image: newsbot.collector:latest
environment:

12
makefile
View File

@ -21,11 +21,9 @@ docker-run: ## Runs the docker compose
docker-migrate: ## Runs the migrations stored in the Docker image
docker run -it --env-file .env ghcr.io/jtom38/newsbot.collector:master /app/goose --dir "/app/migrations" up
migrate-dev: ## Apply sql migrations to dev db
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" up
ef-build: ## Builds migration artifact
dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force
migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down
migrate-refresh: ## Rolls back all migrations
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" reset
ef-migrate: ## Runs migrations based on the newest artifact
dotnet ef migrations bundle --project "Newsbot.Collector.Database" --force
./efbundle --connection "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"