Compare commits

..

No commits in common. "0797fb85029f2fd42e31a8e846d23365fe10222a" and "731b2cf9b7debbd6cd96bf7ed8908fa43faafc5d" have entirely different histories.

27 changed files with 431 additions and 414 deletions

View File

@ -14,20 +14,21 @@ FROM build AS publish
RUN dotnet publish -c Release -o /app/publish RUN dotnet publish -c Release -o /app/publish
RUN dotnet publish -o build RUN dotnet publish -o build
#--self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true
RUN ls build RUN ls build
FROM build as ef
RUN dotnet tool install dotnet-ef --tool-path /usr/bin
RUN dotnet ef migrations bundle --project "Newsbot.Collector.Database"
FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app
ENV ASPNETCORE_URLS=http://*:5000 ENV ASPNETCORE_URLS=http://*:5000
ENV DOTNET_URLS=http://*:5000 ENV DOTNET_URLS=http://*:5000
#RUN apt-get install chromium -y
WORKDIR /app WORKDIR /app
#RUN mkdir /migrations
COPY --from=publish /app/build /app COPY --from=publish /app/build /app
COPY --from=ef /app/efbundle /app #COPY --from=build ./app/Newsbot.Collector.Database/Migrations /app/migrations
#COPY --from=goose /go/bin/goose /app
CMD [ "dotnet", "Newsbot.Collector.Api.dll" ] CMD [ "dotnet", "Newsbot.Collector.Api.dll" ]

View File

@ -43,7 +43,7 @@ public class ArticlesController : ControllerBase
public ArticleDetailsDto GetDetailsById(Guid id) public ArticleDetailsDto GetDetailsById(Guid id)
{ {
var item = _articles.GetById(id); var item = _articles.GetById(id);
var sourceItem = _sources.GetById(item.SourceId); var sourceItem = _sources.GetByID(item.SourceID);
return ArticleDetailsDto.Convert(item, sourceItem); return ArticleDetailsDto.Convert(item, sourceItem);
} }

View File

@ -3,7 +3,6 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Dto; using Newsbot.Collector.Domain.Dto;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
@ -41,12 +40,12 @@ public class DiscordWebHookController : ControllerBase
{ {
var exists = _webhooks.GetByUrl(url); var exists = _webhooks.GetByUrl(url);
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (exists.Id != Guid.Empty) if (exists.ID != Guid.Empty)
{ {
return DiscordWebHookDto.Convert(exists); return DiscordWebHookDto.Convert(exists);
} }
var res = _webhooks.New(new DiscordWebhookEntity var res = _webhooks.New(new DiscordWebHookModel
{ {
Url = url, Url = url,
Server = server, Server = server,
@ -73,7 +72,7 @@ public class DiscordWebHookController : ControllerBase
[HttpGet("{id}")] [HttpGet("{id}")]
public DiscordWebHookDto GetById(Guid id) public DiscordWebHookDto GetById(Guid id)
{ {
var res = _webhooks.GetById(id); var res = _webhooks.GetByID(id);
return DiscordWebHookDto.Convert(res); return DiscordWebHookDto.Convert(res);
} }

View File

@ -3,7 +3,6 @@ using Microsoft.Extensions.Options;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Dto; using Newsbot.Collector.Domain.Dto;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.HtmlParser; using Newsbot.Collector.Services.HtmlParser;
@ -40,10 +39,10 @@ public class SourcesController : ControllerBase
} }
[HttpGet("by/type")] [HttpGet("by/type")]
public IEnumerable<SourceDto> GetByType(string type, int page) public IEnumerable<SourceDto> GetByType(string type)
{ {
var res = new List<SourceDto>(); var res = new List<SourceDto>();
var temp = _sources.ListByType(type, page); var temp = _sources.ListByType(type);
foreach (var item in temp) res.Add(SourceDto.Convert(item)); foreach (var item in temp) res.Add(SourceDto.Convert(item));
return res; return res;
} }
@ -52,7 +51,7 @@ public class SourcesController : ControllerBase
public SourceDto NewReddit(string name) public SourceDto NewReddit(string name)
{ {
var res = _sources.GetByNameAndType(name, SourceTypes.Reddit); var res = _sources.GetByNameAndType(name, SourceTypes.Reddit);
if (res.Id != Guid.Empty) return SourceDto.Convert(res); if (res.ID != Guid.Empty) return SourceDto.Convert(res);
var uri = new Uri($"https://reddit.com/r/{name}"); var uri = new Uri($"https://reddit.com/r/{name}");
@ -62,7 +61,7 @@ public class SourcesController : ControllerBase
}); });
pageReader.Parse(); pageReader.Parse();
var item = _sources.New(new SourceEntity var item = _sources.New(new SourceModel
{ {
Site = SourceTypes.Reddit, Site = SourceTypes.Reddit,
Name = name, Name = name,
@ -75,11 +74,11 @@ public class SourcesController : ControllerBase
// Not all subreddits have an Icon, so we only want to add a record when it has one. // Not all subreddits have an Icon, so we only want to add a record when it has one.
if (pageReader.Data.Header.Image != "") if (pageReader.Data.Header.Image != "")
_icons.New(new IconEntity() _icons.New(new IconModel
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
FileName = pageReader.Data.Header.Image, FileName = pageReader.Data.Header.Image,
SourceId = item.Id SourceId = item.ID
}); });
return SourceDto.Convert(item); return SourceDto.Convert(item);
} }
@ -88,9 +87,9 @@ public class SourcesController : ControllerBase
public SourceDto NewRss(string name, string url) public SourceDto NewRss(string name, string url)
{ {
var res = _sources.GetByNameAndType(name, SourceTypes.Rss); var res = _sources.GetByNameAndType(name, SourceTypes.Rss);
if (res.Id != Guid.Empty) return SourceDto.Convert(res); if (res.ID != Guid.Empty) return SourceDto.Convert(res);
var m = new SourceEntity var m = new SourceModel
{ {
Site = SourceTypes.Rss, Site = SourceTypes.Rss,
Name = name, Name = name,
@ -108,7 +107,7 @@ public class SourcesController : ControllerBase
public SourceDto NewYoutube(string url) public SourceDto NewYoutube(string url)
{ {
var res = _sources.GetByUrl(url); var res = _sources.GetByUrl(url);
if (res.Id != Guid.Empty) return SourceDto.Convert(res); if (res.ID != Guid.Empty) return SourceDto.Convert(res);
var htmlClient = new HtmlPageReader(new HtmlPageReaderOptions var htmlClient = new HtmlPageReader(new HtmlPageReaderOptions
{ {
@ -116,7 +115,7 @@ public class SourcesController : ControllerBase
}); });
htmlClient.Parse(); htmlClient.Parse();
var item = _sources.New(new SourceEntity var item = _sources.New(new SourceModel
{ {
Site = SourceTypes.YouTube, Site = SourceTypes.YouTube,
Type = SourceTypes.YouTube, Type = SourceTypes.YouTube,
@ -128,11 +127,11 @@ public class SourcesController : ControllerBase
YoutubeId = htmlClient.Data.Header.YoutubeChannelID ?? "" YoutubeId = htmlClient.Data.Header.YoutubeChannelID ?? ""
}); });
_icons.New(new IconEntity() _icons.New(new IconModel
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
FileName = htmlClient.Data.Header.Image, FileName = htmlClient.Data.Header.Image,
SourceId = item.Id SourceId = item.ID
}); });
return SourceDto.Convert(item); return SourceDto.Convert(item);
@ -142,9 +141,9 @@ public class SourcesController : ControllerBase
public SourceDto NewTwitch(string name) public SourceDto NewTwitch(string name)
{ {
var res = _sources.GetByNameAndType(name, SourceTypes.Twitch); var res = _sources.GetByNameAndType(name, SourceTypes.Twitch);
if (res.Id != Guid.Empty) return SourceDto.Convert(res); if (res.ID != Guid.Empty) return SourceDto.Convert(res);
var item = _sources.New(new SourceEntity var item = _sources.New(new SourceModel
{ {
Site = SourceTypes.Twitch, Site = SourceTypes.Twitch,
Type = SourceTypes.Twitch, Type = SourceTypes.Twitch,
@ -163,7 +162,7 @@ public class SourcesController : ControllerBase
//if (!url.Contains("github.com")) return new SourceDto(); //if (!url.Contains("github.com")) return new SourceDto();
var res = _sources.GetByUrl(url); var res = _sources.GetByUrl(url);
if (res.Id != Guid.Empty) return SourceDto.Convert(res); if (res.ID != Guid.Empty) return SourceDto.Convert(res);
var slice = url.Split('/'); var slice = url.Split('/');
@ -173,7 +172,7 @@ public class SourcesController : ControllerBase
}); });
pageReader.Parse(); pageReader.Parse();
var item = _sources.New(new SourceEntity var item = _sources.New(new SourceModel
{ {
Site = SourceTypes.CodeProject, Site = SourceTypes.CodeProject,
Type = SourceTypes.CodeProject, Type = SourceTypes.CodeProject,
@ -184,11 +183,11 @@ public class SourcesController : ControllerBase
Tags = $"{slice[2]},{slice[3]},{slice[4]}" Tags = $"{slice[2]},{slice[3]},{slice[4]}"
}); });
_icons.New(new IconEntity() _icons.New(new IconModel
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
FileName = pageReader.Data.Header.Image, FileName = pageReader.Data.Header.Image,
SourceId = item.Id SourceId = item.ID
}); });
return SourceDto.Convert(item); return SourceDto.Convert(item);
@ -197,7 +196,7 @@ public class SourcesController : ControllerBase
[HttpGet("{id}")] [HttpGet("{id}")]
public SourceDto GetById(Guid id) public SourceDto GetById(Guid id)
{ {
var item = _sources.GetById(id); var item = _sources.GetByID(id);
return SourceDto.Convert(item); return SourceDto.Convert(item);
} }

View File

@ -43,8 +43,8 @@ public class SubscriptionsController : ControllerBase
public SubscriptionDetailsDto GetDetailsById(Guid id) public SubscriptionDetailsDto GetDetailsById(Guid id)
{ {
var sub = _subscription.GetById(id); var sub = _subscription.GetById(id);
var webhook = _discord.GetById(sub.DiscordWebHookId); var webhook = _discord.GetByID(sub.DiscordWebHookId);
var source = _sources.GetById(sub.SourceId); var source = _sources.GetByID(sub.SourceId);
return SubscriptionDetailsDto.Convert(sub, source, webhook); return SubscriptionDetailsDto.Convert(sub, source, webhook);
} }
@ -82,11 +82,11 @@ public class SubscriptionsController : ControllerBase
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists); if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetById(discordId); var discord = _discord.GetByID(discordId);
if (discord.Id == Guid.Empty) return new BadRequestResult(); if (discord.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetById(sourceId); var source = _sources.GetByID(sourceId);
if (source.Id == Guid.Empty) return new BadRequestResult(); if (source.ID == Guid.Empty) return new BadRequestResult();
var item = _subscription.New(new SubscriptionModel var item = _subscription.New(new SubscriptionModel
{ {
@ -110,11 +110,11 @@ public class SubscriptionsController : ControllerBase
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists); if (exists.Id != Guid.Empty) return SubscriptionDto.Convert(exists);
var discord = _discord.GetById(discordId); var discord = _discord.GetByID(discordId);
if (discord.Id == Guid.Empty) return new BadRequestResult(); if (discord.ID == Guid.Empty) return new BadRequestResult();
var source = _sources.GetById(sourceId); var source = _sources.GetByID(sourceId);
if (source.Id == Guid.Empty) return new BadRequestResult(); if (source.ID == Guid.Empty) return new BadRequestResult();
var sub = _subscription.New(new SubscriptionModel var sub = _subscription.New(new SubscriptionModel
{ {

View File

@ -30,41 +30,61 @@ public class ArticlesTable : IArticlesRepository
_context = new DatabaseContext(conn); _context = new DatabaseContext(conn);
} }
public List<ArticlesEntity> List(int page = 0, int count = 25) public async Task<List<ArticlesEntity>> ListAsync(int page = 0, int count = 25)
{ {
using var context = new DatabaseContext(_connectionString); await using var context = new DatabaseContext(_connectionString);
var query = context.Articles var query = context.Articles
.Skip(page * count)
.OrderBy(d => d.PubDate) .OrderBy(d => d.PubDate)
.Take(25); .Take(25);
Console.WriteLine(query.ToQueryString()); Console.WriteLine(query.ToQueryString());
return query.ToList(); await query.ToListAsync();
return await query.ToListAsync();
} }
public ArticlesEntity GetById(Guid id) public List<ArticlesEntity> List(int page = 0, int count = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var query = context.Articles var res = conn.Query<ArticlesEntity>(@"select * from Articles
.FirstOrDefault(d => d.Id.Equals(id)); Order By PubDate Desc
query ??= new ArticlesEntity(); Offset @Page
return query; Fetch Next @Count Rows Only", new
} {
Page = page * count,
public ArticlesEntity GetByUrl(string url) Count = count
{ })
using var context = new DatabaseContext(_connectionString); .ToList();
var res = context.Articles.FirstOrDefault(d => d.Url!.Equals(url));
res ??= new ArticlesEntity();
return res; return res;
} }
public List<ArticlesEntity> ListBySourceId(Guid id, int page, int count) public ArticlesModel GetById(Guid ID)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Articles var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID });
.Skip(page * count) if (res.Count() == 0) return new ArticlesModel();
.Where(d => d.SourceId.Equals(id)); return res.First();
return res.ToList(); }
public ArticlesModel GetByUrl(string url)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
if (res.Count() == 0) return new ArticlesModel();
return res.First();
}
public List<ArticlesModel> ListBySourceId(Guid id, int page, int count)
{
using var conn = OpenConnection(_connectionString);
var query = @"Select * from articles
where sourceid = @sourceid
Offset @page
Fetch next @count rows only";
return conn.Query<ArticlesModel>(query, new
{
sourceid = id,
page = page * count,
count
}).ToList();
} }
public ArticlesEntity New(ArticlesEntity model) public ArticlesEntity New(ArticlesEntity model)
@ -106,17 +126,12 @@ public class ArticlesTable : IArticlesRepository
public void DeleteAllBySourceId(Guid sourceId) public void DeleteAllBySourceId(Guid sourceId)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Articles var res = conn.Execute("Delete from articles where sourceid = '@id'", new
.Where(d => d.SourceId.Equals(sourceId))
.ToList();
foreach (var item in res)
{ {
context.Articles.Remove(item); sourceId
} });
if (res == 0) throw new Exception($"No records where deleted that linked to SourceId = '{sourceId}'");
context.SaveChanges();
} }
private IDbConnection OpenConnection(string connectionString) private IDbConnection OpenConnection(string connectionString)

View File

@ -1,6 +1,5 @@
using System.Data; using System.Data;
using Dapper; using Dapper;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Npgsql; using Npgsql;
@ -23,28 +22,33 @@ public class DiscordQueueTable : IDiscordQueueRepository
return conn; return conn;
} }
public void New(DiscordQueueEntity model) public void New(DiscordQueueModel model)
{ {
model.Id = new Guid(); using var conn = OpenConnection(_connectionString);
var query = "Insert into DiscordQueue(ID, ArticleId) Values (@id, @articleid);";
using var context = new DatabaseContext(_connectionString); conn.Execute(query, new
var res = context.DiscordQueue.Add(model); {
context.SaveChanges(); id = Guid.NewGuid(),
articleid = model.ArticleID
});
} }
public void Delete(Guid id) public void Delete(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordQueue.FirstOrDefault(d => d.Id.Equals(id)); var query = "Delete From DiscordQueue Where ID = @id;";
res ??= new DiscordQueueEntity(); conn.Execute(query, new
context.DiscordQueue.Remove(res); {
context.SaveChanges(); id = id
});
} }
public List<DiscordQueueEntity> List(int limit = 25) public List<DiscordQueueModel> List(int limit = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordQueue.Take(limit).ToList(); var query = "Select * from DiscordQueue LIMIT @limit;";
return res; return conn.Query<DiscordQueueModel>(query, new {
limit = limit
}).ToList();
} }
} }

View File

@ -2,7 +2,7 @@ using System.Data;
using Dapper; using Dapper;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models;
using Npgsql; using Npgsql;
namespace Newsbot.Collector.Database.Repositories; namespace Newsbot.Collector.Database.Repositories;
@ -22,104 +22,103 @@ public class DiscordWebhooksTable : IDiscordWebHooksRepository
_connectionString = connstr; _connectionString = connstr;
} }
public DiscordWebhookEntity New(DiscordWebhookEntity model) public DiscordWebHookModel New(DiscordWebHookModel model)
{ {
model.Id = new Guid(); var uid = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
using var context = new DatabaseContext(_connectionString); var query =
context.DiscordWebhooks.Add(model); "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);";
context.SaveChanges(); conn.Execute(query, new
{
id = uid,
url = model.Url,
server = model.Server,
channel = model.Channel,
enabled = model.Enabled
});
model.ID = uid;
return model; return model;
} }
public DiscordWebhookEntity GetById(Guid id) public DiscordWebHookModel GetByID(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordWebhooks.FirstOrDefault(d => d.Id.Equals(id)); var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;";
res ??= new DiscordWebhookEntity(); return conn.Query<DiscordWebHookModel>(query, new
return res; {
id
}).First();
} }
public DiscordWebhookEntity GetByUrl(string url) public DiscordWebHookModel GetByUrl(string url)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordWebhooks.FirstOrDefault(d => d.Url.Equals(url)); var query = "Select * From DiscordWebHooks Where url = @url;";
res ??= new DiscordWebhookEntity(); try
return res; {
var res = conn.QueryFirst<DiscordWebHookModel>(query, new
{
url
});
return res;
}
catch
{
return new DiscordWebHookModel();
}
} }
public List<DiscordWebhookEntity> List(int page, int count = 25) public List<DiscordWebHookModel> List(int page, int count = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordWebhooks var query = @"Select * From DiscordWebHooks
.Skip(page * count) Offset @offset Fetch Next @count Rows Only;";
.Take(count) return conn.Query<DiscordWebHookModel>(query, new
.ToList(); {
res ??= new List<DiscordWebhookEntity>(); offset = page * count, count
return res; }).ToList();
} }
public List<DiscordWebhookEntity> ListByServer(string server, int limit = 25) public List<DiscordWebHookModel> ListByServer(string server, int limit = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordWebhooks var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;";
.Where(d => d.Server.Equals(server)) return conn.Query<DiscordWebHookModel>(query, new
.Take(limit) {
.ToList(); server, limit
res ??= new List<DiscordWebhookEntity>(); }).ToList();
return res;
} }
public List<DiscordWebhookEntity> ListByServerAndChannel(string server, string channel, int limit = 25) public List<DiscordWebHookModel> ListByServerAndChannel(string server, string channel, int limit = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.DiscordWebhooks var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;";
.Where(s => s.Server.Equals(server)) return conn.Query<DiscordWebHookModel>(query, new
.Where(c => c.Channel.Equals(channel)) {
.Take(limit) server,
.ToList(); channel,
res ??= new List<DiscordWebhookEntity>(); limit
return res; }).ToList();
} }
public int Disable(Guid id) public int Disable(Guid id)
{ {
var res = GetById(id); using var conn = OpenConnection(_connectionString);
using var context = new DatabaseContext(_connectionString); var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;";
return conn.Execute(query, new
res.Enabled = true;
context.DiscordWebhooks.Update(res);
try
{ {
context.SaveChanges(); id
return 1; });
}
catch(Exception ex)
{
Console.WriteLine($"Failed to update DiscordWebhook ID = {id}. {ex.Message}");
return 0;
}
} }
public int Enable(Guid id) public int Enable(Guid id)
{ {
var res = GetById(id); using var conn = OpenConnection(_connectionString);
using var context = new DatabaseContext(_connectionString); var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;";
return conn.Execute(query, new
res.Enabled = false;
context.DiscordWebhooks.Update(res);
try
{ {
context.SaveChanges(); id
return 1; });
}
catch(Exception ex)
{
Console.WriteLine($"Failed to update DiscordWebhook ID = {id}. {ex.Message}");
return 0;
}
} }
private IDbConnection OpenConnection(string connectionString) private IDbConnection OpenConnection(string connectionString)

View File

@ -1,7 +1,6 @@
using System.Data; using System.Data;
using Dapper; using Dapper;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Npgsql; using Npgsql;
@ -24,29 +23,37 @@ public class IconsTable : IIconsRepository
_connectionString = connstr; _connectionString = connstr;
} }
public void New(IconEntity model) public void New(IconModel model)
{ {
using var context = new DatabaseContext(_connectionString);
model.Id = Guid.NewGuid(); model.Id = Guid.NewGuid();
context.Icons.Add(model); using var conn = OpenConnection(_connectionString);
context.SaveChanges(); var q = @"Insert Into icons (id, filename, site, sourceid) values (@Id,@FileName, @Site, @SourceId)";
conn.Execute(q, model);
} }
public IconEntity GetById(Guid id) public IconModel GetById(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Icons.FirstOrDefault(f => f.Id.Equals(id)); var query = "Select * From icons where ID = @id Limit 1;";
res ??= new IconEntity(); var res = conn.Query<IconModel>(query, new
return res; {
id
});
if (!res.Any()) return new IconModel();
return res.First();
} }
public IconEntity GetBySourceId(Guid id) public IconModel GetBySourceId(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Icons.FirstOrDefault(f => f.SourceId.Equals(id)); var query = "Select * From icons where sourceid = @id Limit 1;";
res ??= new IconEntity(); var res = conn.Query<IconModel>(query, new
return res; {
id
});
if (!res.Any()) return new IconModel();
return res.First();
} }
private IDbConnection OpenConnection(string connectionString) private IDbConnection OpenConnection(string connectionString)

View File

@ -1,7 +1,6 @@
using System.Data; using System.Data;
using Dapper; using Dapper;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Npgsql; using Npgsql;
@ -24,153 +23,157 @@ public class SourcesTable : ISourcesRepository
_connectionString = connstr; _connectionString = connstr;
} }
public SourceEntity New(SourceEntity model) public SourceModel New(SourceModel model)
{ {
model.Id = Guid.NewGuid(); model.ID = Guid.NewGuid();
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
context.Sources.Add(model); var query =
try "Insert Into Sources (ID, Site, Name, Source, Type, Value, Enabled, Url, Tags, YoutubeId) Values (@id ,@site,@name,@source,@type,@value,@enabled,@url,@tags,@youtubeid);";
conn.Execute(query, new
{ {
context.SaveChanges(); id = model.ID,
} model.Site,
catch (Exception ex) model.Name,
{ model.Source,
Console.WriteLine($"Failed to save "); model.Type,
} model.Value,
model.Enabled,
model.Url,
model.Tags,
model.YoutubeId
});
return model; return model;
} }
public SourceEntity GetById(Guid id) public SourceModel GetByID(Guid ID)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources.FirstOrDefault(f => f.Id.Equals(id)); var query = "Select * From Sources where ID = @id Limit 1;";
res ??= new SourceEntity(); var res = conn.Query<SourceModel>(query, new
return res; {
id = ID
});
if (res.Count() == 0) return new SourceModel();
return res.First();
} }
public SourceEntity GetById(string id) public SourceModel GetByID(string ID)
{ {
var uid = Guid.Parse(id); var uid = Guid.Parse(ID);
return GetById(uid); return GetByID(uid);
} }
public SourceEntity GetByName(string name) public SourceModel GetByName(string Name)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources.FirstOrDefault(f => f.Name.Equals(name)); var query = "Select * from Sources where name = @name Limit 1;";
res ??= new SourceEntity(); var res = conn.Query<SourceModel>(query, new
return res; {
name = Name
});
if (res.Count() == 0) return new SourceModel();
return res.First();
} }
public SourceEntity GetByNameAndType(string name, string type) public SourceModel GetByNameAndType(string name, string type)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources var query = "Select * from Sources WHERE name = @name and type = @type;";
.Where(f => f.Name.Equals(name)) var res = conn.Query<SourceModel>(query, new
.FirstOrDefault(f => f.Type.Equals(type)); {
res ??= new SourceEntity(); name, type
return res; });
if (res.Count() == 0) return new SourceModel();
return res.First();
} }
public SourceEntity GetByUrl(string url) public SourceModel GetByUrl(string url)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources var query = "Select * from Sources WHERE url = @url;";
.FirstOrDefault(f => f.Url.Equals(url)); var res = conn.Query<SourceModel>(query, new
res ??= new SourceEntity(); {
return res; url
});
if (res.ToList().Count == 0) return new SourceModel();
return res.First();
} }
public List<SourceEntity> List(int page = 0, int count = 100) public List<SourceModel> List(int page = 0, int count = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources var query = @"Select * From Sources
.Skip(page * count) Offset @page
.Take(count) Fetch Next @count Rows Only;";
.ToList(); return conn.Query<SourceModel>(query, new
return res; {
page = page * count, count
}).ToList();
} }
public List<SourceEntity> ListBySource(string source, int page = 0, int limit = 25) public List<SourceModel> ListBySource(string source, int limit = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources var query = "Select * From Sources where Source = @source Limit @limit;";
.Where(f => f.Source.Equals(source)) return conn.Query<SourceModel>(query, new
.Skip(page * limit) {
.Take(limit) source, limit
.ToList(); }).ToList();
return res;
} }
public List<SourceEntity> ListByType(string type,int page = 0, int limit = 25) public List<SourceModel> ListByType(string type, int limit = 25)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = context.Sources var query = "Select * From Sources where Type = @type Limit @limit;";
.Where(f => f.Type.Equals(type)) return conn.Query<SourceModel>(query, new
.Skip(page * limit) {
.Take(limit) type, limit
.ToList(); }).ToList();
return res;
} }
public int Disable(Guid id) public int Disable(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = GetById(id); var query = "Update Sources Set Enabled = FALSE where ID = @id;";
res.Enabled = false; return conn.Execute(query, new
context.Sources.Update(res);
try
{ {
context.SaveChanges(); id
return 1; });
}
catch
{
return 0;
}
} }
public int Enable(Guid id) public int Enable(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = GetById(id); var query = "Update Sources Set Enabled = TRUE where ID = @id;";
res.Enabled = true; return conn.Execute(query, new
context.Sources.Update(res);
try
{ {
context.SaveChanges(); id
return 1; });
}
catch
{
return 0;
}
} }
public void Delete(Guid id) public void Delete(Guid id)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = GetById(id); var query = "Delete From sources where id = @id;";
context.Sources.Remove(res); var res = conn.Execute(query, new
context.SaveChanges(); {
id
});
if (res == 0) throw new Exception("Nothing was deleted");
} }
public int UpdateYoutubeId(Guid id, string youtubeId) public int UpdateYoutubeId(Guid id, string youtubeId)
{ {
using var context = new DatabaseContext(_connectionString); using var conn = OpenConnection(_connectionString);
var res = GetById(id); var query = "Update Sources Set youtubeid = @youtubeId where ID = @id;";
res.YoutubeId = youtubeId; return conn.Execute(query, new
context.Sources.Update(res);
try
{ {
context.SaveChanges(); id, youtubeId
return 1; });
}
catch
{
return 0;
}
} }
private IDbConnection OpenConnection(string connectionString) private IDbConnection OpenConnection(string connectionString)

View File

@ -1,11 +1,10 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto; namespace Newsbot.Collector.Domain.Dto;
public class ArticleDetailsDto public class ArticleDetailsDto
{ {
public Guid Id { get; set; } public Guid ID { get; set; }
public string[]? Tags { get; set; } public string[]? Tags { get; set; }
public string? Title { get; set; } public string? Title { get; set; }
public string? Url { get; set; } public string? Url { get; set; }
@ -20,14 +19,14 @@ public class ArticleDetailsDto
public SourceDto? Source { get; set; } public SourceDto? Source { get; set; }
public static ArticleDetailsDto Convert(ArticlesEntity article, SourceEntity source) public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source)
{ {
return new ArticleDetailsDto return new ArticleDetailsDto
{ {
Id = article.Id, ID = article.ID,
Tags = article.Tags.Split(','), Tags = article.Tags.Split(','),
Title = article.Title, Title = article.Title,
Url = article.Url, Url = article.URL,
PubDate = article.PubDate, PubDate = article.PubDate,
Video = article.Video, Video = article.Video,
VideoHeight = article.VideoHeight, VideoHeight = article.VideoHeight,

View File

@ -1,4 +1,4 @@
using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto; namespace Newsbot.Collector.Domain.Dto;
@ -10,11 +10,11 @@ public class DiscordWebHookDto
public string? Channel { get; set; } public string? Channel { get; set; }
public bool Enabled { get; set; } public bool Enabled { get; set; }
public static DiscordWebHookDto Convert(DiscordWebhookEntity model) public static DiscordWebHookDto Convert(DiscordWebHookModel model)
{ {
return new DiscordWebHookDto return new DiscordWebHookDto
{ {
ID = model.Id, ID = model.ID,
Url = model.Url, Url = model.Url,
Server = model.Server, Server = model.Server,
Channel = model.Channel, Channel = model.Channel,

View File

@ -1,5 +1,4 @@
using System.Net.NetworkInformation; using System.Net.NetworkInformation;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto; namespace Newsbot.Collector.Domain.Dto;
@ -17,10 +16,10 @@ public class SourceDto
public string[]? Tags { get; set; } public string[]? Tags { get; set; }
public bool Deleted { get; set; } public bool Deleted { get; set; }
public static SourceDto Convert(SourceEntity model) { public static SourceDto Convert(SourceModel model) {
return new SourceDto return new SourceDto
{ {
ID = model.Id, ID = model.ID,
Site = model.Site, Site = model.Site,
Name = model.Name, Name = model.Name,
Source = model.Source, Source = model.Source,

View File

@ -1,4 +1,3 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Dto; namespace Newsbot.Collector.Domain.Dto;
@ -11,8 +10,8 @@ public class SubscriptionDetailsDto
public SourceDto? Source { get; set; } public SourceDto? Source { get; set; }
public DiscordWebHookDto? DiscordWebHook { get; set; } public DiscordWebHookDto? DiscordWebHook { get; set; }
public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceEntity source, public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source,
DiscordWebhookEntity discord) DiscordWebHookModel discord)
{ {
return new SubscriptionDetailsDto return new SubscriptionDetailsDto
{ {

View File

@ -6,9 +6,9 @@ namespace Newsbot.Collector.Domain.Interfaces;
public interface IArticlesRepository : ITableRepository public interface IArticlesRepository : ITableRepository
{ {
List<ArticlesEntity> List(int page, int count); List<ArticlesEntity> List(int page, int count);
List<ArticlesEntity> ListBySourceId(Guid id, int page = 0, int count = 25); List<ArticlesModel> ListBySourceId(Guid id, int page = 0, int count = 25);
ArticlesEntity GetById(Guid id); ArticlesModel GetById(Guid ID);
ArticlesEntity GetByUrl(string url); ArticlesModel GetByUrl(string url);
ArticlesEntity New(ArticlesEntity model); ArticlesEntity New(ArticlesEntity model);
void DeleteAllBySourceId(Guid sourceId); void DeleteAllBySourceId(Guid sourceId);
} }

View File

@ -1,10 +1,10 @@
using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces; namespace Newsbot.Collector.Domain.Interfaces;
public interface IDiscordQueueRepository public interface IDiscordQueueRepository
{ {
void New(DiscordQueueEntity model); void New(DiscordQueueModel model);
void Delete(Guid id); void Delete(Guid id);
List<DiscordQueueEntity> List(int limit); List<DiscordQueueModel> List(int limit);
} }

View File

@ -1,17 +1,18 @@
using Newsbot.Collector.Domain.Entities; using Microsoft.VisualBasic;
using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces; namespace Newsbot.Collector.Domain.Interfaces;
public interface IDiscordWebHooksRepository public interface IDiscordWebHooksRepository
{ {
DiscordWebhookEntity New(DiscordWebhookEntity model); DiscordWebHookModel New(DiscordWebHookModel model);
DiscordWebhookEntity GetById(Guid id); DiscordWebHookModel GetByID(Guid id);
DiscordWebhookEntity GetByUrl(string url); DiscordWebHookModel GetByUrl(string url);
List<DiscordWebhookEntity> List(int page, int count = 25); List<DiscordWebHookModel> List(int page, int count = 25);
List<DiscordWebhookEntity> ListByServer(string server, int limit); List<DiscordWebHookModel> ListByServer(string server, int limit);
List<DiscordWebhookEntity> ListByServerAndChannel(string server, string channel, int limit); List<DiscordWebHookModel> ListByServerAndChannel(string server, string channel, int limit);
int Disable(Guid id); int Disable(Guid id);
int Enable(Guid id); int Enable(Guid id);

View File

@ -1,11 +1,11 @@
using Newsbot.Collector.Domain.Entities; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces; namespace Newsbot.Collector.Domain.Interfaces;
public interface IIconsRepository public interface IIconsRepository
{ {
public void New(IconEntity model); public void New(IconModel model);
public IconEntity GetById(Guid id); public IconModel GetById(Guid id);
public IconEntity GetBySourceId(Guid id); public IconModel GetBySourceId(Guid id);
} }

View File

@ -1,19 +1,18 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Domain.Interfaces; namespace Newsbot.Collector.Domain.Interfaces;
public interface ISourcesRepository public interface ISourcesRepository
{ {
public SourceEntity New(SourceEntity model); public SourceModel New(SourceModel model);
public SourceEntity GetById(Guid id); public SourceModel GetByID(Guid ID);
public SourceEntity GetById(string id); public SourceModel GetByID(string ID);
public SourceEntity GetByName(string name); public SourceModel GetByName(string name);
public SourceEntity GetByNameAndType(string name, string type); public SourceModel GetByNameAndType(string name, string type);
SourceEntity GetByUrl(string url); SourceModel GetByUrl(string url);
public List<SourceEntity> List(int page, int count); public List<SourceModel> List(int page, int count);
public List<SourceEntity> ListBySource(string source,int page, int limit); public List<SourceModel> ListBySource(string source, int limit);
public List<SourceEntity> ListByType(string type,int page, int limit = 25); public List<SourceModel> ListByType(string type, int limit = 25);
public int Disable(Guid id); public int Disable(Guid id);
public int Enable(Guid id); public int Enable(Guid id);
public void Delete(Guid id); public void Delete(Guid id);

View File

@ -3,7 +3,6 @@ using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Domain.Models.Config;
@ -61,10 +60,10 @@ public class CodeProjectWatcherJob
private void Execute() private void Execute()
{ {
var sources = _source.ListByType(SourceTypes.CodeProject, 0, 100); var sources = _source.ListByType(SourceTypes.CodeProject);
// query sources for things to pull // query sources for things to pull
var items = new List<ArticlesEntity>(); var items = new List<ArticlesModel>();
foreach (var source in sources) foreach (var source in sources)
{ {
@ -75,14 +74,14 @@ public class CodeProjectWatcherJob
foreach (var item in items) foreach (var item in items)
{ {
_articles.New(item); _articles.New(item);
_queue.New(new DiscordQueueEntity() _queue.New(new DiscordQueueModel
{ {
ArticleId = item.Id ArticleID = item.ID
}); });
} }
} }
public IEnumerable<ArticlesEntity> CheckForReleases(SourceEntity source) public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source)
{ {
var url = new Uri(source.Url); var url = new Uri(source.Url);
var links = new List<string> var links = new List<string>
@ -111,10 +110,10 @@ public class CodeProjectWatcherJob
url.AbsoluteUri); url.AbsoluteUri);
} }
return new List<ArticlesEntity>(); return new List<ArticlesModel>();
} }
public IEnumerable<ArticlesEntity> CheckForCommits(SourceEntity source) public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source)
{ {
var url = new Uri(source.Url); var url = new Uri(source.Url);
var links = new List<string> var links = new List<string>
@ -137,19 +136,19 @@ public class CodeProjectWatcherJob
url.AbsoluteUri); url.AbsoluteUri);
} }
return new List<ArticlesEntity>(); return new List<ArticlesModel>();
} }
private IEnumerable<ArticlesEntity> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceEntity source, private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
bool isRelease, bool isCommit) bool isRelease, bool isCommit)
{ {
var items = new List<ArticlesEntity>(); var items = new List<ArticlesModel>();
foreach (var item in feed) foreach (var item in feed)
{ {
var itemUrl = item.Links[0].Uri.AbsoluteUri; var itemUrl = item.Links[0].Uri.AbsoluteUri;
var exits = _articles.GetByUrl(itemUrl); var exits = _articles.GetByUrl(itemUrl);
if (exits.Id != Guid.Empty) continue; if (exits.ID != Guid.Empty) continue;
var parser = new HtmlPageReader(new HtmlPageReaderOptions var parser = new HtmlPageReader(new HtmlPageReaderOptions
{ {
@ -157,12 +156,12 @@ public class CodeProjectWatcherJob
}); });
parser.Parse(); parser.Parse();
var a = new ArticlesEntity var a = new ArticlesModel
{ {
SourceId = source.Id, SourceID = source.ID,
Tags = source.Tags, Tags = source.Tags,
Title = item.Title.Text, Title = item.Title.Text,
Url = itemUrl, URL = itemUrl,
PubDate = item.LastUpdatedTime.DateTime, PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image, Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text, Description = item.Title.Text,

View File

@ -1,5 +1,4 @@
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
@ -84,44 +83,45 @@ public class DiscordNotificationJob
_logger.Information($"{JobName} - Loop has been completed."); _logger.Information($"{JobName} - Loop has been completed.");
} }
public void ProcessQueueItem(DiscordQueueEntity request) public void ProcessQueueItem(DiscordQueueModel request)
{ {
_logger.Debug($"{JobName} - Processing {request.Id}"); _logger.Debug($"{JobName} - Processing {request.ID}");
// Get all details on the article in the queue // Get all details on the article in the queue
var articleDetails = _article.GetById(request.ArticleId); var articleDetails = _article.GetById(request.ArticleID);
// Get the details of the source // Get the details of the source
var sourceDetails = _sources.GetById(articleDetails.SourceId); var sourceDetails = _sources.GetByID(articleDetails.SourceID);
if (sourceDetails.Id == Guid.Empty) if (sourceDetails.ID == Guid.Empty)
{ {
_logger.Error( _logger.Error(
$"{JobName} - Article ({articleDetails.Id}) was linked to a empty Source ID. Removing from the queue."); $"{JobName} - Article ({articleDetails.ID}) was linked to a empty Source ID. Removing from the queue.");
_queue.Delete(request.Id); _queue.Delete(request.ID);
return; return;
} }
var sourceIcon = new IconEntity(); var sourceIcon = new IconModel();
try try
{ {
sourceIcon = _icons.GetBySourceId(sourceDetails.Id); sourceIcon = _icons.GetBySourceId(sourceDetails.ID);
} }
catch catch
{ {
_logger.Warning("{JobName} - Source ID \'{SourceDetailsId}\' is missing an icon", JobName, _logger.Warning("{JobName} - Source ID \'{SourceDetailsId}\' is missing an icon", JobName,
sourceDetails.Id); sourceDetails.ID);
} }
// Find all the subscriptions for that source // Find all the subscriptions for that source
var allSubscriptions = _subs.ListBySourceID(sourceDetails.Id); var allSubscriptions = _subs.ListBySourceID(sourceDetails.ID);
foreach (var sub in allSubscriptions) foreach (var sub in allSubscriptions)
SendSubscriptionNotification(request.Id, articleDetails, sourceDetails, sourceIcon, sub); SendSubscriptionNotification(request.ID, articleDetails, sourceDetails, sourceIcon, sub);
_logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.Id); _logger.Debug("{JobName} - Removing {RequestId} from the queue", JobName, request.ID);
_queue.Delete(request.Id); _queue.Delete(request.ID);
} }
public void SendSubscriptionNotification(Guid requestId, ArticlesEntity articleDetails, SourceEntity sourceDetails, IconEntity sourceIcon, SubscriptionModel sub) public void SendSubscriptionNotification(Guid requestId, ArticlesModel articleDetails, SourceModel sourceDetails,
IconModel sourceIcon, SubscriptionModel sub)
{ {
// Check if the subscription code flags // Check if the subscription code flags
// If the article is a code commit and the subscription does not want them, skip. // If the article is a code commit and the subscription does not want them, skip.
@ -131,7 +131,7 @@ public class DiscordNotificationJob
if (articleDetails.CodeIsRelease && !sub.CodeAllowReleases) throw new MessageTypeNotRequestedException("Message was a code release and was not requested by the subscription"); if (articleDetails.CodeIsRelease && !sub.CodeAllowReleases) throw new MessageTypeNotRequestedException("Message was a code release and was not requested by the subscription");
// find the discord webhooks we need to post to // find the discord webhooks we need to post to
var discordDetails = _webhook.GetById(sub.DiscordWebHookId); var discordDetails = _webhook.GetByID(sub.DiscordWebHookId);
if (discordDetails.Enabled == false) return; if (discordDetails.Enabled == false) return;
var client = new DiscordWebhookClient(discordDetails.Url); var client = new DiscordWebhookClient(discordDetails.Url);
@ -143,15 +143,15 @@ public class DiscordNotificationJob
{ {
_logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message); _logger.Error("Failed to post message to Discord. {ErrorMessage}", e.Message);
_logger.Debug("Queue Record: {RequestId}", requestId); _logger.Debug("Queue Record: {RequestId}", requestId);
_logger.Debug("Article: {ArticleDetailsId}", articleDetails.Id); _logger.Debug("Article: {ArticleDetailsId}", articleDetails.ID);
_logger.Debug("Source: {SourceDetailsId}", sourceDetails.Id); _logger.Debug("Source: {SourceDetailsId}", sourceDetails.ID);
_logger.Debug("Subscription: {SubId}", sub.Id); _logger.Debug("Subscription: {SubId}", sub.Id);
} }
Thread.Sleep(3000); Thread.Sleep(3000);
} }
public DiscordMessage GenerateDiscordMessage(SourceEntity source, ArticlesEntity article, IconEntity icon) public DiscordMessage GenerateDiscordMessage(SourceModel source, ArticlesModel article, IconModel icon)
{ {
var embed = new DiscordMessageEmbed var embed = new DiscordMessageEmbed
{ {
@ -172,13 +172,13 @@ public class DiscordNotificationJob
new() new()
{ {
Name = "Link", Name = "Link",
Value = article.Url, Value = article.URL,
Inline = false Inline = false
} }
} }
}; };
if (article.Url is not null && article.Url != "") embed.Url = article.Url; if (article.URL is not null && article.URL != "") embed.Url = article.URL;
if (article.Thumbnail != "") if (article.Thumbnail != "")
embed.Image = new DiscordMessageEmbedImage embed.Image = new DiscordMessageEmbedImage

View File

@ -2,7 +2,6 @@ using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Domain.Models.Config;
@ -57,17 +56,17 @@ public class RssWatcherJob
public void Execute() public void Execute()
{ {
var articles = new List<ArticlesEntity>(); var articles = new List<ArticlesModel>();
_logger.Information($"{JobName} - Requesting sources"); _logger.Information($"{JobName} - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss, 0, 100); var sources = _source.ListByType(SourceTypes.Rss);
_logger.Information("{JobName} - Got {SourcesCount} back", JobName, sources.Count); _logger.Information($"{JobName} - Got {sources.Count} back");
foreach (var source in sources) foreach (var source in sources)
{ {
_logger.Information("{JobName} - Starting to process \'{SourceName}\'", JobName, source.Name); _logger.Information($"{JobName} - Starting to process '{source.Name}'");
_logger.Information($"{JobName} - Starting to request feed to be processed"); _logger.Information($"{JobName} - Starting to request feed to be processed");
var results = Collect(source.Url, source.Id); var results = Collect(source.Url, source.ID);
_logger.Information($"{JobName} - Collected {results.Count} posts"); _logger.Information($"{JobName} - Collected {results.Count} posts");
articles.AddRange(results); articles.AddRange(results);
@ -79,9 +78,9 @@ public class RssWatcherJob
_logger.Information($"{JobName} - Done!"); _logger.Information($"{JobName} - Done!");
} }
public List<ArticlesEntity> Collect(string url, Guid sourceId, int sleep = 3000) public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
{ {
var collectedPosts = new List<ArticlesEntity>(); var collectedPosts = new List<ArticlesModel>();
using var reader = XmlReader.Create(url); using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
@ -100,15 +99,15 @@ public class RssWatcherJob
}); });
meta.Parse(); meta.Parse();
var article = new ArticlesEntity var article = new ArticlesModel
{ {
Title = post.Title.Text, Title = post.Title.Text,
Tags = FetchTags(post), Tags = FetchTags(post),
Url = articleUrl, URL = articleUrl,
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image, Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description, Description = meta.Data.Header.Description,
SourceId = sourceId SourceID = sourceId
}; };
collectedPosts.Add(article); collectedPosts.Add(article);
@ -120,20 +119,20 @@ public class RssWatcherJob
return collectedPosts; return collectedPosts;
} }
public void UpdateDatabase(List<ArticlesEntity> items) public void UpdateDatabase(List<ArticlesModel> items)
{ {
foreach (var item in items) foreach (var item in items)
{ {
if (item.Url is null) if (item.URL is null)
{ {
Log.Warning("RSS Watcher collected a blank url and was skipped"); Log.Warning("RSS Watcher collected a blank url and was skipped.");
continue; continue;
} }
var p = _articles.New(item); var p = _articles.New(item);
_queue.New(new DiscordQueueEntity _queue.New(new DiscordQueueModel
{ {
ArticleId = p.Id ArticleID = p.ID
}); });
} }
} }
@ -141,7 +140,7 @@ public class RssWatcherJob
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)
{ {
var isKnown = _articles.GetByUrl(url); var isKnown = _articles.GetByUrl(url);
if (isKnown.Url == url) return true; if (isKnown.URL == url) return true;
return false; return false;
} }

View File

@ -2,7 +2,6 @@ using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.HtmlParser; using Newsbot.Collector.Services.HtmlParser;
@ -67,11 +66,11 @@ public class YoutubeWatcherJob
if (channelId == "") if (channelId == "")
{ {
channelId = GetChannelId(source.Url); channelId = GetChannelId(source.Url);
_source.UpdateYoutubeId(source.Id, channelId); _source.UpdateYoutubeId(source.ID, channelId);
} }
// Make sure we have a Icon for the channel // Make sure we have a Icon for the channel
var icon = _icons.GetBySourceId(source.Id); var icon = _icons.GetBySourceId(source.ID);
if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V"); if (icon.Id == Guid.Empty) Console.WriteLine("I was triggered :V");
_logger.Information($"{JobName} - Checking '{source.Name}'"); _logger.Information($"{JobName} - Checking '{source.Name}'");
@ -83,9 +82,9 @@ public class YoutubeWatcherJob
{ {
_logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found"); _logger.Debug($"{JobName} - {video.AuthorName} '{video.Title}' was found");
_articles.New(video); _articles.New(video);
_queue.New(new DiscordQueueEntity _queue.New(new DiscordQueueModel
{ {
ArticleId = video.Id ArticleID = video.ID
}); });
} }
} }
@ -110,9 +109,9 @@ public class YoutubeWatcherJob
return id; return id;
} }
private List<ArticlesEntity> CheckFeed(string url, SourceEntity source) private List<ArticlesModel> CheckFeed(string url, SourceModel source)
{ {
var videos = new List<ArticlesEntity>(); var videos = new List<ArticlesModel>();
using var reader = XmlReader.Create(url); using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
@ -127,17 +126,17 @@ public class YoutubeWatcherJob
}); });
videoDetails.Parse(); videoDetails.Parse();
var article = new ArticlesEntity var article = new ArticlesModel
{ {
//Todo add the icon //Todo add the icon
AuthorName = post.Authors[0].Name, AuthorName = post.Authors[0].Name,
Title = post.Title.Text, Title = post.Title.Text,
Tags = FetchTags(post), Tags = FetchTags(post),
Url = articleUrl, URL = articleUrl,
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = videoDetails.Data.Header.Image, Thumbnail = videoDetails.Data.Header.Image,
Description = videoDetails.Data.Header.Description, Description = videoDetails.Data.Header.Description,
SourceId = source.Id, SourceID = source.ID,
Video = "true" Video = "true"
}; };
@ -152,7 +151,7 @@ public class YoutubeWatcherJob
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)
{ {
var isKnown = _articles.GetByUrl(url); var isKnown = _articles.GetByUrl(url);
if (isKnown.Url == url) return true; if (isKnown.URL == url) return true;
return false; return false;
} }

View File

@ -1,6 +1,5 @@
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config; using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Jobs;
@ -19,9 +18,9 @@ public class CodeProjectWatcherJobTests
FeaturePullCommits = true, FeaturePullCommits = true,
FeaturePullReleases = true FeaturePullReleases = true
}); });
var results = client.CheckForReleases(new SourceEntity var results = client.CheckForReleases(new SourceModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
Url = "https://github.com/jtom38/dvb", Url = "https://github.com/jtom38/dvb",
Type = SourceTypes.CodeProject, Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject, Site = SourceTypes.CodeProject,
@ -45,9 +44,9 @@ public class CodeProjectWatcherJobTests
FeaturePullCommits = true, FeaturePullCommits = true,
FeaturePullReleases = true FeaturePullReleases = true
}); });
var results = client.CheckForReleases(new SourceEntity var results = client.CheckForReleases(new SourceModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
Url = "https://github.com/python/cpython", Url = "https://github.com/python/cpython",
Type = SourceTypes.CodeProject, Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject, Site = SourceTypes.CodeProject,
@ -71,9 +70,9 @@ public class CodeProjectWatcherJobTests
FeaturePullCommits = true, FeaturePullCommits = true,
FeaturePullReleases = true FeaturePullReleases = true
}); });
var results = client.CheckForCommits(new SourceEntity var results = client.CheckForCommits(new SourceModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
Url = "https://github.com/jtom38/dvb", Url = "https://github.com/jtom38/dvb",
Type = SourceTypes.CodeProject, Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject, Site = SourceTypes.CodeProject,

View File

@ -1,4 +1,3 @@
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Jobs;
using Newsbot.Collector.Services.Notifications.Discord; using Newsbot.Collector.Services.Notifications.Discord;
@ -14,9 +13,9 @@ public class DiscordNotificationJobTest
var webhookClient = new DiscordWebhookClient(uri); var webhookClient = new DiscordWebhookClient(uri);
var client = new DiscordNotificationJob(); var client = new DiscordNotificationJob();
var msg = client.GenerateDiscordMessage(new SourceEntity var msg = client.GenerateDiscordMessage(new SourceModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
Site = "Unit Test", Site = "Unit Test",
Source = "placeholder", Source = "placeholder",
Type = "a", Type = "a",
@ -25,17 +24,17 @@ public class DiscordNotificationJobTest
Url = "https://github.com", Url = "https://github.com",
Tags = "Unit, Testing" Tags = "Unit, Testing"
}, },
new ArticlesEntity new ArticlesModel
{ {
Tags = "more,unit,testing", Tags = "more,unit,testing",
Title = "Nope not real", Title = "Nope not real",
Url = "https://github.com/jtom38", URL = "https://github.com/jtom38",
PubDate = DateTime.Now, PubDate = DateTime.Now,
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
Description = "Please work", Description = "Please work",
AuthorName = "No one knows" AuthorName = "No one knows"
}, },
new IconEntity() new IconModel
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png"
@ -51,11 +50,11 @@ public class DiscordNotificationJobTest
{ {
client.SendSubscriptionNotification( client.SendSubscriptionNotification(
new Guid(), new Guid(),
new ArticlesEntity new ArticlesModel
{ {
Tags = "more,unit,testing", Tags = "more,unit,testing",
Title = "Nope not real", Title = "Nope not real",
Url = "https://github.com/jtom38", URL = "https://github.com/jtom38",
PubDate = DateTime.Now, PubDate = DateTime.Now,
Thumbnail = Thumbnail =
"https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
@ -63,9 +62,9 @@ public class DiscordNotificationJobTest
AuthorName = "No one knows", AuthorName = "No one knows",
CodeIsCommit = true CodeIsCommit = true
}, },
new SourceEntity new SourceModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
Site = "Unit Test", Site = "Unit Test",
Source = "placeholder", Source = "placeholder",
Type = "a", Type = "a",
@ -74,7 +73,7 @@ public class DiscordNotificationJobTest
Url = "https://github.com", Url = "https://github.com",
Tags = "Unit, Testing" Tags = "Unit, Testing"
}, },
new IconEntity() new IconModel
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png" FileName = "https://www.redditstatic.com/desktop2x/img/favicon/android-icon-192x192.png"

View File

@ -1,6 +1,5 @@
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Tests.Tables; namespace Newsbot.Collector.Tests.Tables;
@ -24,7 +23,7 @@ public class ArticlesTableTests
{ {
var cfg = GetConfiguration(); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg); var client = new ArticlesTable(cfg);
client.List(0, 25); client.List();
} }
[Fact] [Fact]
@ -35,7 +34,7 @@ public class ArticlesTableTests
var cfg = GetConfiguration(); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg); var client = new ArticlesTable(cfg);
var res = client.GetById(uid); var res = client.GetById(uid);
if (!res.Id.Equals(uid)) if (!res.ID.Equals(uid))
{ {
Assert.Fail("Incorrect record or not found"); Assert.Fail("Incorrect record or not found");
} }
@ -46,14 +45,14 @@ public class ArticlesTableTests
{ {
var cfg = GetConfiguration(); var cfg = GetConfiguration();
var client = new ArticlesTable(cfg); var client = new ArticlesTable(cfg);
var m = new ArticlesEntity var m = new ArticlesModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
SourceId = Guid.NewGuid(), SourceID = Guid.NewGuid(),
Tags = "thing, thing2", Tags = "thing, thing2",
Title = "Unit Testing!", Title = "Unit Testing!",
Url = "https://google.com", URL = "https://google.com",
PubDate = DateTime.Now.ToUniversalTime(), PubDate = DateTime.Now.ToLocalTime(),
}; };
client.New(m); client.New(m);
} }

View File

@ -1,5 +1,4 @@
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Entities;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
namespace Newsbot.Collector.Tests.Tables; namespace Newsbot.Collector.Tests.Tables;
@ -10,9 +9,9 @@ public class SourcesTableTests
public void NewRecordTest() public void NewRecordTest()
{ {
var client = new SourcesTable(""); var client = new SourcesTable("");
var m = new SourceEntity var m = new SourceModel
{ {
Id = Guid.NewGuid(), ID = Guid.NewGuid(),
Site = "Testing", Site = "Testing",
Name = "Testing", Name = "Testing",
Source = "Testing", Source = "Testing",