Features/more rss improvements (#6)
* exposing connectionStrings to controllers * First controller added to start testing * corrected param to be page not age * new model to map connection strings to for the controllers * HelloWorldJob uses options now to make hangfire happy * improved the html reader to find some rss feeds and start to extract the body of the content * moved html parser to its own namespace and make a sub client to process theh header * helpful vsc changes * updated rss watcher to include the sourceId so it can be added to the db call * updated tests to reflect changes * updated gitignore to avoid trash and moved over my makefile * More routes and added serilog * adding more database calls for the controllers * Updated interfaces for the tables * Added Serilog to jobs * removed default files * Added more routes and added DTO * Added DTO objects and SourceType Consts for easy usage * updated discord model name to follow the pattern * updated formatting * new dto objects and Subscriptions repo interface * added subscription db and api calls * focusing on the twitter tags as most sites focus on them * updated test to pull a html based feed
This commit is contained in:
parent
17e97b4e09
commit
521940ca4f
2
.gitignore
vendored
2
.gitignore
vendored
@ -3,6 +3,8 @@
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
out/
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
|
6
.vscode/launch.json
vendored
6
.vscode/launch.json
vendored
@ -24,6 +24,12 @@
|
||||
},
|
||||
"sourceFileMap": {
|
||||
"/Views": "${workspaceFolder}/Views"
|
||||
},
|
||||
"logging": {
|
||||
"engineLogging": false,
|
||||
"moduleLoad": false,
|
||||
"exceptions": false,
|
||||
"browserStdOut": false
|
||||
}
|
||||
},
|
||||
{
|
||||
|
15
.vscode/settings.json
vendored
15
.vscode/settings.json
vendored
@ -4,5 +4,18 @@
|
||||
"**/bin": false
|
||||
},
|
||||
"csharp.inlayHints.types.enabled": true,
|
||||
"omnisharp.enableImportCompletion": true
|
||||
"omnisharp.enableImportCompletion": true,
|
||||
"sqltools.connections": [
|
||||
{
|
||||
"previewLimit": 50,
|
||||
"server": "localhost",
|
||||
"port": 5432,
|
||||
"driver": "PostgreSQL",
|
||||
"name": "localhost",
|
||||
"database": "postgres",
|
||||
"username": "postgres",
|
||||
"password": "postgres"
|
||||
}
|
||||
],
|
||||
"editor.formatOnType": true
|
||||
}
|
65
Newsbot.Collector.Api/Controllers/ArticlesController.cs
Normal file
65
Newsbot.Collector.Api/Controllers/ArticlesController.cs
Normal file
@ -0,0 +1,65 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Dto;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/articles")]
|
||||
public class ArticlesController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<ArticlesController> _logger;
|
||||
private readonly ConnectionStrings _settings;
|
||||
private readonly IArticlesRepository _articles;
|
||||
private readonly ISourcesRepository _sources;
|
||||
|
||||
public ArticlesController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings.Value;
|
||||
_articles = new ArticlesTable(_settings.Database);
|
||||
_sources = new SourcesTable(_settings.Database);
|
||||
}
|
||||
|
||||
[HttpGet(Name = "GetArticles")]
|
||||
public IEnumerable<ArticleDto> Get()
|
||||
{
|
||||
var res = new List<ArticleDto>();
|
||||
var items = _articles.List(0, 25);
|
||||
foreach (var item in items)
|
||||
{
|
||||
res.Add(ArticleDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
public ArticleDto GetById(Guid id)
|
||||
{
|
||||
var item = _articles.GetById(id);
|
||||
return ArticleDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpGet("{id}/details")]
|
||||
public ArticleDetailsDto GetDetailsById(Guid id)
|
||||
{
|
||||
var item = _articles.GetById(id);
|
||||
var sourceItem = _sources.GetByID(item.SourceID);
|
||||
return ArticleDetailsDto.Convert(item, sourceItem);
|
||||
}
|
||||
|
||||
[HttpGet("by/{sourceid}")]
|
||||
public IEnumerable<ArticleDto> GetBySourceID(Guid sourceid, int page = 0, int count = 25)
|
||||
{
|
||||
var res = new List<ArticleDto>();
|
||||
var items = _articles.ListBySourceId(sourceid, page, count);
|
||||
foreach (var item in items)
|
||||
{
|
||||
res.Add(ArticleDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
}
|
@ -0,0 +1,66 @@
|
||||
using System.Net;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/discord/webhooks")]
|
||||
public class DiscordWebHookController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<DiscordWebHookController> _logger;
|
||||
private readonly ConnectionStrings _settings;
|
||||
private readonly IDiscordWebHooksRepository _webhooks;
|
||||
|
||||
public DiscordWebHookController(ILogger<DiscordWebHookController> logger, IOptions<ConnectionStrings> settings)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings.Value;
|
||||
_webhooks = new DiscordWebhooksTable(_settings.Database);
|
||||
}
|
||||
|
||||
[HttpGet(Name = "GetDiscordWebhooks")]
|
||||
public IEnumerable<DiscordWebHookModel> Get(int page)
|
||||
{
|
||||
return _webhooks.List(page);
|
||||
}
|
||||
|
||||
[HttpPost(Name = "New")]
|
||||
public DiscordWebHookModel New(string url, string server, string channel)
|
||||
{
|
||||
return _webhooks.New(new DiscordWebHookModel
|
||||
{
|
||||
Url = url,
|
||||
Server = server,
|
||||
Channel = channel,
|
||||
Enabled = true,
|
||||
});
|
||||
}
|
||||
|
||||
[HttpGet("by/serverAndChannel")]
|
||||
public IEnumerable<DiscordWebHookModel> GetByServerAndChannel(string server, string channel)
|
||||
{
|
||||
return _webhooks.ListByServerAndChannel(server, channel, 25);
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
public DiscordWebHookModel GetById(Guid id)
|
||||
{
|
||||
return _webhooks.GetByID(id);
|
||||
}
|
||||
|
||||
[HttpPost("{id}/disable")]
|
||||
public void DisableById(Guid id)
|
||||
{
|
||||
_webhooks.Disable(id);
|
||||
}
|
||||
|
||||
[HttpPost("{id}/enable")]
|
||||
public void EnableById(Guid id)
|
||||
{
|
||||
_webhooks.Enable(id);
|
||||
}
|
||||
}
|
133
Newsbot.Collector.Api/Controllers/SourcesController.cs
Normal file
133
Newsbot.Collector.Api/Controllers/SourcesController.cs
Normal file
@ -0,0 +1,133 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Consts;
|
||||
using Newsbot.Collector.Domain.Dto;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/sources")]
|
||||
public class SourcesController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<SourcesController> _logger;
|
||||
private readonly ConnectionStrings _settings;
|
||||
private readonly ISourcesRepository _sources;
|
||||
|
||||
public SourcesController(ILogger<SourcesController> logger, IOptions<ConnectionStrings> settings)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings.Value;
|
||||
_sources = new SourcesTable(_settings.Database);
|
||||
}
|
||||
|
||||
[HttpGet(Name = "GetSources")]
|
||||
public IEnumerable<SourceDto> Get(int page)
|
||||
{
|
||||
var res = new List<SourceDto>();
|
||||
var temp = _sources.List(page, 25);
|
||||
foreach (var item in temp)
|
||||
{
|
||||
res.Add(SourceDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
[HttpGet("by/type")]
|
||||
public IEnumerable<SourceDto> GetByType(string type)
|
||||
{
|
||||
var res = new List<SourceDto>();
|
||||
var temp = _sources.ListByType(type);
|
||||
foreach (var item in temp)
|
||||
{
|
||||
res.Add(SourceDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
[HttpPost("new/reddit")]
|
||||
public SourceDto NewReddit(string name, string url)
|
||||
{
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.Reddit,
|
||||
Name = name,
|
||||
Type = SourceTypes.Reddit,
|
||||
Source = "feed",
|
||||
Enabled = true,
|
||||
Url = url,
|
||||
Tags = $"{SourceTypes.Reddit}, {name}"
|
||||
});
|
||||
return SourceDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpPost("new/rss")]
|
||||
public SourceDto NewRss(string name, string url)
|
||||
{
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.Rss,
|
||||
Name = name,
|
||||
Type = SourceTypes.Rss,
|
||||
Source = "feed",
|
||||
Enabled = true,
|
||||
Url = url,
|
||||
Tags = $"{SourceTypes.Rss}, {name}"
|
||||
});
|
||||
return SourceDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpPost("new/youtube")]
|
||||
public SourceDto NewYoutube(string name, string url)
|
||||
{
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.YouTube,
|
||||
Type = SourceTypes.YouTube,
|
||||
Name = name,
|
||||
Source = "feed",
|
||||
Url = url,
|
||||
Enabled = true,
|
||||
Tags = $"{SourceTypes.YouTube}, {name}"
|
||||
});
|
||||
|
||||
return SourceDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpPost("new/twitch")]
|
||||
public SourceDto NewTwitch(string name)
|
||||
{
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.Twitch,
|
||||
Type = SourceTypes.Twitch,
|
||||
Name = name,
|
||||
Url = $"https://twitch.tv/{name}",
|
||||
Source = "api",
|
||||
Enabled = true,
|
||||
Tags = $"{SourceTypes.Twitch}, {name}"
|
||||
});
|
||||
return SourceDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
public SourceDto GetById(Guid id)
|
||||
{
|
||||
var item = _sources.GetByID(id);
|
||||
return SourceDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpPost("{id}/disable")]
|
||||
public void Disable(Guid id)
|
||||
{
|
||||
_sources.Disable(id);
|
||||
}
|
||||
|
||||
[HttpPost("{id}/enable")]
|
||||
public void Enable(Guid id)
|
||||
{
|
||||
_sources.Enable(id);
|
||||
}
|
||||
}
|
98
Newsbot.Collector.Api/Controllers/SubscriptionsController.cs
Normal file
98
Newsbot.Collector.Api/Controllers/SubscriptionsController.cs
Normal file
@ -0,0 +1,98 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Dto;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/subscriptions")]
|
||||
public class SubscriptionsController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<ArticlesController> _logger;
|
||||
private readonly ConnectionStrings _settings;
|
||||
private readonly ISubscriptionRepository _subscription;
|
||||
private readonly IDiscordWebHooksRepository _discord;
|
||||
private readonly ISourcesRepository _sources;
|
||||
public SubscriptionsController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings.Value;
|
||||
_subscription = new SubscriptionsTable(_settings.Database);
|
||||
_discord = new DiscordWebhooksTable(_settings.Database);
|
||||
_sources = new SourcesTable(_settings.Database);
|
||||
}
|
||||
|
||||
[HttpGet(Name = "ListSubscriptions")]
|
||||
public IEnumerable<SubscriptionDto> List(int page)
|
||||
{
|
||||
var res = new List<SubscriptionDto>();
|
||||
var items = _subscription.List(page);
|
||||
foreach (var item in items)
|
||||
{
|
||||
res.Add(SubscriptionDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
public SubscriptionDto GetById(Guid id)
|
||||
{
|
||||
return SubscriptionDto.Convert(_subscription.GetById(id));
|
||||
}
|
||||
|
||||
[HttpGet("{id}/details")]
|
||||
public SubscriptionDetailsDto GetDetailsById(Guid id)
|
||||
{
|
||||
var sub = _subscription.GetById(id);
|
||||
var webhook = _discord.GetByID(sub.DiscordWebHookID);
|
||||
var source = _sources.GetByID(sub.SourceID);
|
||||
|
||||
return SubscriptionDetailsDto.Convert(sub, source, webhook);
|
||||
}
|
||||
|
||||
[HttpPost("{id}/delete")]
|
||||
public void DeleteById(Guid id)
|
||||
{
|
||||
_subscription.Delete(id);
|
||||
}
|
||||
|
||||
[HttpGet("by/discordid")]
|
||||
public IEnumerable<SubscriptionDto> GetByDiscordId(Guid id)
|
||||
{
|
||||
var res = new List<SubscriptionDto>();
|
||||
var items = _subscription.ListByWebhook(id);
|
||||
foreach (var item in items)
|
||||
{
|
||||
res.Add(SubscriptionDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
[HttpGet("by/sourceid")]
|
||||
public IEnumerable<SubscriptionDto> GetBySourceId(Guid id)
|
||||
{
|
||||
var res = new List<SubscriptionDto>();
|
||||
var items = _subscription.ListBySourceID(id);
|
||||
foreach (var item in items)
|
||||
{
|
||||
res.Add(SubscriptionDto.Convert(item));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
[HttpPost("new")]
|
||||
public SubscriptionDto New(Guid sourceId, Guid discordId)
|
||||
{
|
||||
var item = _subscription.New(new SubscriptionModel
|
||||
{
|
||||
ID = Guid.NewGuid(),
|
||||
SourceID = sourceId,
|
||||
DiscordWebHookID = discordId
|
||||
});
|
||||
|
||||
return SubscriptionDto.Convert(item);
|
||||
}
|
||||
}
|
@ -1,32 +0,0 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace Newsbot.Collector.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("[controller]")]
|
||||
public class WeatherForecastController : ControllerBase
|
||||
{
|
||||
private static readonly string[] Summaries = new[]
|
||||
{
|
||||
"Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
|
||||
};
|
||||
|
||||
private readonly ILogger<WeatherForecastController> _logger;
|
||||
|
||||
public WeatherForecastController(ILogger<WeatherForecastController> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[HttpGet(Name = "GetWeatherForecast")]
|
||||
public IEnumerable<WeatherForecast> Get()
|
||||
{
|
||||
return Enumerable.Range(1, 5).Select(index => new WeatherForecast
|
||||
{
|
||||
Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
|
||||
TemperatureC = Random.Shared.Next(-20, 55),
|
||||
Summary = Summaries[Random.Shared.Next(Summaries.Length)]
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
}
|
@ -11,6 +11,8 @@
|
||||
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
|
||||
<PackageReference Include="Serilog" Version="2.12.0" />
|
||||
<PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
|
||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
|
||||
</ItemGroup>
|
||||
|
||||
|
@ -1,24 +1,37 @@
|
||||
using Hangfire;
|
||||
using Hangfire.MemoryStorage;
|
||||
using Serilog;
|
||||
using Newsbot.Collector.Services.Jobs;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Domain.Consts;
|
||||
|
||||
Log.Logger = new LoggerConfiguration()
|
||||
.WriteTo.Console()
|
||||
.CreateLogger();
|
||||
|
||||
Log.Information("Starting up");
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// Add services to the container.
|
||||
// Define Logger
|
||||
builder.Host.UseSerilog(); // <-- Add this line
|
||||
|
||||
// Build the conifg
|
||||
var config = GetConfiguration();
|
||||
builder.Configuration.AddConfiguration(config);
|
||||
|
||||
// Configure Hangfire
|
||||
builder.Services.AddHangfire(f => f.UseMemoryStorage());
|
||||
builder.Services.AddHangfireServer();
|
||||
GlobalConfiguration.Configuration.UseSerilogLogProvider();
|
||||
|
||||
builder.Services.AddControllers();
|
||||
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
|
||||
builder.Services.AddEndpointsApiExplorer();
|
||||
builder.Services.AddSwaggerGen();
|
||||
|
||||
builder.Services.Configure<ConnectionStrings>(config.GetSection("ConnectionStrings"));
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
// Configure the HTTP request pipeline.
|
||||
@ -31,7 +44,7 @@ if (app.Environment.IsDevelopment())
|
||||
app.UseHttpsRedirection();
|
||||
|
||||
app.UseHangfireDashboard();
|
||||
SetupRecurringJobs(config);
|
||||
SetupRecurringJobs(config, Log.Logger);
|
||||
|
||||
app.UseAuthorization();
|
||||
|
||||
@ -39,6 +52,7 @@ app.MapControllers();
|
||||
|
||||
app.Run();
|
||||
|
||||
|
||||
static IConfiguration GetConfiguration()
|
||||
{
|
||||
return new ConfigurationBuilder()
|
||||
@ -47,24 +61,14 @@ static IConfiguration GetConfiguration()
|
||||
.Build();
|
||||
}
|
||||
|
||||
static void SetupRecurringJobs(IConfiguration configuration)
|
||||
static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
|
||||
{
|
||||
var databaseConnectionString = configuration.GetConnectionString("database");
|
||||
if (databaseConnectionString is null)
|
||||
{
|
||||
databaseConnectionString = "";
|
||||
}
|
||||
|
||||
RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
|
||||
{
|
||||
Message = "Hello from the background!"
|
||||
}), "0/2 * * * *");
|
||||
//RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(config), "15 0-23 * * *");
|
||||
|
||||
var c = new RssWatcherJob();
|
||||
BackgroundJob.Enqueue(() => c.InitAndExecute(new RssWatcherJobOptions
|
||||
}), "0/1 * * * *");
|
||||
RecurringJob.AddOrUpdate<RssWatcherJob>("RSS", x => x.InitAndExecute(new RssWatcherJobOptions
|
||||
{
|
||||
ConnectionString = databaseConnectionString
|
||||
}));
|
||||
|
||||
ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
|
||||
}), "15 0-23 * * *");
|
||||
}
|
||||
|
@ -1,12 +0,0 @@
|
||||
namespace Newsbot.Collector.Api;
|
||||
|
||||
public class WeatherForecast
|
||||
{
|
||||
public DateOnly Date { get; set; }
|
||||
|
||||
public int TemperatureC { get; set; }
|
||||
|
||||
public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
|
||||
|
||||
public string? Summary { get; set; }
|
||||
}
|
@ -41,7 +41,12 @@ public class ArticlesTable : IArticlesRepository
|
||||
var res = conn.Query<ArticlesModel>(@"select * from articles
|
||||
Order By PubDate Desc
|
||||
Offset @Page
|
||||
Fetch Next @Count Rows Only", new { Page = page * count, Count = count }).ToList();
|
||||
Fetch Next @Count Rows Only", new
|
||||
{
|
||||
Page = page * count,
|
||||
Count = count
|
||||
})
|
||||
.ToList();
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -67,6 +72,21 @@ public class ArticlesTable : IArticlesRepository
|
||||
return res.First();
|
||||
}
|
||||
|
||||
public List<ArticlesModel> ListBySourceId(Guid id, int page, int count)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = @"Select * from articles
|
||||
where sourceid = @sourceid
|
||||
Offset @page
|
||||
Fetch next @count rows only";
|
||||
return conn.Query<ArticlesModel>(query, new
|
||||
{
|
||||
sourceid = id,
|
||||
page = page * count,
|
||||
count = count
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public ArticlesModel New(ArticlesModel model)
|
||||
{
|
||||
model.ID = Guid.NewGuid();
|
||||
|
@ -1,21 +1,22 @@
|
||||
using System.Data;
|
||||
using Dapper;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Npgsql;
|
||||
|
||||
namespace Newsbot.Collector.Database.Repositories;
|
||||
|
||||
public class WebhooksTable
|
||||
public class DiscordWebhooksTable : IDiscordWebHooksRepository
|
||||
{
|
||||
private string _connectionString;
|
||||
|
||||
public WebhooksTable(string connectionString)
|
||||
public DiscordWebhooksTable(string connectionString)
|
||||
{
|
||||
_connectionString = connectionString;
|
||||
}
|
||||
|
||||
public WebhooksTable(IConfiguration configuration)
|
||||
public DiscordWebhooksTable(IConfiguration configuration)
|
||||
{
|
||||
var connstr = configuration.GetConnectionString("database");
|
||||
if (connstr is null)
|
||||
@ -32,66 +33,71 @@ public class WebhooksTable
|
||||
return conn;
|
||||
}
|
||||
|
||||
public void New(DiscordWebHook model)
|
||||
public DiscordWebHookModel New(DiscordWebHookModel model)
|
||||
{
|
||||
var uid = Guid.NewGuid();
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Insert Into DiscordWebHooks (ID, Url, Server, Channel, Enabled) Values (@id, @url, @server, @channel, @enabled);";
|
||||
conn.Execute(query, new
|
||||
{
|
||||
id = model.ID,
|
||||
id = uid,
|
||||
url = model.Url,
|
||||
server = model.Server,
|
||||
channel = model.Channel,
|
||||
enabled = model.Enabled
|
||||
});
|
||||
model.ID = uid;
|
||||
return model;
|
||||
}
|
||||
|
||||
public DiscordWebHook GetByID(Guid ID)
|
||||
public DiscordWebHookModel GetByID(Guid id)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * from DiscordWebHooks Where ID = @id LIMIT 1;";
|
||||
return conn.Query<DiscordWebHook>(query, new
|
||||
return conn.Query<DiscordWebHookModel>(query, new
|
||||
{
|
||||
id = ID
|
||||
id = id
|
||||
}).First();
|
||||
}
|
||||
|
||||
public DiscordWebHook GetByUrl(string url)
|
||||
public DiscordWebHookModel GetByUrl(string url)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From DiscordWebHooks Where url = @url;";
|
||||
return conn.QueryFirst<DiscordWebHook>(query, new
|
||||
return conn.QueryFirst<DiscordWebHookModel>(query, new
|
||||
{
|
||||
url = url
|
||||
});
|
||||
}
|
||||
|
||||
public List<DiscordWebHook> List(int limit = 25)
|
||||
public List<DiscordWebHookModel> List(int page, int count = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From DiscordWebHooks @limit;";
|
||||
return conn.Query<DiscordWebHook>(query, new
|
||||
var query = @"Select * From DiscordWebHooks
|
||||
Offset @offset Fetch Next @count Rows Only;";
|
||||
return conn.Query<DiscordWebHookModel>(query, new
|
||||
{
|
||||
limit = limit
|
||||
offset = page * count,
|
||||
count = count
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public List<DiscordWebHook> ListByServer(string server, int limit = 25)
|
||||
public List<DiscordWebHookModel> ListByServer(string server, int limit = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From DiscordWebHooks Where Server = @id Limit @limit;";
|
||||
return conn.Query<DiscordWebHook>(query, new
|
||||
return conn.Query<DiscordWebHookModel>(query, new
|
||||
{
|
||||
server = server,
|
||||
limit = limit
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public List<DiscordWebHook> ListByServerAndChannel(string server, string channel, int limit = 25)
|
||||
public List<DiscordWebHookModel> ListByServerAndChannel(string server, string channel, int limit = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "SELECT * FROM DiscordWebHooks WHERE Server = @server and Channel = @channel Limit @limit;";
|
||||
return conn.Query<DiscordWebHook>(query, new
|
||||
return conn.Query<DiscordWebHookModel>(query, new
|
||||
{
|
||||
server = server,
|
||||
channel = channel,
|
||||
@ -99,23 +105,23 @@ public class WebhooksTable
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public int Disable(Guid ID)
|
||||
public int Disable(Guid id)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Update discordwebhooks Set Enabled = FALSE where ID = @id;";
|
||||
return conn.Execute(query, new
|
||||
{
|
||||
id = ID
|
||||
id = id
|
||||
});
|
||||
}
|
||||
|
||||
public int Enable(Guid ID)
|
||||
public int Enable(Guid id)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Update discordwebhooks Set Enabled = TRUE where ID = @id;";
|
||||
return conn.Execute(query, new
|
||||
{
|
||||
id = ID
|
||||
id = id
|
||||
});
|
||||
}
|
||||
}
|
@ -107,13 +107,16 @@ public class SourcesTable : ISourcesRepository
|
||||
return res.First();
|
||||
}
|
||||
|
||||
public List<SourceModel> List(int limit = 25)
|
||||
public List<SourceModel> List(int page = 0, int count = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From Sources Limit @limit;";
|
||||
var query = @"Select * From Sources
|
||||
Offset @page
|
||||
Fetch Next @count Rows Only;";
|
||||
return conn.Query<SourceModel>(query, new
|
||||
{
|
||||
limit = 25
|
||||
page = page * count,
|
||||
count = count
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
|
@ -1,12 +1,13 @@
|
||||
using System.Data;
|
||||
using Dapper;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Npgsql;
|
||||
|
||||
namespace Newsbot.Collector.Database.Repositories;
|
||||
|
||||
public class SubscriptionsTable
|
||||
public class SubscriptionsTable : ISubscriptionRepository
|
||||
{
|
||||
private string _connectionString;
|
||||
|
||||
@ -32,65 +33,100 @@ public class SubscriptionsTable
|
||||
return conn;
|
||||
}
|
||||
|
||||
public void New(SubscriptionModel model)
|
||||
public SubscriptionModel New(SubscriptionModel model)
|
||||
{
|
||||
model.ID = Guid.NewGuid();
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values (@id, @webhookid, @sourceid);";
|
||||
conn.Execute(query, new
|
||||
{
|
||||
id = Guid.NewGuid(),
|
||||
id = model.ID,
|
||||
webhookid = model.DiscordWebHookID,
|
||||
sourceid = model.SourceID
|
||||
});
|
||||
return model;
|
||||
}
|
||||
|
||||
public List<SubscriptionModel> List(int limit = 25)
|
||||
public List<SubscriptionModel> List(int page = 0, int count = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From subscriptions Limit @limit;";
|
||||
var query = @"Select * From subscriptions
|
||||
Offset @page Fetch Next @count Rows Only;";
|
||||
return conn.Query<SubscriptionModel>(query, new
|
||||
{
|
||||
limit = limit,
|
||||
page = page * count,
|
||||
count = count
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
// todo add paging
|
||||
public List<SubscriptionModel> ListBySourceID(Guid sourceID)
|
||||
public List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From subscriptions where sourceid = @sourceid";
|
||||
var query = @"Select * From subscriptions
|
||||
Offset @page Fetch Next @count Rows Only
|
||||
Where sourceid = @sourceid";
|
||||
return conn.Query<SubscriptionModel>(query, new
|
||||
{
|
||||
sourceid = sourceID
|
||||
page = page * count,
|
||||
count = count,
|
||||
sourceid = id
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public List<SubscriptionModel> GetByWebhookAndSource(Guid webhookId, Guid sourceId)
|
||||
public List<SubscriptionModel> ListByWebhook(Guid id, int page = 0, int count = 25)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From subscriptions Where discordwebhookid = @webhookid and sourceid = @sourceid;";
|
||||
var query = @"Select * From subscriptions
|
||||
Offset @page Fetch Next @count Rows Only
|
||||
Where discordwebhookid = @webhookid";
|
||||
return conn.Query<SubscriptionModel>(query, new
|
||||
{
|
||||
page = page * count,
|
||||
count = count,
|
||||
webhookid = id,
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public SubscriptionModel GetById(Guid id)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = @"Select * From subscriptions Where id = @id;";
|
||||
var res = conn.Query<SubscriptionModel>(query, new
|
||||
{
|
||||
id = id,
|
||||
});
|
||||
if (res.Count() == 0)
|
||||
{
|
||||
return new SubscriptionModel();
|
||||
}
|
||||
return res.First();
|
||||
}
|
||||
|
||||
public SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = @"Select * From subscriptions
|
||||
Where discordwebhookid = @webhookid
|
||||
and sourceid = @sourceid;";
|
||||
var res = conn.Query<SubscriptionModel>(query, new
|
||||
{
|
||||
webhookid = webhookId,
|
||||
sourceid = sourceId,
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public List<SubscriptionModel> ListByWebhook(Guid webhookId)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * From subscriptions Where discordwebhookid = @webhookid";
|
||||
return conn.Query<SubscriptionModel>(query, new
|
||||
});
|
||||
if (res.Count() == 0)
|
||||
{
|
||||
webhookid = webhookId,
|
||||
}).ToList();
|
||||
return new SubscriptionModel();
|
||||
}
|
||||
return res.First();
|
||||
}
|
||||
|
||||
public void Delete(Guid id)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Delete From subscriptions Where id = @id;";
|
||||
conn.Execute(query, new {
|
||||
conn.Execute(query, new
|
||||
{
|
||||
id = id
|
||||
});
|
||||
}
|
||||
|
@ -0,0 +1,9 @@
|
||||
namespace Newsbot.Collector.Domain.Consts;
|
||||
|
||||
/// <summary>
|
||||
/// This class contains const entries to access keys within IConfiguration.
|
||||
/// </summary>
|
||||
public class ConfigConnectionStringConst
|
||||
{
|
||||
public const string Database = "ConnectionStrings:Database";
|
||||
}
|
9
Newsbot.Collector.Domain/Consts/ConfigRedditConst.cs
Normal file
9
Newsbot.Collector.Domain/Consts/ConfigRedditConst.cs
Normal file
@ -0,0 +1,9 @@
|
||||
namespace Newsbot.Collector.Domain.Consts;
|
||||
|
||||
public class ConfigRedditConst
|
||||
{
|
||||
public const string IsEnabled = "Reddit:IsEnabled";
|
||||
public const string PullHot = "Reddit:PullHot";
|
||||
public const string PullNsfw = "Reddit:PullNsfw";
|
||||
public const string PullTop = "Reddit:PullTop";
|
||||
}
|
11
Newsbot.Collector.Domain/Consts/ConfigTwitchConst.cs
Normal file
11
Newsbot.Collector.Domain/Consts/ConfigTwitchConst.cs
Normal file
@ -0,0 +1,11 @@
|
||||
namespace Newsbot.Collector.Domain.Consts;
|
||||
|
||||
/// <summary>
|
||||
/// This class contains const entries to access keys within IConfiguration.
|
||||
/// </summary>
|
||||
public class ConfigTwitchConst
|
||||
{
|
||||
public const string IsEnabled = "Twitch:IsEnabled";
|
||||
public const string ClientID = "Twitch:ClientID";
|
||||
public const string ClientSecret = "Twitch:ClientSecret";
|
||||
}
|
10
Newsbot.Collector.Domain/Consts/ConfigYoutubeConst.cs
Normal file
10
Newsbot.Collector.Domain/Consts/ConfigYoutubeConst.cs
Normal file
@ -0,0 +1,10 @@
|
||||
namespace Newsbot.Collector.Domain.Consts;
|
||||
|
||||
/// <summary>
|
||||
/// This class contains const entries to access keys within IConfiguration.
|
||||
/// </summary>
|
||||
public class ConfigYoutubeConst
|
||||
{
|
||||
public const string IsEnable = "Youtube:IsEnabled";
|
||||
public const string Debug = "Youtube:Debug";
|
||||
}
|
41
Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs
Normal file
41
Newsbot.Collector.Domain/Dto/ArticleDetailsDto.cs
Normal file
@ -0,0 +1,41 @@
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class ArticleDetailsDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public string[]? Tags { get; set; }
|
||||
public string? Title { get; set; }
|
||||
public string? Url { get; set; }
|
||||
public DateTime PubDate { get; set; }
|
||||
public string? Video { get; set; }
|
||||
public int VideoHeight { get; set; }
|
||||
public int VideoWidth { get; set; }
|
||||
public string? Thumbnail { get; set; }
|
||||
public string? Description { get; set; }
|
||||
public string? AuthorName { get; set; }
|
||||
public string? AuthorImage { get; set; }
|
||||
|
||||
public SourceDto? Source { get; set; }
|
||||
|
||||
public static ArticleDetailsDto Convert(ArticlesModel article, SourceModel source)
|
||||
{
|
||||
return new ArticleDetailsDto
|
||||
{
|
||||
ID = article.ID,
|
||||
Tags = article.Tags.Split(','),
|
||||
Title = article.Title,
|
||||
Url = article.URL,
|
||||
PubDate = article.PubDate,
|
||||
Video = article.Video,
|
||||
VideoHeight = article.VideoHeight,
|
||||
VideoWidth = article.VideoWidth,
|
||||
Thumbnail = article.Thumbnail,
|
||||
Description = article.Description,
|
||||
AuthorName = article.AuthorName,
|
||||
AuthorImage = article.AuthorImage,
|
||||
Source = SourceDto.Convert(source)
|
||||
};
|
||||
}
|
||||
}
|
39
Newsbot.Collector.Domain/Dto/ArticleDto.cs
Normal file
39
Newsbot.Collector.Domain/Dto/ArticleDto.cs
Normal file
@ -0,0 +1,39 @@
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class ArticleDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public Guid SourceID { get; set; }
|
||||
public string[]? Tags { get; set; }
|
||||
public string? Title { get; set; }
|
||||
public string? Url { get; set; }
|
||||
public DateTime PubDate { get; set; }
|
||||
public string? Video { get; set; }
|
||||
public int VideoHeight { get; set; }
|
||||
public int VideoWidth { get; set; }
|
||||
public string? Thumbnail { get; set; }
|
||||
public string? Description { get; set; }
|
||||
public string? AuthorName { get; set; }
|
||||
public string? AuthorImage { get; set; }
|
||||
public static ArticleDto Convert(ArticlesModel article)
|
||||
{
|
||||
return new ArticleDto
|
||||
{
|
||||
ID = article.ID,
|
||||
SourceID = article.SourceID,
|
||||
Tags = article.Tags.Split(','),
|
||||
Title = article.Title,
|
||||
Url = article.URL,
|
||||
PubDate = article.PubDate,
|
||||
Video = article.Video,
|
||||
VideoHeight = article.VideoHeight,
|
||||
VideoWidth = article.VideoWidth,
|
||||
Thumbnail = article.Thumbnail,
|
||||
Description = article.Description,
|
||||
AuthorName = article.AuthorName,
|
||||
AuthorImage = article.AuthorImage,
|
||||
};
|
||||
}
|
||||
}
|
24
Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs
Normal file
24
Newsbot.Collector.Domain/Dto/DiscordWebHookDto.cs
Normal file
@ -0,0 +1,24 @@
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class DiscordWebHookDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public string? Url { get; set; }
|
||||
public string? Server { get; set; }
|
||||
public string? Channel { get; set; }
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public static DiscordWebHookDto Convert(DiscordWebHookModel model)
|
||||
{
|
||||
return new DiscordWebHookDto
|
||||
{
|
||||
ID = model.ID,
|
||||
Url = model.Url,
|
||||
Server = model.Server,
|
||||
Channel = model.Channel,
|
||||
Enabled = model.Enabled,
|
||||
};
|
||||
}
|
||||
}
|
34
Newsbot.Collector.Domain/Dto/SourceDto.cs
Normal file
34
Newsbot.Collector.Domain/Dto/SourceDto.cs
Normal file
@ -0,0 +1,34 @@
|
||||
using System.Net.NetworkInformation;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class SourceDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public string? Site { get; set; }
|
||||
public string? Name { get; set; }
|
||||
public string? Source { get; set; }
|
||||
public string? Type { get; set; }
|
||||
public string? Value { get; set; }
|
||||
public bool Enabled { get; set; }
|
||||
public string? Url { get; set; }
|
||||
public string[]? Tags { get; set; }
|
||||
public bool Deleted { get; set; }
|
||||
|
||||
public static SourceDto Convert(SourceModel model) {
|
||||
return new SourceDto
|
||||
{
|
||||
ID = model.ID,
|
||||
Site = model.Site,
|
||||
Name = model.Name,
|
||||
Source = model.Source,
|
||||
Type = model.Type,
|
||||
Value = model.Value,
|
||||
Enabled = model.Enabled,
|
||||
Url = model.Url,
|
||||
Tags = model.Tags.Split(','),
|
||||
Deleted = model.Deleted
|
||||
};
|
||||
}
|
||||
}
|
20
Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs
Normal file
20
Newsbot.Collector.Domain/Dto/SubscriptionDetailsDto.cs
Normal file
@ -0,0 +1,20 @@
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class SubscriptionDetailsDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public SourceDto? Source { get; set; }
|
||||
public DiscordWebHookDto? DiscordWebHook { get; set; }
|
||||
|
||||
public static SubscriptionDetailsDto Convert(SubscriptionModel subscription, SourceModel source, DiscordWebHookModel discord)
|
||||
{
|
||||
return new SubscriptionDetailsDto
|
||||
{
|
||||
ID = subscription.ID,
|
||||
Source = SourceDto.Convert(source),
|
||||
DiscordWebHook = DiscordWebHookDto.Convert(discord)
|
||||
};
|
||||
}
|
||||
}
|
20
Newsbot.Collector.Domain/Dto/SubscriptionDto.cs
Normal file
20
Newsbot.Collector.Domain/Dto/SubscriptionDto.cs
Normal file
@ -0,0 +1,20 @@
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Dto;
|
||||
|
||||
public class SubscriptionDto
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public Guid SourceID { get; set; }
|
||||
public Guid DiscordWebHookID { get; set; }
|
||||
|
||||
public static SubscriptionDto Convert(SubscriptionModel model)
|
||||
{
|
||||
return new SubscriptionDto
|
||||
{
|
||||
ID = model.ID,
|
||||
SourceID = model.SourceID,
|
||||
DiscordWebHookID = model.DiscordWebHookID
|
||||
};
|
||||
}
|
||||
}
|
@ -0,0 +1,18 @@
|
||||
namespace Newsbot.Collector.Domain.Exceptions;
|
||||
|
||||
public class MissingHeaderValueException : Exception
|
||||
{
|
||||
public MissingHeaderValueException()
|
||||
{
|
||||
}
|
||||
|
||||
public MissingHeaderValueException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public MissingHeaderValueException(string message, Exception inner)
|
||||
: base(message, inner)
|
||||
{
|
||||
}
|
||||
}
|
@ -4,7 +4,8 @@ namespace Newsbot.Collector.Domain.Interfaces;
|
||||
|
||||
public interface IArticlesRepository : ITableRepository
|
||||
{
|
||||
List<ArticlesModel>List(int age, int count);
|
||||
List<ArticlesModel>List(int page, int count);
|
||||
List<ArticlesModel>ListBySourceId(Guid id, int page = 0, int count = 25);
|
||||
ArticlesModel GetById(Guid ID);
|
||||
ArticlesModel GetByUrl(string url);
|
||||
ArticlesModel New(ArticlesModel model);
|
||||
|
@ -0,0 +1,19 @@
|
||||
using Microsoft.VisualBasic;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Interfaces;
|
||||
|
||||
public interface IDiscordWebHooksRepository
|
||||
{
|
||||
DiscordWebHookModel New(DiscordWebHookModel model);
|
||||
|
||||
DiscordWebHookModel GetByID(Guid id);
|
||||
DiscordWebHookModel GetByUrl(string url);
|
||||
|
||||
List<DiscordWebHookModel> List(int page, int count = 25);
|
||||
List<DiscordWebHookModel> ListByServer(string server, int limit);
|
||||
List<DiscordWebHookModel> ListByServerAndChannel(string server, string channel, int limit);
|
||||
|
||||
int Disable(Guid id);
|
||||
int Enable(Guid id);
|
||||
}
|
@ -10,7 +10,7 @@ public interface ISourcesRepository
|
||||
public SourceModel GetByID(string ID);
|
||||
public SourceModel GetByName(string name);
|
||||
public SourceModel GetByNameAndSource(string name, string source);
|
||||
public List<SourceModel> List(int limit);
|
||||
public List<SourceModel> List(int page, int count);
|
||||
public List<SourceModel> ListBySource(string source, int limit);
|
||||
public List<SourceModel> ListByType(string type, int limit = 25);
|
||||
public int Disable(Guid ID);
|
||||
|
@ -0,0 +1,17 @@
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
namespace Newsbot.Collector.Domain.Interfaces;
|
||||
|
||||
public interface ISubscriptionRepository
|
||||
{
|
||||
SubscriptionModel New(SubscriptionModel model);
|
||||
|
||||
List<SubscriptionModel> List(int page = 0, int count = 25);
|
||||
List<SubscriptionModel> ListBySourceID(Guid id, int page = 0, int count = 25);
|
||||
List<SubscriptionModel> ListByWebhook(Guid id, int page = 0, int count = 25);
|
||||
|
||||
SubscriptionModel GetById(Guid id);
|
||||
SubscriptionModel GetByWebhookAndSource(Guid webhookId, Guid sourceId);
|
||||
|
||||
void Delete(Guid id);
|
||||
}
|
@ -13,4 +13,9 @@ public class RedditConfigModel
|
||||
public bool PullHot { get; set; }
|
||||
public bool PullNsfw { get; set; }
|
||||
public bool PullTop { get; set; }
|
||||
}
|
||||
|
||||
public class ConnectionStrings
|
||||
{
|
||||
public string Database { get; set; } = "";
|
||||
}
|
@ -31,7 +31,7 @@ public class DiscordQueueModel
|
||||
public Guid ArticleID { get; set; }
|
||||
}
|
||||
|
||||
public class DiscordWebHook
|
||||
public class DiscordWebHookModel
|
||||
{
|
||||
public Guid ID { get; set; }
|
||||
public string Url { get; set; } = "";
|
||||
|
@ -1,129 +0,0 @@
|
||||
using System.Data;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Xml;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace Newsbot.Collector.Services;
|
||||
|
||||
public class HtmlData
|
||||
{
|
||||
public HtmlHeaderData Header { get; set; } = new HtmlHeaderData();
|
||||
}
|
||||
|
||||
public class HtmlHeaderData
|
||||
{
|
||||
public HtmlMetaData Meta { get; set; } = new HtmlMetaData();
|
||||
}
|
||||
|
||||
public class HtmlMetaData
|
||||
{
|
||||
public string Title { get; set; } = "";
|
||||
public string Description { get; set; } = "";
|
||||
public string Image { get; set; } = "";
|
||||
public string Url { get; set; } = "";
|
||||
public string PageType { get; set; } = "";
|
||||
//public string Color { get; set; }
|
||||
}
|
||||
|
||||
public class HtmlPageReader
|
||||
{
|
||||
|
||||
public HtmlData Data { get; set; }
|
||||
|
||||
private const string XPathMetaTag = "//head/meta";
|
||||
|
||||
private string _siteContent;
|
||||
|
||||
public HtmlPageReader(string pageUrl)
|
||||
{
|
||||
_siteContent = ReadSiteContent(pageUrl);
|
||||
var tags = CollectMetaTags();
|
||||
|
||||
Data = new HtmlData();
|
||||
Data.Header.Meta.Title = GetMetaTitle();
|
||||
Data.Header.Meta.Description = GetDescription();
|
||||
Data.Header.Meta.Image = GetImage();
|
||||
Data.Header.Meta.Url = GetUrl();
|
||||
Data.Header.Meta.PageType = GetPageType();
|
||||
}
|
||||
|
||||
private string ReadSiteContent(string url)
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
var html = client.GetStringAsync(url);
|
||||
html.Wait();
|
||||
|
||||
var content = html.Result.ToString();
|
||||
return content;
|
||||
}
|
||||
|
||||
private List<HtmlNode> CollectMetaTags()
|
||||
{
|
||||
var htmlDoc = new HtmlDocument();
|
||||
htmlDoc.LoadHtml(_siteContent);
|
||||
|
||||
var tags = htmlDoc.DocumentNode.SelectNodes(XPathMetaTag).ToList();
|
||||
|
||||
return tags;
|
||||
}
|
||||
|
||||
public string GetTagValue(string Tag)
|
||||
{
|
||||
var tags = CollectMetaTags();
|
||||
|
||||
foreach (var meta in tags)
|
||||
{
|
||||
//Console.WriteLine($"Name={meta.Attributes[0].Name} & Value={meta.Attributes[0].Value}");
|
||||
if (meta.Attributes[0].Value.Contains(Tag) == false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return meta.Attributes[1].Value;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private string FindFirstResult(string[] tags)
|
||||
{
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
var res = GetTagValue(tag);
|
||||
if (res == "")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public string GetMetaTitle()
|
||||
{
|
||||
string[] tags = new string[] { "og:title", "twitter:title", "title" };
|
||||
return FindFirstResult(tags);
|
||||
}
|
||||
|
||||
public string GetDescription()
|
||||
{
|
||||
string[] tags = new string[] { "description", "og:description" };
|
||||
return FindFirstResult(tags);
|
||||
}
|
||||
|
||||
public string GetImage()
|
||||
{
|
||||
string[] tags = new string[] { "image", "og:image", "twitter:image" };
|
||||
return FindFirstResult(tags);
|
||||
}
|
||||
|
||||
public string GetUrl()
|
||||
{
|
||||
string[] tags = new string[] { "url", "og:url", "twitter:url" };
|
||||
return FindFirstResult(tags);
|
||||
}
|
||||
|
||||
public string GetPageType()
|
||||
{
|
||||
string[] tags = new string[] { "og:type", "type" };
|
||||
return FindFirstResult(tags);
|
||||
}
|
||||
}
|
186
Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs
Normal file
186
Newsbot.Collector.Services/HtmlParser/HeadParserClient.cs
Normal file
@ -0,0 +1,186 @@
|
||||
using HtmlAgilityPack;
|
||||
using Newsbot.Collector.Domain.Exceptions;
|
||||
|
||||
namespace Newsbot.Collector.Services.HtmlParser;
|
||||
|
||||
public class HeadParserClient
|
||||
{
|
||||
private const string XPathMetaTag = "//head/meta";
|
||||
private const string XPathLinkTag = "//head/link";
|
||||
|
||||
public HeadParserModel Data { get; set; }
|
||||
|
||||
private string _htmlContent;
|
||||
|
||||
public HeadParserClient(string htmlContent)
|
||||
{
|
||||
_htmlContent = htmlContent;
|
||||
Data = new HeadParserModel();
|
||||
}
|
||||
|
||||
public void Parse()
|
||||
{
|
||||
Data.Title = GetMetaTitle();
|
||||
Data.Description = GetMetaDescription();
|
||||
Data.Image = GetMetaImage();
|
||||
Data.Url = GetMetaUrl();
|
||||
Data.PageType = GetMetaPageType();
|
||||
Data.ColorTheme = GetMetaColorTheme();
|
||||
|
||||
Data.FeedUri = GetSiteFeed();
|
||||
}
|
||||
|
||||
private List<HtmlNode> CollectMetaTags()
|
||||
{
|
||||
var htmlDoc = new HtmlDocument();
|
||||
htmlDoc.LoadHtml(_htmlContent);
|
||||
|
||||
var tags = htmlDoc.DocumentNode.SelectNodes(XPathMetaTag).ToList();
|
||||
|
||||
return tags;
|
||||
}
|
||||
|
||||
private List<HtmlNode> CollectLinkTags()
|
||||
{
|
||||
var htmlDoc = new HtmlDocument();
|
||||
htmlDoc.LoadHtml(_htmlContent);
|
||||
var links = htmlDoc.DocumentNode.SelectNodes(XPathLinkTag).ToList();
|
||||
return links;
|
||||
}
|
||||
|
||||
private string GetTagValue(string Tag, List<HtmlNode> html)
|
||||
{
|
||||
foreach (var meta in html)
|
||||
{
|
||||
//Console.WriteLine($"Name={meta.Attributes[0].Name} & Value={meta.Attributes[0].Value}");
|
||||
if (meta.Attributes[0].Value.Contains(Tag) == false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return meta.Attributes[1].Value;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private string FindFirstResult(string[] tags, List<HtmlNode> htmlTags)
|
||||
{
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
var res = GetTagValue(tag, htmlTags);
|
||||
if (res == "")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public string GetMetaTitle()
|
||||
{
|
||||
var htmlTags = CollectMetaTags();
|
||||
string[] tags = new string[] { "twitter:title", "og:title", "title" };
|
||||
return FindFirstResult(tags, htmlTags);
|
||||
}
|
||||
|
||||
public string GetMetaDescription()
|
||||
{
|
||||
var htmlTags = CollectMetaTags();
|
||||
string[] tags = new string[] { "twitter:description", "og:description", "description" };
|
||||
return FindFirstResult(tags, htmlTags);
|
||||
}
|
||||
|
||||
public string GetMetaImage()
|
||||
{
|
||||
var htmlTags = CollectMetaTags();
|
||||
string[] tags = new string[] { "twitter:image", "og:image", "image" };
|
||||
return FindFirstResult(tags, htmlTags);
|
||||
}
|
||||
|
||||
public string GetMetaUrl()
|
||||
{
|
||||
var htmlTags = CollectMetaTags();
|
||||
string[] tags = new string[] { "twitter:url", "og:url", "url" };
|
||||
return FindFirstResult(tags, htmlTags);
|
||||
}
|
||||
|
||||
public string GetMetaPageType()
|
||||
{
|
||||
var htmlTags = CollectMetaTags();
|
||||
string[] tags = new string[] { "og:type", "type" };
|
||||
return FindFirstResult(tags, htmlTags);
|
||||
}
|
||||
|
||||
public string GetMetaColorTheme()
|
||||
{
|
||||
var htmlTags = CollectMetaTags();
|
||||
string[] tags = new string[] { "theme-color" };
|
||||
return FindFirstResult(tags, htmlTags);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This will parse the headers looking for known keys that will contain a RSS feed link.
|
||||
/// If the feed is not found, this will throw an exception (MissingHeaderValueException).
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public string GetSiteFeed()
|
||||
{
|
||||
var htmlTags = CollectLinkTags();
|
||||
var tags = new string[] { "alternate" };
|
||||
try
|
||||
{
|
||||
var attr = FindFirstAttribute(tags, htmlTags);
|
||||
foreach (var item in attr)
|
||||
{
|
||||
if (item.Name != "href")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var uri = item.Value;
|
||||
if (uri.StartsWith("//"))
|
||||
{
|
||||
uri = uri.Replace("//", "https://");
|
||||
}
|
||||
|
||||
return uri;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
catch
|
||||
{
|
||||
// not found
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private HtmlAttributeCollection FindFirstAttribute(string[] tags, List<HtmlNode> htmlTags)
|
||||
{
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
try
|
||||
{
|
||||
var res = GetValidAttribute(tag, htmlTags);
|
||||
return res;
|
||||
}
|
||||
catch (MissingHeaderValueException)
|
||||
{
|
||||
// Nothing was found in the given tag but we will keep looking till we finish all the entries.
|
||||
}
|
||||
}
|
||||
throw new MissingHeaderValueException("Unable to find the requested value");
|
||||
}
|
||||
|
||||
private HtmlAttributeCollection GetValidAttribute(string Tag, List<HtmlNode> html)
|
||||
{
|
||||
foreach (var meta in html)
|
||||
{
|
||||
if (meta.Attributes[0].Value.Contains(Tag) == false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return meta.Attributes;
|
||||
}
|
||||
throw new MissingHeaderValueException("Site does not expose requested tag.");
|
||||
}
|
||||
}
|
71
Newsbot.Collector.Services/HtmlParser/HtmlPageReader.cs
Normal file
71
Newsbot.Collector.Services/HtmlParser/HtmlPageReader.cs
Normal file
@ -0,0 +1,71 @@
|
||||
using HtmlAgilityPack;
|
||||
using Newsbot.Collector.Domain.Exceptions;
|
||||
|
||||
namespace Newsbot.Collector.Services.HtmlParser;
|
||||
|
||||
public class HtmlPageReader
|
||||
{
|
||||
|
||||
public HtmlData Data { get; set; }
|
||||
|
||||
private HeadParserClient _headClient;
|
||||
|
||||
private string _siteContent;
|
||||
|
||||
public HtmlPageReader(string pageUrl)
|
||||
{
|
||||
_siteContent = ReadSiteContent(pageUrl);
|
||||
_headClient = new HeadParserClient(_siteContent);
|
||||
|
||||
Data = new HtmlData();
|
||||
}
|
||||
|
||||
public void Parse()
|
||||
{
|
||||
_headClient.Parse();
|
||||
Data.Header = _headClient.Data;
|
||||
}
|
||||
|
||||
private string ReadSiteContent(string url)
|
||||
{
|
||||
using var client = new HttpClient();
|
||||
var html = client.GetStringAsync(url);
|
||||
html.Wait();
|
||||
|
||||
var content = html.Result.ToString();
|
||||
return content;
|
||||
}
|
||||
|
||||
public string GetSiteContent()
|
||||
{
|
||||
return _siteContent;
|
||||
}
|
||||
|
||||
public List<HtmlNode> CollectPostContent()
|
||||
{
|
||||
var htmlDoc = new HtmlDocument();
|
||||
htmlDoc.LoadHtml(_siteContent);
|
||||
var links = htmlDoc.DocumentNode.SelectNodes("//div[contains(@class, 'article-text')]").ToList();
|
||||
|
||||
if (links.Count == 0)
|
||||
{
|
||||
throw new Exception("Unable to parse body. Tag is unkown.");
|
||||
}
|
||||
|
||||
if (links.Count >= 2)
|
||||
{
|
||||
throw new Exception("Too many results back for the body");
|
||||
}
|
||||
|
||||
var content = new List<string>();
|
||||
foreach (var item in links[0].ChildNodes)
|
||||
{
|
||||
if (item.Name == "p")
|
||||
{
|
||||
content.Add(item.InnerText);
|
||||
}
|
||||
}
|
||||
|
||||
return links;
|
||||
}
|
||||
}
|
18
Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs
Normal file
18
Newsbot.Collector.Services/HtmlParser/HtmlParserModels.cs
Normal file
@ -0,0 +1,18 @@
|
||||
namespace Newsbot.Collector.Services.HtmlParser;
|
||||
|
||||
public class HeadParserModel
|
||||
{
|
||||
public string Title { get; set; } = "";
|
||||
public string Description { get; set; } = "";
|
||||
public string Image { get; set; } = "";
|
||||
public string Url { get; set; } = "";
|
||||
public string PageType { get; set; } = "";
|
||||
public string ColorTheme { get; set; } = "";
|
||||
|
||||
public string? FeedUri { get; set; }
|
||||
}
|
||||
|
||||
public class HtmlData
|
||||
{
|
||||
public HeadParserModel Header { get; set; } = new HeadParserModel();
|
||||
}
|
42
Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs
Normal file
42
Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs
Normal file
@ -0,0 +1,42 @@
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
|
||||
namespace Newsbot.Collector.Services.Jobs;
|
||||
|
||||
public class GithubWatcherJobOptions
|
||||
{
|
||||
public string ConnectionString { get; set; } = "";
|
||||
public bool FeaturePullReleases { get; set; } = false;
|
||||
public bool FeaturePullCommits { get; set; } = false;
|
||||
}
|
||||
|
||||
public class GithubWatcherJob
|
||||
{
|
||||
private IArticlesRepository _articles;
|
||||
private IDiscordQueueRepository _queue;
|
||||
private ISourcesRepository _source;
|
||||
|
||||
public GithubWatcherJob()
|
||||
{
|
||||
_articles = new ArticlesTable("");
|
||||
_queue = new DiscordQueueTable("");
|
||||
_source = new SourcesTable("");
|
||||
}
|
||||
|
||||
private void Init(GithubWatcherJobOptions options)
|
||||
{
|
||||
_articles = new ArticlesTable(options.ConnectionString);
|
||||
_queue = new DiscordQueueTable(options.ConnectionString);
|
||||
_source = new SourcesTable(options.ConnectionString);
|
||||
}
|
||||
|
||||
public void InitAndExecute(GithubWatcherJobOptions options)
|
||||
{
|
||||
Init(options);
|
||||
|
||||
// query sources for things to pull
|
||||
|
||||
// query */release.atom
|
||||
// query */commits.atom
|
||||
}
|
||||
}
|
@ -1,5 +1,4 @@
|
||||
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Serilog;
|
||||
|
||||
namespace Newsbot.Collector.Services.Jobs;
|
||||
|
||||
@ -10,21 +9,22 @@ public class HelloWorldJobOptions
|
||||
|
||||
public class HelloWorldJob
|
||||
{
|
||||
private HelloWorldJobOptions _options;
|
||||
|
||||
public HelloWorldJob(HelloWorldJobOptions options)
|
||||
private string _message;
|
||||
|
||||
public HelloWorldJob()
|
||||
{
|
||||
_options = options;
|
||||
_message = "";
|
||||
}
|
||||
|
||||
public void InitAndExecute(HelloWorldJobOptions options)
|
||||
{
|
||||
_options = options;
|
||||
_message = options.Message;
|
||||
Execute();
|
||||
}
|
||||
|
||||
private void Execute()
|
||||
{
|
||||
Console.WriteLine(_options.Message);
|
||||
Log.Information("Starting 'HelloWorldJob'");
|
||||
Log.Information(_message);
|
||||
}
|
||||
}
|
@ -1,4 +1,3 @@
|
||||
using System.Runtime.InteropServices;
|
||||
using System.ServiceModel.Syndication;
|
||||
using System.Xml;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
@ -6,6 +5,8 @@ using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Consts;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Services.HtmlParser;
|
||||
using Serilog;
|
||||
|
||||
namespace Newsbot.Collector.Services.Jobs;
|
||||
|
||||
@ -31,24 +32,29 @@ public class RssWatcherJob : IHangfireJob
|
||||
|
||||
public void InitAndExecute(RssWatcherJobOptions options)
|
||||
{
|
||||
Console.WriteLine("Job was triggered");
|
||||
Console.WriteLine("Setting up the job");
|
||||
Log.Information("RssWatcherJob - Job was triggered");
|
||||
Log.Information("RssWatcherJob - Setting up the job");
|
||||
Init(options.ConnectionString);
|
||||
|
||||
var articles = new List<ArticlesModel>();
|
||||
|
||||
Console.WriteLine("Requesting sources");
|
||||
Log.Information("RssWatcherJob - Requesting sources");
|
||||
var sources = _source.ListByType(SourceTypes.Rss);
|
||||
Console.WriteLine($"Got {sources.Count()} back");
|
||||
Log.Information($"RssWatcherJob - Got {sources.Count()} back");
|
||||
foreach (var source in sources)
|
||||
{
|
||||
Console.WriteLine("Starting to request feed to be processed");
|
||||
var results = Collect(source.Url);
|
||||
Log.Information($"RssWatcherJob - Starting to proces '{source.Name}'");
|
||||
Log.Information("RssWatcherJob - Starting to request feed to be processed");
|
||||
var results = Collect(source.Url, source.ID);
|
||||
|
||||
Log.Information($"RssWatcherJob - Collected {results.Count()} posts");
|
||||
articles.AddRange(results);
|
||||
}
|
||||
|
||||
Log.Information("RssWatcherJob - Sending posts over to the database");
|
||||
UpdateDatabase(articles);
|
||||
|
||||
Log.Information("RssWatcherJob - Done!");
|
||||
}
|
||||
|
||||
public void InitAndExecute(IConfiguration config)
|
||||
@ -66,7 +72,7 @@ public class RssWatcherJob : IHangfireJob
|
||||
var sources = _source.ListByType(SourceTypes.Rss);
|
||||
foreach (var source in sources)
|
||||
{
|
||||
var results = Collect(source.Url);
|
||||
var results = Collect(source.Url, source.ID);
|
||||
|
||||
articles.AddRange(results);
|
||||
}
|
||||
@ -81,7 +87,7 @@ public class RssWatcherJob : IHangfireJob
|
||||
_source = new SourcesTable(connectionString);
|
||||
}
|
||||
|
||||
public List<ArticlesModel> Collect(string url, int sleep = 3000)
|
||||
public List<ArticlesModel> Collect(string url, Guid SourceID, int sleep = 3000)
|
||||
{
|
||||
var CollectedPosts = new List<ArticlesModel>();
|
||||
|
||||
@ -100,6 +106,7 @@ public class RssWatcherJob : IHangfireJob
|
||||
}
|
||||
|
||||
var meta = new HtmlPageReader(articleUrl);
|
||||
meta.Parse();
|
||||
|
||||
var article = new ArticlesModel
|
||||
{
|
||||
@ -107,8 +114,9 @@ public class RssWatcherJob : IHangfireJob
|
||||
Tags = FetchTags(post),
|
||||
URL = articleUrl,
|
||||
PubDate = post.PublishDate.DateTime,
|
||||
Thumbnail = meta.Data.Header.Meta.Image,
|
||||
Description = meta.Data.Header.Meta.Description,
|
||||
Thumbnail = meta.Data.Header.Image,
|
||||
Description = meta.Data.Header.Description,
|
||||
SourceID = SourceID
|
||||
};
|
||||
|
||||
CollectedPosts.Add(article);
|
||||
@ -123,7 +131,7 @@ public class RssWatcherJob : IHangfireJob
|
||||
{
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (IsThisUrlKnown(item.URL) == false)
|
||||
if (IsThisUrlKnown(item.URL) == true)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
@ -7,6 +7,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
|
||||
<PackageReference Include="Serilog" Version="2.12.0" />
|
||||
<PackageReference Include="System.ServiceModel.Syndication" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
|
@ -27,7 +27,7 @@ public class RssWatcherJobTest
|
||||
{
|
||||
var url = "https://www.engadget.com/rss.xml";
|
||||
var client = new RssWatcherJob();
|
||||
var items = client.Collect(url);
|
||||
var items = client.Collect(url, Guid.NewGuid(), 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@ -36,6 +36,28 @@ public class RssWatcherJobTest
|
||||
var url = "https://www.engadget.com/rss.xml";
|
||||
var client = new RssWatcherJob();
|
||||
client.Init(ConnectionString());
|
||||
client.Collect(url, 0);
|
||||
var items = client.Collect(url, Guid.NewGuid(), 0);
|
||||
client.UpdateDatabase(items);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanReadHtmlDrivenFeedPage()
|
||||
{
|
||||
var url = "https://www.howtogeek.com/feed/";
|
||||
var client = new RssWatcherJob();
|
||||
client.Init(ConnectionString());
|
||||
var items = client.Collect(url, Guid.NewGuid(), 0);
|
||||
Console.WriteLine('k');
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InitAndExecuteTest()
|
||||
{
|
||||
var client = new RssWatcherJob();
|
||||
client.InitAndExecute(new RssWatcherJobOptions
|
||||
{
|
||||
ConnectionString = ConnectionString()
|
||||
});
|
||||
|
||||
}
|
||||
}
|
38
Newsbot.Collector.Tests/Services/HtmlPageReaderTests.cs
Normal file
38
Newsbot.Collector.Tests/Services/HtmlPageReaderTests.cs
Normal file
@ -0,0 +1,38 @@
|
||||
using Newsbot.Collector.Services.HtmlParser;
|
||||
|
||||
namespace Newsbot.Collector.Tests.Services;
|
||||
|
||||
public class HtmlPageReaderTests
|
||||
{
|
||||
[Fact]
|
||||
public void BaseSiteContainsRssFeed()
|
||||
{
|
||||
var client = new HtmlPageReader("https://dotnettutorials.net/");
|
||||
var headClient = new HeadParserClient(client.GetSiteContent());
|
||||
var feedUri = headClient.GetSiteFeed();
|
||||
if (feedUri == "")
|
||||
{
|
||||
Assert.Fail("Failed to find the RSS feed");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SiteDoesNotReturnRssFeed()
|
||||
{
|
||||
var client = new HtmlPageReader("https://www.engadget.com/");
|
||||
var headClient = new HeadParserClient(client.GetSiteContent());
|
||||
var feedUri = headClient.GetSiteFeed();
|
||||
if (feedUri == "")
|
||||
{
|
||||
Assert.Fail("");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanFindBodyOfTheArticle()
|
||||
{
|
||||
var client = new HtmlPageReader("https://www.engadget.com/apple-is-convinced-my-dog-is-stalking-me-143100932.html");
|
||||
var c = client.CollectPostContent();
|
||||
Console.WriteLine(c);
|
||||
}
|
||||
}
|
25
makefile
Normal file
25
makefile
Normal file
@ -0,0 +1,25 @@
|
||||
.PHONY: help
|
||||
help: ## Shows this help command
|
||||
@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
build: ## builds the application with the current go runtime
|
||||
dotnet restore
|
||||
dotnet build
|
||||
|
||||
pub: ## Generate artifacts
|
||||
dotnet restore
|
||||
dotnet publish -o out
|
||||
rm ./out/appsettings.json
|
||||
|
||||
docker-build: ## Generates the docker image
|
||||
docker build -t "newsbot.collector" .
|
||||
docker image ls | grep newsbot.collector
|
||||
|
||||
docker-run: ## Runs the docker compose
|
||||
docker compose up
|
||||
|
||||
migrate-dev: ## Apply sql migrations to dev db
|
||||
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" up
|
||||
|
||||
migrate-dev-down: ## revert sql migrations to dev db
|
||||
goose -dir "./Newsbot.Collector.Database/Migrations" postgres "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" down
|
Loading…
Reference in New Issue
Block a user