diff --git a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
index 0f5fc65..c4fb0d5 100644
--- a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
+++ b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
@@ -1,29 +1,34 @@
-
- net7.0
- enable
- enable
-
+
+ net7.0
+ enable
+ enable
+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
diff --git a/Newsbot.Collector.Api/Program.cs b/Newsbot.Collector.Api/Program.cs
index 5f11a88..3222673 100644
--- a/Newsbot.Collector.Api/Program.cs
+++ b/Newsbot.Collector.Api/Program.cs
@@ -6,22 +6,28 @@ using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs;
using Serilog;
-
-Log.Logger = new LoggerConfiguration()
- .WriteTo.Console()
- .CreateLogger();
-
-Log.Information("Starting up");
+using ILogger = Serilog.ILogger;
var builder = WebApplication.CreateBuilder(args);
// Define Logger
-builder.Host.UseSerilog(); // <-- Add this line
+builder.Host.UseSerilog();
// Build the config
var config = GetConfiguration();
builder.Configuration.AddConfiguration(config);
+Log.Logger = new LoggerConfiguration()
+ .WriteTo.Console()
+ .WriteTo.OpenTelemetry(
+ config.GetValue(ConfigConnectionStringConst.OpenTelemetry) ?? "",
+ resourceAttributes: new Dictionary
+ {
+ { "service.name", "newsbot-collector-api" }
+ })
+ .CreateLogger();
+
+Log.Information("Starting up");
// Configure Hangfire
builder.Services.AddHangfire(f => f.UseMemoryStorage());
builder.Services.AddHangfireServer();
@@ -40,11 +46,11 @@ builder.Services.Configure(config.GetSection("ConnectionStrin
var app = builder.Build();
// Configure the HTTP request pipeline.
-//if (app.Environment.IsDevelopment())
-//{
-app.UseSwagger();
-app.UseSwaggerUI();
-//}
+if (config.GetValue("EnableSwagger"))
+{
+ app.UseSwagger();
+ app.UseSwaggerUI();
+}
app.UseHttpsRedirection();
@@ -71,7 +77,7 @@ static IConfiguration GetConfiguration()
.Build();
}
-static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
+static void SetupRecurringJobs(IConfiguration configuration, ILogger logger)
{
//RecurringJob.AddOrUpdate("Example", x => x.InitAndExecute(new HelloWorldJobOptions
//{
@@ -83,8 +89,9 @@ static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger log
ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "15 0-23 * * *");
- RecurringJob.AddOrUpdate("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions
- {
- DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
- }), "5/10 * * * *");
-}
+ RecurringJob.AddOrUpdate("Discord Alerts", x =>
+ x.InitAndExecute(new DiscordNotificationJobOptions
+ {
+ DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
+ }), "5/10 * * * *");
+}
\ No newline at end of file
diff --git a/Newsbot.Collector.Api/appsettings.template.json b/Newsbot.Collector.Api/appsettings.template.json
index 63e9531..58449fc 100644
--- a/Newsbot.Collector.Api/appsettings.template.json
+++ b/Newsbot.Collector.Api/appsettings.template.json
@@ -5,27 +5,28 @@
"Microsoft.AspNetCore": "Warning"
}
},
- "Config": {
- "ServerAddress": "",
- "SqlConnectionString": "",
- "Reddit": {
- "IsEnabled": false,
- "PullHot": false,
- "PullNsfw": false,
- "PullTop": false
- },
- "Youtube": {
- "IsEnabled": false,
- "Debug": false
- },
- "Twitch": {
- "IsEnabled": false,
- "ClientID": "",
- "ClientSecret": ""
- },
- "FFXIV": {
- "IsEnabled": false
- }
+ "ConnectionStrings": {
+ "Database": "",
+ "OpenTelemetry": ""
},
- "AllowedHosts": "*"
+ "Reddit": {
+ "IsEnabled": false,
+ "PullHot": false,
+ "PullNsfw": false,
+ "PullTop": false
+ },
+ "Youtube": {
+ "IsEnabled": false,
+ "Debug": false
+ },
+ "Twitch": {
+ "IsEnabled": false,
+ "ClientID": "",
+ "ClientSecret": ""
+ },
+ "FinalFantasyXiv": {
+ "IsEnabled": false
+ },
+ "AllowedHosts": "*",
+ "EnableSwagger": true
}
\ No newline at end of file
diff --git a/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs b/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs
index c8a2995..85f9db6 100644
--- a/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs
+++ b/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs
@@ -1,9 +1,10 @@
namespace Newsbot.Collector.Domain.Consts;
///
-/// This class contains const entries to access keys within IConfiguration.
+/// This class contains const entries to access keys within IConfiguration.
///
-public class ConfigConnectionStringConst
+public static class ConfigConnectionStringConst
{
public const string Database = "ConnectionStrings:Database";
+ public const string OpenTelemetry = "ConnectionStrings:OpenTelemetry";
}
\ No newline at end of file
diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
index 77bda4d..4436837 100644
--- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
+++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
@@ -1,6 +1,5 @@
using System.ServiceModel.Syndication;
using System.Xml;
-using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces;
@@ -12,14 +11,15 @@ namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJobOptions
{
- public string ConnectionString { get; set; } = "";
+ public string? ConnectionString { get; init; }
+ public string? OpenTelemetry { get; init; }
}
// This class was made to work with Hangfire and it does not support constructors.
-public class RssWatcherJob : IHangfireJob
+public class RssWatcherJob
{
-
private IArticlesRepository _articles;
+ private ILogger _logger;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
@@ -28,73 +28,67 @@ public class RssWatcherJob : IHangfireJob
_articles = new ArticlesTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
+ _logger = GetLogger("");
}
public void InitAndExecute(RssWatcherJobOptions options)
{
- Log.Information("RssWatcherJob - Job was triggered");
- Log.Information("RssWatcherJob - Setting up the job");
- Init(options.ConnectionString);
+ Init(options);
+ _logger.Information("RssWatcherJob - Job was triggered");
+ _logger.Information("RssWatcherJob - Setting up the job");
+
+ Execute();
+ }
+
+ private ILogger GetLogger(string connectionString)
+ {
+ return Log.Logger = new LoggerConfiguration()
+ .WriteTo.Console()
+ .WriteTo.OpenTelemetry(
+ connectionString,
+ resourceAttributes: new Dictionary
+ {
+ { "service.name", "newsbot-collector-api" },
+ { "Job", "RssWatcherJob" }
+ })
+ .CreateLogger();
+ }
+
+ public void Init(RssWatcherJobOptions options)
+ {
+ _articles = new ArticlesTable(options.ConnectionString ?? "");
+ _queue = new DiscordQueueTable(options.ConnectionString ?? "");
+ _source = new SourcesTable(options.ConnectionString ?? "");
+ _logger = GetLogger(options.OpenTelemetry ?? "");
+ }
+
+ public void Execute()
+ {
var articles = new List();
- Log.Information("RssWatcherJob - Requesting sources");
+ _logger.Information("RssWatcherJob - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss);
- Log.Information($"RssWatcherJob - Got {sources.Count()} back");
+ _logger.Information($"RssWatcherJob - Got {sources.Count} back");
foreach (var source in sources)
{
- Log.Information($"RssWatcherJob - Starting to proces '{source.Name}'");
- Log.Information("RssWatcherJob - Starting to request feed to be processed");
+ _logger.Information($"RssWatcherJob - Starting to process '{source.Name}'");
+ _logger.Information("RssWatcherJob - Starting to request feed to be processed");
var results = Collect(source.Url, source.ID);
- Log.Information($"RssWatcherJob - Collected {results.Count()} posts");
+ _logger.Information($"RssWatcherJob - Collected {results.Count} posts");
articles.AddRange(results);
}
- Log.Information("RssWatcherJob - Sending posts over to the database");
+ _logger.Information("RssWatcherJob - Sending posts over to the database");
UpdateDatabase(articles);
- Log.Information("RssWatcherJob - Done!");
+ _logger.Information("RssWatcherJob - Done!");
}
- public void InitAndExecute(IConfiguration config)
+ public List Collect(string url, Guid sourceId, int sleep = 3000)
{
- // reach out to the db and find all the rss feeds
- var connectionString = config.GetConnectionString("database");
- if (connectionString is null)
- {
- connectionString = "";
- }
- Init(connectionString);
-
- var articles = new List();
-
- var sources = _source.ListByType(SourceTypes.Rss);
- foreach (var source in sources)
- {
- if (source.Enabled == false)
- {
- continue;
- }
-
- var results = Collect(source.Url, source.ID);
-
- articles.AddRange(results);
- }
-
- UpdateDatabase(articles);
- }
-
- public void Init(string connectionString)
- {
- _articles = new ArticlesTable(connectionString);
- _queue = new DiscordQueueTable(connectionString);
- _source = new SourcesTable(connectionString);
- }
-
- public List Collect(string url, Guid SourceID, int sleep = 3000)
- {
- var CollectedPosts = new List();
+ var collectedPosts = new List();
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
@@ -105,10 +99,7 @@ public class RssWatcherJob : IHangfireJob
// Check if we have seen the url before
// If we have, skip and save the site bandwidth
- if (IsThisUrlKnown(articleUrl) == true)
- {
- continue;
- }
+ if (IsThisUrlKnown(articleUrl)) continue;
var meta = new HtmlPageReader(articleUrl);
meta.Parse();
@@ -121,15 +112,16 @@ public class RssWatcherJob : IHangfireJob
PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description,
- SourceID = SourceID
+ SourceID = sourceId
};
- CollectedPosts.Add(article);
+ collectedPosts.Add(article);
// try to not be too greedy
Thread.Sleep(sleep);
}
- return CollectedPosts;
+
+ return collectedPosts;
}
public void UpdateDatabase(List items)
@@ -138,7 +130,7 @@ public class RssWatcherJob : IHangfireJob
{
if (item.URL is null)
{
- Log.Warning($"RSS Watcher collected a blank url and was skipped.");
+ Log.Warning("RSS Watcher collected a blank url and was skipped.");
continue;
}
@@ -153,20 +145,14 @@ public class RssWatcherJob : IHangfireJob
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
- if (isKnown.URL == url)
- {
- return true;
- }
+ if (isKnown.URL == url) return true;
return false;
}
private string FetchTags(SyndicationItem post)
{
- string result = "";
- foreach (var tag in post.Categories)
- {
- result += $"{tag.Name},";
- }
+ var result = "";
+ foreach (var tag in post.Categories) result += $"{tag.Name},";
return result;
}
}
\ No newline at end of file
diff --git a/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj b/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj
index 56f2c55..6501693 100644
--- a/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj
+++ b/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj
@@ -1,20 +1,25 @@
-
-
-
-
+
+
+
+
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-
- net7.0
- enable
- enable
-
+
+ net7.0
+ enable
+ enable
+