From ac6bdaa1848b3332a419bd1d38bb57f47e7c1461 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sun, 19 Mar 2023 22:54:17 -0700 Subject: [PATCH] Added otel support and pushed data to my server. (#12) --- .../Newsbot.Collector.Api.csproj | 51 ++++---- Newsbot.Collector.Api/Program.cs | 43 +++--- .../appsettings.template.json | 45 +++---- .../Consts/ConfigConnectionStringsConst.cs | 5 +- .../Jobs/RssWatcherJob.cs | 122 ++++++++---------- .../Newsbot.Collector.Services.csproj | 33 +++-- 6 files changed, 152 insertions(+), 147 deletions(-) diff --git a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj index 0f5fc65..c4fb0d5 100644 --- a/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj +++ b/Newsbot.Collector.Api/Newsbot.Collector.Api.csproj @@ -1,29 +1,34 @@ - - net7.0 - enable - enable - + + net7.0 + enable + enable + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - + + + + + diff --git a/Newsbot.Collector.Api/Program.cs b/Newsbot.Collector.Api/Program.cs index 5f11a88..3222673 100644 --- a/Newsbot.Collector.Api/Program.cs +++ b/Newsbot.Collector.Api/Program.cs @@ -6,22 +6,28 @@ using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Services.Jobs; using Serilog; - -Log.Logger = new LoggerConfiguration() - .WriteTo.Console() - .CreateLogger(); - -Log.Information("Starting up"); +using ILogger = Serilog.ILogger; var builder = WebApplication.CreateBuilder(args); // Define Logger -builder.Host.UseSerilog(); // <-- Add this line +builder.Host.UseSerilog(); // Build the config var config = GetConfiguration(); builder.Configuration.AddConfiguration(config); +Log.Logger = new LoggerConfiguration() + .WriteTo.Console() + .WriteTo.OpenTelemetry( + config.GetValue(ConfigConnectionStringConst.OpenTelemetry) ?? "", + resourceAttributes: new Dictionary + { + { "service.name", "newsbot-collector-api" } + }) + .CreateLogger(); + +Log.Information("Starting up"); // Configure Hangfire builder.Services.AddHangfire(f => f.UseMemoryStorage()); builder.Services.AddHangfireServer(); @@ -40,11 +46,11 @@ builder.Services.Configure(config.GetSection("ConnectionStrin var app = builder.Build(); // Configure the HTTP request pipeline. -//if (app.Environment.IsDevelopment()) -//{ -app.UseSwagger(); -app.UseSwaggerUI(); -//} +if (config.GetValue("EnableSwagger")) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} app.UseHttpsRedirection(); @@ -71,7 +77,7 @@ static IConfiguration GetConfiguration() .Build(); } -static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger) +static void SetupRecurringJobs(IConfiguration configuration, ILogger logger) { //RecurringJob.AddOrUpdate("Example", x => x.InitAndExecute(new HelloWorldJobOptions //{ @@ -83,8 +89,9 @@ static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger log ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" }), "15 0-23 * * *"); - RecurringJob.AddOrUpdate("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions - { - DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" - }), "5/10 * * * *"); -} + RecurringJob.AddOrUpdate("Discord Alerts", x => + x.InitAndExecute(new DiscordNotificationJobOptions + { + DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" + }), "5/10 * * * *"); +} \ No newline at end of file diff --git a/Newsbot.Collector.Api/appsettings.template.json b/Newsbot.Collector.Api/appsettings.template.json index 63e9531..58449fc 100644 --- a/Newsbot.Collector.Api/appsettings.template.json +++ b/Newsbot.Collector.Api/appsettings.template.json @@ -5,27 +5,28 @@ "Microsoft.AspNetCore": "Warning" } }, - "Config": { - "ServerAddress": "", - "SqlConnectionString": "", - "Reddit": { - "IsEnabled": false, - "PullHot": false, - "PullNsfw": false, - "PullTop": false - }, - "Youtube": { - "IsEnabled": false, - "Debug": false - }, - "Twitch": { - "IsEnabled": false, - "ClientID": "", - "ClientSecret": "" - }, - "FFXIV": { - "IsEnabled": false - } + "ConnectionStrings": { + "Database": "", + "OpenTelemetry": "" }, - "AllowedHosts": "*" + "Reddit": { + "IsEnabled": false, + "PullHot": false, + "PullNsfw": false, + "PullTop": false + }, + "Youtube": { + "IsEnabled": false, + "Debug": false + }, + "Twitch": { + "IsEnabled": false, + "ClientID": "", + "ClientSecret": "" + }, + "FinalFantasyXiv": { + "IsEnabled": false + }, + "AllowedHosts": "*", + "EnableSwagger": true } \ No newline at end of file diff --git a/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs b/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs index c8a2995..85f9db6 100644 --- a/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs +++ b/Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs @@ -1,9 +1,10 @@ namespace Newsbot.Collector.Domain.Consts; /// -/// This class contains const entries to access keys within IConfiguration. +/// This class contains const entries to access keys within IConfiguration. /// -public class ConfigConnectionStringConst +public static class ConfigConnectionStringConst { public const string Database = "ConnectionStrings:Database"; + public const string OpenTelemetry = "ConnectionStrings:OpenTelemetry"; } \ No newline at end of file diff --git a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs index 77bda4d..4436837 100644 --- a/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/RssWatcherJob.cs @@ -1,6 +1,5 @@ using System.ServiceModel.Syndication; using System.Xml; -using Microsoft.Extensions.Configuration; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Interfaces; @@ -12,14 +11,15 @@ namespace Newsbot.Collector.Services.Jobs; public class RssWatcherJobOptions { - public string ConnectionString { get; set; } = ""; + public string? ConnectionString { get; init; } + public string? OpenTelemetry { get; init; } } // This class was made to work with Hangfire and it does not support constructors. -public class RssWatcherJob : IHangfireJob +public class RssWatcherJob { - private IArticlesRepository _articles; + private ILogger _logger; private IDiscordQueueRepository _queue; private ISourcesRepository _source; @@ -28,73 +28,67 @@ public class RssWatcherJob : IHangfireJob _articles = new ArticlesTable(""); _queue = new DiscordQueueTable(""); _source = new SourcesTable(""); + _logger = GetLogger(""); } public void InitAndExecute(RssWatcherJobOptions options) { - Log.Information("RssWatcherJob - Job was triggered"); - Log.Information("RssWatcherJob - Setting up the job"); - Init(options.ConnectionString); + Init(options); + _logger.Information("RssWatcherJob - Job was triggered"); + _logger.Information("RssWatcherJob - Setting up the job"); + + Execute(); + } + + private ILogger GetLogger(string connectionString) + { + return Log.Logger = new LoggerConfiguration() + .WriteTo.Console() + .WriteTo.OpenTelemetry( + connectionString, + resourceAttributes: new Dictionary + { + { "service.name", "newsbot-collector-api" }, + { "Job", "RssWatcherJob" } + }) + .CreateLogger(); + } + + public void Init(RssWatcherJobOptions options) + { + _articles = new ArticlesTable(options.ConnectionString ?? ""); + _queue = new DiscordQueueTable(options.ConnectionString ?? ""); + _source = new SourcesTable(options.ConnectionString ?? ""); + _logger = GetLogger(options.OpenTelemetry ?? ""); + } + + public void Execute() + { var articles = new List(); - Log.Information("RssWatcherJob - Requesting sources"); + _logger.Information("RssWatcherJob - Requesting sources"); var sources = _source.ListByType(SourceTypes.Rss); - Log.Information($"RssWatcherJob - Got {sources.Count()} back"); + _logger.Information($"RssWatcherJob - Got {sources.Count} back"); foreach (var source in sources) { - Log.Information($"RssWatcherJob - Starting to proces '{source.Name}'"); - Log.Information("RssWatcherJob - Starting to request feed to be processed"); + _logger.Information($"RssWatcherJob - Starting to process '{source.Name}'"); + _logger.Information("RssWatcherJob - Starting to request feed to be processed"); var results = Collect(source.Url, source.ID); - Log.Information($"RssWatcherJob - Collected {results.Count()} posts"); + _logger.Information($"RssWatcherJob - Collected {results.Count} posts"); articles.AddRange(results); } - Log.Information("RssWatcherJob - Sending posts over to the database"); + _logger.Information("RssWatcherJob - Sending posts over to the database"); UpdateDatabase(articles); - Log.Information("RssWatcherJob - Done!"); + _logger.Information("RssWatcherJob - Done!"); } - public void InitAndExecute(IConfiguration config) + public List Collect(string url, Guid sourceId, int sleep = 3000) { - // reach out to the db and find all the rss feeds - var connectionString = config.GetConnectionString("database"); - if (connectionString is null) - { - connectionString = ""; - } - Init(connectionString); - - var articles = new List(); - - var sources = _source.ListByType(SourceTypes.Rss); - foreach (var source in sources) - { - if (source.Enabled == false) - { - continue; - } - - var results = Collect(source.Url, source.ID); - - articles.AddRange(results); - } - - UpdateDatabase(articles); - } - - public void Init(string connectionString) - { - _articles = new ArticlesTable(connectionString); - _queue = new DiscordQueueTable(connectionString); - _source = new SourcesTable(connectionString); - } - - public List Collect(string url, Guid SourceID, int sleep = 3000) - { - var CollectedPosts = new List(); + var collectedPosts = new List(); using var reader = XmlReader.Create(url); var feed = SyndicationFeed.Load(reader); @@ -105,10 +99,7 @@ public class RssWatcherJob : IHangfireJob // Check if we have seen the url before // If we have, skip and save the site bandwidth - if (IsThisUrlKnown(articleUrl) == true) - { - continue; - } + if (IsThisUrlKnown(articleUrl)) continue; var meta = new HtmlPageReader(articleUrl); meta.Parse(); @@ -121,15 +112,16 @@ public class RssWatcherJob : IHangfireJob PubDate = post.PublishDate.DateTime, Thumbnail = meta.Data.Header.Image, Description = meta.Data.Header.Description, - SourceID = SourceID + SourceID = sourceId }; - CollectedPosts.Add(article); + collectedPosts.Add(article); // try to not be too greedy Thread.Sleep(sleep); } - return CollectedPosts; + + return collectedPosts; } public void UpdateDatabase(List items) @@ -138,7 +130,7 @@ public class RssWatcherJob : IHangfireJob { if (item.URL is null) { - Log.Warning($"RSS Watcher collected a blank url and was skipped."); + Log.Warning("RSS Watcher collected a blank url and was skipped."); continue; } @@ -153,20 +145,14 @@ public class RssWatcherJob : IHangfireJob private bool IsThisUrlKnown(string url) { var isKnown = _articles.GetByUrl(url); - if (isKnown.URL == url) - { - return true; - } + if (isKnown.URL == url) return true; return false; } private string FetchTags(SyndicationItem post) { - string result = ""; - foreach (var tag in post.Categories) - { - result += $"{tag.Name},"; - } + var result = ""; + foreach (var tag in post.Categories) result += $"{tag.Name},"; return result; } } \ No newline at end of file diff --git a/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj b/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj index 56f2c55..6501693 100644 --- a/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj +++ b/Newsbot.Collector.Services/Newsbot.Collector.Services.csproj @@ -1,20 +1,25 @@ - - - - + + + + - - - - - + + + + + + + + + + - - net7.0 - enable - enable - + + net7.0 + enable + enable +