Added otel support and pushed data to my server. (#12)

Newsbot.Collector.Api/Newsbot.Collector.Api.csproj

@@ -15,8 +15,13 @@
         <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
         <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
         <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+        <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.4.0" />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.4.0" />
+        <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.4.0" />
+        <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc9.14" />
         <PackageReference Include="Serilog" Version="2.12.0" />
         <PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
+        <PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="1.0.0-dev-00113" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
     </ItemGroup>
 

Newsbot.Collector.Api/Program.cs

@@ -6,22 +6,28 @@ using Newsbot.Collector.Domain.Consts;
 using Newsbot.Collector.Domain.Models;
 using Newsbot.Collector.Services.Jobs;
 using Serilog;
-
+using ILogger = Serilog.ILogger;
-Log.Logger = new LoggerConfiguration()
-    .WriteTo.Console()
-    .CreateLogger();
-
-Log.Information("Starting up");
 
 var builder = WebApplication.CreateBuilder(args);
 
 // Define Logger
-builder.Host.UseSerilog(); // <-- Add this line
+builder.Host.UseSerilog();
 
 // Build the config
 var config = GetConfiguration();
 builder.Configuration.AddConfiguration(config);
 
+Log.Logger = new LoggerConfiguration()
+    .WriteTo.Console()
+    .WriteTo.OpenTelemetry(
+        config.GetValue<string>(ConfigConnectionStringConst.OpenTelemetry) ?? "",
+        resourceAttributes: new Dictionary<string, object>
+        {
+            { "service.name", "newsbot-collector-api" }
+        })
+    .CreateLogger();
+
+Log.Information("Starting up");
 // Configure Hangfire
 builder.Services.AddHangfire(f => f.UseMemoryStorage());
 builder.Services.AddHangfireServer();
@@ -40,11 +46,11 @@ builder.Services.Configure<ConnectionStrings>(config.GetSection("ConnectionStrin
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.
-//if (app.Environment.IsDevelopment())
+if (config.GetValue<bool>("EnableSwagger"))
-//{
+{
     app.UseSwagger();
     app.UseSwaggerUI();
-//}
+}
 
 app.UseHttpsRedirection();
 
@@ -71,7 +77,7 @@ static IConfiguration GetConfiguration()
         .Build();
 }
 
-static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
+static void SetupRecurringJobs(IConfiguration configuration, ILogger logger)
 {
     //RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
     //{
@@ -83,7 +89,8 @@ static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger log
         ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
     }), "15 0-23 * * *");
 
-    RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions
+    RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
+        x.InitAndExecute(new DiscordNotificationJobOptions
         {
             DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
         }), "5/10 * * * *");

Newsbot.Collector.Api/appsettings.template.json

@@ -5,9 +5,10 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "Config": {
+  "ConnectionStrings": {
-    "ServerAddress": "",
+    "Database": "",
-    "SqlConnectionString": "",
+    "OpenTelemetry": ""
+  },
   "Reddit": {
     "IsEnabled": false,
     "PullHot": false,
@@ -23,9 +24,9 @@
     "ClientID": "",
     "ClientSecret": ""
   },
-    "FFXIV": {
+  "FinalFantasyXiv": {
     "IsEnabled": false
-    }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "EnableSwagger": true
 }

Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs

@@ -3,7 +3,8 @@ namespace Newsbot.Collector.Domain.Consts;
 /// <summary>
 ///     This class contains const entries to access keys within IConfiguration.
 /// </summary>
-public class ConfigConnectionStringConst
+public static class ConfigConnectionStringConst
 {
     public const string Database = "ConnectionStrings:Database";
+    public const string OpenTelemetry = "ConnectionStrings:OpenTelemetry";
 }

Newsbot.Collector.Services/Jobs/RssWatcherJob.cs

@@ -1,6 +1,5 @@
 using System.ServiceModel.Syndication;
 using System.Xml;
-using Microsoft.Extensions.Configuration;
 using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Consts;
 using Newsbot.Collector.Domain.Interfaces;
@@ -12,14 +11,15 @@ namespace Newsbot.Collector.Services.Jobs;
 
 public class RssWatcherJobOptions
 {
-    public string ConnectionString { get; set; } = "";
+    public string? ConnectionString { get; init; }
+    public string? OpenTelemetry { get; init; }
 }
 
 // This class was made to work with Hangfire and it does not support constructors.
-public class RssWatcherJob : IHangfireJob
+public class RssWatcherJob
 {
-
     private IArticlesRepository _articles;
+    private ILogger _logger;
     private IDiscordQueueRepository _queue;
     private ISourcesRepository _source;
 
@@ -28,73 +28,67 @@ public class RssWatcherJob : IHangfireJob
         _articles = new ArticlesTable("");
         _queue = new DiscordQueueTable("");
         _source = new SourcesTable("");
+        _logger = GetLogger("");
     }
 
     public void InitAndExecute(RssWatcherJobOptions options)
     {
-        Log.Information("RssWatcherJob - Job was triggered");
+        Init(options);
-        Log.Information("RssWatcherJob - Setting up the job");
-        Init(options.ConnectionString);
 
+        _logger.Information("RssWatcherJob - Job was triggered");
+        _logger.Information("RssWatcherJob - Setting up the job");
+
+        Execute();
+    }
+
+    private ILogger GetLogger(string connectionString)
+    {
+        return Log.Logger = new LoggerConfiguration()
+            .WriteTo.Console()
+            .WriteTo.OpenTelemetry(
+                connectionString,
+                resourceAttributes: new Dictionary<string, object>
+                {
+                    { "service.name", "newsbot-collector-api" },
+                    { "Job", "RssWatcherJob" }
+                })
+            .CreateLogger();
+    }
+
+    public void Init(RssWatcherJobOptions options)
+    {
+        _articles = new ArticlesTable(options.ConnectionString ?? "");
+        _queue = new DiscordQueueTable(options.ConnectionString ?? "");
+        _source = new SourcesTable(options.ConnectionString ?? "");
+        _logger = GetLogger(options.OpenTelemetry ?? "");
+    }
+
+    public void Execute()
+    {
         var articles = new List<ArticlesModel>();
 
-        Log.Information("RssWatcherJob - Requesting sources");
+        _logger.Information("RssWatcherJob - Requesting sources");
         var sources = _source.ListByType(SourceTypes.Rss);
-        Log.Information($"RssWatcherJob - Got {sources.Count()} back");
+        _logger.Information($"RssWatcherJob - Got {sources.Count} back");
         foreach (var source in sources)
         {
-            Log.Information($"RssWatcherJob - Starting to proces '{source.Name}'");
+            _logger.Information($"RssWatcherJob - Starting to process '{source.Name}'");
-            Log.Information("RssWatcherJob - Starting to request feed to be processed");
+            _logger.Information("RssWatcherJob - Starting to request feed to be processed");
             var results = Collect(source.Url, source.ID);
 
-            Log.Information($"RssWatcherJob - Collected {results.Count()} posts");
+            _logger.Information($"RssWatcherJob - Collected {results.Count} posts");
             articles.AddRange(results);
         }
 
-        Log.Information("RssWatcherJob - Sending posts over to the database");
+        _logger.Information("RssWatcherJob - Sending posts over to the database");
         UpdateDatabase(articles);
 
-        Log.Information("RssWatcherJob - Done!");
+        _logger.Information("RssWatcherJob - Done!");
     }
 
-    public void InitAndExecute(IConfiguration config)
+    public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
     {
-        // reach out to the db and find all the rss feeds
+        var collectedPosts = new List<ArticlesModel>();
-        var connectionString = config.GetConnectionString("database");
-        if (connectionString is null)
-        {
-            connectionString = "";
-        }
-        Init(connectionString);
-
-        var articles = new List<ArticlesModel>();
-
-        var sources = _source.ListByType(SourceTypes.Rss);
-        foreach (var source in sources)
-        {
-            if (source.Enabled == false)
-            {
-                continue;
-            }
-
-            var results = Collect(source.Url, source.ID);
-
-            articles.AddRange(results);
-        }
-
-        UpdateDatabase(articles);
-    }
-
-    public void Init(string connectionString)
-    {
-        _articles = new ArticlesTable(connectionString);
-        _queue = new DiscordQueueTable(connectionString);
-        _source = new SourcesTable(connectionString);
-    }
-
-    public List<ArticlesModel> Collect(string url, Guid SourceID, int sleep = 3000)
-    {
-        var CollectedPosts = new List<ArticlesModel>();
 
         using var reader = XmlReader.Create(url);
         var feed = SyndicationFeed.Load(reader);
@@ -105,10 +99,7 @@ public class RssWatcherJob : IHangfireJob
 
             // Check if we have seen the url before
             // If we have, skip and save the site bandwidth
-            if (IsThisUrlKnown(articleUrl) == true)
+            if (IsThisUrlKnown(articleUrl)) continue;
-            {
-                continue;
-            }
 
             var meta = new HtmlPageReader(articleUrl);
             meta.Parse();
@@ -121,15 +112,16 @@ public class RssWatcherJob : IHangfireJob
                 PubDate = post.PublishDate.DateTime,
                 Thumbnail = meta.Data.Header.Image,
                 Description = meta.Data.Header.Description,
-                SourceID = SourceID
+                SourceID = sourceId
             };
 
-            CollectedPosts.Add(article);
+            collectedPosts.Add(article);
 
             // try to not be too greedy
             Thread.Sleep(sleep);
         }
-        return CollectedPosts;
+
+        return collectedPosts;
     }
 
     public void UpdateDatabase(List<ArticlesModel> items)
@@ -138,7 +130,7 @@ public class RssWatcherJob : IHangfireJob
         {
             if (item.URL is null)
             {
-                Log.Warning($"RSS Watcher collected a blank url and was skipped.");
+                Log.Warning("RSS Watcher collected a blank url and was skipped.");
                 continue;
             }
 
@@ -153,20 +145,14 @@ public class RssWatcherJob : IHangfireJob
     private bool IsThisUrlKnown(string url)
     {
         var isKnown = _articles.GetByUrl(url);
-        if (isKnown.URL == url)
+        if (isKnown.URL == url) return true;
-        {
-            return true;
-        }
         return false;
     }
 
     private string FetchTags(SyndicationItem post)
     {
-        string result = "";
+        var result = "";
-        foreach (var tag in post.Categories)
+        foreach (var tag in post.Categories) result += $"{tag.Name},";
-        {
-            result += $"{tag.Name},";
-        }
         return result;
     }
 }

Newsbot.Collector.Services/Newsbot.Collector.Services.csproj

@@ -6,8 +6,13 @@
     </ItemGroup>
 
     <ItemGroup>
+        <PackageReference Include="Google.Apis.YouTube.v3" Version="1.60.0.2945"/>
         <PackageReference Include="HtmlAgilityPack" Version="1.11.46"/>
+        <PackageReference Include="Selenium.WebDriver" Version="4.8.1"/>
+        <PackageReference Include="Selenium.WebDriver.GeckoDriver" Version="0.32.2"/>
         <PackageReference Include="Serilog" Version="2.12.0"/>
+        <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0"/>
+        <PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="1.0.0-dev-00113"/>
         <PackageReference Include="System.ServiceModel.Syndication" Version="7.0.0"/>
     </ItemGroup>