jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Added otel support and pushed data to my server. (#12)

Browse Source
This commit is contained in:
James Tombleson 2023-03-19 22:54:17 -07:00 committed by GitHub
parent 607df99bfb
commit ac6bdaa184
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
View File

@ -1,29 +1,34 @@
<Project Sdk="Microsoft.NET.Sdk.Web"> <Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="AspNetCore.HealthChecks.NpgSql" Version="6.0.2" /> <PackageReference Include="AspNetCore.HealthChecks.NpgSql" Version="6.0.2" />
<PackageReference Include="AspNetCore.HealthChecks.UI" Version="6.0.5" /> <PackageReference Include="AspNetCore.HealthChecks.UI" Version="6.0.5" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="6.0.5" /> <PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="6.0.5" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Core" Version="6.0.5" /> <PackageReference Include="AspNetCore.HealthChecks.UI.Core" Version="6.0.5" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" /> <PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" /> <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" /> <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Serilog" Version="2.12.0" /> <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.4.0" />
<PackageReference Include="Serilog.AspNetCore" Version="6.1.0" /> <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.4.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" /> <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.4.0" />
</ItemGroup> <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc9.14" />
<PackageReference Include="Serilog" Version="2.12.0" />
<PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
<PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="1.0.0-dev-00113" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj" /> <ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Services\Newsbot.Collector.Services.csproj" /> <ProjectReference Include="..\Newsbot.Collector.Services\Newsbot.Collector.Services.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" /> <ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

41
Newsbot.Collector.Api/Program.cs
View File

@ -6,22 +6,28 @@ using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models; using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs; using Newsbot.Collector.Services.Jobs;
using Serilog; using Serilog;
using ILogger = Serilog.ILogger;
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
Log.Information("Starting up");
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
// Define Logger // Define Logger
builder.Host.UseSerilog(); // <-- Add this line builder.Host.UseSerilog();
// Build the config // Build the config
var config = GetConfiguration(); var config = GetConfiguration();
builder.Configuration.AddConfiguration(config); builder.Configuration.AddConfiguration(config);
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.OpenTelemetry(
config.GetValue<string>(ConfigConnectionStringConst.OpenTelemetry) ?? "",
resourceAttributes: new Dictionary<string, object>
{
{ "service.name", "newsbot-collector-api" }
})
.CreateLogger();
Log.Information("Starting up");
// Configure Hangfire // Configure Hangfire
builder.Services.AddHangfire(f => f.UseMemoryStorage()); builder.Services.AddHangfire(f => f.UseMemoryStorage());
builder.Services.AddHangfireServer(); builder.Services.AddHangfireServer();
@ -40,11 +46,11 @@ builder.Services.Configure<ConnectionStrings>(config.GetSection("ConnectionStrin
var app = builder.Build(); var app = builder.Build();
// Configure the HTTP request pipeline. // Configure the HTTP request pipeline.
//if (app.Environment.IsDevelopment()) if (config.GetValue<bool>("EnableSwagger"))
//{ {
app.UseSwagger(); app.UseSwagger();
app.UseSwaggerUI(); app.UseSwaggerUI();
//} }
app.UseHttpsRedirection(); app.UseHttpsRedirection();
@ -71,7 +77,7 @@ static IConfiguration GetConfiguration()
.Build(); .Build();
} }
static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger) static void SetupRecurringJobs(IConfiguration configuration, ILogger logger)
{ {
//RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions //RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
//{ //{
@ -83,8 +89,9 @@ static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger log
ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "15 0-23 * * *"); }), "15 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
{ x.InitAndExecute(new DiscordNotificationJobOptions
DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? "" {
}), "5/10 * * * *"); DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "5/10 * * * *");
} }

45
Newsbot.Collector.Api/appsettings.template.json
View File

@ -5,27 +5,28 @@
"Microsoft.AspNetCore": "Warning" "Microsoft.AspNetCore": "Warning"
} }
}, },
"Config": { "ConnectionStrings": {
"ServerAddress": "", "Database": "",
"SqlConnectionString": "", "OpenTelemetry": ""
"Reddit": {
"IsEnabled": false,
"PullHot": false,
"PullNsfw": false,
"PullTop": false
},
"Youtube": {
"IsEnabled": false,
"Debug": false
},
"Twitch": {
"IsEnabled": false,
"ClientID": "",
"ClientSecret": ""
},
"FFXIV": {
"IsEnabled": false
}
}, },
"AllowedHosts": "*" "Reddit": {
"IsEnabled": false,
"PullHot": false,
"PullNsfw": false,
"PullTop": false
},
"Youtube": {
"IsEnabled": false,
"Debug": false
},
"Twitch": {
"IsEnabled": false,
"ClientID": "",
"ClientSecret": ""
},
"FinalFantasyXiv": {
"IsEnabled": false
},
"AllowedHosts": "*",
"EnableSwagger": true
} }

5
Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs
View File

@ -1,9 +1,10 @@
namespace Newsbot.Collector.Domain.Consts; namespace Newsbot.Collector.Domain.Consts;
/// <summary> /// <summary>
/// This class contains const entries to access keys within IConfiguration. /// This class contains const entries to access keys within IConfiguration.
/// </summary> /// </summary>
public class ConfigConnectionStringConst public static class ConfigConnectionStringConst
{ {
public const string Database = "ConnectionStrings:Database"; public const string Database = "ConnectionStrings:Database";
public const string OpenTelemetry = "ConnectionStrings:OpenTelemetry";
} }

122
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -1,6 +1,5 @@
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Xml; using System.Xml;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts; using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Interfaces;
@ -12,14 +11,15 @@ namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJobOptions public class RssWatcherJobOptions
{ {
public string ConnectionString { get; set; } = ""; public string? ConnectionString { get; init; }
public string? OpenTelemetry { get; init; }
} }
// This class was made to work with Hangfire and it does not support constructors. // This class was made to work with Hangfire and it does not support constructors.
public class RssWatcherJob : IHangfireJob public class RssWatcherJob
{ {
private IArticlesRepository _articles; private IArticlesRepository _articles;
private ILogger _logger;
private IDiscordQueueRepository _queue; private IDiscordQueueRepository _queue;
private ISourcesRepository _source; private ISourcesRepository _source;
@ -28,73 +28,67 @@ public class RssWatcherJob : IHangfireJob
_articles = new ArticlesTable(""); _articles = new ArticlesTable("");
_queue = new DiscordQueueTable(""); _queue = new DiscordQueueTable("");
_source = new SourcesTable(""); _source = new SourcesTable("");
_logger = GetLogger("");
} }
public void InitAndExecute(RssWatcherJobOptions options) public void InitAndExecute(RssWatcherJobOptions options)
{ {
Log.Information("RssWatcherJob - Job was triggered"); Init(options);
Log.Information("RssWatcherJob - Setting up the job");
Init(options.ConnectionString);
_logger.Information("RssWatcherJob - Job was triggered");
_logger.Information("RssWatcherJob - Setting up the job");
Execute();
}
private ILogger GetLogger(string connectionString)
{
return Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.OpenTelemetry(
connectionString,
resourceAttributes: new Dictionary<string, object>
{
{ "service.name", "newsbot-collector-api" },
{ "Job", "RssWatcherJob" }
})
.CreateLogger();
}
public void Init(RssWatcherJobOptions options)
{
_articles = new ArticlesTable(options.ConnectionString ?? "");
_queue = new DiscordQueueTable(options.ConnectionString ?? "");
_source = new SourcesTable(options.ConnectionString ?? "");
_logger = GetLogger(options.OpenTelemetry ?? "");
}
public void Execute()
{
var articles = new List<ArticlesModel>(); var articles = new List<ArticlesModel>();
Log.Information("RssWatcherJob - Requesting sources"); _logger.Information("RssWatcherJob - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss); var sources = _source.ListByType(SourceTypes.Rss);
Log.Information($"RssWatcherJob - Got {sources.Count()} back"); _logger.Information($"RssWatcherJob - Got {sources.Count} back");
foreach (var source in sources) foreach (var source in sources)
{ {
Log.Information($"RssWatcherJob - Starting to proces '{source.Name}'"); _logger.Information($"RssWatcherJob - Starting to process '{source.Name}'");
Log.Information("RssWatcherJob - Starting to request feed to be processed"); _logger.Information("RssWatcherJob - Starting to request feed to be processed");
var results = Collect(source.Url, source.ID); var results = Collect(source.Url, source.ID);
Log.Information($"RssWatcherJob - Collected {results.Count()} posts"); _logger.Information($"RssWatcherJob - Collected {results.Count} posts");
articles.AddRange(results); articles.AddRange(results);
} }
Log.Information("RssWatcherJob - Sending posts over to the database"); _logger.Information("RssWatcherJob - Sending posts over to the database");
UpdateDatabase(articles); UpdateDatabase(articles);
Log.Information("RssWatcherJob - Done!"); _logger.Information("RssWatcherJob - Done!");
} }
public void InitAndExecute(IConfiguration config) public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
{ {
// reach out to the db and find all the rss feeds var collectedPosts = new List<ArticlesModel>();
var connectionString = config.GetConnectionString("database");
if (connectionString is null)
{
connectionString = "";
}
Init(connectionString);
var articles = new List<ArticlesModel>();
var sources = _source.ListByType(SourceTypes.Rss);
foreach (var source in sources)
{
if (source.Enabled == false)
{
continue;
}
var results = Collect(source.Url, source.ID);
articles.AddRange(results);
}
UpdateDatabase(articles);
}
public void Init(string connectionString)
{
_articles = new ArticlesTable(connectionString);
_queue = new DiscordQueueTable(connectionString);
_source = new SourcesTable(connectionString);
}
public List<ArticlesModel> Collect(string url, Guid SourceID, int sleep = 3000)
{
var CollectedPosts = new List<ArticlesModel>();
using var reader = XmlReader.Create(url); using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader); var feed = SyndicationFeed.Load(reader);
@ -105,10 +99,7 @@ public class RssWatcherJob : IHangfireJob
// Check if we have seen the url before // Check if we have seen the url before
// If we have, skip and save the site bandwidth // If we have, skip and save the site bandwidth
if (IsThisUrlKnown(articleUrl) == true) if (IsThisUrlKnown(articleUrl)) continue;
{
continue;
}
var meta = new HtmlPageReader(articleUrl); var meta = new HtmlPageReader(articleUrl);
meta.Parse(); meta.Parse();
@ -121,15 +112,16 @@ public class RssWatcherJob : IHangfireJob
PubDate = post.PublishDate.DateTime, PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image, Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description, Description = meta.Data.Header.Description,
SourceID = SourceID SourceID = sourceId
}; };
CollectedPosts.Add(article); collectedPosts.Add(article);
// try to not be too greedy // try to not be too greedy
Thread.Sleep(sleep); Thread.Sleep(sleep);
} }
return CollectedPosts;
return collectedPosts;
} }
public void UpdateDatabase(List<ArticlesModel> items) public void UpdateDatabase(List<ArticlesModel> items)
@ -138,7 +130,7 @@ public class RssWatcherJob : IHangfireJob
{ {
if (item.URL is null) if (item.URL is null)
{ {
Log.Warning($"RSS Watcher collected a blank url and was skipped."); Log.Warning("RSS Watcher collected a blank url and was skipped.");
continue; continue;
} }
@ -153,20 +145,14 @@ public class RssWatcherJob : IHangfireJob
private bool IsThisUrlKnown(string url) private bool IsThisUrlKnown(string url)
{ {
var isKnown = _articles.GetByUrl(url); var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url) if (isKnown.URL == url) return true;
{
return true;
}
return false; return false;
} }
private string FetchTags(SyndicationItem post) private string FetchTags(SyndicationItem post)
{ {
string result = ""; var result = "";
foreach (var tag in post.Categories) foreach (var tag in post.Categories) result += $"{tag.Name},";
{
result += $"{tag.Name},";
}
return result; return result;
} }
} }

33
Newsbot.Collector.Services/Newsbot.Collector.Services.csproj
View File

@ -1,20 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj" /> <ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj"/>
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" /> <ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj"/>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" /> <PackageReference Include="Google.Apis.YouTube.v3" Version="1.60.0.2945"/>
<PackageReference Include="Serilog" Version="2.12.0" /> <PackageReference Include="HtmlAgilityPack" Version="1.11.46"/>
<PackageReference Include="System.ServiceModel.Syndication" Version="7.0.0" /> <PackageReference Include="Selenium.WebDriver" Version="4.8.1"/>
</ItemGroup> <PackageReference Include="Selenium.WebDriver.GeckoDriver" Version="0.32.2"/>
<PackageReference Include="Serilog" Version="2.12.0"/>
<PackageReference Include="Serilog.Sinks.Console" Version="4.1.0"/>
<PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="1.0.0-dev-00113"/>
<PackageReference Include="System.ServiceModel.Syndication" Version="7.0.0"/>
</ItemGroup>
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
</Project> </Project>