jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Added otel support and pushed data to my server. (#12)

Browse Source
This commit is contained in:
James Tombleson 2023-03-19 22:54:17 -07:00 committed by GitHub
parent 607df99bfb
commit ac6bdaa184
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
Newsbot.Collector.Api/Newsbot.Collector.Api.csproj
View File

@ -1,29 +1,34 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AspNetCore.HealthChecks.NpgSql" Version="6.0.2" />
<PackageReference Include="AspNetCore.HealthChecks.UI" Version="6.0.5" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="6.0.5" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Core" Version="6.0.5" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Serilog" Version="2.12.0" />
<PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AspNetCore.HealthChecks.NpgSql" Version="6.0.2" />
<PackageReference Include="AspNetCore.HealthChecks.UI" Version="6.0.5" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="6.0.5" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Core" Version="6.0.5" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.33" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.4.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.4.0" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.4.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc9.14" />
<PackageReference Include="Serilog" Version="2.12.0" />
<PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
<PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="1.0.0-dev-00113" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Services\Newsbot.Collector.Services.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Services\Newsbot.Collector.Services.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" />
</ItemGroup>
</Project>

43
Newsbot.Collector.Api/Program.cs
View File

@ -6,22 +6,28 @@ using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Services.Jobs;
using Serilog;
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
Log.Information("Starting up");
using ILogger = Serilog.ILogger;
var builder = WebApplication.CreateBuilder(args);
// Define Logger
builder.Host.UseSerilog(); // <-- Add this line
builder.Host.UseSerilog();
// Build the config
var config = GetConfiguration();
builder.Configuration.AddConfiguration(config);
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.OpenTelemetry(
config.GetValue<string>(ConfigConnectionStringConst.OpenTelemetry) ?? "",
resourceAttributes: new Dictionary<string, object>
{
{ "service.name", "newsbot-collector-api" }
})
.CreateLogger();
Log.Information("Starting up");
// Configure Hangfire
builder.Services.AddHangfire(f => f.UseMemoryStorage());
builder.Services.AddHangfireServer();
@ -40,11 +46,11 @@ builder.Services.Configure<ConnectionStrings>(config.GetSection("ConnectionStrin
var app = builder.Build();
// Configure the HTTP request pipeline.
//if (app.Environment.IsDevelopment())
//{
app.UseSwagger();
app.UseSwaggerUI();
//}
if (config.GetValue<bool>("EnableSwagger"))
{
app.UseSwagger();
app.UseSwaggerUI();
}
app.UseHttpsRedirection();
@ -71,7 +77,7 @@ static IConfiguration GetConfiguration()
.Build();
}
static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger logger)
static void SetupRecurringJobs(IConfiguration configuration, ILogger logger)
{
//RecurringJob.AddOrUpdate<HelloWorldJob>("Example", x => x.InitAndExecute(new HelloWorldJobOptions
//{
@ -83,8 +89,9 @@ static void SetupRecurringJobs(IConfiguration configuration, Serilog.ILogger log
ConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "15 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x => x.InitAndExecute(new DiscordNotificationJobOptions
{
DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "5/10 * * * *");
}
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
x.InitAndExecute(new DiscordNotificationJobOptions
{
DatabaseConnectionString = configuration.GetSection(ConfigConnectionStringConst.Database).Value ?? ""
}), "5/10 * * * *");
}

45
Newsbot.Collector.Api/appsettings.template.json
View File

@ -5,27 +5,28 @@
"Microsoft.AspNetCore": "Warning"
}
},
"Config": {
"ServerAddress": "",
"SqlConnectionString": "",
"Reddit": {
"IsEnabled": false,
"PullHot": false,
"PullNsfw": false,
"PullTop": false
},
"Youtube": {
"IsEnabled": false,
"Debug": false
},
"Twitch": {
"IsEnabled": false,
"ClientID": "",
"ClientSecret": ""
},
"FFXIV": {
"IsEnabled": false
}
"ConnectionStrings": {
"Database": "",
"OpenTelemetry": ""
},
"AllowedHosts": "*"
"Reddit": {
"IsEnabled": false,
"PullHot": false,
"PullNsfw": false,
"PullTop": false
},
"Youtube": {
"IsEnabled": false,
"Debug": false
},
"Twitch": {
"IsEnabled": false,
"ClientID": "",
"ClientSecret": ""
},
"FinalFantasyXiv": {
"IsEnabled": false
},
"AllowedHosts": "*",
"EnableSwagger": true
}

5
Newsbot.Collector.Domain/Consts/ConfigConnectionStringsConst.cs
View File

@ -1,9 +1,10 @@
namespace Newsbot.Collector.Domain.Consts;
/// <summary>
/// This class contains const entries to access keys within IConfiguration.
/// This class contains const entries to access keys within IConfiguration.
/// </summary>
public class ConfigConnectionStringConst
public static class ConfigConnectionStringConst
{
public const string Database = "ConnectionStrings:Database";
public const string OpenTelemetry = "ConnectionStrings:OpenTelemetry";
}

122
Newsbot.Collector.Services/Jobs/RssWatcherJob.cs
View File

@ -1,6 +1,5 @@
using System.ServiceModel.Syndication;
using System.Xml;
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces;
@ -12,14 +11,15 @@ namespace Newsbot.Collector.Services.Jobs;
public class RssWatcherJobOptions
{
public string ConnectionString { get; set; } = "";
public string? ConnectionString { get; init; }
public string? OpenTelemetry { get; init; }
}
// This class was made to work with Hangfire and it does not support constructors.
public class RssWatcherJob : IHangfireJob
public class RssWatcherJob
{
private IArticlesRepository _articles;
private ILogger _logger;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
@ -28,73 +28,67 @@ public class RssWatcherJob : IHangfireJob
_articles = new ArticlesTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
_logger = GetLogger("");
}
public void InitAndExecute(RssWatcherJobOptions options)
{
Log.Information("RssWatcherJob - Job was triggered");
Log.Information("RssWatcherJob - Setting up the job");
Init(options.ConnectionString);
Init(options);
_logger.Information("RssWatcherJob - Job was triggered");
_logger.Information("RssWatcherJob - Setting up the job");
Execute();
}
private ILogger GetLogger(string connectionString)
{
return Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.OpenTelemetry(
connectionString,
resourceAttributes: new Dictionary<string, object>
{
{ "service.name", "newsbot-collector-api" },
{ "Job", "RssWatcherJob" }
})
.CreateLogger();
}
public void Init(RssWatcherJobOptions options)
{
_articles = new ArticlesTable(options.ConnectionString ?? "");
_queue = new DiscordQueueTable(options.ConnectionString ?? "");
_source = new SourcesTable(options.ConnectionString ?? "");
_logger = GetLogger(options.OpenTelemetry ?? "");
}
public void Execute()
{
var articles = new List<ArticlesModel>();
Log.Information("RssWatcherJob - Requesting sources");
_logger.Information("RssWatcherJob - Requesting sources");
var sources = _source.ListByType(SourceTypes.Rss);
Log.Information($"RssWatcherJob - Got {sources.Count()} back");
_logger.Information($"RssWatcherJob - Got {sources.Count} back");
foreach (var source in sources)
{
Log.Information($"RssWatcherJob - Starting to proces '{source.Name}'");
Log.Information("RssWatcherJob - Starting to request feed to be processed");
_logger.Information($"RssWatcherJob - Starting to process '{source.Name}'");
_logger.Information("RssWatcherJob - Starting to request feed to be processed");
var results = Collect(source.Url, source.ID);
Log.Information($"RssWatcherJob - Collected {results.Count()} posts");
_logger.Information($"RssWatcherJob - Collected {results.Count} posts");
articles.AddRange(results);
}
Log.Information("RssWatcherJob - Sending posts over to the database");
_logger.Information("RssWatcherJob - Sending posts over to the database");
UpdateDatabase(articles);
Log.Information("RssWatcherJob - Done!");
_logger.Information("RssWatcherJob - Done!");
}
public void InitAndExecute(IConfiguration config)
public List<ArticlesModel> Collect(string url, Guid sourceId, int sleep = 3000)
{
// reach out to the db and find all the rss feeds
var connectionString = config.GetConnectionString("database");
if (connectionString is null)
{
connectionString = "";
}
Init(connectionString);
var articles = new List<ArticlesModel>();
var sources = _source.ListByType(SourceTypes.Rss);
foreach (var source in sources)
{
if (source.Enabled == false)
{
continue;
}
var results = Collect(source.Url, source.ID);
articles.AddRange(results);
}
UpdateDatabase(articles);
}
public void Init(string connectionString)
{
_articles = new ArticlesTable(connectionString);
_queue = new DiscordQueueTable(connectionString);
_source = new SourcesTable(connectionString);
}
public List<ArticlesModel> Collect(string url, Guid SourceID, int sleep = 3000)
{
var CollectedPosts = new List<ArticlesModel>();
var collectedPosts = new List<ArticlesModel>();
using var reader = XmlReader.Create(url);
var feed = SyndicationFeed.Load(reader);
@ -105,10 +99,7 @@ public class RssWatcherJob : IHangfireJob
// Check if we have seen the url before
// If we have, skip and save the site bandwidth
if (IsThisUrlKnown(articleUrl) == true)
{
continue;
}
if (IsThisUrlKnown(articleUrl)) continue;
var meta = new HtmlPageReader(articleUrl);
meta.Parse();
@ -121,15 +112,16 @@ public class RssWatcherJob : IHangfireJob
PubDate = post.PublishDate.DateTime,
Thumbnail = meta.Data.Header.Image,
Description = meta.Data.Header.Description,
SourceID = SourceID
SourceID = sourceId
};
CollectedPosts.Add(article);
collectedPosts.Add(article);
// try to not be too greedy
Thread.Sleep(sleep);
}
return CollectedPosts;
return collectedPosts;
}
public void UpdateDatabase(List<ArticlesModel> items)
@ -138,7 +130,7 @@ public class RssWatcherJob : IHangfireJob
{
if (item.URL is null)
{
Log.Warning($"RSS Watcher collected a blank url and was skipped.");
Log.Warning("RSS Watcher collected a blank url and was skipped.");
continue;
}
@ -153,20 +145,14 @@ public class RssWatcherJob : IHangfireJob
private bool IsThisUrlKnown(string url)
{
var isKnown = _articles.GetByUrl(url);
if (isKnown.URL == url)
{
return true;
}
if (isKnown.URL == url) return true;
return false;
}
private string FetchTags(SyndicationItem post)
{
string result = "";
foreach (var tag in post.Categories)
{
result += $"{tag.Name},";
}
var result = "";
foreach (var tag in post.Categories) result += $"{tag.Name},";
return result;
}
}

33
Newsbot.Collector.Services/Newsbot.Collector.Services.csproj
View File

@ -1,20 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj" />
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Newsbot.Collector.Domain\Newsbot.Collector.Domain.csproj"/>
<ProjectReference Include="..\Newsbot.Collector.Database\Newsbot.Collector.Database.csproj"/>
</ItemGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
<PackageReference Include="Serilog" Version="2.12.0" />
<PackageReference Include="System.ServiceModel.Syndication" Version="7.0.0" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Google.Apis.YouTube.v3" Version="1.60.0.2945"/>
<PackageReference Include="HtmlAgilityPack" Version="1.11.46"/>
<PackageReference Include="Selenium.WebDriver" Version="4.8.1"/>
<PackageReference Include="Selenium.WebDriver.GeckoDriver" Version="0.32.2"/>
<PackageReference Include="Serilog" Version="2.12.0"/>
<PackageReference Include="Serilog.Sinks.Console" Version="4.1.0"/>
<PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="1.0.0-dev-00113"/>
<PackageReference Include="System.ServiceModel.Syndication" Version="7.0.0"/>
</ItemGroup>
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>