jtom38/Newsbot.Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases 1 Wiki Activity

Features/add code projects watcher (#26)

Browse Source 
* Migration added to update Articles to define if Release or Commit

* CodeProjectWatcher Job was created from GithubWatcher as this will target services like gitlab and also gitea.

* article model was updated to reflect migration changes

* Added CodeProjects to startup

* Seed was updated with CodeProjects and some new defaults

* Added Delete call for Sources

* Added a route to cleanup all records based on SourceId

* Added CodeProject const values to load from config

* minor changes to the rss controller

* Added codeprojects to the routes to trigger the job
This commit is contained in:
James Tombleson 2023-04-10 22:59:13 -07:00 committed by GitHub
parent a25c44d8cc
commit 117653c001
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 421 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Newsbot.Collector.Api/BackgroundJobs.cs
View File

@ -1,5 +1,6 @@
using Hangfire;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api;
@ -23,6 +24,15 @@ public static class BackgroundJobs
IsEnabled = configuration.GetValue<bool>(ConfigConst.YoutubeIsEnable)
}), "20 0-23 * * *");
RecurringJob.AddOrUpdate<CodeProjectWatcherJob>("CodeProjects", x =>
x.InitAndExecute(new CodeProjectWatcherJobOptions
{
ConnectionStrings = configuration.GetSection(ConfigConst.SectionConnectionStrings)
.Get<ConfigSectionConnectionStrings>(),
FeaturePullCommits = true,
FeaturePullReleases = true
}), "25 0-23 * * *");
RecurringJob.AddOrUpdate<DiscordNotificationJob>("Discord Alerts", x =>
x.InitAndExecute(new DiscordNotificationJobOptions
{

13
Newsbot.Collector.Api/Controllers/ArticlesController.cs
View File

@ -11,8 +11,8 @@ namespace Newsbot.Collector.Api.Controllers;
[Route("api/articles")]
public class ArticlesController : ControllerBase
{
private readonly ILogger<ArticlesController> _logger;
private readonly IArticlesRepository _articles;
private readonly ILogger<ArticlesController> _logger;
private readonly ISourcesRepository _sources;
public ArticlesController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
@ -27,14 +27,12 @@ public class ArticlesController : ControllerBase
{
var res = new List<ArticleDto>();
var items = _articles.List(0, 25);
foreach (var item in items)
{
res.Add(ArticleDto.Convert(item));
}
foreach (var item in items) res.Add(ArticleDto.Convert(item));
return res;
}
[HttpGet("{id:guid}")]
[EndpointDescription("Returns the article based on the Id value given.")]
public ArticleDto GetById(Guid id)
{
var item = _articles.GetById(id);
@ -54,10 +52,7 @@ public class ArticlesController : ControllerBase
{
var res = new List<ArticleDto>();
var items = _articles.ListBySourceId(sourceId, page, count);
foreach (var item in items)
{
res.Add(ArticleDto.Convert(item));
}
foreach (var item in items) res.Add(ArticleDto.Convert(item));
return res;
}
}

33
Newsbot.Collector.Api/Controllers/CodeProjectController.cs Normal file
View File

@ -0,0 +1,33 @@
using Hangfire;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Api.Controllers;
[ApiController]
[Route("api/codeprojects")]
public class CodeProjectController
{
private readonly ConfigSectionConnectionStrings _connectionStrings;
private readonly ILogger<CodeProjectController> _logger;
public CodeProjectController(ILogger<CodeProjectController> logger,
IOptions<ConfigSectionConnectionStrings> settings)
{
_logger = logger;
_connectionStrings = settings.Value;
}
[HttpPost("check")]
public void PullNow()
{
BackgroundJob.Enqueue<CodeProjectWatcherJob>(x => x.InitAndExecute(new CodeProjectWatcherJobOptions
{
ConnectionStrings = _connectionStrings,
FeaturePullReleases = true,
FeaturePullCommits = true
}));
}
}

6
Newsbot.Collector.Api/Controllers/RssController.cs
View File

@ -1,8 +1,6 @@
using Hangfire;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
@ -13,10 +11,10 @@ namespace Newsbot.Collector.Api.Controllers;
public class RssController
{
private readonly ConfigSectionConnectionStrings _connectionStrings;
private readonly ILogger<RssController> _logger;
private readonly ConfigSectionRssModel _rssConfig;
private readonly ILogger<SourcesController> _logger;
public RssController(ILogger<SourcesController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
public RssController(ILogger<RssController> logger, IOptions<ConfigSectionConnectionStrings> connectionStrings,
IOptions<ConfigSectionRssModel> rss)
{
_logger = logger;

22
Newsbot.Collector.Api/Controllers/SourcesController.cs
View File

@ -13,10 +13,11 @@ namespace Newsbot.Collector.Api.Controllers;
[Route("api/sources")]
public class SourcesController : ControllerBase
{
private readonly IIconsRepository _icons;
private readonly ILogger<SourcesController> _logger;
private readonly IArticlesRepository _articles;
//private readonly ConnectionStrings _settings;
private readonly IIconsRepository _icons;
private readonly ILogger<SourcesController> _logger;
private readonly ISourcesRepository _sources;
public SourcesController(ILogger<SourcesController> logger, IOptions<ConnectionStrings> settings)
@ -25,6 +26,7 @@ public class SourcesController : ControllerBase
//_settings = settings.Value;
_sources = new SourcesTable(settings.Value.Database);
_icons = new IconsTable(settings.Value.Database);
_articles = new ArticlesTable(settings.Value.Database);
}
[HttpGet(Name = "GetSources")]
@ -154,10 +156,10 @@ public class SourcesController : ControllerBase
return SourceDto.Convert(item);
}
[HttpPost("new/github")]
[HttpPost("new/codeproject")]
public SourceDto NewGithub(string url)
{
if (!url.Contains("github.com")) return new SourceDto();
//if (!url.Contains("github.com")) return new SourceDto();
var res = _sources.GetByUrl(url);
if (res.ID != Guid.Empty) return SourceDto.Convert(res);
@ -172,13 +174,13 @@ public class SourcesController : ControllerBase
var item = _sources.New(new SourceModel
{
Site = SourceTypes.GitHub,
Type = SourceTypes.GitHub,
Site = SourceTypes.CodeProject,
Type = SourceTypes.CodeProject,
Name = $"{slice[3]}/{slice[4]}",
Url = url,
Source = "feed",
Enabled = true,
Tags = $"{SourceTypes.GitHub}, {slice[3]}, {slice[4]}"
Tags = $"{slice[2]},{slice[3]},{slice[4]}"
});
_icons.New(new IconModel
@ -209,4 +211,10 @@ public class SourcesController : ControllerBase
{
_sources.Enable(id);
}
[HttpDelete("{id}")]
public void Delete(Guid id, bool purgeOrphanedRecords)
{
_sources.Delete(id);
}
}

16
Newsbot.Collector.Database/Migrations/20230326223545_icon_sourceid.sql
View File

@ -1,13 +1,21 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
ALTER TABLE icons
ADD COLUMN SourceId uuid;
ALTER TABLE sources
ADD COLUMN IconUri uuid;
ALTER TABLE articles
ADD COLUMN CodeIsRelease bool,
ADD COLUMN CodeIsCommit bool;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
SELECT 'down SQL query';
ALTER TABLE icons
DROP COLUMN SourceId;
ALTER TABLE sources
DROP COLUMN IconUri;
Alter TABLE articles
DROP COLUMN CodeIsRelease,
DROP COLUMN CodeIsCommit
-- +goose StatementEnd

58
Newsbot.Collector.Database/Repositories/ArticlesTable.cs
View File

@ -9,8 +9,7 @@ namespace Newsbot.Collector.Database.Repositories;
public class ArticlesTable : IArticlesRepository
{
private string _connectionString;
private readonly string _connectionString;
public ArticlesTable(string connectionString)
{
@ -20,21 +19,11 @@ public class ArticlesTable : IArticlesRepository
public ArticlesTable(IConfiguration configuration)
{
var conn = configuration.GetConnectionString("database");
if (conn is null)
{
conn = "";
}
if (conn is null) conn = "";
_connectionString = conn;
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
public List<ArticlesModel> List(int page = 0, int count = 25)
{
using var conn = OpenConnection(_connectionString);
@ -42,22 +31,19 @@ public class ArticlesTable : IArticlesRepository
Order By PubDate Desc
Offset @Page
Fetch Next @Count Rows Only", new
{
Page = page * count,
Count = count
})
.ToList();
{
Page = page * count,
Count = count
})
.ToList();
return res;
}
public ArticlesModel GetById(Guid ID)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID = ID });
if (res.Count() == 0)
{
return new ArticlesModel();
}
var res = conn.Query<ArticlesModel>("select * from articles where ID = @ID", new { ID });
if (res.Count() == 0) return new ArticlesModel();
return res.First();
}
@ -65,10 +51,7 @@ public class ArticlesTable : IArticlesRepository
{
using var conn = OpenConnection(_connectionString);
var res = conn.Query<ArticlesModel>("select * from articles where Url = @Url Limit 1", new { Url = url });
if (res.Count() == 0)
{
return new ArticlesModel();
}
if (res.Count() == 0) return new ArticlesModel();
return res.First();
}
@ -83,7 +66,7 @@ public class ArticlesTable : IArticlesRepository
{
sourceid = id,
page = page * count,
count = count
count
}).ToList();
}
@ -92,7 +75,8 @@ public class ArticlesTable : IArticlesRepository
model.ID = Guid.NewGuid();
using var conn = OpenConnection(_connectionString);
var q = "INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
var q =
"INSERT INTO Articles (id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage) Values (@id, @sourceid, @tags, @title, @url, @pubdate, @video, @videoheight, @videowidth, @thumbnail, @description, @authorname, @authorimage);";
var res = conn.Execute(q, new
{
id = model.ID,
@ -112,4 +96,20 @@ public class ArticlesTable : IArticlesRepository
return model;
}
public void DeleteAllBySourceId(Guid sourceId)
{
using var conn = OpenConnection(_connectionString);
var res = conn.Execute("Delete from articles where sourceid = '@id'", new
{
sourceId
});
if (res == 0) throw new Exception($"No records where deleted that linked to SourceId = '{sourceId}'");
}
private IDbConnection OpenConnection(string connectionString)
{
var conn = new NpgsqlConnection(_connectionString);
conn.Open();
return conn;
}
}

11
Newsbot.Collector.Database/Repositories/SourcesTable.cs
View File

@ -155,6 +155,17 @@ public class SourcesTable : ISourcesRepository
});
}
public void Delete(Guid id)
{
using var conn = OpenConnection(_connectionString);
var query = "Delete From sources where id = @id;";
var res = conn.Execute(query, new
{
id
});
if (res == 0) throw new Exception("Nothing was deleted");
}
public int UpdateYoutubeId(Guid id, string youtubeId)
{
using var conn = OpenConnection(_connectionString);

3
Newsbot.Collector.Domain/Consts/ConfigConst.cs
View File

@ -10,6 +10,7 @@ public class ConfigConst
public const string SectionRss = "Rss";
public const string SectionTwitch = "Twitch";
public const string SectionYoutube = "Youtube";
public const string SectionCodeProjects = "CodeProjects";
public const string SectionNotificationsDiscord = "Notifications:Discord";
public const string ConnectionStringDatabase = "ConnectionStrings:Database";
@ -29,6 +30,8 @@ public class ConfigConst
public const string YoutubeIsEnable = "Youtube:IsEnabled";
public const string YoutubeDebug = "Youtube:Debug";
public const string CodeProjectsIsEnabled = "CodeProjects:IsEnabled";
public const string EnableSwagger = "EnableSwagger";
public const string DiscordNotificationsEnabled = "Notifications:Discord:IsEnabled";

3
Newsbot.Collector.Domain/Consts/SourcesTypes.cs
View File

@ -7,6 +7,5 @@ public class SourceTypes
public const string YouTube = "youtube";
public const string Twitch = "twitch";
public const string FinalFantasyXiv = "ffxiv";
public const string GitHub = "github";
public const string CodeProject = "code";
}

5
Newsbot.Collector.Domain/Interfaces/IArticlesRepository.cs
View File

@ -4,9 +4,10 @@ namespace Newsbot.Collector.Domain.Interfaces;
public interface IArticlesRepository : ITableRepository
{
List<ArticlesModel>List(int page, int count);
List<ArticlesModel>ListBySourceId(Guid id, int page = 0, int count = 25);
List<ArticlesModel> List(int page, int count);
List<ArticlesModel> ListBySourceId(Guid id, int page = 0, int count = 25);
ArticlesModel GetById(Guid ID);
ArticlesModel GetByUrl(string url);
ArticlesModel New(ArticlesModel model);
void DeleteAllBySourceId(Guid sourceId);
}

1
Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs
View File

@ -15,5 +15,6 @@ public interface ISourcesRepository
public List<SourceModel> ListByType(string type, int limit = 25);
public int Disable(Guid id);
public int Enable(Guid id);
public void Delete(Guid id);
public int UpdateYoutubeId(Guid id, string youtubeId);
}

2
Newsbot.Collector.Domain/Models/DatabaseModel.cs
View File

@ -15,6 +15,8 @@ public class ArticlesModel
public string Description { get; set; } = "";
public string AuthorName { get; set; } = "";
public string? AuthorImage { get; set; }
public bool CodeIsRelease { get; set; }
public bool CodeIsCommit { get; set; }
}
public class AuthorModel

178
Newsbot.Collector.Services/Jobs/CodeProjectWatcherJob.cs Normal file
View File

@ -0,0 +1,178 @@
using System.Collections;
using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.HtmlParser;
using Serilog;
namespace Newsbot.Collector.Services.Jobs;
public class CodeProjectWatcherJobOptions
{
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
//public string ConnectionString { get; set; } = "";
public bool FeaturePullReleases { get; set; } = false;
public bool FeaturePullCommits { get; set; } = false;
//public bool PullIssues { get; set; } = false;
}
public class CodeProjectWatcherJob
{
private const string JobName = "CodeProjectWatcher";
private IArticlesRepository _articles;
private ILogger _logger;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
public CodeProjectWatcherJob()
{
_articles = new ArticlesTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
_logger = JobLogger.GetLogger("", JobName);
}
public CodeProjectWatcherJob(CodeProjectWatcherJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
}
public void InitAndExecute(CodeProjectWatcherJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
_logger = JobLogger.GetLogger(options.ConnectionStrings.OpenTelemetry ?? "", JobName);
Execute();
}
private void Execute()
{
var sources = _source.ListByType(SourceTypes.CodeProject);
// query sources for things to pull
var items = new List<ArticlesModel>();
foreach (var source in sources)
{
items.AddRange(CheckForReleases(source));
items.AddRange(CheckForCommits(source));
}
foreach (var item in items)
{
_articles.New(item);
_queue.New(new DiscordQueueModel
{
ArticleID = item.ID
});
}
}
public IEnumerable<ArticlesModel> CheckForReleases(SourceModel source)
{
var url = new Uri(source.Url);
var links = new List<string>
{
$"{url.AbsoluteUri}/releases.atom",
$"{url.AbsoluteUri}/tags.atom" //github converts tags as releases
};
foreach (var link in links)
try
{
using var reader = XmlReader.Create(link);
var client = SyndicationFeed.Load(reader);
return ProcessFeed(client.Items, source, true, false);
//if (link.EndsWith("tags.atom"))
//{
// return ProcessFeed(client.Items, source, false, true, false);
//}
}
catch
{
_logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}. Might not have anything", JobName,
url.AbsoluteUri);
}
return new List<ArticlesModel>();
}
public IEnumerable<ArticlesModel> CheckForCommits(SourceModel source)
{
var url = new Uri(source.Url);
var links = new List<string>
{
$"{url.AbsoluteUri}/commits/main.atom",
$"{url.AbsoluteUri}/commits/master.atom"
};
foreach (var link in links)
try
{
using var reader = XmlReader.Create(link);
var client = SyndicationFeed.Load(reader);
return ProcessFeed(client.Items, source, false, true);
}
catch
{
_logger.Debug("{JobName} - Does not respond to {UrlAbsoluteUri}. Might not have anything", JobName,
url.AbsoluteUri);
}
return new List<ArticlesModel>();
}
private IEnumerable<ArticlesModel> ProcessFeed(IEnumerable<SyndicationItem> feed, SourceModel source,
bool isRelease, bool isCommit)
{
var items = new List<ArticlesModel>();
foreach (var item in feed)
{
var itemUrl = item.Links[0].Uri.AbsoluteUri;
var exits = _articles.GetByUrl(itemUrl);
if (exits.ID != Guid.Empty) continue;
var parser = new HtmlPageReader(new HtmlPageReaderOptions
{
Url = itemUrl
});
parser.Parse();
var a = new ArticlesModel
{
SourceID = source.ID,
Tags = source.Tags,
Title = item.Title.Text,
URL = itemUrl,
PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image,
Description = item.Title.Text,
AuthorName = item.Authors[0].Name ?? "",
AuthorImage = item.Authors[0].Uri ?? "",
CodeIsRelease = isRelease,
CodeIsCommit = isCommit,
};
items.Add(a);
}
return items;
}
}

115
Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs
View File

@ -1,115 +0,0 @@
using System.ServiceModel.Syndication;
using System.Xml;
using Newsbot.Collector.Database.Repositories;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Interfaces;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.HtmlParser;
namespace Newsbot.Collector.Services.Jobs;
public class GithubWatcherJobOptions
{
public ConfigSectionConnectionStrings? ConnectionStrings { get; set; }
//public string ConnectionString { get; set; } = "";
public bool FeaturePullReleases { get; set; } = false;
public bool FeaturePullCommits { get; set; } = false;
//public bool PullIssues { get; set; } = false;
}
public class GithubWatcherJob
{
private IArticlesRepository _articles;
private IDiscordQueueRepository _queue;
private ISourcesRepository _source;
public GithubWatcherJob()
{
_articles = new ArticlesTable("");
_queue = new DiscordQueueTable("");
_source = new SourcesTable("");
}
public void InitAndExecute(GithubWatcherJobOptions options)
{
options.ConnectionStrings ??= new ConfigSectionConnectionStrings();
_articles = new ArticlesTable(options.ConnectionStrings.Database ?? "");
_queue = new DiscordQueueTable(options.ConnectionStrings.Database ?? "");
_source = new SourcesTable(options.ConnectionStrings.Database ?? "");
Execute();
}
private void Execute()
{
_source.ListBySource(SourceTypes.GitHub, 25);
// query sources for things to pull
var items = new List<ArticlesModel>();
items.AddRange(Collect(new Uri("https://github.com/jtom38/dvb")));
// query */commits/master.atom
// query */commits/main.atom
}
public List<ArticlesModel> Collect(Uri url)
{
var items = new List<ArticlesModel>();
var placeHolderId = Guid.NewGuid();
// query */release.atom
items.AddRange(CollectItems($"{url.AbsoluteUri}/releases.atom", placeHolderId));
items.AddRange(CollectItems($"{url.AbsoluteUri}/master.atom", placeHolderId));
return items;
}
private List<ArticlesModel> CollectItems(string baseUrl, Guid sourceId)
{
var items = new List<ArticlesModel>();
using var reader = XmlReader.Create(baseUrl);
var client = SyndicationFeed.Load(reader);
foreach (var item in client.Items)
{
var itemUrl = item.Links[0].Uri.AbsoluteUri;
var exits = _articles.GetByUrl(itemUrl);
if (exits.ID != Guid.Empty) continue;
var parser = new HtmlPageReader(new HtmlPageReaderOptions
{
Url = itemUrl
});
parser.Parse();
try
{
var a = new ArticlesModel
{
SourceID = sourceId,
Tags = "github",
Title = item.Title.Text,
URL = itemUrl,
//PubDate = item.LastUpdatedTime.DateTime,
Thumbnail = parser.Data.Header.Image,
Description = $"'dvb' has released '{item.Title.Text}'!",
AuthorName = item.Authors[0].Name ?? "",
AuthorImage = item.Authors[0].Uri ?? ""
};
items.Add(a);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
return items;
}
}

88
Newsbot.Collector.Tests/Jobs/CodeProjectWatcherJobTests.cs Normal file
View File

@ -0,0 +1,88 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Consts;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Tests.Jobs;
public class CodeProjectWatcherJobTests
{
[Fact]
public void CanReturnReleases()
{
var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions
{
ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings)
.Get<ConfigSectionConnectionStrings>(),
FeaturePullCommits = true,
FeaturePullReleases = true
});
var results = client.CheckForReleases(new SourceModel
{
ID = Guid.NewGuid(),
Url = "https://github.com/jtom38/dvb",
Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject,
Name = "jtom38/dvb",
Source = "feed",
Enabled = true
});
if (!results.Any())
{
Assert.Fail("Expected at least one item");
}
}
[Fact]
public void CollectsTagsButNoReleases()
{
var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions
{
ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings)
.Get<ConfigSectionConnectionStrings>(),
FeaturePullCommits = true,
FeaturePullReleases = true
});
var results = client.CheckForReleases(new SourceModel
{
ID = Guid.NewGuid(),
Url = "https://github.com/python/cpython",
Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject,
Name = "python.cpython",
Source = "feed",
Enabled = true
});
if (!results.Any())
{
Assert.Fail("Expected at least one item");
}
}
[Fact]
public void CollectsCommits()
{
var client = new CodeProjectWatcherJob(new CodeProjectWatcherJobOptions
{
ConnectionStrings = TestHelper.LoadConfig().GetSection(ConfigConst.SectionConnectionStrings)
.Get<ConfigSectionConnectionStrings>(),
FeaturePullCommits = true,
FeaturePullReleases = true
});
var results = client.CheckForCommits(new SourceModel
{
ID = Guid.NewGuid(),
Url = "https://github.com/jtom38/dvb",
Type = SourceTypes.CodeProject,
Site = SourceTypes.CodeProject,
Name = "jtom38/dvb",
Source = "feed",
Enabled = true
});
if (!results.Any())
{
Assert.Fail("Expected at least one item");
}
}
}

46
Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs
View File

@ -1,46 +0,0 @@
using Microsoft.Extensions.Configuration;
using Newsbot.Collector.Domain.Models;
using Newsbot.Collector.Domain.Models.Config;
using Newsbot.Collector.Services.Jobs;
namespace Newsbot.Collector.Tests.Jobs;
public class GithubWatcherJobTests
{
private IConfiguration GetConfiguration()
{
var inMemorySettings = new Dictionary<string, string>
{
{
"ConnectionStrings:database",
"Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"
}
};
IConfiguration configuration = new ConfigurationBuilder()
.AddInMemoryCollection(inMemorySettings)
.Build();
return configuration;
}
private string ConnectionString()
{
return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
}
[Fact]
public void CanPullAFeed()
{
var client = new GithubWatcherJob();
client.InitAndExecute(new GithubWatcherJobOptions
{
ConnectionStrings = new ConfigSectionConnectionStrings
{
Database = ConnectionString()
},
FeaturePullCommits = true,
FeaturePullReleases = true
});
client.Collect(new Uri("https://github.com/jtom38/dvb"));
}
}

24
seed.ps1
View File

@ -50,6 +50,20 @@ function New-TwitchSource {
return $res
}
function New-CodeProject {
param (
[string] $Url
)
$urlEncoded = [uri]::EscapeDataString($Url)
[string] $param = "url=$urlEncoded"
[string] $uri = "$ApiServer/api/sources/new/codeproject?$param"
Write-Host "Adding CodeProject '$url'"
$res = Invoke-RestMethod -Method Post -Uri $uri
return $res
}
function New-DiscordWebhook {
param (
[string] $Server,
@ -94,6 +108,11 @@ $youtubeLinusTechTips = New-YoutubeSource -Url "https://www.youtube.com/@LinusTe
$youtubeFireship = New-YoutubeSource -Url "https://www.youtube.com/@Fireship"
$youtubeClimateTown = New-YoutubeSource -Url "https://www.youtube.com/c/ClimateTown"
$codeDotnet = New-CodeProject -Url "https://github.com/dotnet/runtime"
#$codePython = New-CodeProject -Url "https://github.com/python/cpython"
#$codeGolang = New-CodeProject -Url "https://github.com/golang/go"
#$codePowerShell = New-CodeProject -Url "https://github.com/PowerShell/PowerShell"
#$codeLinux = New-CodeProject -Url "https://github.com/torvalds/linux"
$twitchNintendo = New-TwitchSource -Name "Nintendo"
@ -102,6 +121,7 @@ $sky = New-DiscordWebhook -Server "Let's Mosley" -Channel "bot test" -url $secre
New-Subscription -SourceId $redditDadJokes.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $redditSteamDeck.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssSteamDeck.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssFaysHaremporium.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $miharuMonitor.id
@ -109,9 +129,13 @@ New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $sky.id
New-Subscription -SourceId $rssOmgLinux.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssEngadget.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $rssArsTechnica.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $codeDotnet.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeLinusTechTips.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeFireship.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $youtubeClimateTown.id -DiscordWebhookId $miharuMonitor.id
New-Subscription -SourceId $twitchNintendo.id -DiscordWebhookId $miharuMonitor.id