Features/pulling GitHub (#9)
* Still working though it but looking good on releases * added the example discord message test * updated source repo to return an existing record before a new is added * updated the sources repo interface * updated new routes to check for existing records * starting to migrate the seed out of the sql migrations * A new seed script was made to reload the db from the api * Docker image works locally * Adding CI to build docker image * ... disabled swagger so I can test docker * Added more to the github job but its not finished. Isnt pulling sources yet. * cleaned up formatting * Controller updates to look for existing records when requesting a new one * null check cleanup * namespace fix
This commit is contained in:
GitHub
parent
aa53b1eeeb
commit
799668a059
|
|
@ -0,0 +1,3 @@
|
|||
appsettings.json
|
||||
**/bin/
|
||||
**/obj/
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
name: Docker
|
||||
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
on:
|
||||
#schedule:
|
||||
# - cron: '21 19 * * *'
|
||||
push:
|
||||
branches: [ master ]
|
||||
# Publish semver tags as releases.
|
||||
tags: [ 'v*.*.*' ]
|
||||
#pull_request:
|
||||
# branches: [ master ]
|
||||
|
||||
env:
|
||||
# Use docker.io for Docker Hub if empty
|
||||
REGISTRY: ghcr.io
|
||||
# github.repository as <account>/<repo>
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into registry ${{ env.REGISTRY }}
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Extract metadata (tags, labels) for Docker
|
||||
# https://github.com/docker/metadata-action
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
#images: ${{ env.REGISTRY }}/newsbot.worker
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
|
||||
with:
|
||||
context: .
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
seed.secrects.json
|
||||
out/
|
||||
|
||||
# User-specific files
|
||||
|
|
@ -15,6 +16,9 @@ out/
|
|||
appsettings.Development.json
|
||||
appsettings.json
|
||||
|
||||
# Ignore submodules
|
||||
DiscordWebhookClient/*
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
|
|
@ -353,3 +357,4 @@ MigrationBackup/
|
|||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
seed.secrets.json
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
FROM golang:latest as goose
|
||||
|
||||
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/sdk:7.0.103 as build
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN dotnet restore
|
||||
RUN dotnet build
|
||||
|
||||
FROM build AS publish
|
||||
RUN dotnet publish -c Release -o /app/publish
|
||||
|
||||
RUN dotnet publish -o build
|
||||
#--self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true
|
||||
RUN ls build
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app
|
||||
|
||||
ENV ASPNETCORE_URLS=http://*:5000
|
||||
ENV DOTNET_URLS=http://*:5000
|
||||
|
||||
#RUN apt-get install chromium -y
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN mkdir /migrations
|
||||
COPY --from=publish /app/build /app
|
||||
COPY --from=build ./app/Newsbot.Collector.Database/Migrations/ /app/migrations
|
||||
COPY --from=goose /go/bin/goose /app
|
||||
|
||||
ENTRYPOINT [ "dotnet", "Newsbot.Collector.Api.dll" ]
|
||||
|
|
@ -12,16 +12,14 @@ namespace Newsbot.Collector.Api.Controllers;
|
|||
public class ArticlesController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<ArticlesController> _logger;
|
||||
private readonly ConnectionStrings _settings;
|
||||
private readonly IArticlesRepository _articles;
|
||||
private readonly ISourcesRepository _sources;
|
||||
|
||||
public ArticlesController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings.Value;
|
||||
_articles = new ArticlesTable(_settings.Database);
|
||||
_sources = new SourcesTable(_settings.Database);
|
||||
_articles = new ArticlesTable(settings.Value.Database);
|
||||
_sources = new SourcesTable(settings.Value.Database);
|
||||
}
|
||||
|
||||
[HttpGet(Name = "GetArticles")]
|
||||
|
|
@ -36,14 +34,14 @@ public class ArticlesController : ControllerBase
|
|||
return res;
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
[HttpGet("{id:guid}")]
|
||||
public ArticleDto GetById(Guid id)
|
||||
{
|
||||
var item = _articles.GetById(id);
|
||||
return ArticleDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpGet("{id}/details")]
|
||||
[HttpGet("{id:guid}/details")]
|
||||
public ArticleDetailsDto GetDetailsById(Guid id)
|
||||
{
|
||||
var item = _articles.GetById(id);
|
||||
|
|
@ -51,11 +49,11 @@ public class ArticlesController : ControllerBase
|
|||
return ArticleDetailsDto.Convert(item, sourceItem);
|
||||
}
|
||||
|
||||
[HttpGet("by/{sourceid}")]
|
||||
public IEnumerable<ArticleDto> GetBySourceID(Guid sourceid, int page = 0, int count = 25)
|
||||
[HttpGet("by/{sourceId:guid}")]
|
||||
public IEnumerable<ArticleDto> GetBySourceId(Guid sourceId, int page = 0, int count = 25)
|
||||
{
|
||||
var res = new List<ArticleDto>();
|
||||
var items = _articles.ListBySourceId(sourceid, page, count);
|
||||
var items = _articles.ListBySourceId(sourceId, page, count);
|
||||
foreach (var item in items)
|
||||
{
|
||||
res.Add(ArticleDto.Convert(item));
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ using System.Net;
|
|||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Dto;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
|
||||
|
|
@ -23,33 +24,56 @@ public class DiscordWebHookController : ControllerBase
|
|||
}
|
||||
|
||||
[HttpGet(Name = "GetDiscordWebhooks")]
|
||||
public IEnumerable<DiscordWebHookModel> Get(int page)
|
||||
public IEnumerable<DiscordWebHookDto> Get(int page)
|
||||
{
|
||||
return _webhooks.List(page);
|
||||
var items = new List<DiscordWebHookDto>();
|
||||
var res = _webhooks.List(page);
|
||||
foreach (var item in res)
|
||||
{
|
||||
items.Add(DiscordWebHookDto.Convert(item));
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
[HttpPost(Name = "New")]
|
||||
public DiscordWebHookModel New(string url, string server, string channel)
|
||||
public DiscordWebHookDto New(string url, string server, string channel)
|
||||
{
|
||||
return _webhooks.New(new DiscordWebHookModel
|
||||
var exists = _webhooks.GetByUrl(url);
|
||||
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
|
||||
if (exists is not null)
|
||||
{
|
||||
return DiscordWebHookDto.Convert(exists);
|
||||
}
|
||||
|
||||
var res = _webhooks.New(new DiscordWebHookModel
|
||||
{
|
||||
Url = url,
|
||||
Server = server,
|
||||
Channel = channel,
|
||||
Enabled = true,
|
||||
});
|
||||
|
||||
return DiscordWebHookDto.Convert(res);
|
||||
}
|
||||
|
||||
[HttpGet("by/serverAndChannel")]
|
||||
public IEnumerable<DiscordWebHookModel> GetByServerAndChannel(string server, string channel)
|
||||
public IEnumerable<DiscordWebHookDto> GetByServerAndChannel(string server, string channel)
|
||||
{
|
||||
return _webhooks.ListByServerAndChannel(server, channel, 25);
|
||||
var items = new List<DiscordWebHookDto>();
|
||||
var res = _webhooks.ListByServerAndChannel(server, channel, 25);
|
||||
|
||||
foreach (var item in res)
|
||||
{
|
||||
items.Add(DiscordWebHookDto.Convert(item));
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
public DiscordWebHookModel GetById(Guid id)
|
||||
public DiscordWebHookDto GetById(Guid id)
|
||||
{
|
||||
return _webhooks.GetByID(id);
|
||||
var res = _webhooks.GetByID(id);
|
||||
return DiscordWebHookDto.Convert(res);
|
||||
}
|
||||
|
||||
[HttpPost("{id}/disable")]
|
||||
|
|
|
|||
|
|
@ -50,6 +50,12 @@ public class SourcesController : ControllerBase
|
|||
[HttpPost("new/reddit")]
|
||||
public SourceDto NewReddit(string name, string url)
|
||||
{
|
||||
var res = _sources.GetByNameAndType(name, SourceTypes.Reddit);
|
||||
if (res.ID != Guid.Empty)
|
||||
{
|
||||
return SourceDto.Convert(res);
|
||||
}
|
||||
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.Reddit,
|
||||
|
|
@ -66,7 +72,13 @@ public class SourcesController : ControllerBase
|
|||
[HttpPost("new/rss")]
|
||||
public SourceDto NewRss(string name, string url)
|
||||
{
|
||||
var item = _sources.New(new SourceModel
|
||||
var res = _sources.GetByNameAndType(name, SourceTypes.Rss);
|
||||
if (res.ID != Guid.Empty)
|
||||
{
|
||||
return SourceDto.Convert(res);
|
||||
}
|
||||
|
||||
var m = new SourceModel
|
||||
{
|
||||
Site = SourceTypes.Rss,
|
||||
Name = name,
|
||||
|
|
@ -75,13 +87,20 @@ public class SourcesController : ControllerBase
|
|||
Enabled = true,
|
||||
Url = url,
|
||||
Tags = $"{SourceTypes.Rss}, {name}"
|
||||
});
|
||||
};
|
||||
var item = _sources.New(m);
|
||||
return SourceDto.Convert(item);
|
||||
}
|
||||
|
||||
[HttpPost("new/youtube")]
|
||||
public SourceDto NewYoutube(string name, string url)
|
||||
{
|
||||
var res = _sources.GetByNameAndType(name, SourceTypes.YouTube);
|
||||
if (res.ID != Guid.Empty)
|
||||
{
|
||||
return SourceDto.Convert(res);
|
||||
}
|
||||
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.YouTube,
|
||||
|
|
@ -99,6 +118,12 @@ public class SourcesController : ControllerBase
|
|||
[HttpPost("new/twitch")]
|
||||
public SourceDto NewTwitch(string name)
|
||||
{
|
||||
var res = _sources.GetByNameAndType(name, SourceTypes.Twitch);
|
||||
if (res.ID != Guid.Empty)
|
||||
{
|
||||
return SourceDto.Convert(res);
|
||||
}
|
||||
|
||||
var item = _sources.New(new SourceModel
|
||||
{
|
||||
Site = SourceTypes.Twitch,
|
||||
|
|
|
|||
|
|
@ -12,17 +12,15 @@ namespace Newsbot.Collector.Api.Controllers;
|
|||
public class SubscriptionsController : ControllerBase
|
||||
{
|
||||
private readonly ILogger<ArticlesController> _logger;
|
||||
private readonly ConnectionStrings _settings;
|
||||
private readonly ISubscriptionRepository _subscription;
|
||||
private readonly IDiscordWebHooksRepository _discord;
|
||||
private readonly ISourcesRepository _sources;
|
||||
public SubscriptionsController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings.Value;
|
||||
_subscription = new SubscriptionsTable(_settings.Database);
|
||||
_discord = new DiscordWebhooksTable(_settings.Database);
|
||||
_sources = new SourcesTable(_settings.Database);
|
||||
_subscription = new SubscriptionsTable(settings.Value.Database);
|
||||
_discord = new DiscordWebhooksTable(settings.Value.Database);
|
||||
_sources = new SourcesTable(settings.Value.Database);
|
||||
}
|
||||
|
||||
[HttpGet(Name = "ListSubscriptions")]
|
||||
|
|
@ -71,7 +69,7 @@ public class SubscriptionsController : ControllerBase
|
|||
return res;
|
||||
}
|
||||
|
||||
[HttpGet("by/sourceid")]
|
||||
[HttpGet("by/sourceId")]
|
||||
public IEnumerable<SubscriptionDto> GetBySourceId(Guid id)
|
||||
{
|
||||
var res = new List<SubscriptionDto>();
|
||||
|
|
@ -83,9 +81,16 @@ public class SubscriptionsController : ControllerBase
|
|||
return res;
|
||||
}
|
||||
|
||||
[HttpPost("new")]
|
||||
[HttpPost(Name = "New Subscription")]
|
||||
public SubscriptionDto New(Guid sourceId, Guid discordId)
|
||||
{
|
||||
var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
|
||||
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
|
||||
if (exists is not null)
|
||||
{
|
||||
return SubscriptionDto.Convert(exists);
|
||||
}
|
||||
|
||||
var item = _subscription.New(new SubscriptionModel
|
||||
{
|
||||
ID = Guid.NewGuid(),
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ var builder = WebApplication.CreateBuilder(args);
|
|||
// Define Logger
|
||||
builder.Host.UseSerilog(); // <-- Add this line
|
||||
|
||||
// Build the conifg
|
||||
// Build the config
|
||||
var config = GetConfiguration();
|
||||
builder.Configuration.AddConfiguration(config);
|
||||
|
||||
|
|
@ -35,11 +35,11 @@ builder.Services.Configure<ConnectionStrings>(config.GetSection("ConnectionStrin
|
|||
var app = builder.Build();
|
||||
|
||||
// Configure the HTTP request pipeline.
|
||||
if (app.Environment.IsDevelopment())
|
||||
{
|
||||
app.UseSwagger();
|
||||
app.UseSwaggerUI();
|
||||
}
|
||||
//if (app.Environment.IsDevelopment())
|
||||
//{
|
||||
app.UseSwagger();
|
||||
app.UseSwaggerUI();
|
||||
//}
|
||||
|
||||
app.UseHttpsRedirection();
|
||||
|
||||
|
|
|
|||
|
|
@ -18,11 +18,7 @@ public class DiscordWebhooksTable : IDiscordWebHooksRepository
|
|||
|
||||
public DiscordWebhooksTable(IConfiguration configuration)
|
||||
{
|
||||
var connstr = configuration.GetConnectionString("database");
|
||||
if (connstr is null)
|
||||
{
|
||||
connstr = "";
|
||||
}
|
||||
var connstr = configuration.GetConnectionString("database") ?? "";
|
||||
_connectionString = connstr;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -90,14 +90,14 @@ public class SourcesTable : ISourcesRepository
|
|||
return res.First();
|
||||
}
|
||||
|
||||
public SourceModel GetByNameAndSource(string name, string source)
|
||||
public SourceModel GetByNameAndType(string name, string type)
|
||||
{
|
||||
using var conn = OpenConnection(_connectionString);
|
||||
var query = "Select * from Sources WHERE name = @name and source = @source;";
|
||||
var query = "Select * from Sources WHERE name = @name and type = @type;";
|
||||
var res = conn.Query<SourceModel>(query, new
|
||||
{
|
||||
name = name,
|
||||
source = source
|
||||
type = type
|
||||
});
|
||||
|
||||
if (res.Count() == 0)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ public interface ISourcesRepository
|
|||
public SourceModel GetByID(Guid ID);
|
||||
public SourceModel GetByID(string ID);
|
||||
public SourceModel GetByName(string name);
|
||||
public SourceModel GetByNameAndSource(string name, string source);
|
||||
public SourceModel GetByNameAndType(string name, string type);
|
||||
public List<SourceModel> List(int page, int count);
|
||||
public List<SourceModel> ListBySource(string source, int limit);
|
||||
public List<SourceModel> ListByType(string type, int limit = 25);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
using System.ServiceModel.Syndication;
|
||||
using System.Xml;
|
||||
using Newsbot.Collector.Database.Repositories;
|
||||
using Newsbot.Collector.Domain.Interfaces;
|
||||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Services.HtmlParser;
|
||||
|
||||
namespace Newsbot.Collector.Services.Jobs;
|
||||
|
||||
|
|
@ -8,6 +12,7 @@ public class GithubWatcherJobOptions
|
|||
public string ConnectionString { get; set; } = "";
|
||||
public bool FeaturePullReleases { get; set; } = false;
|
||||
public bool FeaturePullCommits { get; set; } = false;
|
||||
public bool PullIssues { get; set; } = false;
|
||||
}
|
||||
|
||||
public class GithubWatcherJob
|
||||
|
|
@ -23,7 +28,7 @@ public class GithubWatcherJob
|
|||
_source = new SourcesTable("");
|
||||
}
|
||||
|
||||
private void Init(GithubWatcherJobOptions options)
|
||||
public void Init(GithubWatcherJobOptions options)
|
||||
{
|
||||
_articles = new ArticlesTable(options.ConnectionString);
|
||||
_queue = new DiscordQueueTable(options.ConnectionString);
|
||||
|
|
@ -33,10 +38,72 @@ public class GithubWatcherJob
|
|||
public void InitAndExecute(GithubWatcherJobOptions options)
|
||||
{
|
||||
Init(options);
|
||||
Execute();
|
||||
}
|
||||
|
||||
private void Execute()
|
||||
{
|
||||
// query sources for things to pull
|
||||
var items = new List<ArticlesModel>();
|
||||
|
||||
items.AddRange(Collect(new Uri("https://github.com/jtom38/dvb")));
|
||||
|
||||
// query */commits/master.atom
|
||||
// query */commits/main.atom
|
||||
}
|
||||
|
||||
public List<ArticlesModel> Collect(Uri url)
|
||||
{
|
||||
var items = new List<ArticlesModel>();
|
||||
|
||||
Guid placeHolderId = Guid.NewGuid();
|
||||
// query */release.atom
|
||||
// query */commits.atom
|
||||
items.AddRange(CollectItems($"{url.AbsoluteUri}/releases.atom", placeHolderId));
|
||||
items.AddRange(CollectItems($"{url.AbsoluteUri}/master.atom", placeHolderId));
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private List<ArticlesModel> CollectItems(string baseUrl, Guid sourceId)
|
||||
{
|
||||
var items = new List<ArticlesModel>();
|
||||
|
||||
using var reader = XmlReader.Create(baseUrl);
|
||||
var client = SyndicationFeed.Load(reader);
|
||||
|
||||
foreach (var item in client.Items)
|
||||
{
|
||||
var itemUrl = item.Links[0].Uri.AbsoluteUri;
|
||||
var exits = _articles.GetByUrl(itemUrl);
|
||||
if (exits.ID != Guid.Empty)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var parser = new HtmlPageReader(itemUrl);
|
||||
parser.Parse();
|
||||
|
||||
try
|
||||
{
|
||||
var a = new ArticlesModel
|
||||
{
|
||||
SourceID = sourceId,
|
||||
Tags = "github",
|
||||
Title = item.Title.Text,
|
||||
URL = itemUrl,
|
||||
//PubDate = item.LastUpdatedTime.DateTime,
|
||||
Thumbnail = parser.Data.Header.Image,
|
||||
Description = $"'dvb' has released '{item.Title.Text}'!",
|
||||
AuthorName = item.Authors[0].Name ?? "",
|
||||
AuthorImage = item.Authors[0].Uri ?? ""
|
||||
};
|
||||
items.Add(a);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
return items;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
using Newsbot.Collector.Domain.Models;
|
||||
using Newsbot.Collector.Services.Jobs;
|
||||
using Newsbot.Collector.Services.Notifications.Discord;
|
||||
|
||||
namespace Newsbot.Collector.Tests.Jobs;
|
||||
|
||||
public class DiscordNotificationJobTest
|
||||
{
|
||||
|
||||
[Fact]
|
||||
public void PostTestMessage()
|
||||
{
|
||||
var uri = "";
|
||||
var webhookClient = new DiscordWebhookClient(uri);
|
||||
|
||||
var client = new DiscordNotificationJob();
|
||||
var msg = client.GenerateDiscordMessage(new SourceModel
|
||||
{
|
||||
ID = Guid.NewGuid(),
|
||||
Site = "Unit Test",
|
||||
Source = "placeholder",
|
||||
Type = "a",
|
||||
Value = "a",
|
||||
Enabled = true,
|
||||
Url = "https://github.com",
|
||||
Tags = "Unit, Testing",
|
||||
},
|
||||
new ArticlesModel
|
||||
{
|
||||
Tags = "more,unit,testing",
|
||||
Title = "Nope not real",
|
||||
URL = "https://github.com/jtom38",
|
||||
PubDate = DateTime.Now,
|
||||
Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
|
||||
Description = "Please work",
|
||||
AuthorName = "No one knows"
|
||||
});
|
||||
webhookClient.SendMessage(msg);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
using Microsoft.Extensions.Configuration;
|
||||
using Newsbot.Collector.Services.Jobs;
|
||||
|
||||
namespace Newsbot.Collector.Tests.Jobs;
|
||||
|
||||
public class GithubWatcherJobTests
|
||||
{
|
||||
private IConfiguration GetConfiguration()
|
||||
{
|
||||
var inMemorySettings = new Dictionary<string, string> {
|
||||
{"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
|
||||
};
|
||||
|
||||
IConfiguration configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(inMemorySettings)
|
||||
.Build();
|
||||
return configuration;
|
||||
}
|
||||
|
||||
private string ConnectionString()
|
||||
{
|
||||
return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanPullAFeed()
|
||||
{
|
||||
var client = new GithubWatcherJob();
|
||||
client.Init(new GithubWatcherJobOptions
|
||||
{
|
||||
ConnectionString = ConnectionString(),
|
||||
FeaturePullCommits = true,
|
||||
FeaturePullReleases = true
|
||||
});
|
||||
client.Collect(new Uri("https://github.com/jtom38/dvb"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
version: "3"
|
||||
|
||||
networks:
|
||||
newsbot:
|
||||
|
||||
volumes:
|
||||
db:
|
||||
|
||||
services:
|
||||
#db:
|
||||
# image: postgres:latest
|
||||
# #ports:
|
||||
# # - "5432:5432"
|
||||
# networks:
|
||||
# - newsbot
|
||||
# volumes:
|
||||
# - db:/var/lib/postgresql/data
|
||||
|
||||
api:
|
||||
image: newsbot.collector:latest
|
||||
environment:
|
||||
# Used for database migrations
|
||||
GOOSE_DRIVER: "postgres"
|
||||
GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
|
||||
|
||||
SERVER_ADDRESS: "localhost"
|
||||
|
||||
Logging__LogLevel__Default: "Information"
|
||||
Logging__LogLevel__Microsoft.AspNetCore: "Warning"
|
||||
Logging__LogLevel__Hangfire: "Information"
|
||||
|
||||
ConnectionStrings__Database: "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"
|
||||
|
||||
# Enable/Disable Reddit monitoring
|
||||
Reddit__IsEnabled: false
|
||||
Reddit__PullHot: true
|
||||
Reddit__PullNsfw: true
|
||||
Reddit__PullTop: true
|
||||
|
||||
# Enable/Disable YouTube monitoring
|
||||
Youtube__IsEnabled: false
|
||||
|
||||
TWITCH__IsEnabled: false
|
||||
# Set your Twitch Developer ID and Secrets here and they will be used to collect updates.
|
||||
TWITCH__ClientID: ""
|
||||
TWITCH__ClientSecret: ""
|
||||
|
||||
# If you want to collect news on Final Fantasy XIV, set this to true
|
||||
FFXIV__IsEnabled: false
|
||||
ports:
|
||||
- "5001:5000"
|
||||
networks:
|
||||
- newsbot
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
param (
|
||||
[string] $ApiServer = "http://localhost:5011",
|
||||
[string] $JsonSecrets = "./seed.secrets.json"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function NewRedditSource {
|
||||
param (
|
||||
[string] $Name,
|
||||
[string] $Url
|
||||
)
|
||||
$urlEncoded = [uri]::EscapeDataString($Url)
|
||||
|
||||
$param = "name=$Name&url=$urlEncoded"
|
||||
$uri = "$ApiServer/api/sources/new/reddit?$param"
|
||||
$res = Invoke-RestMethod -Method Post -Uri $uri
|
||||
return $res
|
||||
}
|
||||
|
||||
function NewRssSource {
|
||||
param (
|
||||
[string] $Name,
|
||||
[string] $Url
|
||||
)
|
||||
$urlEncoded = [uri]::EscapeDataString($Url)
|
||||
$param = "name=$Name&url=$urlEncoded"
|
||||
[string] $uri = "$ApiServer/api/sources/new/rss?$param"
|
||||
$res = Invoke-RestMethod -Method Post -Uri $uri
|
||||
return $res
|
||||
}
|
||||
|
||||
function NewYoutubeSource {
|
||||
param (
|
||||
[string] $Name,
|
||||
[string] $Url
|
||||
)
|
||||
$urlEncoded = [uri]::EscapeDataString($Url)
|
||||
[string] $param = "name=$Name&url=$urlEncoded"
|
||||
[string] $uri = "$ApiServer/api/sources/new/youtube?$param"
|
||||
$res = Invoke-RestMethod -Method Post -Uri $uri
|
||||
return $res
|
||||
}
|
||||
|
||||
function NewTwitchSource {
|
||||
param (
|
||||
[string] $Name
|
||||
)
|
||||
[string] $param = "name=$Name"
|
||||
[string] $uri = "$ApiServer/api/sources/new/twitch?$param"
|
||||
$res = Invoke-RestMethod -Method Post -Uri $uri
|
||||
return $res
|
||||
}
|
||||
|
||||
function New-DiscordWebhook {
|
||||
param (
|
||||
[string] $Server,
|
||||
[string] $Channel,
|
||||
[string] $Url
|
||||
)
|
||||
$urlEncoded = [uri]::EscapeDataString($Url)
|
||||
[string] $param = "server=$Server&channel=$Channel&url=$urlEncoded"
|
||||
[string] $uri = "$ApiServer/api/discord/webhooks?$param"
|
||||
Write-Host $uri
|
||||
$res = Invoke-RestMethod -Method Post -Uri $uri
|
||||
return $res
|
||||
}
|
||||
|
||||
function New-Subscription {
|
||||
param (
|
||||
[string] $SourceId,
|
||||
[string] $DiscordWebhookId
|
||||
)
|
||||
[string] $param = "sourceId=$SourceId&discordId=$DiscordWebhookId"
|
||||
[string] $uri = "$ApiServer/api/subscriptions?$param"
|
||||
$res = Invoke-RestMethod -Method Post -Uri $uri
|
||||
return $res
|
||||
}
|
||||
|
||||
# Load Secrets file
|
||||
$secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json
|
||||
|
||||
$redditDadJokes = NewRedditSource -Name "dadjokes" -Url "https://reddit.com/r/dadjokes"
|
||||
$redditSteamDeck = NewRedditSource -Name "steamdeck" -Url "https://reddit.com/r/steamdeck"
|
||||
|
||||
$rssSteamDeck = NewRssSource -Name "Steampowered - Steam Deck" -Url "https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107"
|
||||
$rssFaysHaremporium = NewRssSource -Name "Fay's Haremporium" -Url "https://blog.nyxstudios.moe/rss/"
|
||||
$rssPodcastLetsMosley = NewRssSource -Name "Let's Mosley" -Url "https://anchor.fm/s/6c7aa4c4/podcast/rss"
|
||||
|
||||
$youtubeGameGrumps = NewYoutubeSource -Name "Game Grumps" -Url "https://www.youtube.com/user/GameGrumps"
|
||||
$youtubeCityPlannerPlays = NewYoutubeSource -Name "City Planner Plays" -Url "https://www.youtube.com/c/cityplannerplays"
|
||||
|
||||
$twitchNintendo = NewTwitchSource -Name "Nintendo"
|
||||
$twitchNintendo.id
|
||||
|
||||
$miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01
|
||||
|
||||
New-Subscription -SourceId $redditDadJokes.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $redditSteamDeck.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $rssSteamDeck.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $rssFaysHaremporium.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id
|
||||
New-Subscription -SourceId $twitchNintendo.id -DiscordWebhookId $miharuMonitor.id
|
||||
Loading…
Reference in New Issue