From 799668a05916effb23b2d9f59e3fef027f6f48cf Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sat, 11 Mar 2023 10:43:06 -0800 Subject: [PATCH] Features/pulling GitHub (#9) * Still working though it but looking good on releases * added the example discord message test * updated source repo to return an existing record before a new is added * updated the sources repo interface * updated new routes to check for existing records * starting to migrate the seed out of the sql migrations * A new seed script was made to reload the db from the api * Docker image works locally * Adding CI to build docker image * ... disabled swagger so I can test docker * Added more to the github job but its not finished. Isnt pulling sources yet. * cleaned up formatting * Controller updates to look for existing records when requesting a new one * null check cleanup * namespace fix --- .DockerIgnore | 3 + .github/workflows/build-docker-image.yml | 64 +++++++++++ .gitignore | 5 + Dockerfile | 34 ++++++ .../Controllers/ArticlesController.cs | 16 ++- .../Controllers/DiscordWebHooksController.cs | 40 +++++-- .../Controllers/SourcesController.cs | 29 ++++- .../Controllers/SubscriptionsController.cs | 19 ++-- Newsbot.Collector.Api/Program.cs | 12 +- .../Repositories/DiscordWebhooksTable.cs | 6 +- .../Repositories/SourcesTable.cs | 6 +- .../Interfaces/ISourcesRepository.cs | 2 +- .../Jobs/GithubWatcherJob.cs | 71 +++++++++++- .../Jobs/DiscordNotificationJobTest.cs | 40 +++++++ .../Jobs/GithubWatcherJobTests.cs | 37 ++++++ docker-compose.yaml | 53 +++++++++ seed.ps1 | 105 ++++++++++++++++++ 17 files changed, 499 insertions(+), 43 deletions(-) create mode 100644 .DockerIgnore create mode 100644 .github/workflows/build-docker-image.yml create mode 100644 Dockerfile create mode 100644 Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs create mode 100644 Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs create mode 100644 docker-compose.yaml create mode 100644 seed.ps1 diff --git a/.DockerIgnore b/.DockerIgnore new file mode 100644 index 0000000..07c6bb4 --- /dev/null +++ b/.DockerIgnore @@ -0,0 +1,3 @@ +appsettings.json +**/bin/ +**/obj/ \ No newline at end of file diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml new file mode 100644 index 0000000..7b4e750 --- /dev/null +++ b/.github/workflows/build-docker-image.yml @@ -0,0 +1,64 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + #schedule: + # - cron: '21 19 * * *' + push: + branches: [ master ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + #pull_request: + # branches: [ master ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + #images: ${{ env.REGISTRY }}/newsbot.worker + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index e5466fc..b73bc23 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore +seed.secrects.json out/ # User-specific files @@ -15,6 +16,9 @@ out/ appsettings.Development.json appsettings.json +# Ignore submodules +DiscordWebhookClient/* + # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs @@ -353,3 +357,4 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ +seed.secrets.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6ed4123 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM golang:latest as goose + +RUN go install github.com/pressly/goose/v3/cmd/goose@latest + +FROM mcr.microsoft.com/dotnet/sdk:7.0.103 as build + +COPY . /app +WORKDIR /app + +RUN dotnet restore +RUN dotnet build + +FROM build AS publish +RUN dotnet publish -c Release -o /app/publish + +RUN dotnet publish -o build +#--self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true +RUN ls build + +FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app + +ENV ASPNETCORE_URLS=http://*:5000 +ENV DOTNET_URLS=http://*:5000 + +#RUN apt-get install chromium -y + +WORKDIR /app + +RUN mkdir /migrations +COPY --from=publish /app/build /app +COPY --from=build ./app/Newsbot.Collector.Database/Migrations/ /app/migrations +COPY --from=goose /go/bin/goose /app + +ENTRYPOINT [ "dotnet", "Newsbot.Collector.Api.dll" ] \ No newline at end of file diff --git a/Newsbot.Collector.Api/Controllers/ArticlesController.cs b/Newsbot.Collector.Api/Controllers/ArticlesController.cs index b5fefc3..0867e01 100644 --- a/Newsbot.Collector.Api/Controllers/ArticlesController.cs +++ b/Newsbot.Collector.Api/Controllers/ArticlesController.cs @@ -12,16 +12,14 @@ namespace Newsbot.Collector.Api.Controllers; public class ArticlesController : ControllerBase { private readonly ILogger _logger; - private readonly ConnectionStrings _settings; private readonly IArticlesRepository _articles; private readonly ISourcesRepository _sources; public ArticlesController(ILogger logger, IOptions settings) { _logger = logger; - _settings = settings.Value; - _articles = new ArticlesTable(_settings.Database); - _sources = new SourcesTable(_settings.Database); + _articles = new ArticlesTable(settings.Value.Database); + _sources = new SourcesTable(settings.Value.Database); } [HttpGet(Name = "GetArticles")] @@ -36,14 +34,14 @@ public class ArticlesController : ControllerBase return res; } - [HttpGet("{id}")] + [HttpGet("{id:guid}")] public ArticleDto GetById(Guid id) { var item = _articles.GetById(id); return ArticleDto.Convert(item); } - [HttpGet("{id}/details")] + [HttpGet("{id:guid}/details")] public ArticleDetailsDto GetDetailsById(Guid id) { var item = _articles.GetById(id); @@ -51,11 +49,11 @@ public class ArticlesController : ControllerBase return ArticleDetailsDto.Convert(item, sourceItem); } - [HttpGet("by/{sourceid}")] - public IEnumerable GetBySourceID(Guid sourceid, int page = 0, int count = 25) + [HttpGet("by/{sourceId:guid}")] + public IEnumerable GetBySourceId(Guid sourceId, int page = 0, int count = 25) { var res = new List(); - var items = _articles.ListBySourceId(sourceid, page, count); + var items = _articles.ListBySourceId(sourceId, page, count); foreach (var item in items) { res.Add(ArticleDto.Convert(item)); diff --git a/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs b/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs index 33807c1..5ba1d9c 100644 --- a/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs +++ b/Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs @@ -2,6 +2,7 @@ using System.Net; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Newsbot.Collector.Database.Repositories; +using Newsbot.Collector.Domain.Dto; using Newsbot.Collector.Domain.Interfaces; using Newsbot.Collector.Domain.Models; @@ -23,33 +24,56 @@ public class DiscordWebHookController : ControllerBase } [HttpGet(Name = "GetDiscordWebhooks")] - public IEnumerable Get(int page) + public IEnumerable Get(int page) { - return _webhooks.List(page); + var items = new List(); + var res = _webhooks.List(page); + foreach (var item in res) + { + items.Add(DiscordWebHookDto.Convert(item)); + } + return items; } [HttpPost(Name = "New")] - public DiscordWebHookModel New(string url, string server, string channel) + public DiscordWebHookDto New(string url, string server, string channel) { - return _webhooks.New(new DiscordWebHookModel + var exists = _webhooks.GetByUrl(url); + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (exists is not null) + { + return DiscordWebHookDto.Convert(exists); + } + + var res = _webhooks.New(new DiscordWebHookModel { Url = url, Server = server, Channel = channel, Enabled = true, }); + + return DiscordWebHookDto.Convert(res); } [HttpGet("by/serverAndChannel")] - public IEnumerable GetByServerAndChannel(string server, string channel) + public IEnumerable GetByServerAndChannel(string server, string channel) { - return _webhooks.ListByServerAndChannel(server, channel, 25); + var items = new List(); + var res = _webhooks.ListByServerAndChannel(server, channel, 25); + + foreach (var item in res) + { + items.Add(DiscordWebHookDto.Convert(item)); + } + return items; } [HttpGet("{id}")] - public DiscordWebHookModel GetById(Guid id) + public DiscordWebHookDto GetById(Guid id) { - return _webhooks.GetByID(id); + var res = _webhooks.GetByID(id); + return DiscordWebHookDto.Convert(res); } [HttpPost("{id}/disable")] diff --git a/Newsbot.Collector.Api/Controllers/SourcesController.cs b/Newsbot.Collector.Api/Controllers/SourcesController.cs index 5e8a039..b09cdc7 100644 --- a/Newsbot.Collector.Api/Controllers/SourcesController.cs +++ b/Newsbot.Collector.Api/Controllers/SourcesController.cs @@ -50,6 +50,12 @@ public class SourcesController : ControllerBase [HttpPost("new/reddit")] public SourceDto NewReddit(string name, string url) { + var res = _sources.GetByNameAndType(name, SourceTypes.Reddit); + if (res.ID != Guid.Empty) + { + return SourceDto.Convert(res); + } + var item = _sources.New(new SourceModel { Site = SourceTypes.Reddit, @@ -66,7 +72,13 @@ public class SourcesController : ControllerBase [HttpPost("new/rss")] public SourceDto NewRss(string name, string url) { - var item = _sources.New(new SourceModel + var res = _sources.GetByNameAndType(name, SourceTypes.Rss); + if (res.ID != Guid.Empty) + { + return SourceDto.Convert(res); + } + + var m = new SourceModel { Site = SourceTypes.Rss, Name = name, @@ -75,13 +87,20 @@ public class SourcesController : ControllerBase Enabled = true, Url = url, Tags = $"{SourceTypes.Rss}, {name}" - }); + }; + var item = _sources.New(m); return SourceDto.Convert(item); } [HttpPost("new/youtube")] public SourceDto NewYoutube(string name, string url) { + var res = _sources.GetByNameAndType(name, SourceTypes.YouTube); + if (res.ID != Guid.Empty) + { + return SourceDto.Convert(res); + } + var item = _sources.New(new SourceModel { Site = SourceTypes.YouTube, @@ -99,6 +118,12 @@ public class SourcesController : ControllerBase [HttpPost("new/twitch")] public SourceDto NewTwitch(string name) { + var res = _sources.GetByNameAndType(name, SourceTypes.Twitch); + if (res.ID != Guid.Empty) + { + return SourceDto.Convert(res); + } + var item = _sources.New(new SourceModel { Site = SourceTypes.Twitch, diff --git a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs index 38eaccd..b490e4e 100644 --- a/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs +++ b/Newsbot.Collector.Api/Controllers/SubscriptionsController.cs @@ -12,17 +12,15 @@ namespace Newsbot.Collector.Api.Controllers; public class SubscriptionsController : ControllerBase { private readonly ILogger _logger; - private readonly ConnectionStrings _settings; private readonly ISubscriptionRepository _subscription; private readonly IDiscordWebHooksRepository _discord; private readonly ISourcesRepository _sources; public SubscriptionsController(ILogger logger, IOptions settings) { _logger = logger; - _settings = settings.Value; - _subscription = new SubscriptionsTable(_settings.Database); - _discord = new DiscordWebhooksTable(_settings.Database); - _sources = new SourcesTable(_settings.Database); + _subscription = new SubscriptionsTable(settings.Value.Database); + _discord = new DiscordWebhooksTable(settings.Value.Database); + _sources = new SourcesTable(settings.Value.Database); } [HttpGet(Name = "ListSubscriptions")] @@ -71,7 +69,7 @@ public class SubscriptionsController : ControllerBase return res; } - [HttpGet("by/sourceid")] + [HttpGet("by/sourceId")] public IEnumerable GetBySourceId(Guid id) { var res = new List(); @@ -83,9 +81,16 @@ public class SubscriptionsController : ControllerBase return res; } - [HttpPost("new")] + [HttpPost(Name = "New Subscription")] public SubscriptionDto New(Guid sourceId, Guid discordId) { + var exists = _subscription.GetByWebhookAndSource(discordId, sourceId); + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (exists is not null) + { + return SubscriptionDto.Convert(exists); + } + var item = _subscription.New(new SubscriptionModel { ID = Guid.NewGuid(), diff --git a/Newsbot.Collector.Api/Program.cs b/Newsbot.Collector.Api/Program.cs index 75f0334..1fe67e6 100644 --- a/Newsbot.Collector.Api/Program.cs +++ b/Newsbot.Collector.Api/Program.cs @@ -16,7 +16,7 @@ var builder = WebApplication.CreateBuilder(args); // Define Logger builder.Host.UseSerilog(); // <-- Add this line -// Build the conifg +// Build the config var config = GetConfiguration(); builder.Configuration.AddConfiguration(config); @@ -35,11 +35,11 @@ builder.Services.Configure(config.GetSection("ConnectionStrin var app = builder.Build(); // Configure the HTTP request pipeline. -if (app.Environment.IsDevelopment()) -{ - app.UseSwagger(); - app.UseSwaggerUI(); -} +//if (app.Environment.IsDevelopment()) +//{ +app.UseSwagger(); +app.UseSwaggerUI(); +//} app.UseHttpsRedirection(); diff --git a/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs b/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs index 0af712a..0594d5e 100644 --- a/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs +++ b/Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs @@ -18,11 +18,7 @@ public class DiscordWebhooksTable : IDiscordWebHooksRepository public DiscordWebhooksTable(IConfiguration configuration) { - var connstr = configuration.GetConnectionString("database"); - if (connstr is null) - { - connstr = ""; - } + var connstr = configuration.GetConnectionString("database") ?? ""; _connectionString = connstr; } diff --git a/Newsbot.Collector.Database/Repositories/SourcesTable.cs b/Newsbot.Collector.Database/Repositories/SourcesTable.cs index e05e9bd..4657f68 100644 --- a/Newsbot.Collector.Database/Repositories/SourcesTable.cs +++ b/Newsbot.Collector.Database/Repositories/SourcesTable.cs @@ -90,14 +90,14 @@ public class SourcesTable : ISourcesRepository return res.First(); } - public SourceModel GetByNameAndSource(string name, string source) + public SourceModel GetByNameAndType(string name, string type) { using var conn = OpenConnection(_connectionString); - var query = "Select * from Sources WHERE name = @name and source = @source;"; + var query = "Select * from Sources WHERE name = @name and type = @type;"; var res = conn.Query(query, new { name = name, - source = source + type = type }); if (res.Count() == 0) diff --git a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs index 9eed6ef..1fa8527 100644 --- a/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs +++ b/Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs @@ -9,7 +9,7 @@ public interface ISourcesRepository public SourceModel GetByID(Guid ID); public SourceModel GetByID(string ID); public SourceModel GetByName(string name); - public SourceModel GetByNameAndSource(string name, string source); + public SourceModel GetByNameAndType(string name, string type); public List List(int page, int count); public List ListBySource(string source, int limit); public List ListByType(string type, int limit = 25); diff --git a/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs b/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs index 41cac9b..be61d52 100644 --- a/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs +++ b/Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs @@ -1,5 +1,9 @@ +using System.ServiceModel.Syndication; +using System.Xml; using Newsbot.Collector.Database.Repositories; using Newsbot.Collector.Domain.Interfaces; +using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Services.HtmlParser; namespace Newsbot.Collector.Services.Jobs; @@ -8,6 +12,7 @@ public class GithubWatcherJobOptions public string ConnectionString { get; set; } = ""; public bool FeaturePullReleases { get; set; } = false; public bool FeaturePullCommits { get; set; } = false; + public bool PullIssues { get; set; } = false; } public class GithubWatcherJob @@ -23,7 +28,7 @@ public class GithubWatcherJob _source = new SourcesTable(""); } - private void Init(GithubWatcherJobOptions options) + public void Init(GithubWatcherJobOptions options) { _articles = new ArticlesTable(options.ConnectionString); _queue = new DiscordQueueTable(options.ConnectionString); @@ -33,10 +38,72 @@ public class GithubWatcherJob public void InitAndExecute(GithubWatcherJobOptions options) { Init(options); + Execute(); + } + private void Execute() + { // query sources for things to pull + var items = new List(); + items.AddRange(Collect(new Uri("https://github.com/jtom38/dvb"))); + + // query */commits/master.atom + // query */commits/main.atom + } + + public List Collect(Uri url) + { + var items = new List(); + + Guid placeHolderId = Guid.NewGuid(); // query */release.atom - // query */commits.atom + items.AddRange(CollectItems($"{url.AbsoluteUri}/releases.atom", placeHolderId)); + items.AddRange(CollectItems($"{url.AbsoluteUri}/master.atom", placeHolderId)); + + return items; + } + + private List CollectItems(string baseUrl, Guid sourceId) + { + var items = new List(); + + using var reader = XmlReader.Create(baseUrl); + var client = SyndicationFeed.Load(reader); + + foreach (var item in client.Items) + { + var itemUrl = item.Links[0].Uri.AbsoluteUri; + var exits = _articles.GetByUrl(itemUrl); + if (exits.ID != Guid.Empty) + { + continue; + } + + var parser = new HtmlPageReader(itemUrl); + parser.Parse(); + + try + { + var a = new ArticlesModel + { + SourceID = sourceId, + Tags = "github", + Title = item.Title.Text, + URL = itemUrl, + //PubDate = item.LastUpdatedTime.DateTime, + Thumbnail = parser.Data.Header.Image, + Description = $"'dvb' has released '{item.Title.Text}'!", + AuthorName = item.Authors[0].Name ?? "", + AuthorImage = item.Authors[0].Uri ?? "" + }; + items.Add(a); + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + return items; } } \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs new file mode 100644 index 0000000..38175ac --- /dev/null +++ b/Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs @@ -0,0 +1,40 @@ +using Newsbot.Collector.Domain.Models; +using Newsbot.Collector.Services.Jobs; +using Newsbot.Collector.Services.Notifications.Discord; + +namespace Newsbot.Collector.Tests.Jobs; + +public class DiscordNotificationJobTest +{ + + [Fact] + public void PostTestMessage() + { + var uri = ""; + var webhookClient = new DiscordWebhookClient(uri); + + var client = new DiscordNotificationJob(); + var msg = client.GenerateDiscordMessage(new SourceModel + { + ID = Guid.NewGuid(), + Site = "Unit Test", + Source = "placeholder", + Type = "a", + Value = "a", + Enabled = true, + Url = "https://github.com", + Tags = "Unit, Testing", + }, + new ArticlesModel + { + Tags = "more,unit,testing", + Title = "Nope not real", + URL = "https://github.com/jtom38", + PubDate = DateTime.Now, + Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg", + Description = "Please work", + AuthorName = "No one knows" + }); + webhookClient.SendMessage(msg); + } +} \ No newline at end of file diff --git a/Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs b/Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs new file mode 100644 index 0000000..b916d73 --- /dev/null +++ b/Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.Configuration; +using Newsbot.Collector.Services.Jobs; + +namespace Newsbot.Collector.Tests.Jobs; + +public class GithubWatcherJobTests +{ + private IConfiguration GetConfiguration() + { + var inMemorySettings = new Dictionary { + {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"} + }; + + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + return configuration; + } + + private string ConnectionString() + { + return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"; + } + + [Fact] + public void CanPullAFeed() + { + var client = new GithubWatcherJob(); + client.Init(new GithubWatcherJobOptions + { + ConnectionString = ConnectionString(), + FeaturePullCommits = true, + FeaturePullReleases = true + }); + client.Collect(new Uri("https://github.com/jtom38/dvb")); + } +} \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..296abb8 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,53 @@ +version: "3" + +networks: + newsbot: + +volumes: + db: + +services: + #db: + # image: postgres:latest + # #ports: + # # - "5432:5432" + # networks: + # - newsbot + # volumes: + # - db:/var/lib/postgresql/data + + api: + image: newsbot.collector:latest + environment: + # Used for database migrations + GOOSE_DRIVER: "postgres" + GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" + + SERVER_ADDRESS: "localhost" + + Logging__LogLevel__Default: "Information" + Logging__LogLevel__Microsoft.AspNetCore: "Warning" + Logging__LogLevel__Hangfire: "Information" + + ConnectionStrings__Database: "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable" + + # Enable/Disable Reddit monitoring + Reddit__IsEnabled: false + Reddit__PullHot: true + Reddit__PullNsfw: true + Reddit__PullTop: true + + # Enable/Disable YouTube monitoring + Youtube__IsEnabled: false + + TWITCH__IsEnabled: false + # Set your Twitch Developer ID and Secrets here and they will be used to collect updates. + TWITCH__ClientID: "" + TWITCH__ClientSecret: "" + + # If you want to collect news on Final Fantasy XIV, set this to true + FFXIV__IsEnabled: false + ports: + - "5001:5000" + networks: + - newsbot \ No newline at end of file diff --git a/seed.ps1 b/seed.ps1 new file mode 100644 index 0000000..1dbb74e --- /dev/null +++ b/seed.ps1 @@ -0,0 +1,105 @@ +param ( + [string] $ApiServer = "http://localhost:5011", + [string] $JsonSecrets = "./seed.secrets.json" +) + +$ErrorActionPreference = 'Stop' + +function NewRedditSource { + param ( + [string] $Name, + [string] $Url + ) + $urlEncoded = [uri]::EscapeDataString($Url) + + $param = "name=$Name&url=$urlEncoded" + $uri = "$ApiServer/api/sources/new/reddit?$param" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + +function NewRssSource { + param ( + [string] $Name, + [string] $Url + ) + $urlEncoded = [uri]::EscapeDataString($Url) + $param = "name=$Name&url=$urlEncoded" + [string] $uri = "$ApiServer/api/sources/new/rss?$param" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + +function NewYoutubeSource { + param ( + [string] $Name, + [string] $Url + ) + $urlEncoded = [uri]::EscapeDataString($Url) + [string] $param = "name=$Name&url=$urlEncoded" + [string] $uri = "$ApiServer/api/sources/new/youtube?$param" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + +function NewTwitchSource { + param ( + [string] $Name + ) + [string] $param = "name=$Name" + [string] $uri = "$ApiServer/api/sources/new/twitch?$param" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + +function New-DiscordWebhook { + param ( + [string] $Server, + [string] $Channel, + [string] $Url + ) + $urlEncoded = [uri]::EscapeDataString($Url) + [string] $param = "server=$Server&channel=$Channel&url=$urlEncoded" + [string] $uri = "$ApiServer/api/discord/webhooks?$param" + Write-Host $uri + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + +function New-Subscription { + param ( + [string] $SourceId, + [string] $DiscordWebhookId + ) + [string] $param = "sourceId=$SourceId&discordId=$DiscordWebhookId" + [string] $uri = "$ApiServer/api/subscriptions?$param" + $res = Invoke-RestMethod -Method Post -Uri $uri + return $res +} + +# Load Secrets file +$secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json + +$redditDadJokes = NewRedditSource -Name "dadjokes" -Url "https://reddit.com/r/dadjokes" +$redditSteamDeck = NewRedditSource -Name "steamdeck" -Url "https://reddit.com/r/steamdeck" + +$rssSteamDeck = NewRssSource -Name "Steampowered - Steam Deck" -Url "https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107" +$rssFaysHaremporium = NewRssSource -Name "Fay's Haremporium" -Url "https://blog.nyxstudios.moe/rss/" +$rssPodcastLetsMosley = NewRssSource -Name "Let's Mosley" -Url "https://anchor.fm/s/6c7aa4c4/podcast/rss" + +$youtubeGameGrumps = NewYoutubeSource -Name "Game Grumps" -Url "https://www.youtube.com/user/GameGrumps" +$youtubeCityPlannerPlays = NewYoutubeSource -Name "City Planner Plays" -Url "https://www.youtube.com/c/cityplannerplays" + +$twitchNintendo = NewTwitchSource -Name "Nintendo" +$twitchNintendo.id + +$miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01 + +New-Subscription -SourceId $redditDadJokes.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $redditSteamDeck.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $rssSteamDeck.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $rssFaysHaremporium.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id +New-Subscription -SourceId $twitchNintendo.id -DiscordWebhookId $miharuMonitor.id