Features/pulling GitHub (#9)

* Still working though it but looking good on releases

* added the example discord message test

* updated source repo to return an existing record before a new is added

* updated the sources repo interface

* updated new routes to check for existing records

* starting to migrate the seed out of the sql migrations

* A new seed script was made to reload the db from the api

* Docker image works locally

* Adding CI to build docker image

* ... disabled swagger so I can test docker

* Added more to the github job but its not finished.  Isnt pulling sources yet.

* cleaned up formatting

* Controller updates to look for existing records when requesting a new one

* null check cleanup

* namespace fix

.DockerIgnore

@@ -0,0 +1,3 @@
+appsettings.json
+**/bin/
+**/obj/

.github/workflows/build-docker-image.yml

@@ -0,0 +1,64 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  #schedule:
+  #  - cron: '21 19 * * *'
+  push:
+    branches: [ master ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  #pull_request:
+  #  branches: [ master ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          #images: ${{ env.REGISTRY }}/newsbot.worker
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -3,6 +3,7 @@
 ##
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
 
+seed.secrects.json
 out/
 
 # User-specific files
@@ -15,6 +16,9 @@ out/
 appsettings.Development.json
 appsettings.json
 
+# Ignore submodules
+DiscordWebhookClient/*
+
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs
 
@@ -353,3 +357,4 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+seed.secrets.json

Dockerfile

@@ -0,0 +1,34 @@
+FROM golang:latest as goose
+
+RUN go install github.com/pressly/goose/v3/cmd/goose@latest
+
+FROM mcr.microsoft.com/dotnet/sdk:7.0.103 as build
+
+COPY . /app
+WORKDIR /app
+
+RUN dotnet restore
+RUN dotnet build
+
+FROM build AS publish
+RUN dotnet publish -c Release -o /app/publish
+
+RUN dotnet publish -o build
+#--self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true
+RUN ls build
+
+FROM mcr.microsoft.com/dotnet/aspnet:7.0.3 as app
+
+ENV ASPNETCORE_URLS=http://*:5000
+ENV DOTNET_URLS=http://*:5000
+
+#RUN apt-get install chromium -y
+
+WORKDIR /app
+
+RUN mkdir /migrations
+COPY --from=publish /app/build /app
+COPY --from=build ./app/Newsbot.Collector.Database/Migrations/ /app/migrations
+COPY --from=goose /go/bin/goose /app
+
+ENTRYPOINT [ "dotnet", "Newsbot.Collector.Api.dll" ]

Newsbot.Collector.Api/Controllers/ArticlesController.cs

@@ -12,16 +12,14 @@ namespace Newsbot.Collector.Api.Controllers;
 public class ArticlesController : ControllerBase
 {
     private readonly ILogger<ArticlesController> _logger;
-    private readonly ConnectionStrings _settings;
     private readonly IArticlesRepository _articles;
     private readonly ISourcesRepository _sources;
 
     public ArticlesController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
     {
         _logger = logger;
-        _settings = settings.Value;
-        _articles = new ArticlesTable(_settings.Database);
-        _sources = new SourcesTable(_settings.Database);
+        _articles = new ArticlesTable(settings.Value.Database);
+        _sources = new SourcesTable(settings.Value.Database);
     }
 
     [HttpGet(Name = "GetArticles")]
@@ -36,14 +34,14 @@ public class ArticlesController : ControllerBase
         return res;
     }
 
-    [HttpGet("{id}")]
+    [HttpGet("{id:guid}")]
     public ArticleDto GetById(Guid id)
     {
         var item = _articles.GetById(id);
         return ArticleDto.Convert(item);
     }
 
-    [HttpGet("{id}/details")]
+    [HttpGet("{id:guid}/details")]
     public ArticleDetailsDto GetDetailsById(Guid id)
     {
         var item = _articles.GetById(id);
@@ -51,11 +49,11 @@ public class ArticlesController : ControllerBase
         return ArticleDetailsDto.Convert(item, sourceItem);
     }
 
-    [HttpGet("by/{sourceid}")]
-    public IEnumerable<ArticleDto> GetBySourceID(Guid sourceid, int page = 0, int count = 25)
+    [HttpGet("by/{sourceId:guid}")]
+    public IEnumerable<ArticleDto> GetBySourceId(Guid sourceId, int page = 0, int count = 25)
     {
         var res = new List<ArticleDto>();
-        var items = _articles.ListBySourceId(sourceid, page, count);
+        var items = _articles.ListBySourceId(sourceId, page, count);
         foreach (var item in items)
         {
             res.Add(ArticleDto.Convert(item));

Newsbot.Collector.Api/Controllers/DiscordWebHooksController.cs

@@ -2,6 +2,7 @@ using System.Net;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Newsbot.Collector.Database.Repositories;
+using Newsbot.Collector.Domain.Dto;
 using Newsbot.Collector.Domain.Interfaces;
 using Newsbot.Collector.Domain.Models;
 
@@ -23,33 +24,56 @@ public class DiscordWebHookController : ControllerBase
     }
 
     [HttpGet(Name = "GetDiscordWebhooks")]
-    public IEnumerable<DiscordWebHookModel> Get(int page)
+    public IEnumerable<DiscordWebHookDto> Get(int page)
     {
-        return _webhooks.List(page);
+        var items = new List<DiscordWebHookDto>();
+        var res = _webhooks.List(page);
+        foreach (var item in res)
+        {
+            items.Add(DiscordWebHookDto.Convert(item));
+        }
+        return items;
     }
 
     [HttpPost(Name = "New")]
-    public DiscordWebHookModel New(string url, string server, string channel)
+    public DiscordWebHookDto New(string url, string server, string channel)
     {
-        return _webhooks.New(new DiscordWebHookModel
+        var exists = _webhooks.GetByUrl(url);
+        // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
+        if (exists is not null)
+        {
+            return DiscordWebHookDto.Convert(exists);
+        }
+
+        var res = _webhooks.New(new DiscordWebHookModel
         {
             Url = url,
             Server = server,
             Channel = channel,
             Enabled = true,
         });
+
+        return DiscordWebHookDto.Convert(res);
     }
 
     [HttpGet("by/serverAndChannel")]
-    public IEnumerable<DiscordWebHookModel> GetByServerAndChannel(string server, string channel)
+    public IEnumerable<DiscordWebHookDto> GetByServerAndChannel(string server, string channel)
     {
-        return _webhooks.ListByServerAndChannel(server, channel, 25);
+        var items = new List<DiscordWebHookDto>();
+        var res = _webhooks.ListByServerAndChannel(server, channel, 25);
+
+        foreach (var item in res)
+        {
+            items.Add(DiscordWebHookDto.Convert(item));
+        }
+        return items;
     }
 
     [HttpGet("{id}")]
-    public DiscordWebHookModel GetById(Guid id)
+    public DiscordWebHookDto GetById(Guid id)
     {
-        return _webhooks.GetByID(id);
+        var res = _webhooks.GetByID(id);
+        return DiscordWebHookDto.Convert(res);
     }
 
     [HttpPost("{id}/disable")]

Newsbot.Collector.Api/Controllers/SourcesController.cs

@@ -50,6 +50,12 @@ public class SourcesController : ControllerBase
     [HttpPost("new/reddit")]
     public SourceDto NewReddit(string name, string url)
     {
+        var res = _sources.GetByNameAndType(name, SourceTypes.Reddit);
+        if (res.ID != Guid.Empty)
+        {
+            return SourceDto.Convert(res);
+        }
+
         var item = _sources.New(new SourceModel
         {
             Site = SourceTypes.Reddit,
@@ -66,7 +72,13 @@ public class SourcesController : ControllerBase
     [HttpPost("new/rss")]
     public SourceDto NewRss(string name, string url)
     {
-        var item = _sources.New(new SourceModel
+        var res = _sources.GetByNameAndType(name, SourceTypes.Rss);
+        if (res.ID != Guid.Empty)
+        {
+            return SourceDto.Convert(res);
+        }
+
+        var m = new SourceModel
         {
             Site = SourceTypes.Rss,
             Name = name,
@@ -75,13 +87,20 @@ public class SourcesController : ControllerBase
             Enabled = true,
             Url = url,
             Tags = $"{SourceTypes.Rss}, {name}"
-        });
+        };
+        var item = _sources.New(m);
         return SourceDto.Convert(item);
     }
 
     [HttpPost("new/youtube")]
     public SourceDto NewYoutube(string name, string url)
     {
+        var res = _sources.GetByNameAndType(name, SourceTypes.YouTube);
+        if (res.ID != Guid.Empty)
+        {
+            return SourceDto.Convert(res);
+        }
+
         var item = _sources.New(new SourceModel
         {
             Site = SourceTypes.YouTube,
@@ -99,6 +118,12 @@ public class SourcesController : ControllerBase
     [HttpPost("new/twitch")]
     public SourceDto NewTwitch(string name)
     {
+        var res = _sources.GetByNameAndType(name, SourceTypes.Twitch);
+        if (res.ID != Guid.Empty)
+        {
+            return SourceDto.Convert(res);
+        }
+
         var item = _sources.New(new SourceModel
         {
             Site = SourceTypes.Twitch,

Newsbot.Collector.Api/Controllers/SubscriptionsController.cs

@@ -12,17 +12,15 @@ namespace Newsbot.Collector.Api.Controllers;
 public class SubscriptionsController : ControllerBase
 {
     private readonly ILogger<ArticlesController> _logger;
-    private readonly ConnectionStrings _settings;
     private readonly ISubscriptionRepository _subscription;
     private readonly IDiscordWebHooksRepository _discord;
     private readonly ISourcesRepository _sources;
     public SubscriptionsController(ILogger<ArticlesController> logger, IOptions<ConnectionStrings> settings)
     {
         _logger = logger;
-        _settings = settings.Value;
-        _subscription = new SubscriptionsTable(_settings.Database);
-        _discord = new DiscordWebhooksTable(_settings.Database);
-        _sources = new SourcesTable(_settings.Database);
+        _subscription = new SubscriptionsTable(settings.Value.Database);
+        _discord = new DiscordWebhooksTable(settings.Value.Database);
+        _sources = new SourcesTable(settings.Value.Database);
     }
 
     [HttpGet(Name = "ListSubscriptions")]
@@ -71,7 +69,7 @@ public class SubscriptionsController : ControllerBase
         return res;
     }
 
-    [HttpGet("by/sourceid")]
+    [HttpGet("by/sourceId")]
     public IEnumerable<SubscriptionDto> GetBySourceId(Guid id)
     {
         var res = new List<SubscriptionDto>();
@@ -83,9 +81,16 @@ public class SubscriptionsController : ControllerBase
         return res;
     }
 
-    [HttpPost("new")]
+    [HttpPost(Name = "New Subscription")]
     public SubscriptionDto New(Guid sourceId, Guid discordId)
     {
+        var exists = _subscription.GetByWebhookAndSource(discordId, sourceId);
+        // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
+        if (exists is not null)
+        {
+            return SubscriptionDto.Convert(exists);
+        }
+
         var item = _subscription.New(new SubscriptionModel
         {
             ID = Guid.NewGuid(),

Newsbot.Collector.Api/Program.cs

@@ -16,7 +16,7 @@ var builder = WebApplication.CreateBuilder(args);
 // Define Logger
 builder.Host.UseSerilog(); // <-- Add this line
 
-// Build the conifg
+// Build the config
 var config = GetConfiguration();
 builder.Configuration.AddConfiguration(config);
 
@@ -35,11 +35,11 @@ builder.Services.Configure<ConnectionStrings>(config.GetSection("ConnectionStrin
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.
-if (app.Environment.IsDevelopment())
-{
+//if (app.Environment.IsDevelopment())
+//{
 app.UseSwagger();
 app.UseSwaggerUI();
-}
+//}
 
 app.UseHttpsRedirection();
 

Newsbot.Collector.Database/Repositories/DiscordWebhooksTable.cs

@@ -18,11 +18,7 @@ public class DiscordWebhooksTable : IDiscordWebHooksRepository
 
     public DiscordWebhooksTable(IConfiguration configuration)
     {
-        var connstr = configuration.GetConnectionString("database");
-        if (connstr is null)
-        {
-            connstr = "";
-        }
+        var connstr = configuration.GetConnectionString("database") ?? "";
         _connectionString = connstr;
     }
 

Newsbot.Collector.Database/Repositories/SourcesTable.cs

@@ -90,14 +90,14 @@ public class SourcesTable : ISourcesRepository
         return res.First();
     }
 
-    public SourceModel GetByNameAndSource(string name, string source)
+    public SourceModel GetByNameAndType(string name, string type)
     {
         using var conn = OpenConnection(_connectionString);
-        var query = "Select * from Sources WHERE name = @name and source = @source;";
+        var query = "Select * from Sources WHERE name = @name and type = @type;";
         var res = conn.Query<SourceModel>(query, new
         {
             name = name,
-            source = source
+            type = type
         });
 
         if (res.Count() == 0)

Newsbot.Collector.Domain/Interfaces/ISourcesRepository.cs

@@ -9,7 +9,7 @@ public interface ISourcesRepository
     public SourceModel GetByID(Guid ID);
     public SourceModel GetByID(string ID);
     public SourceModel GetByName(string name);
-    public SourceModel GetByNameAndSource(string name, string source);
+    public SourceModel GetByNameAndType(string name, string type);
     public List<SourceModel> List(int page, int count);
     public List<SourceModel> ListBySource(string source, int limit);
     public List<SourceModel> ListByType(string type, int limit = 25);

Newsbot.Collector.Services/Jobs/GithubWatcherJob.cs

@@ -1,5 +1,9 @@
+using System.ServiceModel.Syndication;
+using System.Xml;
 using Newsbot.Collector.Database.Repositories;
 using Newsbot.Collector.Domain.Interfaces;
+using Newsbot.Collector.Domain.Models;
+using Newsbot.Collector.Services.HtmlParser;
 
 namespace Newsbot.Collector.Services.Jobs;
 
@@ -8,6 +12,7 @@ public class GithubWatcherJobOptions
     public string ConnectionString { get; set; } = "";
     public bool FeaturePullReleases { get; set; } = false;
     public bool FeaturePullCommits { get; set; } = false;
+    public bool PullIssues { get; set; } = false;
 }
 
 public class GithubWatcherJob
@@ -23,7 +28,7 @@ public class GithubWatcherJob
         _source = new SourcesTable("");
     }
 
-    private void Init(GithubWatcherJobOptions options)
+    public void Init(GithubWatcherJobOptions options)
     {
         _articles = new ArticlesTable(options.ConnectionString);
         _queue = new DiscordQueueTable(options.ConnectionString);
@@ -33,10 +38,72 @@ public class GithubWatcherJob
     public void InitAndExecute(GithubWatcherJobOptions options)
     {
         Init(options);
+        Execute();
+    }
 
+    private void Execute()
+    {
         // query sources for things to pull
+        var items = new List<ArticlesModel>();
 
+        items.AddRange(Collect(new Uri("https://github.com/jtom38/dvb")));
+
+        // query */commits/master.atom
+        // query */commits/main.atom
+    }
+
+    public List<ArticlesModel> Collect(Uri url)
+    {
+        var items = new List<ArticlesModel>();
+
+        Guid placeHolderId = Guid.NewGuid();
         // query */release.atom
-        // query */commits.atom
+        items.AddRange(CollectItems($"{url.AbsoluteUri}/releases.atom", placeHolderId));
+        items.AddRange(CollectItems($"{url.AbsoluteUri}/master.atom", placeHolderId));
+
+        return items;
+    }
+
+    private List<ArticlesModel> CollectItems(string baseUrl, Guid sourceId)
+    {
+        var items = new List<ArticlesModel>();
+
+        using var reader = XmlReader.Create(baseUrl);
+        var client = SyndicationFeed.Load(reader);
+
+        foreach (var item in client.Items)
+        {
+            var itemUrl = item.Links[0].Uri.AbsoluteUri;
+            var exits = _articles.GetByUrl(itemUrl);
+            if (exits.ID != Guid.Empty)
+            {
+                continue;
+            }
+
+            var parser = new HtmlPageReader(itemUrl);
+            parser.Parse();
+
+            try
+            {
+                var a = new ArticlesModel
+                {
+                    SourceID = sourceId,
+                    Tags = "github",
+                    Title = item.Title.Text,
+                    URL = itemUrl,
+                    //PubDate = item.LastUpdatedTime.DateTime,
+                    Thumbnail = parser.Data.Header.Image,
+                    Description = $"'dvb' has released '{item.Title.Text}'!",
+                    AuthorName = item.Authors[0].Name ?? "",
+                    AuthorImage = item.Authors[0].Uri ?? ""
+                };
+                items.Add(a);
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e);
+            }
+        }
+        return items;
     }
 }

Newsbot.Collector.Tests/Jobs/DiscordNotificationJobTest.cs

@@ -0,0 +1,40 @@
+using Newsbot.Collector.Domain.Models;
+using Newsbot.Collector.Services.Jobs;
+using Newsbot.Collector.Services.Notifications.Discord;
+
+namespace Newsbot.Collector.Tests.Jobs;
+
+public class DiscordNotificationJobTest
+{
+
+    [Fact]
+    public void PostTestMessage()
+    {
+        var uri = "";
+        var webhookClient = new DiscordWebhookClient(uri);
+
+        var client = new DiscordNotificationJob();
+        var msg = client.GenerateDiscordMessage(new SourceModel
+        {
+            ID = Guid.NewGuid(),
+            Site = "Unit Test",
+            Source = "placeholder",
+            Type = "a",
+            Value = "a",
+            Enabled = true,
+            Url = "https://github.com",
+            Tags = "Unit, Testing",
+        },
+        new ArticlesModel
+        {
+            Tags = "more,unit,testing",
+            Title = "Nope not real",
+            URL = "https://github.com/jtom38",
+            PubDate = DateTime.Now,
+            Thumbnail = "https://cdn.arstechnica.net/wp-content/uploads/2023/03/GettyImages-944827400-800x534.jpg",
+            Description = "Please work",
+            AuthorName = "No one knows"
+        });
+        webhookClient.SendMessage(msg);
+    }
+}

Newsbot.Collector.Tests/Jobs/GithubWatcherJobTests.cs

@@ -0,0 +1,37 @@
+using Microsoft.Extensions.Configuration;
+using Newsbot.Collector.Services.Jobs;
+
+namespace Newsbot.Collector.Tests.Jobs;
+
+public class GithubWatcherJobTests
+{
+    private IConfiguration GetConfiguration()
+    {
+        var inMemorySettings = new Dictionary<string, string> {
+            {"ConnectionStrings:database", "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"}
+        };
+
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(inMemorySettings)
+            .Build();
+        return configuration;
+    }
+
+    private string ConnectionString()
+    {
+        return "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable";
+    }
+
+    [Fact]
+    public void CanPullAFeed()
+    {
+        var client = new GithubWatcherJob();
+        client.Init(new GithubWatcherJobOptions
+        {
+            ConnectionString = ConnectionString(),
+            FeaturePullCommits = true,
+            FeaturePullReleases = true
+        });
+        client.Collect(new Uri("https://github.com/jtom38/dvb"));
+    }
+}

docker-compose.yaml

@@ -0,0 +1,53 @@
+version: "3"
+
+networks:
+  newsbot:
+    
+volumes:
+  db:
+
+services:
+  #db:
+  #  image: postgres:latest
+  #  #ports:
+  #  #  - "5432:5432"
+  #  networks:
+  #    - newsbot
+  #  volumes:
+  #    - db:/var/lib/postgresql/data
+
+  api:
+    image: newsbot.collector:latest
+    environment:
+      # Used for database migrations
+      GOOSE_DRIVER: "postgres"
+      GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
+
+      SERVER_ADDRESS: "localhost"
+
+      Logging__LogLevel__Default: "Information"
+      Logging__LogLevel__Microsoft.AspNetCore: "Warning"
+      Logging__LogLevel__Hangfire: "Information"
+
+      ConnectionStrings__Database: "Host=localhost;Username=postgres;Password=postgres;Database=postgres;sslmode=disable"
+      
+      # Enable/Disable Reddit monitoring
+      Reddit__IsEnabled: false
+      Reddit__PullHot: true
+      Reddit__PullNsfw: true
+      Reddit__PullTop: true
+
+      # Enable/Disable YouTube monitoring
+      Youtube__IsEnabled: false
+
+      TWITCH__IsEnabled: false
+      # Set your Twitch Developer ID and Secrets here and they will be used to collect updates.
+      TWITCH__ClientID: ""
+      TWITCH__ClientSecret: ""
+
+      # If you want to collect news on Final Fantasy XIV, set this to true
+      FFXIV__IsEnabled: false
+    ports:
+      - "5001:5000"
+    networks:
+      - newsbot

seed.ps1

@@ -0,0 +1,105 @@
+param (
+    [string] $ApiServer = "http://localhost:5011",
+    [string] $JsonSecrets = "./seed.secrets.json"
+)
+
+$ErrorActionPreference = 'Stop'
+
+function NewRedditSource {
+    param (
+        [string] $Name,
+        [string] $Url
+    )
+    $urlEncoded = [uri]::EscapeDataString($Url)
+
+    $param = "name=$Name&url=$urlEncoded"
+    $uri = "$ApiServer/api/sources/new/reddit?$param"
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
+function NewRssSource {
+    param (
+        [string] $Name,
+        [string] $Url
+    )
+    $urlEncoded = [uri]::EscapeDataString($Url)
+    $param = "name=$Name&url=$urlEncoded"
+    [string] $uri = "$ApiServer/api/sources/new/rss?$param"
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
+function NewYoutubeSource {
+    param (
+        [string] $Name,
+        [string] $Url
+    )
+    $urlEncoded = [uri]::EscapeDataString($Url)
+    [string] $param = "name=$Name&url=$urlEncoded"
+    [string] $uri = "$ApiServer/api/sources/new/youtube?$param"
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
+function NewTwitchSource {
+    param (
+        [string] $Name
+    )
+    [string] $param = "name=$Name"
+    [string] $uri = "$ApiServer/api/sources/new/twitch?$param"
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
+function New-DiscordWebhook {
+    param (
+        [string] $Server,
+        [string] $Channel,
+        [string] $Url
+    )
+    $urlEncoded = [uri]::EscapeDataString($Url)
+    [string] $param = "server=$Server&channel=$Channel&url=$urlEncoded"
+    [string] $uri = "$ApiServer/api/discord/webhooks?$param"
+    Write-Host $uri
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
+function New-Subscription {
+    param (
+        [string] $SourceId,
+        [string] $DiscordWebhookId
+    )
+    [string] $param = "sourceId=$SourceId&discordId=$DiscordWebhookId"
+    [string] $uri = "$ApiServer/api/subscriptions?$param"
+    $res = Invoke-RestMethod -Method Post -Uri $uri
+    return $res
+}
+
+# Load Secrets file
+$secrets = Get-Content $JsonSecrets -Raw | ConvertFrom-Json
+
+$redditDadJokes = NewRedditSource -Name "dadjokes" -Url "https://reddit.com/r/dadjokes"
+$redditSteamDeck = NewRedditSource -Name "steamdeck" -Url "https://reddit.com/r/steamdeck"
+
+$rssSteamDeck = NewRssSource -Name "Steampowered - Steam Deck" -Url "https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107"
+$rssFaysHaremporium = NewRssSource -Name "Fay's Haremporium" -Url "https://blog.nyxstudios.moe/rss/"
+$rssPodcastLetsMosley = NewRssSource -Name "Let's Mosley" -Url "https://anchor.fm/s/6c7aa4c4/podcast/rss"
+
+$youtubeGameGrumps = NewYoutubeSource -Name "Game Grumps" -Url "https://www.youtube.com/user/GameGrumps"
+$youtubeCityPlannerPlays = NewYoutubeSource -Name "City Planner Plays" -Url "https://www.youtube.com/c/cityplannerplays"
+
+$twitchNintendo = NewTwitchSource -Name "Nintendo"
+$twitchNintendo.id
+
+$miharuMonitor = New-DiscordWebhook -Server "Miharu Monitor" -Channel "dev" -Url $secrets.MiharuMonitor.dev01
+
+New-Subscription -SourceId $redditDadJokes.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $redditSteamDeck.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $rssSteamDeck.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $rssFaysHaremporium.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $rssPodcastLetsMosley.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $youtubeGameGrumps.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $youtubeCityPlannerPlays.id -DiscordWebhookId $miharuMonitor.id
+New-Subscription -SourceId $twitchNintendo.id -DiscordWebhookId $miharuMonitor.id