Merge pull request 'fixed the route to pull sourceById' (#11) from features/sourceById into main

Reviewed-on: #11

.drone.yaml

@@ -0,0 +1,60 @@
+---
+kind: pipeline
+type: docker
+name: buildLatestImage
+
+steps:
+  - name: buildLatestImage
+    image: plugins/docker
+    settings:
+      repo: jtom38/newsbot-collector
+      username: jtom38
+      password:
+        from_secret: DockerPushPat
+trigger:
+  branch:
+    include:
+      - main
+
+  event:
+    exclude:
+      - pull_request
+---
+kind: pipeline
+type: docker
+name: buildReleaseImage
+
+steps:
+  - name: buildReleaseImage
+    image: plugins/docker
+    settings:
+      repo: jtom38/newsbot-collector
+      username: jtom38
+      password:
+        from_secret: DockerPushPat
+trigger:
+  branch:
+    include:
+      - releases/*
+  ref:
+    include:
+      - refs/tags/**
+  event:
+    exclude:
+      - pull_request
+
+
+---
+kind: pipeline
+type: docker
+name: PullRequestCompileTest
+steps:
+  - name: Compile project
+    image: golang:1.22
+    commands:
+      - go test ./internal/repository
+      - go build ./cmd/server.go
+      - 
+trigger:
+  event:
+    - pull_request

.github/workflows/docker.build.yaml

@@ -1,64 +0,0 @@
-name: Docker
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-on:
-  #schedule:
-  #  - cron: '21 19 * * *'
-  push:
-    branches: [ master ]
-    # Publish semver tags as releases.
-    tags: [ 'v*.*.*' ]
-  #pull_request:
-  #  branches: [ master ]
-
-env:
-  # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
-      - name: Log into registry ${{ env.REGISTRY }}
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
-        id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          #images: ${{ env.REGISTRY }}/newsbot.worker
-
-      # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
-      - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
-        with:
-          context: .
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/go-build.yml

@@ -1,25 +0,0 @@
-name: Go
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.18
-
-    - name: Build
-      run: go build -v ./...
-
-    #- name: Test
-    #  run: go test -v ./...

.gitignore

@@ -1,6 +1,7 @@
 .env
 dev.session.sql
-
+__debug_bin
+server
 .vscode
 
 # Binaries for programs and plugins
@@ -10,10 +11,12 @@ dev.session.sql
 *.so
 *.dylib
 collector
+newsbot.db
 
 # Test binary, built with `go test -c`
 *.test
 
+
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 

.vscode/launch.json

@@ -1,15 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Launch Package",
-            "type": "go",
-            "request": "launch",
-            "mode": "auto",
-            "program": "."
-        }
-    ]
-}

Dockerfile

@@ -1,9 +1,14 @@
-FROM golang:1.18.4 as build
+FROM golang:1.22 as build
 
 COPY . /app
 WORKDIR /app
-RUN go build .
-RUN go install github.com/pressly/goose/v3/cmd/goose@latest
+
+# Always make sure that swagger docs are updated
+RUN go install github.com/swaggo/swag/cmd/swag@latest
+RUN /go/bin/swag init -g cmd/server.go
+
+#RUN go build .
+#RUN go install github.com/pressly/goose/v3/cmd/goose@latest
 
 FROM alpine:latest as app
 
@@ -12,8 +17,7 @@ RUN apk --no-cache add libc6-compat
 RUN apk --no-cache add chromium
 
 RUN mkdir /app && mkdir /app/migrations
-COPY --from=build /app/collector /app
-COPY --from=build /go/bin/goose /app
-COPY ./database/migrations/ /app/migrations
+COPY --from=build /app/server /app
+COPY ./internal/database/migrations/ /app/migrations
 
 CMD [ "/app/collector" ]

api.http

@@ -0,0 +1,6 @@
+### Select Sources fro mthe top
+GET http://localhost:8081/api/v1/sources/
+
+
+### Select Sources by type
+GET http://localhost:8081/api/v1/sources/by/source?source=rss

cmd/server.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+
+	_ "github.com/glebarez/go-sqlite"
+	"github.com/pressly/goose/v3"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/docs"
+	v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
+)
+
+// @title                       NewsBot collector
+// @version                     0.1
+// @BasePath                    /api
+// @securityDefinitions.apikey  Bearer
+// @in                          header
+// @name                        Authorization
+// @description                 Type "Bearer" followed by a space and JWT token.
+func main() {
+	ctx := context.Background()
+
+	cfg := services.NewConfig()
+	configs := services.GetEnvConfig()
+	address := cfg.GetConfig(services.ServerAddress)
+	docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
+
+	db, err := sql.Open("sqlite", "newsbot.db")
+	if err != nil {
+		panic(err)
+	}
+
+	err = migrateDatabase(db)
+	if err != nil {
+		fmt.Print(err)
+	}
+
+	c := cron.NewScheduler(ctx, db)
+	c.Start()
+
+	server := v1.NewServer(ctx, configs, db)
+
+	fmt.Println("API is online and waiting for requests.")
+	fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
+	fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress)
+
+	server.Router.Start(":8081")
+}
+
+func migrateDatabase(db *sql.DB) error {
+	err := goose.SetDialect("sqlite3")
+	if err != nil {
+		panic(err)
+	}
+
+	err = goose.Up(db, "../internal/database/migrations")
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = os.Stat("./migrations")
+	if err == nil {
+
+		err = goose.Up(db, "../internal/database/migrations")
+		if err != nil {
+			panic(err)
+		}
+		
+		return nil
+	}
+
+	_, err = os.Stat("../internal/database/migrations")
+	if err == nil {
+
+		err = goose.Up(db, "../internal/database/migrations")
+		if err != nil {
+			panic(err)
+		}
+
+		return nil
+	}
+
+	return errors.New("failed to find the migration files")
+}

database/db.go

@@ -1,31 +0,0 @@
-// Code generated by sqlc. DO NOT EDIT.
-// versions:
-//   sqlc v1.13.0
-
-package database
-
-import (
-	"context"
-	"database/sql"
-)
-
-type DBTX interface {
-	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
-	PrepareContext(context.Context, string) (*sql.Stmt, error)
-	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
-	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
-}
-
-func New(db DBTX) *Queries {
-	return &Queries{db: db}
-}
-
-type Queries struct {
-	db DBTX
-}
-
-func (q *Queries) WithTx(tx *sql.Tx) *Queries {
-	return &Queries{
-		db: tx,
-	}
-}

database/migrations/20220522083756_init.sql

@@ -1,72 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-CREATE TABLE Articles (
-	ID 			uuid PRIMARY KEY,
-	SourceId 	uuid NOT null,
-	Tags 		TEXT NOT NULL,
-	Title 		TEXT NOT NULL,
-	Url 		TEXT NOT NULL,
-	PubDate 	timestamp NOT NULL,
-	Video 		TEXT,
-	VideoHeight int NOT NULL,
-	VideoWidth 	int NOT NULL,
-	Thumbnail 	TEXT NOT NULL,
-	Description TEXT NOT NULL,
-	AuthorName 	TEXT,
-	AuthorImage TEXT
-);
-
-CREATE Table DiscordQueue ( 
-    ID 			uuid PRIMARY KEY,
-    ArticleId 	uuid NOT NULL
-);
-
-CREATE Table DiscordWebHooks (
-	ID 		uuid PRIMARY KEY,
-	Name 	TEXT NOT NULL, -- Defines webhook purpose
-	Key 	TEXT,
-	Url 	TEXT NOT NULL, -- Webhook Url
-	Server 	TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
-	Channel TEXT NOT NULL, -- Defines the channel its bound to.  Used for refrence
-	Enabled BOOLEAN NOT NULL
-);
-
-CREATE Table Icons (
-	ID 			uuid PRIMARY Key,
-	FileName 	TEXT NOT NULL,
-	Site 		TEXT NOT NULL
-);
-
-Create Table Settings (
-	ID 		uuid PRIMARY Key,
-	Key 	TEXT NOT NULL, -- How you search for a entry
-	Value 	TEXT NOT NULL, -- The value for one
-	Options TEXT -- any notes about the entry
-);
-
-Create Table Sources (
-	ID 		uuid PRIMARY Key,
-	Site 	TEXT NOT NULL, -- Vanity name
-	Name 	TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
-	Source 	TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
-	Type 	TEXT NOT NULL, -- Defines what kind of feed this is.  feed, user, tag
-	Value 	TEXT,
-	Enabled BOOLEAN NOT NULL,
-	Url 	TEXT NOT NULL,
-	Tags 	TEXT NOT NULL
-);
-
-
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-Drop Table Articles;
-Drop Table DiscordQueue;
-Drop Table DiscordWebHooks;
-Drop Table Icons;
-Drop Table Settings;
-Drop Table Sources;
--- +goose StatementEnd

database/migrations/20220529082459_seed.sql

@@ -1,50 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-
--- Enable UUID's
-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-
--- Final Fantasy XIV Entries
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-
--- Reddit Entries
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-
--- Youtube Entries
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-
--- RSS Entries
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-
--- Twitch Entries 
-INSERT INTO sources VALUES 
-(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
---SELECT 'down SQL query';
-
-DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
-DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
-DELETE FROM sources where source = 'ffxiv';
-DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
-DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
-DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
--- +goose StatementEnd

database/migrations/20220619085634_subscriptions.sql

@@ -1,21 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-Create TABLE Subscriptions (
-	ID 			      uuid Primary Key,
-	DiscordWebHookID  uuid Not Null,
-	SourceID         uuid Not Null
-);
-
-ALTER TABLE discordwebhooks drop COLUMN Name;
-ALTER TABLE discordwebhooks drop COLUMN Key;
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-SELECT 'down SQL query';
-Drop Table Subscriptions;
-ALTER TABLE discordwebhooks Add COLUMN Name TEXT;
---ALTER TABLE discordwebhooks Add COLUMN Key TEXT;
--- +goose StatementEnd

database/models.go

@@ -1,72 +0,0 @@
-// Code generated by sqlc. DO NOT EDIT.
-// versions:
-//   sqlc v1.13.0
-
-package database
-
-import (
-	"database/sql"
-	"time"
-
-	"github.com/google/uuid"
-)
-
-type Article struct {
-	ID          uuid.UUID
-	Sourceid    uuid.UUID
-	Tags        string
-	Title       string
-	Url         string
-	Pubdate     time.Time
-	Video       sql.NullString
-	Videoheight int32
-	Videowidth  int32
-	Thumbnail   string
-	Description string
-	Authorname  sql.NullString
-	Authorimage sql.NullString
-}
-
-type Discordqueue struct {
-	ID        uuid.UUID
-	Articleid uuid.UUID
-}
-
-type Discordwebhook struct {
-	ID      uuid.UUID
-	Url     string
-	Server  string
-	Channel string
-	Enabled bool
-}
-
-type Icon struct {
-	ID       uuid.UUID
-	Filename string
-	Site     string
-}
-
-type Setting struct {
-	ID      uuid.UUID
-	Key     string
-	Value   string
-	Options sql.NullString
-}
-
-type Source struct {
-	ID      uuid.UUID
-	Site    string
-	Name    string
-	Source  string
-	Type    string
-	Value   sql.NullString
-	Enabled bool
-	Url     string
-	Tags    string
-}
-
-type Subscription struct {
-	ID               uuid.UUID
-	Discordwebhookid uuid.UUID
-	Sourceid         uuid.UUID
-}

database/schema/query.sql

@@ -1,176 +0,0 @@
-/* Articles */
--- name: GetArticleByID :one
-Select * from Articles
-WHERE ID = $1 LIMIT 1;
-
--- name: GetArticleByUrl :one
-Select * from Articles
-Where Url = $1 LIMIT 1;
-
--- name: ListArticles :many
-Select * From articles Limit $1;
-
--- name: ListArticlesByDate :many
-Select * From articles ORDER BY pubdate desc Limit $1;
-
--- name: GetArticlesBySource :many
-select * from articles
-INNER join sources on articles.sourceid=Sources.ID
-where site = $1;
-
--- name: GetArticlesBySourceId :many
-Select * From articles
-Where sourceid = $1 Limit 50;
-
--- name: GetArticlesBySourceName :many
-select 
-articles.ID, articles.SourceId, articles.Tags, articles.Title, articles.Url, articles.PubDate, articles.Video, articles.VideoHeight, articles.VideoWidth, articles.Thumbnail, articles.Description, articles.AuthorName, articles.AuthorImage, sources.source, sources.name
-From articles
-Left Join sources
-On articles.sourceid = sources.id
-Where name = $1;
-
--- name: CreateArticle :exec
-INSERT INTO Articles 
-(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
-Values
-($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
-
-
-/* DiscordQueue */
--- name: CreateDiscordQueue :exec
-Insert into DiscordQueue
-(ID, ArticleId)
-Values
-($1, $2);
-
--- name: GetDiscordQueueByID :one
-Select * from DiscordQueue
-Where ID = $1 LIMIT 1;
-
--- name: DeleteDiscordQueueItem :exec
-Delete From DiscordQueue Where ID = $1;
-
--- name: ListDiscordQueueItems :many
-Select * from DiscordQueue LIMIT $1;
-
-/* DiscordWebHooks */
--- name: CreateDiscordWebHook :exec
-Insert Into DiscordWebHooks
-(ID, Url, Server, Channel, Enabled)
-Values
-($1, $2, $3, $4, $5);
-
--- name: GetDiscordWebHooksByID :one
-Select * from DiscordWebHooks
-Where ID = $1 LIMIT 1;
-
--- name: ListDiscordWebHooksByServer :many
-Select * From DiscordWebHooks
-Where Server = $1;
-
--- name: GetDiscordWebHookByUrl :one
-Select * From DiscordWebHooks Where url = $1;
-
--- name: ListDiscordWebhooks :many
-Select * From discordwebhooks LIMIT $1;
-
--- name: DeleteDiscordWebHooks :exec
-Delete From discordwebhooks Where ID = $1;
-
-
-/* Icons */
-
--- name: CreateIcon :exec
-INSERT INTO Icons
-(ID, FileName, Site)
-VALUES
-($1,$2,$3);
-
--- name: GetIconByID :one
-Select * FROM Icons
-Where ID = $1 Limit 1;
-
--- name: GetIconBySite :one
-Select * FROM Icons
-Where Site = $1 Limit 1;
-
--- name: DeleteIcon :exec
-Delete From Icons where ID = $1;
-
-/* Settings */
-
--- name: CreateSettings :one
-Insert Into settings
-(ID, Key, Value, OPTIONS)
-Values
-($1,$2,$3,$4)
-RETURNING *;
-
--- name: GetSettingByID :one
-Select * From settings
-Where ID = $1 Limit 1;
-
--- name: GetSettingByKey :one
-Select * From settings Where 
-Key = $1 Limit 1;
-
--- name: GetSettingByValue :one
-Select * From settings Where 
-Value = $1 Limit 1;
-
--- name: DeleteSetting :exec
-Delete From settings Where ID = $1;
-
-/* Sources */
-
--- name: CreateSource :exec
-Insert Into Sources
-(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
-Values
-($1,$2,$3,$4,$5,$6,$7,$8,$9);
-
--- name: GetSourceByID :one
-Select * From Sources where ID = $1 Limit 1;
-
--- name: GetSourceByName :one
-Select * from Sources where name = $1 Limit 1;
-
--- name: ListSources :many
-Select * From Sources Limit $1;
-
--- name: ListSourcesBySource :many
-Select * From Sources where Source = $1;
-
--- name: DeleteSource :exec
-DELETE From sources where id = $1;
-
--- name: DisableSource :exec
-Update Sources Set Enabled = FALSE where ID = $1;
-
--- name: EnableSource :exec
-Update Sources Set Enabled = TRUE where ID = $1;
-
-
-/* Subscriptions */
-
--- name: CreateSubscription :exec
-Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3);
-
--- name: ListSubscriptions :many
-Select * From subscriptions Limit $1;
-
--- name: ListSubscriptionsBySourceId :many
-Select * From subscriptions where sourceid = $1;
-
--- name: QuerySubscriptions :one
-Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1;
-
--- name: GetSubscriptionsBySourceID :many
-Select * From subscriptions Where sourceid = $1;
-
--- name: GetSubscriptionsByDiscordWebHookId :many
-Select * from subscriptions Where discordwebhookid = $1;
-
--- name: DeleteSubscription :exec
-Delete From subscriptions Where id = $1;

database/schema/schema.sql

@@ -1,60 +0,0 @@
-CREATE TABLE Articles (
-	ID 			uuid PRIMARY KEY,
-	SourceId 	uuid NOT null,
-	Tags 		TEXT NOT NULL,
-	Title 		TEXT NOT NULL,
-	Url 		TEXT NOT NULL,
-	PubDate 	timestamp NOT NULL,
-	Video 		TEXT,
-	VideoHeight int NOT NULL,
-	VideoWidth 	int NOT NULL,
-	Thumbnail 	TEXT NOT NULL,
-	Description TEXT NOT NULL,
-	AuthorName 	TEXT,
-	AuthorImage TEXT
-);
-
-CREATE Table DiscordQueue ( 
-    ID 			uuid PRIMARY KEY,
-    ArticleId 	uuid NOT NULL
-);
-
-CREATE Table DiscordWebHooks (
-	ID 		uuid PRIMARY KEY,
-	Url 	TEXT NOT NULL, -- Webhook Url
-	Server 	TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
-	Channel TEXT NOT NULL, -- Defines the channel its bound to.  Used for refrence
-	Enabled BOOLEAN NOT NULL
-);
-
-CREATE Table Icons (
-	ID 			uuid PRIMARY Key,
-	FileName 	TEXT NOT NULL,
-	Site 		TEXT NOT NULL
-);
-
-Create Table Settings (
-	ID 		uuid PRIMARY Key,
-	Key 	TEXT NOT NULL,
-	Value 	TEXT NOT NULL,
-	Options TEXT
-);
-
-Create Table Sources (
-	ID 		uuid PRIMARY Key,
-	Site 	TEXT NOT NULL, -- Vanity name
-	Name 	TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
-	Source 	TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
-	Type 	TEXT NOT NULL, -- Defines what kind of feed this is.  feed, user, tag
-	Value 	TEXT,
-	Enabled BOOLEAN NOT NULL,
-	Url 	TEXT NOT NULL,
-	Tags 	TEXT NOT NULL
-);
-
-/* This table is used to track what the Web Hook wants to have sent by Source */;
-Create TABLE Subscriptions (
-	ID 					uuid Primary Key,
-	DiscordWebHookID  	uuid Not Null,
-	SourceID         	uuid Not Null
-);

domain/const.go

@@ -0,0 +1,9 @@
+package domain
+
+const (
+	SourceCollectorRss     = "rss"
+	SourceCollectorFfxiv   = "ffxiv"
+	SourceCollectorTwitch  = "twitch"
+	SourceCollectorYoutube = "youtube"
+	SourceCollectorReddit  = "reddit"
+)

domain/dto.go

@@ -0,0 +1,64 @@
+package domain
+
+import "time"
+
+type ArticleDto struct {
+	ID             int64     `json:"id"`
+	SourceID       int64     `json:"sourceId"`
+	Tags           string    `json:"tags"`
+	Title          string    `json:"title"`
+	Url            string    `json:"url"`
+	PubDate        time.Time `json:"pubDate"`
+	IsVideo        bool      `json:"isVideo"`
+	Thumbnail      string    `json:"thumbnail"`
+	Description    string    `json:"description"`
+	AuthorName     string    `json:"authorName"`
+	AuthorImageUrl string    `json:"authorImage"`
+}
+
+type DiscordQueueDto struct {
+	ID        int64 `json:"id"`
+	ArticleId int64 `json:"articleId"`
+	SourceId  int64 `json:"sourceId"`
+}
+
+type DiscordWebHookDto struct {
+	ID      int64   `json:"id"`
+	//Name    string `json:"name"`
+	//Key     string `json:"key"`
+	Url     string `json:"url"`
+	Server  string `json:"server"`
+	Channel string `json:"channel"`
+	Enabled bool   `json:"enabled"`
+}
+
+type IconDto struct {
+	ID       int64  `json:"id"`
+	FileName string `json:"fileName"`
+	Site     string `json:"site"`
+}
+
+type SettingDto struct {
+	ID      int64  `json:"id"`
+	Key     string `json:"key"`
+	Value   string `json:"value"`
+	Options string `json:"options"`
+}
+
+type SubscriptionDto struct {
+	ID          int64  `json:"id"`
+	SourceID    int64  `json:"sourceId"`
+	SourceType  string `json:"sourceType"`
+	SourceName  string `json:"sourceName"`
+	DiscordID   int64  `json:"discordId"`
+	DiscordName string `json:"discordName"`
+}
+
+type SourceDto struct {
+	ID          int64  `json:"id"`
+	Source      string `json:"source"`
+	DisplayName string `json:"name"`
+	Url         string `json:"url"`
+	Tags        string `json:"tags"`
+	Enabled     bool   `json:"enabled"`
+}

domain/interfaces/source.go

@@ -1,23 +0,0 @@
-package interfaces
-
-import (
-	"github.com/go-rod/rod"
-	"github.com/mmcdole/gofeed"
-)
-
-type Sources interface {
-	CheckSource() error
-	PullFeed() (*gofeed.Feed, error)
-
-	GetBrowser() *rod.Browser
-	GetPage(parser *rod.Browser, url string) *rod.Page
-
-	ExtractThumbnail(page *rod.Page) (string, error)
-	ExtractPubDate(page *rod.Page) (string, error)
-	ExtractDescription(page *rod.Page) (string, error)
-	ExtractAuthor(page *rod.Page) (string, error)
-	ExtractAuthorImage(page *rod.Page) (string, error)
-	ExtractTags(page *rod.Page) (string, error)
-	ExtractTitle(page *rod.Page) (string, error)
-}
-

domain/model/database.go

@@ -1,90 +0,0 @@
-package model
-
-import (
-	"time"
-)
-
-// Articles represents the model for an Article
-type Articles struct {
-	ID uint `json:"ID"`
-	SourceID uint `json:"sourceId"`
-	Tags string	`json:"tags"`
-	Title string `json:"title"`
-	Url string `json:"url"`
-	PubDate time.Time `json:"pubdate"`
-	Video string `json:"video"`
-	VideoHeight uint16 `json:"videoHeight"`
-	VideoWidth uint16 `json:"videoWidth"`
-	Thumbnail string `json:"thumbnail"`
-	Description string `json:"description"`
-	AuthorName string `json:"authorName"`
-	AuthorImage string `json:"authorImage"`
-}
-
-type DiscordQueue struct {
-	ID uint `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-	ArticleId string `json:"articleId"`
-}
-
-type DiscordWebHooks struct {
-	ID uint `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-
-	Name string 	`json:"name"`
-	Key string 		`json:"key"`
-	Url string 		`json:"url"`
-	Server string 	`json:"server"`
-	Channel string	`json:"channel"`
-	Enabled bool	`json:"enabled"`
-}
-
-type Icons struct {
-	ID uint `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-
-	FileName string	`json:"fileName"`
-	Site string		`json:"site"`
-}
-
-type Settings struct {
-	ID uint `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-
-	Key string		`json:"key"`
-	Value string	`json:"value"`
-	Options string	`json:"options"`
-}
-
-type Sources struct {
-	ID uint 		`json:"ID"`
-	Site string		`json:"site"`
-	Name string		`json:"name"`
-	Source string	`json:"source"`
-	Type string		`json:"type"`
-	Value string	`json:"value"`
-	Enabled bool	`json:"enabled"`
-	Url string		`json:"url"`
-	Tags string		`json:"tags"`
-}
-
-type SourceLinks struct {
-	ID uint `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-	
-	SourceID uint		`json:"sourceId"`
-	SourceType string	`json:"sourceType"`
-	SourceName string	`json:"sourceName"`
-	DiscordID uint	`json:"discordId"`
-	DiscordName string	`json:"discordName"`
-}

domain/model/reddit.go

@@ -1,54 +0,0 @@
-package model
-
-// This is the root Json object.  It does not contain data that we care about though.
-type RedditJsonContent struct {
-	Kind string `json:"kind"`
-	Data RedditJsonContentData `json:"data"`
-}
-
-type RedditJsonContentData struct {
-	After string `json:"after"`
-	Dist int `json:"dist"`
-	Modhash string `json:"modhash"`
-	Children []RedditJsonContentChildren `json:"children"`
-}
-
-type RedditJsonContentChildren struct {
-	Kind string `json:"kind"`
-	Data RedditPost `json:"data"`
-}
-
-// RedditPost contains the information that was posted by a user.
-type RedditPost struct {
-	Subreddit string `json:"subreddit"`
-	Title string `json:"title"`
-	Content string `json:"selftext"`
-	ContentHtml string `json:"selftext_html"`
-	Author string `json:"author"`
-	Permalink string `json:"permalink"`
-	IsVideo bool `json:"is_video"`
-	Media RedditPostMedia `json:"media"`
-	Url string `json:"url"`
-	UrlOverriddenByDest string `json:"url_overridden_by_dest"`
-
-	Thumbnail string `json:"thumbnail"`
-}
-
-// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
-type RedditPostMedia struct {
-	RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
-}
-
-// RedditVideo contains information about the video in the post.
-type RedditPostMediaRedditVideo struct {
-	BitrateKbps int `json:"bitrate_kpbs"`
-	FallBackUrl string `json:"fallback_url"`
-	Height int `json:"height"`
-	Width int `json:"width"`
-	ScrubberMediaUrl string `json:"scrubber_media_url"`
-	DashUrl string `json:"dash_url"`
-	Duration int `json:"duration"`
-	HlsUrl string `json:"hls_url"`
-	IsGif bool `json:"is_gif"`
-	TranscodingStatus string `json:"transcoding_status"`
-}

domain/requests.go

@@ -0,0 +1,27 @@
+package domain
+
+type LoginFormRequest struct {
+	Username string `form:"username"`
+	Password string `form:"password"`
+}
+
+type GetSourceBySourceAndNameParamRequest struct {
+	Name   string `query:"name"`
+	Source string `query:"source"`
+}
+
+type NewSourceParamRequest struct {
+	Name string `query:"name"`
+	Url  string `query:"url"`
+	Tags string `query:"tags"`
+}
+
+type RefreshTokenRequest struct {
+	Username     string `json:"username"`
+	RefreshToken string `json:"refreshToken"`
+}
+
+type UpdateScopesRequest struct {
+	Username string   `json:"username"`
+	Scopes   []string `json:"scopes" validate:"required"`
+}

domain/responses.go

@@ -0,0 +1,38 @@
+package domain
+
+type BaseResponse struct {
+	Message string `json:"message"`
+	IsError bool   `json:"isError"`
+}
+
+type LoginResponse struct {
+	BaseResponse
+	Token        string `json:"token"`
+	Type         string `json:"type"`
+	RefreshToken string `json:"refreshToken"`
+}
+
+type ArticleResponse struct {
+	BaseResponse
+	Payload []ArticleDto `json:"payload"`
+}
+
+type ArticleAndSourceModel struct {
+	Article ArticleDto `json:"article"`
+	Source  SourceDto  `json:"source"`
+}
+
+type ArticleDetailedResponse struct {
+	BaseResponse
+	Payload ArticleAndSourceModel `json:"payload"`
+}
+
+type DiscordWebhookResponse struct {
+	BaseResponse
+	Payload []DiscordWebHookDto `json:"payload"`
+}
+
+type SourcesResponse struct {
+	BaseResponse
+	Payload []SourceDto `json:"payload"`
+}

domain/scopes.go

@@ -0,0 +1,14 @@
+package domain
+
+const (
+	ScopeAll = "newsbot:all"
+
+	ScopeArticleRead    = "newsbot:article:read"
+	ScopeArticleDisable = "newsbot:article:disable"
+
+	ScopeSourceRead   = "newsbot:source:read"
+	ScopeSourceCreate = "newsbot:source:create"
+
+	ScopeDiscordWebHookCreate = "newsbot:discordwebhook:create"
+	ScopeDiscordWebhookRead   = "newsbot:discordwebhook:read"
+)

go.mod

@@ -1,18 +1,48 @@
-module github.com/jtom38/newsbot/collector
+module git.jamestombleson.com/jtom38/newsbot-api
 
-go 1.18
+go 1.22
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
-	github.com/go-chi/chi/v5 v5.0.7
+	github.com/glebarez/go-sqlite v1.22.0
 	github.com/go-rod/rod v0.107.1
-	github.com/google/uuid v1.3.0
+	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/google/uuid v1.6.0
+	github.com/huandu/go-sqlbuilder v1.27.1
 	github.com/joho/godotenv v1.4.0
+	github.com/labstack/echo-jwt/v4 v4.2.0
+	github.com/labstack/echo/v4 v4.12.0
 	github.com/mmcdole/gofeed v1.1.3
 	github.com/nicklaw5/helix/v2 v2.4.0
+	github.com/pressly/goose/v3 v3.20.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/swaggo/http-swagger v1.3.0
-	github.com/swaggo/swag v1.8.2
+	github.com/swaggo/echo-swagger v1.4.1
+	github.com/swaggo/swag v1.8.12
+	golang.org/x/crypto v0.22.0
+)
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/labstack/gommon v0.4.2 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mfridman/interpolate v0.0.2 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/sethvargo/go-retry v0.2.4 // indirect
+	github.com/swaggo/files/v2 v2.0.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasttemplate v1.2.2 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	modernc.org/libc v1.41.0 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/sqlite v1.29.6 // indirect
 )
 
 require (
@@ -22,7 +52,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/spec v0.20.6 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/lib/pq v1.10.6
@@ -30,13 +60,12 @@ require (
 	github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
 	github.com/ysmood/gson v0.7.2 // indirect
 	github.com/ysmood/leakless v0.7.0 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
-	golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.11 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -12,8 +12,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
-github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
+github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -27,11 +31,25 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK
 github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY=
 github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
-github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
-github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
+github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
+github.com/huandu/go-sqlbuilder v1.27.1 h1:7UU/3EMIQYYX8wn+L7BNcGVz1aEs5TPNOVFd7ryrPos=
+github.com/huandu/go-sqlbuilder v1.27.1/go.mod h1:nUVmMitjOmn/zacMLXT0d3Yd3RHoO2K+vy906JzqxMI=
+github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
+github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -44,6 +62,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/labstack/echo-jwt/v4 v4.2.0 h1:odSISV9JgcSCuhgQSV/6Io3i7nUmfM/QkBeR5GVJj5c=
+github.com/labstack/echo-jwt/v4 v4.2.0/go.mod h1:MA2RqdXdEn4/uEglx0HcUOgQSyBaTh5JcaHIan3biwU=
+github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
+github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
+github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
+github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
 github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
 github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -51,6 +75,13 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
+github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
 github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
 github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
 github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
@@ -62,27 +93,41 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nicklaw5/helix/v2 v2.4.0 h1:ZvqCKVqza1eJYyqgTRrZ/xjDq0w/EQVFNkN067Utls0=
 github.com/nicklaw5/helix/v2 v2.4.0/go.mod h1:0ONzvVi1cH+k3a7EDIFNNqxfW0podhf+CqlmFvuexq8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0=
+github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
+github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc=
-github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
-github.com/swaggo/http-swagger v1.3.0 h1:1+6M4qRorIbdyTWTsGrwnb0r9jGK5dcWN82O6oY/yHQ=
-github.com/swaggo/http-swagger v1.3.0/go.mod h1:9glekdg40lwclrrKNRGgj/IMDxpNPZ3kzab4oPcF8EM=
-github.com/swaggo/swag v1.8.2 h1:D4aBiVS2a65zhyk3WFqOUz7Rz0sOaUcgeErcid5uGL4=
-github.com/swaggo/swag v1.8.2/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk=
+github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc=
+github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
+github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
+github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w=
+github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its=
 github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
+github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
 github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
 github.com/ysmood/got v0.29.5 h1:+wMnm8UjoyYFMfeAsr57a1bahWTkloysc0Hxsu2gmnM=
@@ -94,29 +139,39 @@ github.com/ysmood/gson v0.7.2 h1:1iWUvpi5DPvd2j59W7ifRPR9DiAZ3Ga+fmMl1mJrRbM=
 github.com/ysmood/gson v0.7.2/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
 github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
 github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
-golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 h1:PgOr27OhUx2IRqGJ2RxAWI4dJQ7bi9cSrB82uzFzfUA=
-golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
-golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
@@ -125,5 +180,20 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
+modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4=
+modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/database/migrations/20240425083756_init.sql

@@ -0,0 +1,115 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+CREATE TABLE Articles (
+	ID 				INTEGER 	PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 		DATETIME	NOT NULL,
+	UpdatedAt 		DATETIME 	NOT NULL,
+	DeletedAt 		DATETIME	NOT NULL,
+	SourceId 		NUMBER 		NOT NULL,
+	Tags 			TEXT 		NOT NULL,
+	Title 			TEXT 		NOT NULL,
+	Url 			TEXT 		NOT NULL,
+	PubDate 		DATETIME 	NOT NULL,
+	IsVideo 		TEXT		NOT NULL,
+	ThumbnailUrl 	TEXT 		NOT NULL,
+	Description 	TEXT 		NOT NULL,
+	AuthorName 		TEXT		NOT NULL,
+	AuthorImageUrl 	TEXT		NOT NULL
+);
+
+CREATE Table DiscordWebHooks (
+	ID 			INTEGER 	PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 	DATETIME 	NOT NULL,
+	UpdatedAt 	DATETIME 	NOT NULL,
+	DeletedAt 	DATETIME	NOT NULL,
+	UserID 		INTEGER 	NOT NULL,
+	Url 		TEXT 		NOT NULL, -- Webhook Url
+	Server 		TEXT 		NOT NULL, -- Defines the server its bound it. Used for reference
+	Channel 	TEXT 		NOT NULL, -- Defines the channel its bound to.  Used for reference
+	Enabled 	BOOLEAN 	NOT NULL
+);
+
+CREATE Table Icons (
+	ID 			INTEGER	 	PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 	DATETIME 	NOT NULL,
+	UpdatedAt 	DATETIME 	NOT NULL,
+	DeletedAt 	DATETIME,
+	FileName 	TEXT 		NOT NULL,
+	Site 		TEXT 		NOT NULL
+);
+
+Create Table Settings (
+	ID 			INTEGER		PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 	DATETIME 	NOT NULL,
+	UpdatedAt 	DATETIME 	NOT NULL,
+	DeletedAt 	DATETIME,
+	Key 		TEXT 		NOT NULL, -- How you search for a entry
+	Value 		TEXT 		NOT NULL, -- The value for one
+	Options 	TEXT -- any notes about the entry
+);
+
+CREATE Table Sources (
+	ID 			INTEGER		PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 	DATETIME 	NOT NULL,
+	UpdatedAt 	DATETIME 	NOT NULL,
+	DeletedAt 	DATETIME	NOT NULL,
+	DisplayName	TEXT 		NOT NULL, -- Vanity name
+	Source 		TEXT 		NOT NULL, -- Defines the service that will use this record. IE reddit or youtube
+	Enabled 	BOOLEAN 	NOT NULL,
+	Url 		TEXT 		NOT NULL,
+	Tags 		TEXT 		NOT NULL
+);
+
+CREATE TABLE UserSourceSubscriptions (
+	ID 					INTEGER		PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 			DATETIME 	NOT NULL,
+	UpdatedAt 			DATETIME 	NOT NULL,
+	DeletedAt 			DATETIME 	NOT NULL,
+	UserID				NUMBER		NOT NULL,
+	SourceID 			NUMBER 		NOT NULL
+);
+
+CREATE TABLE AlertDiscord (
+	ID 					INTEGER		PRIMARY KEY AUTOINCREMENT,
+	CreatedAt 			DATETIME 	NOT NULL,
+	UpdatedAt 			DATETIME 	NOT NULL,
+	DeletedAt 			DATETIME 	NOT NULL,
+	UserID				NUMBER 		NOT NULL,
+	SourceID 			NUMBER 		NOT NULL,
+	DiscordWebHookID 	NUMBER 		NOT NULL
+);
+
+CREATE TABLE Users (
+    ID          INTEGER     PRIMARY KEY AUTOINCREMENT,
+    CreatedAt   DATETIME    NOT NULL,
+    UpdatedAt	DATETIME    NOT NULL,
+	DeletedAt 	DATETIME	NOT NULL,
+    Name        TEXT        NOT NULL,
+    Hash        TEXT        NOT NULL,
+	Scopes 		TEXT		NOT NULL
+); 
+
+CREATE TABLE RefreshTokens (
+    ID          INTEGER     PRIMARY KEY AUTOINCREMENT,
+    CreatedAt   DATETIME    NOT NULL,
+    UpdatedAt 	DATETIME    NOT NULL,
+	DeletedAt 	DATETIME	NOT NULL,
+    Username    TEXT        NOT NULL,
+    Token       TEXT        NOT NULL
+);
+
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE AlertDiscord;
+Drop Table Articles;
+Drop Table DiscordWebHooks;
+Drop Table Icons;
+DROP TABLE RefreshTokens;
+Drop Table Sources;
+DROP TABLE Users;
+DROP TABLE UserSourceSubscriptions;
+-- +goose StatementEnd

internal/database/migrations/20240425092459_seed.sql

@@ -0,0 +1,50 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+
+-- Enable UUID's
+--CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Final Fantasy XIV Entries
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES  
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES  
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
+
+-- Reddit Entries
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
+
+-- Youtube Entries
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
+
+-- RSS Entries
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
+
+-- Twitch Entries 
+INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
+("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+--SELECT 'down SQL query';
+
+DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes';
+DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck';
+DELETE FROM sources where Source = 'ffxiv';
+DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo';
+DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps';
+DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck';
+-- +goose StatementEnd

internal/database/migrations/20240521194914_session_token.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE Users ADD SessionToken TEXT; 
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE Users DROP SessionToken;
+-- +goose StatementEnd

internal/domain/models.go

@@ -1,8 +1,6 @@
-package model
+package domain
 
-import (
-	"time"
-)
+import "time"
 
 type CacheItem struct {
 	Key   string
@@ -10,7 +8,7 @@ type CacheItem struct {
 
 	// Group defines what it should be a reference to.
 	// youtube, reddit, ect
-	Group   string
-	Expires time.Time
+	Group     string
+	Expires   time.Time
 	IsTainted bool
-}
+}

internal/domain/modelsReddit.go

@@ -0,0 +1,54 @@
+package domain
+
+// This is the root Json object.  It does not contain data that we care about though.
+type RedditJsonContent struct {
+	Kind string                `json:"kind"`
+	Data RedditJsonContentData `json:"data"`
+}
+
+type RedditJsonContentData struct {
+	After    string                      `json:"after"`
+	Dist     int                         `json:"dist"`
+	Modhash  string                      `json:"modhash"`
+	Children []RedditJsonContentChildren `json:"children"`
+}
+
+type RedditJsonContentChildren struct {
+	Kind string     `json:"kind"`
+	Data RedditPost `json:"data"`
+}
+
+// RedditPost contains the information that was posted by a user.
+type RedditPost struct {
+	Subreddit           string          `json:"subreddit"`
+	Title               string          `json:"title"`
+	Content             string          `json:"selftext"`
+	ContentHtml         string          `json:"selftext_html"`
+	Author              string          `json:"author"`
+	Permalink           string          `json:"permalink"`
+	IsVideo             bool            `json:"is_video"`
+	Media               RedditPostMedia `json:"media"`
+	Url                 string          `json:"url"`
+	UrlOverriddenByDest string          `json:"url_overridden_by_dest"`
+
+	Thumbnail string `json:"thumbnail"`
+}
+
+// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
+type RedditPostMedia struct {
+	RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
+}
+
+// RedditVideo contains information about the video in the post.
+type RedditPostMediaRedditVideo struct {
+	BitrateKbps       int    `json:"bitrate_kpbs"`
+	FallBackUrl       string `json:"fallback_url"`
+	Height            int    `json:"height"`
+	Width             int    `json:"width"`
+	ScrubberMediaUrl  string `json:"scrubber_media_url"`
+	DashUrl           string `json:"dash_url"`
+	Duration          int    `json:"duration"`
+	HlsUrl            string `json:"hls_url"`
+	IsGif             bool   `json:"is_gif"`
+	TranscodingStatus string `json:"transcoding_status"`
+}

internal/dtoconv/dtoconv.go

@@ -0,0 +1,67 @@
+package dtoconv
+
+import (
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+)
+
+func ArticlesToDto(items []entity.ArticleEntity) []domain.ArticleDto {
+	var dtos []domain.ArticleDto
+	for _, item := range items {
+		dtos = append(dtos, ArticleToDto(item))
+	}
+	return dtos
+}
+
+func ArticleToDto(item entity.ArticleEntity) domain.ArticleDto {
+	return domain.ArticleDto{
+		ID:             item.ID,
+		SourceID:       item.SourceID,
+		Tags:           item.Tags,
+		Title:          item.Title,
+		Url:            item.Url,
+		PubDate:        item.PubDate,
+		IsVideo:        item.IsVideo,
+		Thumbnail:      item.Thumbnail,
+		Description:    item.Description,
+		AuthorName:     item.AuthorName,
+		AuthorImageUrl: item.AuthorImageUrl,
+	}
+}
+
+func DiscordWebhooksToDto(items []entity.DiscordWebHookEntity) []domain.DiscordWebHookDto{
+	var dtos []domain.DiscordWebHookDto
+	for _, item := range items {
+		dtos = append(dtos, DiscordWebhookToDto(item))
+	}
+	return dtos
+}
+
+func DiscordWebhookToDto(item entity.DiscordWebHookEntity) domain.DiscordWebHookDto {
+	return domain.DiscordWebHookDto{
+		ID:      item.ID,
+		Server:  item.Server,
+		Channel: item.Channel,
+		Url:     item.Url,
+		Enabled: item.Enabled,
+	}
+}
+
+func SourcesToDto(items []entity.SourceEntity) []domain.SourceDto {
+	var dtos []domain.SourceDto
+	for _, item := range items {
+		dtos = append(dtos, SourceToDto(item))
+	}
+	return dtos
+}
+
+func SourceToDto(item entity.SourceEntity) domain.SourceDto {
+	return domain.SourceDto{
+		ID:          item.ID,
+		Source:      item.Source,
+		DisplayName: item.DisplayName,
+		Url:         item.Url,
+		Tags:        item.Tags,
+		Enabled:     item.Enabled,
+	}
+}

internal/entity/entity.go

@@ -0,0 +1,141 @@
+package entity
+
+import (
+	"time"
+)
+
+// This links a source to a discord webhook.
+// It is owned by a user so they can remove the link
+type AlertDiscordEntity struct {
+	ID               int64
+	CreatedAt        time.Time
+	UpdatedAt        time.Time
+	DeletedAt        time.Time
+	UserID           int64
+	SourceID         int64
+	DiscordWebHookId int64
+}
+
+type ArticleEntity struct {
+	ID             int64
+	CreatedAt      time.Time
+	UpdatedAt      time.Time
+	DeletedAt      time.Time
+	SourceID       int64
+	Tags           string
+	Title          string
+	Url            string
+	PubDate        time.Time
+	IsVideo        bool
+	Thumbnail      string
+	Description    string
+	AuthorName     string
+	AuthorImageUrl string
+}
+
+type DiscordQueueEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+	ArticleId int64
+	SourceId  int64
+}
+
+type DiscordWebHookEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+	UserID    int64
+	Url       string
+	Server    string
+	Channel   string
+	Enabled   bool
+}
+
+type IconEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+	FileName  string
+	Site      string
+}
+
+type SettingEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+	Key       string
+	Value     string
+	Options   string
+}
+
+type SourceEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+
+	// Who will collect from it.  Used
+	// domain.SourceCollector...
+	Source string
+
+	// Human Readable value to state what is getting collected
+	DisplayName string
+
+	// Tells the parser where to look for data
+	Url string
+
+	// Static tags for this defined record
+	Tags string
+
+	// If the record is disabled, then it will be skipped on processing
+	Enabled bool
+}
+
+//type SubscriptionEntity struct {
+//	ID        int64
+//	CreatedAt time.Time
+//	UpdatedAt time.Time
+//	DeletedAt time.Time
+//	UserID    int64
+//	SourceID  int64
+//	//SourceType  string
+//	//SourceName  string
+//	DiscordID int64
+//	//DiscordName string
+//}
+
+// This defines what sources a user wants to follow.
+// These will show up for the user as a front page
+type UserSourceSubscriptionEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+	UserID    int64
+	SourceID  int64
+}
+
+type UserEntity struct {
+	ID           int64
+	CreatedAt    time.Time
+	UpdatedAt    time.Time
+	DeletedAt    time.Time
+	Username     string
+	Hash         string
+	Scopes       string
+	SessionToken string
+}
+
+type RefreshTokenEntity struct {
+	ID        int64
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt time.Time
+	Username  string
+	Token     string
+}

internal/handler/v1/articles.go

@@ -0,0 +1,181 @@
+package v1
+
+import (
+	"net/http"
+	"strconv"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
+	"github.com/labstack/echo/v4"
+)
+
+// ListArticles
+// @Summary   Lists the top 25 records ordering from newest to oldest.
+// @Produce   application/json
+// @Param     page  query  string  false  "page number"
+// @Tags      Articles
+// @Router    /v1/articles [get]
+// @Success   200  {object}  domain.ArticleResponse
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (s *Handler) listArticles(c echo.Context) error {
+	resp := domain.ArticleResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
+	if err != nil {
+		return c.JSON(http.StatusUnauthorized, resp)
+	}
+
+	page, err := strconv.Atoi(c.QueryParam("page"))
+	if err != nil {
+		page = 0
+	}
+
+	res, err := s.repo.Articles.ListByPage(c.Request().Context(), page, 25)
+	if err != nil {
+		return c.JSON(http.StatusInternalServerError, resp)
+	}
+
+	resp.Payload = dtoconv.ArticlesToDto(res)
+	resp.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, resp)
+}
+
+// GetArticle
+// @Summary   Returns an article based on defined ID.
+// @Param     ID  path  string  true  "int"
+// @Produce   application/json
+// @Tags      Articles
+// @Router    /v1/articles/{ID} [get]
+// @Success   200  {object}  domain.ArticleResponse  "OK"
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (s *Handler) getArticle(c echo.Context) error {
+	p := domain.ArticleResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
+	if err != nil {
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id := c.Param("ID")
+	idNumber, err := strconv.Atoi(id)
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	item, err := s.repo.Articles.GetById(c.Request().Context(), int64(idNumber))
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	var dtos []domain.ArticleDto
+	dtos = append(dtos, dtoconv.ArticleToDto(item))
+	p.Payload = dtos
+
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// GetArticleDetails
+// @Summary   Returns an article and source based on defined ID.
+// @Param     ID  path  string  true  "int"
+// @Produce   application/json
+// @Tags      Articles
+// @Router    /v1/articles/{ID}/details [get]
+// @Success   200  {object}  domain.ArticleDetailedResponse  "OK"
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (s *Handler) getArticleDetails(c echo.Context) error {
+	p := domain.ArticleDetailedResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+		Payload: domain.ArticleAndSourceModel{},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
+	if err != nil {
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	article, err := s.repo.Articles.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	source, err := s.repo.Sources.GetById(c.Request().Context(), article.SourceID)
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	p.Payload.Article = dtoconv.ArticleToDto(article)
+	p.Payload.Source = dtoconv.SourceToDto(source)
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// ListArticlesBySourceID
+// @Summary   Finds the articles based on the SourceID provided.  Returns the top 25.
+// @Param     id    query  string  true   "source id"
+// @Param     page  query  int     false  "Page to query"
+// @Produce   application/json
+// @Tags      Articles
+// @Router    /v1/articles/by/sourceId [get]
+// @Success   200  {object}  domain.ArticleResponse  "OK"
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (s *Handler) ListArticlesBySourceId(c echo.Context) error {
+	p := domain.ArticleResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
+	if err != nil {
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.QueryParam("id"))
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// if the page number is missing, default to 0
+	_page, err := strconv.Atoi(c.QueryParam("page"))
+	if err != nil {
+		_page = 0
+	}
+
+	items, err := s.repo.Articles.ListBySource(c.Request().Context(), _page, 25, id, "")
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	p.Payload = dtoconv.ArticlesToDto(items)
+	p.BaseResponse.IsError = false
+
+	return c.JSON(http.StatusOK, p)
+}

internal/handler/v1/discordwebhooks.go

@@ -0,0 +1,443 @@
+package v1
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
+	"github.com/labstack/echo/v4"
+)
+
+// ListDiscordWebhooks
+// @Summary   Returns the top 100
+// @Produce   application/json
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks [get]
+// @Success   200  {object}  domain.DiscordWebhookResponse
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) ListDiscordWebHooks(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
+	if err != nil {
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	res, err := s.repo.DiscordWebHooks.ListByServerName(c.Request().Context(), "")
+	if err != nil {
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Payload = dtoconv.DiscordWebhooksToDto(res)
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// GetDiscordWebHook
+// @Summary   Returns the top 100 entries from the queue to be processed.
+// @Produce   application/json
+// @Param     id  path  int  true  "id"
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks/{id} [get]
+// @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) GetDiscordWebHooksById(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
+	if err != nil {
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	res, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dtos []domain.DiscordWebHookDto
+	dtos = append(dtos, dtoconv.DiscordWebhookToDto(res))
+	p.Payload = dtos
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// GetDiscordWebHookByServerAndChannel
+// @Summary   Returns all the known web hooks based on the Server and Channel given.
+// @Produce   application/json
+// @Param     server   query  string  true  "Fancy Server"
+// @Param     channel  query  string  true  "memes"
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks/by/serverAndChannel [get]
+// @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	_server := c.QueryParam("server")
+	if _server == "" {
+		p.BaseResponse.Message = "server was not defined"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	_channel := c.QueryParam("channel")
+	if _channel == "" {
+		p.BaseResponse.Message = "channel was not defined"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	res, err := s.repo.DiscordWebHooks.ListByServerAndChannel(c.Request().Context(), _server, _channel)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Payload = dtoconv.DiscordWebhooksToDto(res)
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// NewDiscordWebHook
+// @Summary   Creates a new record for a discord web hook to post data to.
+// @Param     url      query  string  true  "url"
+// @Param     server   query  string  true  "Server name"
+// @Param     channel  query  string  true  "Channel name"
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks/new [post]
+// @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) NewDiscordWebHook(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	token, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	_url := c.QueryParam("url")
+	_server := c.QueryParam("server")
+	_channel := c.QueryParam("channel")
+
+	if _url == "" {
+		p.Message = "url is missing a value"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if !strings.Contains(_url, "discord.com/api/webhooks") {
+		p.Message = "invalid url"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if _server == "" {
+		p.Message = "server is missing"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if _channel == "" {
+		p.Message = "channel is missing"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	rows, err := s.repo.DiscordWebHooks.Create(c.Request().Context(), user.ID, _url, _server, _channel, true)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if rows != 1 {
+		p.Message = "data was not written to database"
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err := s.repo.DiscordWebHooks.GetByUrl(c.Request().Context(), _url)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dtos []domain.DiscordWebHookDto
+	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
+
+	p.Payload = dtos
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// DisableDiscordWebHooks
+// @Summary   Disables a Webhook from being used.
+// @Param     id  path  int  true  "id"
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks/{ID}/disable [post]
+// @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) disableDiscordWebHook(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to make sure we can find the record
+	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if record.UserID != s.GetUserIdFromJwtToken(c) {
+		p.BaseResponse.Message = ErrYouDontOwnTheRecord
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// flip the it
+	updated, err := s.repo.DiscordWebHooks.Disable(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	// make sure we got a row updated
+	if updated != 1 {
+		p.BaseResponse.Message = "unexpected number of updates found"
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dtos []domain.DiscordWebHookDto
+	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
+
+	p.Payload = dtos
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// EnableDiscordWebHook
+// @Summary   Enables a source to continue processing.
+// @Param     id  path  int  true  "id"
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks/{ID}/enable [post]
+// @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) enableDiscordWebHook(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to make sure we can find the record
+	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	if record.UserID != s.GetUserIdFromJwtToken(c) {
+		p.BaseResponse.Message = ErrYouDontOwnTheRecord
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	updated, err := s.repo.DiscordWebHooks.Enable(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if updated != 1 {
+		p.BaseResponse.Message = ErrFailedToUpdateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dtos []domain.DiscordWebHookDto
+	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
+
+	p.Payload = dtos
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// DeleteDiscordWebHook
+// @Summary   Deletes a record by ID.
+// @Param     id  path  string  true  "id"
+// @Tags      DiscordWebhook
+// @Router    /v1/discord/webhooks/{ID} [delete]
+// @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure   400  {object}  domain.DiscordWebhookResponse
+// @Failure   500  {object}  domain.DiscordWebhookResponse
+// @Security  Bearer
+func (s *Handler) deleteDiscordWebHook(c echo.Context) error {
+	p := domain.DiscordWebhookResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to make sure we can find the record
+	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	if record.UserID != s.GetUserIdFromJwtToken(c) {
+		p.BaseResponse.Message = ErrYouDontOwnTheRecord
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Soft delete the record
+	updated, err := s.repo.DiscordWebHooks.SoftDelete(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if updated != 1 {
+		p.BaseResponse.Message = ErrFailedToUpdateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dtos []domain.DiscordWebHookDto
+	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
+
+	p.Payload = dtos
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// UpdateDiscordWebHook
+// @Summary  Updates a valid discord webhook ID based on the body given.
+// @Param    id  path  string  true  "id"
+// @Tags     DiscordWebhook
+// @Router   /v1/discord/webhooks/{id} [patch]
+// @Success  200  {object}  domain.DiscordWebhookResponse  "OK"
+// @Failure  400  {object}  domain.BaseResponse
+// @Failure  500  {object}  domain.BaseResponse
+//func (s *Handler) UpdateDiscordWebHook(c echo.Context) error {
+//	id, err := strconv.Atoi(c.Param("ID"))
+//	if err != nil {
+//		return c.JSON(http.StatusInternalServerError, err.Error())
+//	}
+//
+//	// Check to make sure we can find the record
+//	_, err = s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
+//	if err != nil {
+//		return c.JSON(http.StatusInternalServerError, err.Error())
+//	}
+//
+//	// Soft delete the record
+//	updated, err := s.repo.DiscordWebHooks(c.Request().Context(), int64(id))
+//	if err != nil {
+//		return c.JSON(http.StatusInternalServerError, err.Error())
+//	}
+//
+//	_, err = s.Db.GetDiscordQueueByID(c.Request().Context(), uuid)
+//	if err != nil {
+//		return c.JSON(http.StatusInternalServerError, err.Error())
+//	}
+//
+//	return nil
+//}

internal/handler/v1/handler.go

@@ -0,0 +1,197 @@
+package v1
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"github.com/golang-jwt/jwt/v5"
+	echojwt "github.com/labstack/echo-jwt/v4"
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	swagger "github.com/swaggo/echo-swagger"
+
+	_ "git.jamestombleson.com/jtom38/newsbot-api/docs"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+)
+
+type Handler struct {
+	Router *echo.Echo
+	//Db     *database.Queries
+	config services.Configs
+	repo   services.RepositoryService
+}
+
+const (
+	ErrParameterIdMissing = "The requested parameter ID was not found."
+	ErrParameterMissing   = "The requested parameter was not found found:"
+	ErrUnableToParseId    = "Unable to parse the requested ID"
+
+	ErrRecordMissing        = "The requested record was not found"
+	ErrFailedToCreateRecord = "The record was not created due to a database problem"
+	ErrFailedToUpdateRecord = "The requested record was not updated due to a database problem"
+
+	ErrUserUnknown         = "User is unknown"
+	ErrYouDontOwnTheRecord = "The record requested does not belong to you"
+
+	ResponseMessageSuccess = "Success"
+)
+
+var (
+	ErrIdValueMissing        string = "id value is missing"
+	ErrValueNotUuid          string = "a value given was expected to be a uuid but was not correct."
+	ErrNoRecordFound         string = "no record was found."
+	ErrUnableToConvertToJson string = "Unable to convert to json"
+)
+
+func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
+	s := &Handler{
+		config: configs,
+		repo:   services.NewRepositoryService(conn),
+	}
+
+	jwtConfig := echojwt.Config{
+		NewClaimsFunc: func(c echo.Context) jwt.Claims {
+			return new(JwtToken)
+		},
+		SigningKey: []byte(configs.JwtSecret),
+	}
+
+	router := echo.New()
+	router.Pre(middleware.RemoveTrailingSlash())
+	router.Pre(middleware.Logger())
+	router.Pre(middleware.Recover())
+	router.GET("/swagger/*", swagger.WrapHandler)
+
+	v1 := router.Group("/api/v1")
+	articles := v1.Group("/articles")
+	articles.Use(echojwt.WithConfig(jwtConfig))
+	articles.GET("", s.listArticles)
+	articles.GET(":id", s.getArticle)
+	articles.GET(":id/details", s.getArticleDetails)
+	articles.GET("by/source/:id", s.ListArticlesBySourceId)
+
+	//dwh := v1.Group("/discord/webhooks")
+	//dwh.GET("/", s.ListDiscordWebHooks)
+	//dwh.POST("/new", s.NewDiscordWebHook)
+	//dwh.GET("/by/serverAndChannel", s.GetDiscordWebHooksByServerAndChannel)
+	//dwh.GET("/:ID", s.GetDiscordWebHooksById)
+	//dwh.DELETE("/:ID", s.deleteDiscordWebHook)
+	//dwh.POST("/:ID/disable", s.disableDiscordWebHook)
+	//dwh.POST("/:ID/enable", s.enableDiscordWebHook)
+
+	//queue := v1.Group("/queue")
+	//queue.GET("/discord/webhooks", s.ListDiscordWebhookQueue) // TODO this needs to be reworked
+
+	//settings := v1.Group("/settings")
+	//settings.GET("/", s.getSettings)
+
+	sources := v1.Group("/sources")
+	sources.Use(echojwt.WithConfig(jwtConfig))
+	sources.GET("", s.listSources)
+	sources.GET("/by/source", s.listSourcesBySource)
+	sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName)
+	//sources.POST("/new/reddit", s.newRedditSource)
+	//sources.POST("/new/youtube", s.newYoutubeSource)
+	//sources.POST("/new/twitch", s.newTwitchSource)
+	sources.POST("/new/rss", s.newRssSource)
+	sources.GET("/:id", s.getSource)
+	sources.DELETE("/:id", s.deleteSources)
+	sources.POST("/:id/disable", s.disableSource)
+	sources.POST("/:id/enable", s.enableSource)
+
+	users := v1.Group("/users")
+	users.POST("/login", s.AuthLogin)
+	users.POST("/register", s.AuthRegister)
+	users.Use(echojwt.WithConfig(jwtConfig))
+	users.POST("/scopes/add", s.AddScopes)
+	users.POST("/scopes/remove", s.RemoveScopes)
+	users.POST("/refresh/token", s.RefreshJwtToken)
+	users.POST("/refresh/sessionToken", s.NewSessionToken)
+
+	s.Router = router
+	return s
+}
+
+//type ApiStatusModel struct {
+//	StatusCode int    `json:"status"`
+//	Message    string `json:"message"`
+//}
+
+//type ApiError struct {
+//	*ApiStatusModel
+//}
+
+//func (s *Handler) WriteError(c echo.Context, errMessage error, HttpStatusCode int) error {
+//	return c.JSON(HttpStatusCode, domain.BaseResponse{
+//		Message: errMessage.Error(),
+//		IsError: true,
+//	})
+//}
+
+//func (s *Handler) WriteMessage(c echo.Context, msg string, HttpStatusCode int) error {
+//	return c.JSON(HttpStatusCode, domain.BaseResponse{
+//		Message: msg,
+//	})
+//}
+
+//func (s *Handler) InternalServerErrorResponse(c echo.Context, msg string) error {
+//	return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
+//		Message: msg,
+//	})
+//}
+
+//func (s *Handler) UnauthorizedResponse(c echo.Context, msg string) error {
+//	return c.JSON(http.StatusUnauthorized, domain.BaseResponse{
+//		Message: msg,
+//	})
+//}
+
+// If the token is not valid then an json error will be returned.
+// If the token has the wrong scope, a json error will be returned.
+// If the token passes all the checks, it is valid and is returned back to the caller.
+func (s *Handler) ValidateJwtToken(c echo.Context, requiredScope string) (JwtToken, error) {
+	token, err := s.getJwtTokenFromContext(c)
+	if err != nil {
+		return JwtToken{}, errors.New(ErrJwtMissing)
+	}
+
+	err = token.hasExpired()
+	if err != nil {
+		return JwtToken{}, errors.New(ErrJwtExpired)
+	}
+
+	err = token.hasScope(requiredScope)
+	if err != nil {
+		return JwtToken{}, errors.New(ErrJwtScopeMissing)
+	}
+
+	if token.Iss != s.config.ServerAddress {
+		return JwtToken{}, errors.New(ErrJwtInvalidIssuer)
+	}
+
+	// If you are the built in admin account, skip the username and session token check
+	if token.UserName == "admin" {
+		return token, nil
+	}
+
+	user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
+	if err != nil {
+		return JwtToken{}, errors.New("user record not found")
+	}
+
+	if user.SessionToken != token.SessionToken {
+		return JwtToken{}, errors.New("invalid session token")
+	}
+
+	return token, nil
+}
+
+func (s *Handler) GetUserIdFromJwtToken(c echo.Context) int64 {
+	token, err := s.getJwtTokenFromContext(c)
+	if err != nil {
+		return -1
+	}
+
+	return token.GetUserId()
+}

internal/handler/v1/jwt.go

@@ -0,0 +1,130 @@
+package v1
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/labstack/echo/v4"
+)
+
+const (
+	ErrJwtMissing       = "auth token is missing"
+	ErrJwtClaimsMissing = "claims missing on token"
+	ErrJwtExpired       = "auth token has expired"
+	ErrJwtScopeMissing  = "required scope is missing"
+	ErrJwtInvalidIssuer = "incorrect server issued the token"
+)
+
+type JwtToken struct {
+	Exp          time.Time `json:"exp"`
+	Iss          string    `json:"iss"`
+	Authorized   bool      `json:"authorized"`
+	UserName     string    `json:"username"`
+	UserId       int64     `json:"userId"`
+	Scopes       []string  `json:"scopes"`
+	SessionToken string    `json:"sessionToken"`
+	jwt.RegisteredClaims
+}
+
+func (j JwtToken) IsValid(scope string) error {
+	err := j.hasExpired()
+	if err != nil {
+		return err
+	}
+
+	// Check to see if they have the scope to do anything
+	// if they do, let them pass
+	err = j.hasScope(domain.ScopeAll)
+	if err == nil {
+		return nil
+	}
+
+	err = j.hasScope(scope)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (j JwtToken) GetUsername() string {
+	return j.UserName
+}
+
+func (j JwtToken) GetUserId() int64 {
+	return j.UserId
+}
+
+func (j JwtToken) hasExpired() error {
+	// Check to see if the token has expired
+	//hasExpired := j.Exp.Compare(time.Now())
+	hasExpired := time.Now().Compare(j.Exp)
+	if hasExpired == 1 {
+		return errors.New(ErrJwtExpired)
+	}
+	return nil
+}
+
+// This will check the users token to make sure they have the correct scope to access the handler.
+// It will evaluate if you have the admin scope or the required scope for the handler.
+func (j JwtToken) hasScope(scope string) error {
+	// they have the scope to access everything, so let them pass.
+	userScopes := strings.Join(j.Scopes, "")
+	if strings.Contains(domain.ScopeAll, userScopes) {
+		return nil
+	}
+
+	if strings.Contains(userScopes, scope) {
+		return nil
+	}
+
+	return errors.New(ErrJwtScopeMissing)
+}
+
+func (h *Handler) generateJwt(username, issuer, sessionToken string, userScopes []string, userId int64) (string, error) {
+	return h.generateJwtWithExp(username, issuer, sessionToken, userScopes, userId, time.Now().Add(10*time.Minute))
+}
+
+func (h *Handler) generateJwtWithExp(username, issuer, sessionToken string, userScopes []string, userId int64, expiresAt time.Time) (string, error) {
+	secret := []byte(h.config.JwtSecret)
+
+	// Anyone who wants to decrypt the key needs to use the same method
+	token := jwt.New(jwt.SigningMethodHS256)
+	claims := token.Claims.(jwt.MapClaims)
+	claims["exp"] = expiresAt
+	claims["authorized"] = true
+	claims["username"] = username
+	claims["iss"] = issuer
+	claims["userId"] = userId
+	claims["sessionToken"] = sessionToken
+
+	var scopes []string
+	scopes = append(scopes, userScopes...)
+	claims["scopes"] = scopes
+
+	tokenString, err := token.SignedString(secret)
+	if err != nil {
+		return "", err
+	}
+
+	return tokenString, nil
+}
+
+func (h *Handler) getJwtTokenFromContext(c echo.Context) (JwtToken, error) {
+	// Make sure that the request came with a jwtToken
+	token, ok := c.Get("user").(*jwt.Token)
+	if !ok {
+		return JwtToken{}, errors.New(ErrJwtMissing)
+	}
+
+	// Generate the claims from the token
+	claims, ok := token.Claims.(*JwtToken)
+	if !ok {
+		return JwtToken{}, errors.New(ErrJwtClaimsMissing)
+	}
+
+	return *claims, nil
+}

internal/handler/v1/sources.go

@@ -0,0 +1,637 @@
+package v1
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
+	"github.com/labstack/echo/v4"
+)
+
+// ListSources
+// @Summary   Lists the top 50 records
+// @Param     page  query  string  false  "page number"
+// @Produce   application/json
+// @Tags      Source
+// @Router    /v1/sources [get]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse  "Unable to reach SQL or Data problems"
+// @Security  Bearer
+func (s *Handler) listSources(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	page, err := strconv.Atoi(c.QueryParam("page"))
+	if err != nil {
+		page = 0
+	}
+
+	// Default way of showing all sources
+	items, err := s.repo.Sources.List(c.Request().Context(), page, 25)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Payload = dtoconv.SourcesToDto(items)
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// ListSourcesBySource
+// @Summary   Lists the top 50 records based on the name given. Example: reddit
+// @Param     source  query  string  true   "Source Name"
+// @Param     page    query  string  false  "page number"
+// @Produce   application/json
+// @Tags      Source
+// @Router    /v1/sources/by/source [get]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) listSourcesBySource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	source := c.QueryParam("source")
+	if source == "" {
+		p.BaseResponse.Message = fmt.Sprintf("%s source", ErrParameterMissing)
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	page, err := strconv.Atoi(c.QueryParam("page"))
+	if err != nil {
+		page = 0
+	}
+
+	// Shows the list by Sources.source
+	items, err := s.repo.Sources.ListBySource(c.Request().Context(), page, 25, source)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Payload = dtoconv.SourcesToDto(items)
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// GetSource
+// @Summary   Returns a single entity by ID
+// @Param     id  path  int  true  "id"
+// @Produce   application/json
+// @Tags      Source
+// @Router    /v1/sources/{id} [get]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) getSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("id"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// GetSourceByNameAndSource
+// @Summary   Returns a single entity by ID
+// @Param     name    query  string  true  "dadjokes"
+// @Param     source  query  string  true  "reddit"
+// @Produce   application/json
+// @Tags      Source
+// @Router    /v1/sources/by/sourceAndName [get]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (s *Handler) GetSourceBySourceAndName(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	var param domain.GetSourceBySourceAndNameParamRequest
+	err = c.Bind(&param)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), param.Source, param.Name)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// NewRedditSource
+// @Summary   Creates a new reddit source to monitor.
+// @Param     name  query  string  true  "name"
+// @Param     url   query  string  true  "url"
+// @Tags      Source
+// @Router    /v1/sources/new/reddit [post]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) newRedditSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	var param domain.NewSourceParamRequest
+	err = c.Bind(&param)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if param.Url == "" {
+		p.BaseResponse.Message = "url is missing"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if !strings.Contains(param.Url, "reddit.com") {
+		p.BaseResponse.Message = "invalid url"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to see if we already have this record, if we do, return it.
+	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
+	if err == nil {
+		var dto []domain.SourceDto
+		dto = append(dto, dtoconv.SourceToDto(item))
+		p.Payload = dto
+		p.BaseResponse.IsError = false
+		return c.JSON(http.StatusOK, p)
+	}
+
+	tags := fmt.Sprintf("twitch, %v, %s", param.Name, param.Tags)
+	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorReddit, param.Name, param.Url, tags, true)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if rows != 1 {
+		p.BaseResponse.Message = ErrFailedToCreateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// NewYoutubeSource
+// @Summary   Creates a new youtube source to monitor.
+// @Param     name  query  string  true  "name"
+// @Param     url   query  string  true  "url"
+// @Tags      Source
+// @Router    /v1/sources/new/youtube [post]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) newYoutubeSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+		},
+	}
+
+	// Validate the jwt
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	var param domain.NewSourceParamRequest
+	err = c.Bind(&param)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if param.Url == "" {
+		p.BaseResponse.Message = "url is missing a value"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	if !strings.Contains(param.Url, "youtube.com") {
+		p.BaseResponse.Message = "invalid url"
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
+	if err == nil {
+		var dto []domain.SourceDto
+		dto = append(dto, dtoconv.SourceToDto(item))
+		p.Payload = dto
+		p.BaseResponse.IsError = false
+		return c.JSON(http.StatusOK, p)
+	}
+
+	tags := fmt.Sprintf("twitch, %v", param.Name)
+	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorYoutube, param.Name, param.Url, tags, true)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if rows != 1 {
+		p.BaseResponse.Message = ErrFailedToCreateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// NewTwitchSource
+// @Summary   Creates a new twitch source to monitor.
+// @Param     name  query  string  true  "name"
+// @Tags      Source
+// @Router    /v1/sources/new/twitch [post]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) newTwitchSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	var param domain.NewSourceParamRequest
+	err = c.Bind(&param)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	tags := fmt.Sprintf("twitch, %v", param.Name)
+	url := fmt.Sprintf("https://twitch.tv/%v", param.Name)
+
+	// Check if the record already exists
+	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
+	if err == nil {
+		var dto []domain.SourceDto
+		dto = append(dto, dtoconv.SourceToDto(item))
+		p.Payload = dto
+		p.BaseResponse.IsError = false
+		return c.JSON(http.StatusOK, p)
+	}
+
+	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorTwitch, param.Name, url, tags, true)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if rows != 1 {
+		p.BaseResponse.Message = ErrFailedToCreateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, _ = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// NewRssSource
+// @Summary   Creates a new rss source to monitor.
+// @Param     name  query  string  true  "Site Name"
+// @Param     url   query  string  true  "RSS Url"
+// @Tags      Source
+// @Router    /v1/sources/new/rss [post]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) newRssSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+
+	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	var param domain.NewSourceParamRequest
+	err = c.Bind(&param)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	if param.Url == "" {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check if the record already exists
+	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
+	if err == nil {
+		var dto []domain.SourceDto
+		dto = append(dto, dtoconv.SourceToDto(item))
+		p.Payload = dto
+		p.BaseResponse.IsError = false
+		return c.JSON(http.StatusOK, p)
+	}
+
+	tags := fmt.Sprintf("rss, %v, %s", param.Name, param.Tags)
+	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorRss, param.Name, param.Url, tags, true)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	if rows != 1 {
+		p.BaseResponse.Message = ErrFailedToCreateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// DeleteSource
+// @Summary   Marks a source as deleted based on its ID value.
+// @Param     id  path  int  true  "id"
+// @Tags      Source
+// @Router    /v1/sources/{id} [POST]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) deleteSources(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to make sure we can find the record
+	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	// Delete the record
+	rows, err := s.repo.Sources.SoftDelete(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+	if rows != 1 {
+		p.BaseResponse.Message = ErrFailedToUpdateRecord
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	// pull the record with its updated value
+	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var items []domain.SourceDto
+	items = append(items, dtoconv.SourceToDto(item))
+	p.Payload = items
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// DisableSource
+// @Summary   Disables a source from processing.
+// @Param     id  path  int  true  "id"
+// @Tags      Source
+// @Router    /v1/sources/{id}/disable [post]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) disableSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to make sure we can find the record
+	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	_, err = s.repo.Sources.Disable(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// EnableSource
+// @Summary   Enables a source to continue processing.
+// @Param     id  path  string  true  "id"
+// @Tags      Source
+// @Router    /v1/sources/{id}/enable [post]
+// @Success   200  {object}  domain.SourcesResponse  "ok"
+// @Failure   400  {object}  domain.SourcesResponse
+// @Failure   500  {object}  domain.SourcesResponse
+// @Security  Bearer
+func (s *Handler) enableSource(c echo.Context) error {
+	p := domain.SourcesResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+	}
+	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+
+	id, err := strconv.Atoi(c.Param("ID"))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	// Check to make sure we can find the record
+	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	_, err = s.repo.Sources.Enable(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	var dto []domain.SourceDto
+	dto = append(dto, dtoconv.SourceToDto(item))
+	p.Payload = dto
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}

internal/handler/v1/users.go

@@ -0,0 +1,351 @@
+package v1
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+
+	"github.com/google/uuid"
+	"github.com/labstack/echo/v4"
+)
+
+const (
+	ErrUserNotFound          = "requested user does not exist"
+	ErrUsernameAlreadyExists = "the requested username already exists"
+)
+
+// @Summary  Creates a new user
+// @Router   /v1/users/register [post]
+// @Param    request  formData  domain.LoginFormRequest  true  "form"
+// @Accepts  x-www-form-urlencoded
+// @Produce   json
+// @Tags      Users
+// @Success  201  {object}  domain.BaseResponse
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+func (h *Handler) AuthRegister(c echo.Context) error {
+	p := domain.BaseResponse{
+		Message: ResponseMessageSuccess,
+		IsError: true,
+	}
+
+	username := c.FormValue("username")
+	password := c.FormValue("password")
+
+	//username := c.QueryParam("username")
+	exists, err := h.repo.Users.GetUser(c.Request().Context(), username)
+	if err != nil {
+		// if we have an err, validate that if its not user not found.
+		// if the user is not found, we can use that name
+		if err.Error() != repository.ErrUserNotFound {
+			p.Message = err.Error()
+			return c.JSON(http.StatusBadRequest, p)
+		}
+	}
+	if exists.Username == username {
+		p.Message = ErrUsernameAlreadyExists
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	//password := c.QueryParam("password")
+	err = h.repo.Users.CheckPasswordForRequirements(password)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	_, err = h.repo.Users.Create(c.Request().Context(), username, password, domain.ScopeArticleRead)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	return c.JSON(http.StatusCreated, p)
+}
+
+// @Summary  Logs into the API and returns a bearer token if successful
+// @Router   /v1/users/login [post]
+// @Param    request  formData  domain.LoginFormRequest  true  "form"
+// @Accepts  x-www-form-urlencoded
+// @Produce   json
+// @Tags      Users
+// @Success   200  {object}  domain.LoginResponse
+// @Failure  400  {object}  domain.LoginResponse
+// @Failure  500  {object}  domain.LoginResponse
+func (h *Handler) AuthLogin(c echo.Context) error {
+	p := domain.LoginResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+		//Token:        jwt,
+		Type: "Bearer",
+		//RefreshToken: refresh,
+	}
+
+	username := c.FormValue("username")
+	password := c.FormValue("password")
+
+	// Check to see if they are trying to login with the admin token
+	if username == "" {
+		return h.createAdminToken(c, password)
+	}
+
+	// check if the user exists
+	user, err := h.repo.Users.GetUser(c.Request().Context(), username)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	// make sure the hash matches
+	err = h.repo.Users.DoesPasswordMatchHash(c.Request().Context(), username, password)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	// TODO think about moving this down some?
+	expiresAt := time.Now().Add(time.Hour * 48)
+	userScopes := strings.Split(user.Scopes, ",")
+
+	jwt, err := h.generateJwtWithExp(username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, expiresAt)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	refresh, err := h.repo.RefreshTokens.Create(c.Request().Context(), username)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Token = jwt
+	p.RefreshToken = refresh
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+func (h *Handler) createAdminToken(c echo.Context, password string) error {
+	p := domain.LoginResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: ResponseMessageSuccess,
+			IsError: true,
+		},
+		//Token: token,
+		Type: "Bearer",
+	}
+
+	// if the admin token is blank, then the admin wanted this disabled.
+	// this will fail right away and not progress.
+	if h.config.AdminSecret == "" {
+		p.BaseResponse.Message = ErrUserNotFound
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	if h.config.AdminSecret != password {
+		p.BaseResponse.Message = ErrUserNotFound
+		return c.JSON(http.StatusBadRequest, p)
+	}
+	var userScopes []string
+	userScopes = append(userScopes, domain.ScopeAll)
+	sessionToken, err := uuid.NewV7()
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	token, err := h.generateJwt("admin", h.config.ServerAddress, sessionToken.String(), userScopes, -1)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Token = token
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// This will take collect some information about the requested refresh, validate and then return a new jwt token if approved.
+// Register
+// @Summary   Generates a new token
+// @Router    /v1/users/refresh/token [post]
+// @Param     request  body  domain.RefreshTokenRequest  true  "body"
+// @Tags      Users
+// @Success  200  {object}  domain.LoginResponse
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (h *Handler) RefreshJwtToken(c echo.Context) error {
+	p := domain.LoginResponse{
+		BaseResponse: domain.BaseResponse{
+			Message: "OK",
+		},
+		//Token:        jwt,
+		Type: "Bearer",
+		//RefreshToken: newRefreshToken,
+	}
+
+	_, err := h.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	// Check the context for the refresh token
+	var request domain.RefreshTokenRequest
+	err = (&echo.DefaultBinder{}).BindBody(c, &request)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	err = h.repo.RefreshTokens.IsRequestValid(c.Request().Context(), request.Username, request.RefreshToken)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	user, err := h.repo.Users.GetUser(c.Request().Context(), request.Username)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+	userScopes := strings.Split(user.Scopes, ",")
+
+	jwt, err := h.generateJwtWithExp(request.Username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, time.Now().Add(time.Hour*48))
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	newRefreshToken, err := h.repo.RefreshTokens.Create(c.Request().Context(), request.Username)
+	if err != nil {
+		p.BaseResponse.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.Token = jwt
+	p.RefreshToken = newRefreshToken
+	p.BaseResponse.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// @Summary   Adds a new scope to a user account
+// @Router    /v1/users/scopes/add [post]
+// @Param     request  body  domain.UpdateScopesRequest  true  "body"
+// @Tags     Users
+// @Accept    json
+// @Produce   json
+// @Success   200  {object}  domain.BaseResponse
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (h *Handler) AddScopes(c echo.Context) error {
+	p := domain.BaseResponse{
+		Message: ResponseMessageSuccess,
+		IsError: true,
+	}
+
+	_, err := h.ValidateJwtToken(c, domain.ScopeAll)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	request := domain.UpdateScopesRequest{}
+	err = (&echo.DefaultBinder{}).BindBody(c, &request)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	err = h.repo.Users.AddScopes(c.Request().Context(), request.Username, request.Scopes)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// @Summary   Adds a new scope to a user account
+// @Router    /v1/users/scopes/remove [post]
+// @Param     request  body  domain.UpdateScopesRequest  true  "body"
+// @Tags     Users
+// @Accept    json
+// @Produce  json
+// @Success   200  {object}  domain.BaseResponse
+// @Failure   400  {object}  domain.BaseResponse
+// @Failure   500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (h *Handler) RemoveScopes(c echo.Context) error {
+	p := domain.BaseResponse{
+		Message: ResponseMessageSuccess,
+		IsError: true,
+	}
+
+	token, err := h.getJwtTokenFromContext(c)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	err = token.IsValid(domain.ScopeAll)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	request := domain.UpdateScopesRequest{}
+	err = (&echo.DefaultBinder{}).BindBody(c, &request)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusBadRequest, p)
+	}
+
+	err = h.repo.Users.RemoveScopes(c.Request().Context(), request.Username, request.Scopes)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.IsError = false
+	return c.JSON(http.StatusOK, p)
+}
+
+// @Summary   Revokes the current session token and replaces it with a new one.
+// @Router    /v1/users/refresh/sessionToken [post]
+// @Tags      Users
+// @Accept    json
+// @Produce  json
+// @Success   200  {object}  domain.BaseResponse
+// @Failure  400  {object}  domain.BaseResponse
+// @Failure  500  {object}  domain.BaseResponse
+// @Security  Bearer
+func (h *Handler) NewSessionToken(c echo.Context) error {
+	p := domain.BaseResponse{
+		Message: ResponseMessageSuccess,
+		IsError: true,
+	}
+
+	token, err := h.getJwtTokenFromContext(c)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusUnauthorized, p)
+	}
+
+	_, err = h.repo.Users.NewSessionToken(c.Request().Context(), token.UserName)
+	if err != nil {
+		p.Message = err.Error()
+		return c.JSON(http.StatusInternalServerError, p)
+	}
+
+	p.IsError = false
+	return c.JSON(http.StatusInternalServerError, p)
+}

internal/repository/alertDiscord.go

@@ -0,0 +1,122 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+type AlertDiscordRepo interface {
+	Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error)
+	SoftDelete(ctx context.Context, id int64) (int64, error)
+	Restore(ctx context.Context, id int64) (int64, error)
+	Delete(ctx context.Context, id int64) (int64, error)
+	ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error)
+}
+
+type alertDiscordRepository struct {
+	conn          *sql.DB
+	defaultLimit  int
+	defaultOffset int
+}
+
+func NewAlertDiscordRepository(conn *sql.DB) alertDiscordRepository {
+	return alertDiscordRepository{
+		conn:          conn,
+		defaultLimit:  50,
+		defaultOffset: 50,
+	}
+}
+
+func (r alertDiscordRepository) Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error) {
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("AlertDiscord")
+	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID", "DiscordWebHookID")
+	queryBuilder.Values(dt, dt, timeZero, userId, sourceId, webhookId)
+	query, args := queryBuilder.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r alertDiscordRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
+	return softDeleteRow(ctx, r.conn, "AlertDiscord", id)
+}
+
+func (r alertDiscordRepository) Restore(ctx context.Context, id int64) (int64, error) {
+	return restoreRow(ctx, r.conn, "AlertDiscord", id)
+}
+
+func (r alertDiscordRepository) Delete(ctx context.Context, id int64) (int64, error) {
+	return deleteFromTable(ctx, r.conn, "AlertDiscord", id)
+}
+
+func (r alertDiscordRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("AlertDiscord")
+	builder.Where(
+		builder.Equal("UserID", userId),
+	)
+	builder.Offset(page * limit)
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.AlertDiscordEntity{}, err
+	}
+
+	data := r.processRows(rows)
+	if len(data) == 0 {
+		return []entity.AlertDiscordEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data, nil
+}
+
+func (ur alertDiscordRepository) processRows(rows *sql.Rows) []entity.AlertDiscordEntity {
+	items := []entity.AlertDiscordEntity{}
+
+	for rows.Next() {
+		var id int64
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt time.Time
+		var userId int64
+		var sourceId int64
+		var webhookId int64
+
+		err := rows.Scan(
+			&id, &createdAt, &updatedAt, &deletedAt,
+			&userId, &sourceId, &webhookId,
+		)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		item := entity.AlertDiscordEntity{
+			ID:               id,
+			CreatedAt:        createdAt,
+			UpdatedAt:        updatedAt,
+			DeletedAt:        deletedAt,
+			UserID:           userId,
+			SourceID:         sourceId,
+			DiscordWebHookId: webhookId,
+		}
+
+		items = append(items, item)
+	}
+
+	return items
+}

internal/repository/alertDiscord_test.go

@@ -0,0 +1,63 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+)
+
+func TestAlertDiscordCreate(t *testing.T) {
+	t.Log(time.Time{})
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	r := repository.NewAlertDiscordRepository(db)
+	created, err := r.Create(context.Background(), 1, 1, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if created != 1 {
+		t.Log("failed to create the record")
+		t.FailNow()
+	}
+}
+
+func TestAlertDiscordCreateAndValidate(t *testing.T) {
+	t.Log(time.Time{})
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	source := repository.NewSourceRepository(db)
+	source.Create(context.Background(), domain.SourceCollectorRss, "Unit Testing", "www.fake.com", "testing,units", true)
+	sourceRecord, _ := source.GetBySourceAndName(context.Background(), domain.SourceCollectorRss, "Unit Testing")
+
+	webhookRepo := repository.NewDiscordWebHookRepository(db)
+	webhookRepo.Create(context.Background(), 999, "discord.com", "Unit Testing", "memes", true)
+	webhook, _ := webhookRepo.GetByUrl(context.Background(), "discord.com")
+	
+	r := repository.NewAlertDiscordRepository(db)
+	r.Create(context.Background(), 999, sourceRecord.ID, webhook.ID)
+	alert, err := r.ListByUser(context.Background(), 0, 10, 999)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+
+	if len(alert) != 1 {
+		t.Error("got the incorrect number of rows back")
+		t.FailNow()
+	}
+}

internal/repository/article.go

@@ -0,0 +1,261 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+const (
+	ArticleOrderByPublishDateDesc = "pubDate desc"
+	ArticleOrderByPublishDateAsc  = "pubDate asc"
+)
+
+type ArticlesRepo interface {
+	GetById(ctx context.Context, id int64) (entity.ArticleEntity, error)
+	GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error)
+	ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error)
+	ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error)
+	ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error)
+	ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error)
+	Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error)
+	CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error)
+}
+
+type ArticleRepository struct {
+	conn          *sql.DB
+	defaultLimit  int
+	defaultOffset int
+}
+
+func NewArticleRepository(conn *sql.DB) ArticleRepository {
+	return ArticleRepository{
+		conn:          conn,
+		defaultLimit:  50,
+		defaultOffset: 50,
+	}
+}
+
+func (ar ArticleRepository) GetById(ctx context.Context, id int64) (entity.ArticleEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("articles").Where(
+		builder.E("id", id),
+	)
+	builder.Limit(1)
+
+	query, args := builder.Build()
+	rows, err := ar.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.ArticleEntity{}, err
+	}
+
+	data := ar.processRows(rows)
+	if len(data) == 0 {
+		return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data[0], nil
+}
+
+func (ar ArticleRepository) GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("articles").Where(
+		builder.E("url", url),
+	)
+	builder.Limit(1)
+
+	query, args := builder.Build()
+	rows, err := ar.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.ArticleEntity{}, err
+	}
+
+	data := ar.processRows(rows)
+	if len(data) == 0 {
+		return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data[0], nil
+}
+
+func (ar ArticleRepository) ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("articles")
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := ar.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.ArticleEntity{}, err
+	}
+
+	data := ar.processRows(rows)
+	if len(data) == 0 {
+		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data, nil
+}
+
+func (ar ArticleRepository) ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("articles")
+	builder.OrderBy(ArticleOrderByPublishDateDesc)
+	builder.Offset(page * limit)
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := ar.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.ArticleEntity{}, err
+	}
+
+	data := ar.processRows(rows)
+	if len(data) == 0 {
+		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data, nil
+}
+
+func (ar ArticleRepository) ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("articles")
+	if orderBy != "" {
+		builder.OrderBy(orderBy)
+	}
+	builder.Offset(page * limit)
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := ar.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.ArticleEntity{}, err
+	}
+
+	data := ar.processRows(rows)
+	if len(data) == 0 {
+		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
+	}
+	return data, nil
+}
+
+func (ar ArticleRepository) ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("articles")
+	builder.JoinWithOption("InnerJoin", "sources", "articles.sourceId=sources.Id")
+	if orderBy != "" {
+		builder.OrderBy(orderBy)
+	}
+	builder.Where(
+		builder.Equal("SourceId", sourceId),
+	)
+	builder.Offset(50)
+	builder.Limit(page * limit)
+
+	query, args := builder.Build()
+	rows, err := ar.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.ArticleEntity{}, err
+	}
+
+	data := ar.processRows(rows)
+	if len(data) == 0 {
+		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
+	}
+	return data, nil
+}
+
+func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error) {
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("articles")
+	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
+	queryBuilder.Values(dt, dt, timeZero, sourceId, tags, title, url, pubDate, isVideo, thumbnailUrl, description, authorName, authorImageUrl)
+	query, args := queryBuilder.Build()
+
+	_, err := ar.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error) {
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("articles")
+	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
+	queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl)
+	query, args := queryBuilder.Build()
+
+	_, err := ar.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (ur ArticleRepository) processRows(rows *sql.Rows) []entity.ArticleEntity {
+	items := []entity.ArticleEntity{}
+
+	for rows.Next() {
+		var id int64
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt time.Time
+		var sourceId int64
+		var tags string
+		var title string
+		var url string
+		var pubDate time.Time
+		var isVideo bool
+		var thumbnail string
+		var description string
+		var authorName string
+		var authorImageUrl string
+		err := rows.Scan(
+			&id, &createdAt, &updatedAt,
+			&deletedAt, &sourceId, &tags,
+			&title, &url, &pubDate,
+			&isVideo, &thumbnail, &description,
+			&authorName, &authorImageUrl)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		item := entity.ArticleEntity{
+			ID:             id,
+			CreatedAt:      createdAt,
+			UpdatedAt:      updatedAt,
+			DeletedAt:      deletedAt,
+			SourceID:       sourceId,
+			Tags:           tags,
+			Title:          title,
+			Url:            url,
+			PubDate:        pubDate,
+			IsVideo:        isVideo,
+			Thumbnail:      thumbnail,
+			Description:    description,
+			AuthorName:     authorName,
+			AuthorImageUrl: authorImageUrl,
+		}
+
+		items = append(items, item)
+	}
+
+	return items
+}

internal/repository/article_test.go

@@ -0,0 +1,156 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+)
+
+const (
+	articleFakeDotCom = "www.fake.com"
+)
+
+func TestCreateArticle(t *testing.T) {
+	t.Log(time.Time{})
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	r := repository.NewArticleRepository(db)
+	created, err := r.Create(context.Background(), 1, "", "unit test", articleFakeDotCom, "", "testing", "", "", time.Now(), false)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if created != 1 {
+		t.Log("failed to create the record")
+		t.FailNow()
+	}
+}
+
+func TestArticleByUrl(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	r := repository.NewArticleRepository(db)
+
+	err = insertFakeArticles(r, "u1", 0)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	article, err := r.GetByUrl(context.Background(), articleFakeDotCom)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if article.Url != "www.fake.com" {
+		t.Log("failed to find the requested record")
+		t.FailNow()
+	}
+}
+
+func TestPullingMultipleArticlesWithLimit(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	r := repository.NewArticleRepository(db)
+	insertFakeArticles(r, "u1", 0)
+	insertFakeArticles(r, "u2", 0)
+	insertFakeArticles(r, "u3", 0)
+	insertFakeArticles(r, "u4", 0)
+
+	items, err := r.ListTop(context.Background(), 3)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if len(items) != 3 {
+		t.Log("expected 3 rows back")
+		t.FailNow()
+	}
+}
+
+func TestPullingMultipleArticlesWithPaging(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	r := repository.NewArticleRepository(db)
+	insertFakeArticles(r, "u1", 0)
+	insertFakeArticles(r, "u2", 0)
+	insertFakeArticles(r, "u3", 0)
+	insertFakeArticles(r, "u4", 0)
+
+	items, err := r.ListByPage(context.Background(), 2, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if items[0].Title != "u2" {
+		t.Log("pulled the wrong record with paging")
+		t.FailNow()
+	}
+}
+
+func TestPullingByPublishDate(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	r := repository.NewArticleRepository(db)
+
+	today := time.Now()
+
+	insertFakeArticles(r, "u1", 0)
+	insertFakeArticles(r, "u1", -1)
+	insertFakeArticles(r, "u1", -2)
+
+	items, err := r.ListByPublishDate(context.Background(), 0, 2, repository.ArticleOrderByPublishDateDesc)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if len(items) != 2 {
+		t.Log("expected two items back")
+		t.FailNow()
+	}
+	pubDate := items[1].PubDate.Day()
+	expectedDay := today.Day() - 1
+	if pubDate != expectedDay {
+		t.Log("expected a record that was 2 days old")
+		t.FailNow()
+	}
+}
+
+//func TestArticleBySource
+
+func insertFakeArticles(r repository.ArticleRepository, title string, daysOld int) error {
+	pubDate := time.Now().AddDate(0,0, daysOld)
+	_, err := r.Create(context.Background(), 1, "", title, articleFakeDotCom, "", "testing", "", "", pubDate, false)
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/repository/common.go

@@ -0,0 +1,71 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	timeZero = time.Time{}
+)
+
+func deleteFromTable(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
+	b := sqlbuilder.NewDeleteBuilder()
+	b.DeleteFrom(tableName)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func restoreRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
+	timeZero := time.Time{}
+	b := sqlbuilder.NewUpdateBuilder()
+	b.Update(tableName)
+	b.Set(
+		b.Assign("UpdatedAt", time.Now()),
+		b.Assign("DeletedAt", timeZero),
+	)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func softDeleteRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
+	now := time.Now()
+	b := sqlbuilder.NewUpdateBuilder()
+	b.Update(tableName)
+	b.Set(
+		b.Assign("UpdatedAt", now),
+		b.Assign("DeletedAt", now),
+	)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}

internal/repository/discordWebHooks.go

@@ -0,0 +1,228 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+type DiscordWebHookRepo interface {
+	Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error)
+	Enable(ctx context.Context, id int64) (int64, error)
+	Disable(ctx context.Context, id int64) (int64, error)
+	SoftDelete(ctx context.Context, id int64) (int64, error)
+	Restore(ctx context.Context, id int64) (int64, error)
+	Delete(ctx context.Context, id int64) (int64, error)
+	GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error)
+	GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error)
+	ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error)
+	ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error)
+}
+
+type discordWebHookRepository struct {
+	conn *sql.DB
+}
+
+func NewDiscordWebHookRepository(conn *sql.DB) discordWebHookRepository {
+	return discordWebHookRepository{
+		conn: conn,
+	}
+}
+
+func (r discordWebHookRepository) Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error) {
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("DiscordWebHooks")
+	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "Url", "Server", "Channel", "Enabled")
+	queryBuilder.Values(dt, dt, timeZero, userId, url, server, channel, enabled)
+	query, args := queryBuilder.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r discordWebHookRepository) Enable(ctx context.Context, id int64) (int64, error) {
+	b := sqlbuilder.NewUpdateBuilder()
+	b.Update("DiscordWebHooks")
+	b.Set(
+		b.Assign("Enabled", true),
+		b.Assign("UpdatedAt", time.Now()),
+	)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r discordWebHookRepository) Disable(ctx context.Context, id int64) (int64, error) {
+	b := sqlbuilder.NewUpdateBuilder()
+	b.Update("DiscordWebHooks")
+	b.Set(
+		b.Assign("Enabled", false),
+		b.Assign("UpdatedAt", time.Now()),
+	)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r discordWebHookRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
+	return softDeleteRow(ctx, r.conn, "DiscordWebHooks", id)
+}
+
+func (r discordWebHookRepository) Restore(ctx context.Context, id int64) (int64, error) {
+	return restoreRow(ctx, r.conn, "DiscordWebHooks", id)
+}
+
+func (r discordWebHookRepository) Delete(ctx context.Context, id int64) (int64, error) {
+	return deleteFromTable(ctx, r.conn, "DiscordWebHooks", id)
+}
+
+func (r discordWebHookRepository) GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("DiscordWebHooks").Where(
+		builder.E("id", id),
+	)
+	builder.Limit(1)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.DiscordWebHookEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return entity.DiscordWebHookEntity{}, err
+	}
+
+	return data[0], nil
+}
+
+func (r discordWebHookRepository) GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("DiscordWebHooks").Where(
+		builder.E("Url", url),
+	)
+	builder.Limit(1)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.DiscordWebHookEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return entity.DiscordWebHookEntity{}, err
+	}
+
+	return data[0], nil
+}
+
+func (r discordWebHookRepository) ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("DiscordWebHooks").Where(
+		builder.E("Server", name),
+	)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.DiscordWebHookEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return []entity.DiscordWebHookEntity{}, err
+	}
+
+	return data, nil
+}
+
+func (r discordWebHookRepository) ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("DiscordWebHooks").Where(
+		builder.Equal("Server", server),
+		builder.Equal("Channel", channel),
+	)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.DiscordWebHookEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return []entity.DiscordWebHookEntity{}, err
+	}
+
+	return data, nil
+}
+
+func (r discordWebHookRepository) processRows(rows *sql.Rows) ([]entity.DiscordWebHookEntity, error) {
+	items := []entity.DiscordWebHookEntity{}
+
+	for rows.Next() {
+		var id int64
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt time.Time
+		var userId int64
+		var url string
+		var server string
+		var channel string
+		var enabled bool
+		err := rows.Scan(
+			&id, &createdAt, &updatedAt,
+			&deletedAt, &userId, &url, &server,
+			&channel, &enabled,
+		)
+		if err != nil {
+			return items, err
+		}
+
+		item := entity.DiscordWebHookEntity{
+			ID:        id,
+			CreatedAt: createdAt,
+			UpdatedAt: updatedAt,
+			DeletedAt: deletedAt,
+			UserID:    userId,
+			Url:       url,
+			Server:    server,
+			Channel:   channel,
+			Enabled:   enabled,
+		}
+
+		items = append(items, item)
+	}
+
+	return items, nil
+}

internal/repository/discordWebHooks_test.go

@@ -0,0 +1,287 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+)
+
+func TestCreateDiscordWebHookRecord(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	r := repository.NewDiscordWebHookRepository(db)
+	created, err := r.Create(context.Background(), 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if created != 1 {
+		t.Log("failed to create the record")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookGetById(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewDiscordWebHookRepository(db)
+	created, err := r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if created != 1 {
+		t.Log("failed to create the record")
+		t.FailNow()
+	}
+
+	item, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.ID != 1 {
+		t.Log("got the wrong record back")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookGetByUrl(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
+	item, err := r.GetByUrl(ctx, "www.discord.com/bad/webhook")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Url != "www.discord.com/bad/webhook" {
+		t.Log("got the wrong record back")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookListByServerName(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	serverName := "Unit Testing"
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, "memes", true)
+
+	item, err := r.ListByServerName(ctx, serverName)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item[0].Server != serverName {
+		t.Log("got the wrong record back")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookListByServerAndChannel(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	serverName := "Unit Testing"
+	channel := "memes"
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
+
+	item, err := r.ListByServerAndChannel(ctx, serverName, channel)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item[0].Server != serverName {
+		t.Log("got the wrong wrong server back")
+		t.FailNow()
+	}
+
+	if item[0].Channel != channel {
+		t.Log("got the wrong channel back")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookEnableRecord(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	serverName := "Unit Testing"
+	channel := "memes"
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, false)
+
+	item, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled != false {
+		t.Log("the initial record was created wrong")
+		t.FailNow()
+	}
+
+	_, err = r.Enable(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled == updated.Enabled {
+		t.Log("failed to update the enabled value")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookDisableRecord(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	serverName := "Unit Testing"
+	channel := "memes"
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
+
+	item, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled != true {
+		t.Log("the initial record was created wrong")
+		t.FailNow()
+	}
+
+	_, err = r.Disable(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled == updated.Enabled {
+		t.Log("failed to update the enabled value")
+		t.FailNow()
+	}
+}
+
+func TestDiscordWebHookSoftDelete(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	serverName := "Unit Testing"
+	channel := "memes"
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
+	_, err = r.SoftDelete(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, _ := r.GetById(ctx, 1)
+	t.Log(updated.DeletedAt)
+}
+
+func TestDiscordWebHookRestore(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	serverName := "Unit Testing"
+	channel := "memes"
+	timeZero := time.Time{}
+
+	r := repository.NewDiscordWebHookRepository(db)
+	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
+	item, _ := r.GetById(ctx, 1)
+	if item.DeletedAt != timeZero {
+		t.Log("DeletedAt was not zero")
+		t.FailNow()
+	}
+	_, _ = r.SoftDelete(ctx, 1)
+	softDeleted, _ := r.GetById(ctx, 1)
+	if softDeleted.ID != 1 {
+		t.Log("record went boom")
+		t.FailNow()
+	}
+
+	_, err = r.Restore(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, _ := r.GetById(ctx, 1)
+	t.Log(updated.DeletedAt)
+}

internal/repository/refreshTokens.go

@@ -0,0 +1,120 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+const (
+	refreshTokenTableName = "RefreshTokens"
+)
+
+type RefreshToken interface {
+	Create(ctx context.Context, username string, token string) (int64, error)
+	GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
+	DeleteById(ctx context.Context, id int64) (int64, error)
+}
+
+type RefreshTokenRepository struct {
+	connection *sql.DB
+}
+
+func NewRefreshTokenRepository(conn *sql.DB) RefreshTokenRepository {
+	return RefreshTokenRepository{
+		connection: conn,
+	}
+}
+
+func (rt RefreshTokenRepository) Create(ctx context.Context, username string, token string) (int64, error) {
+	dt := time.Now()
+	builder := sqlbuilder.NewInsertBuilder()
+	builder.InsertInto(refreshTokenTableName)
+	builder.Cols("Username", "Token", "CreatedAt", "UpdatedAt", "DeletedAt")
+	builder.Values(username, token, dt, dt, time.Time{})
+	query, args := builder.Build()
+
+	_, err := rt.connection.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (rt RefreshTokenRepository) GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*").From(refreshTokenTableName).Where(
+		builder.E("Username", name),
+	)
+
+	query, args := builder.Build()
+	rows, err := rt.connection.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.RefreshTokenEntity{}, err
+	}
+
+	data := rt.processRows(rows)
+	if len(data) == 0 {
+		return entity.RefreshTokenEntity{}, errors.New("no token found for user")
+	}
+
+	return data[0], nil
+}
+
+func (rt RefreshTokenRepository) DeleteById(ctx context.Context, id int64) (int64, error) {
+	builder := sqlbuilder.NewDeleteBuilder()
+	builder.DeleteFrom(refreshTokenTableName)
+	builder.Where(
+		builder.EQ("Id", id),
+	)
+
+	query, args := builder.Build()
+	rows, err := rt.connection.ExecContext(ctx, query, args...)
+	if err != nil {
+		return -1, err
+	}
+
+	return rows.RowsAffected()
+}
+
+func (rd RefreshTokenRepository) processRows(rows *sql.Rows) []entity.RefreshTokenEntity {
+	items := []entity.RefreshTokenEntity{}
+
+	for rows.Next() {
+		var id int64
+		var username string
+		var token string
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt sql.NullTime
+
+		err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &token)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		item := entity.RefreshTokenEntity{
+			ID:        id,
+			Username:  username,
+			Token:     token,
+			CreatedAt: createdAt,
+			UpdatedAt: updatedAt,
+		}
+
+		if deletedAt.Valid {
+			item.DeletedAt = deletedAt.Time
+		}
+
+		items = append(items, item)
+	}
+
+	return items
+}
+
+//func (rt RefreshTokenRepository) Delete()

internal/repository/refreshTokens_test.go

@@ -0,0 +1,93 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+)
+
+func TestRefreshTokenCreate(t *testing.T) {
+	conn, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	client := repository.NewRefreshTokenRepository(conn)
+	rows, err := client.Create(context.Background(), "tester", "BadTokenDontUse")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if rows == 0 {
+		t.Log("expected one row to come back but got 0")
+	}
+}
+
+func TestRefreshTokenGetByUsername(t *testing.T) {
+	conn, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	client := repository.NewRefreshTokenRepository(conn)
+	rows, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if rows != 1 {
+		t.Log("expected a row to be added but not the wrong value back")
+		t.FailNow()
+	}
+
+	model, err := client.GetByUsername(context.Background(), "tester")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if model.Username != "tester" {
+		t.Log("got the wrong user back")
+		t.FailNow()
+	}
+}
+
+func TestRefreshTokenDeleteById(t *testing.T) {
+	conn, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	client := repository.NewRefreshTokenRepository(conn)
+	created, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if created != 1 {
+		t.Log("Unexpected number back for rows created")
+	}
+
+	model, err := client.GetByUsername(context.Background(), "tester")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, err := client.DeleteById(context.Background(), model.ID)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if updated != 1 {
+		t.Log("deleted the wrong number of records")
+		t.FailNow()
+	}
+}

internal/repository/source.go

@@ -0,0 +1,277 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+type Sources interface {
+	Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error)
+	GetById(ctx context.Context, id int64) (entity.SourceEntity, error)
+	GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error)
+	GetBySource(ctx context.Context, source string) (entity.SourceEntity, error)
+	GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error)
+	List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error)
+	ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error)
+	Enable(ctx context.Context, id int64) (int64, error)
+	Disable(ctx context.Context, id int64) (int64, error)
+	SoftDelete(ctx context.Context, id int64) (int64, error)
+	Restore(ctx context.Context, id int64) (int64, error)
+	Delete(ctx context.Context, id int64) (int64, error)
+}
+
+type sourceRepository struct {
+	conn *sql.DB
+}
+
+func NewSourceRepository(conn *sql.DB) sourceRepository {
+	return sourceRepository{
+		conn: conn,
+	}
+}
+
+func (r sourceRepository) Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error) {
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("Sources")
+	queryBuilder.Cols("CreatedAt", "UpdatedAt", "DeletedAt", "DisplayName", "Source", "Url", "Tags", "Enabled")
+	queryBuilder.Values(dt, dt, timeZero, displayName, source, url, tags, enabled)
+	query, args := queryBuilder.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r sourceRepository) GetById(ctx context.Context, id int64) (entity.SourceEntity, error) {
+	b := sqlbuilder.NewSelectBuilder()
+	b.Select("*")
+	b.From("Sources").Where(
+		b.Equal("Id", id),
+	)
+	b.Limit(1)
+	query, args := b.Build()
+
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.SourceEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return entity.SourceEntity{}, err
+	}
+
+	return data[0], nil
+}
+
+func (r sourceRepository) GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error) {
+	b := sqlbuilder.NewSelectBuilder()
+	b.Select("*")
+	b.From("Sources").Where(
+		b.Equal("DisplayName", displayName),
+	)
+	b.Limit(1)
+	query, args := b.Build()
+
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.SourceEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return entity.SourceEntity{}, err
+	}
+
+	return data[0], nil
+}
+
+func (r sourceRepository) GetBySource(ctx context.Context, source string) (entity.SourceEntity, error) {
+	b := sqlbuilder.NewSelectBuilder()
+	b.Select("*")
+	b.From("Sources").Where(
+		b.Equal("Source", source),
+	)
+	b.Limit(1)
+	query, args := b.Build()
+
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.SourceEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return entity.SourceEntity{}, err
+	}
+
+	return data[0], nil
+}
+
+func (r sourceRepository) GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error) {
+	b := sqlbuilder.NewSelectBuilder()
+	b.Select("*")
+	b.From("Sources").Where(
+		b.Equal("Source", source),
+		b.Equal("Name", name),
+	)
+	b.Limit(1)
+	query, args := b.Build()
+
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.SourceEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return entity.SourceEntity{}, err
+	}
+
+	return data[0], nil
+}
+
+func (r sourceRepository) List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("Sources")
+	builder.Offset(page * limit)
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.SourceEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return []entity.SourceEntity{}, err
+	}
+
+	return data, nil
+}
+
+func (r sourceRepository) ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("Sources")
+	builder.Where(
+		builder.Equal("Source", source),
+	)
+	builder.Offset(page * limit)
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.SourceEntity{}, err
+	}
+
+	data, err := r.processRows(rows)
+	if len(data) == 0 {
+		return []entity.SourceEntity{}, err
+	}
+
+	return data, nil
+}
+
+func (r sourceRepository) Enable(ctx context.Context, id int64) (int64, error) {
+	b := sqlbuilder.NewUpdateBuilder()
+	b.Update("Sources")
+	b.Set(
+		b.Assign("Enabled", true),
+		b.Assign("UpdatedAt", time.Now()),
+	)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r sourceRepository) Disable(ctx context.Context, id int64) (int64, error) {
+	b := sqlbuilder.NewUpdateBuilder()
+	b.Update("Sources")
+	b.Set(
+		b.Assign("Enabled", false),
+		b.Assign("UpdatedAt", time.Now()),
+	)
+	b.Where(
+		b.Equal("Id", id),
+	)
+	query, args := b.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r sourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
+	return softDeleteRow(ctx, r.conn, "Sources", id)
+}
+
+func (r sourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
+	return restoreRow(ctx, r.conn, "Sources", id)
+}
+
+func (r sourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
+	return deleteFromTable(ctx, r.conn, "Sources", id)
+}
+
+func (r sourceRepository) processRows(rows *sql.Rows) ([]entity.SourceEntity, error) {
+	items := []entity.SourceEntity{}
+
+	for rows.Next() {
+		var id int64
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt time.Time
+		var displayName string
+		var source string
+		var enabled bool
+		var url string
+		var tags string
+		err := rows.Scan(
+			&id, &createdAt, &updatedAt,
+			&deletedAt, &displayName, &source,
+			&enabled, &url, &tags,
+		)
+		if err != nil {
+			return items, err
+		}
+
+		item := entity.SourceEntity{
+			ID:          id,
+			CreatedAt:   createdAt,
+			UpdatedAt:   updatedAt,
+			DeletedAt:   deletedAt,
+			DisplayName: displayName,
+			Source:      source,
+			Enabled:     enabled,
+			Url:         url,
+			Tags:        tags,
+		}
+
+		items = append(items, item)
+	}
+
+	return items, nil
+}

internal/repository/source_test.go

@@ -0,0 +1,246 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+)
+
+func TestSourceCreate(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+
+	rows, err := r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if rows != 1 {
+		t.Log("failed to create a record, why")
+		t.FailNow()
+	}
+}
+
+func TestSourceGetById(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+
+	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	item, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if item.ID != 1 {
+		t.Log("got no record or the wrong one")
+		t.FailNow()
+	}
+}
+
+func TestSourceGetByDisplayName(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+
+	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	item, err := r.GetByDisplayName(ctx, "Test")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if item.DisplayName != "Test" {
+		t.Log("got no record or the wrong one")
+		t.FailNow()
+	}
+}
+
+func TestSourceGetBySource(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+
+	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	item, err := r.GetBySource(ctx, domain.SourceCollectorRss)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if item.Source != domain.SourceCollectorRss {
+		t.Log("got no record or the wrong one")
+		t.FailNow()
+	}
+}
+
+func TestSourceList(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+
+	items, err := r.List(ctx, 0, 4)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if len(items ) != 4 {
+		t.Log("something bad happened here")
+		t.FailNow()
+	}
+}
+
+func TestSourceListBySource(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+
+	items, err := r.ListBySource(ctx, 0, 4, domain.SourceCollectorRss)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if len(items ) != 4 {
+		t.Log("something bad happened here")
+		t.FailNow()
+	}
+}
+
+func TestSourcesEnableRecord(t *testing.T) {
+	// This depends on the seed migration
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+
+	r := repository.NewSourceRepository(db)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", false)
+	item, err := r.GetByDisplayName(ctx, "Test")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled != false {
+		t.Log("the initial record was created wrong")
+		t.FailNow()
+	}
+
+	_, err = r.Enable(ctx, item.ID)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, err := r.GetById(ctx, item.ID)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled == updated.Enabled {
+		t.Log("failed to update the enabled value")
+		t.FailNow()
+	}
+}
+
+func TestSourcesDisableRecord(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	ctx := context.Background()
+	r := repository.NewSourceRepository(db)
+	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
+	item, err := r.GetByDisplayName(ctx, "Test")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled != true {
+		t.Log("the initial record was created wrong")
+		t.FailNow()
+	}
+
+	_, err = r.Disable(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	updated, err := r.GetById(ctx, 1)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if item.Enabled == updated.Enabled {
+		t.Log("failed to update the enabled value")
+		t.FailNow()
+	}
+}

internal/repository/userSourceSubscription.go

@@ -0,0 +1,120 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+type UserSourceRepo interface {
+	Create(ctx context.Context, userId, sourceId int64) (int64, error)
+	SoftDelete(ctx context.Context, id int64) (int64, error)
+	Restore(ctx context.Context, id int64) (int64, error)
+	Delete(ctx context.Context, id int64) (int64, error)
+	ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error)
+}
+
+type userSourceRepository struct {
+	conn          *sql.DB
+	defaultLimit  int
+	defaultOffset int
+}
+
+func NewUserSourceRepository(conn *sql.DB) userSourceRepository {
+	return userSourceRepository{
+		conn:          conn,
+		defaultLimit:  50,
+		defaultOffset: 50,
+	}
+}
+
+func (r userSourceRepository) Create(ctx context.Context, userId, sourceId int64) (int64, error) {
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("UserSourceSubscriptions")
+	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID")
+	queryBuilder.Values(dt, dt, timeZero, userId, sourceId)
+	query, args := queryBuilder.Build()
+
+	_, err := r.conn.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (r userSourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
+	return softDeleteRow(ctx, r.conn, "UserSourceSubscriptions", id)
+}
+
+func (r userSourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
+	return restoreRow(ctx, r.conn, "UserSourceSubscriptions", id)
+}
+
+func (r userSourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
+	return deleteFromTable(ctx, r.conn, "UserSourceSubscriptions", id)
+}
+
+func (r userSourceRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*")
+	builder.From("UserSourceSubscriptions")
+	builder.Where(
+		builder.Equal("UserID", userId),
+	)
+	builder.Offset(page * limit)
+	builder.Limit(limit)
+
+	query, args := builder.Build()
+	rows, err := r.conn.QueryContext(ctx, query, args...)
+	if err != nil {
+		return []entity.UserSourceSubscriptionEntity{}, err
+	}
+
+	data := r.processRows(rows)
+	if len(data) == 0 {
+		return []entity.UserSourceSubscriptionEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data, nil
+}
+
+func (ur userSourceRepository) processRows(rows *sql.Rows) []entity.UserSourceSubscriptionEntity {
+	items := []entity.UserSourceSubscriptionEntity{}
+
+	for rows.Next() {
+		var id int64
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt time.Time
+		var userId int64
+		var sourceId int64
+
+		err := rows.Scan(
+			&id, &createdAt, &updatedAt, &deletedAt,
+			&userId, &sourceId,
+		)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		item := entity.UserSourceSubscriptionEntity{
+			ID:        id,
+			CreatedAt: createdAt,
+			UpdatedAt: updatedAt,
+			DeletedAt: deletedAt,
+			UserID:    userId,
+			SourceID:  sourceId,
+		}
+
+		items = append(items, item)
+	}
+
+	return items
+}

internal/repository/users.go

@@ -0,0 +1,192 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+
+	"github.com/huandu/go-sqlbuilder"
+	"golang.org/x/crypto/bcrypt"
+)
+
+const (
+	usersTableName  string = "users"
+	ErrUserNotFound string = "requested user was not found"
+)
+
+type Users interface {
+	GetByName(ctx context.Context, name string) (entity.UserEntity, error)
+	Create(ctx context.Context, name, password, sessionTOken, scope string) (int64, error)
+	Update(ctx context.Context, id int, entity entity.UserEntity) error
+	UpdatePassword(ctx context.Context, name, password string) error
+	CheckUserHash(ctx context.Context, name, password string) error
+	UpdateScopes(ctx context.Context, name, scope string) error
+	UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error)
+}
+
+// Creates a new instance of UserRepository with the bound sql
+func NewUserRepository(conn *sql.DB) userRepository {
+	return userRepository{
+		connection: conn,
+	}
+}
+
+type userRepository struct {
+	connection *sql.DB
+}
+
+func (ur userRepository) GetByName(ctx context.Context, name string) (entity.UserEntity, error) {
+	builder := sqlbuilder.NewSelectBuilder()
+	builder.Select("*").From("users").Where(
+		builder.E("Name", name),
+	)
+	query, args := builder.Build()
+
+	rows, err := ur.connection.QueryContext(ctx, query, args...)
+	if err != nil {
+		return entity.UserEntity{}, err
+	}
+
+	data := ur.processRows(rows)
+	if len(data) == 0 {
+		return entity.UserEntity{}, errors.New(ErrUserNotFound)
+	}
+
+	return data[0], nil
+}
+
+func (ur userRepository) Create(ctx context.Context, name, password, sessionToken, scope string) (int64, error) {
+	passwordBytes := []byte(password)
+	hash, err := bcrypt.GenerateFromPassword(passwordBytes, bcrypt.DefaultCost)
+	if err != nil {
+		return 0, err
+	}
+
+	dt := time.Now()
+	queryBuilder := sqlbuilder.NewInsertBuilder()
+	queryBuilder.InsertInto("users")
+	queryBuilder.Cols("Name", "Hash", "UpdatedAt", "CreatedAt", "DeletedAt", "Scopes", "SessionToken")
+	queryBuilder.Values(name, string(hash), dt, dt, time.Time{}, scope, sessionToken)
+	query, args := queryBuilder.Build()
+
+	_, err = ur.connection.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (ur userRepository) Update(ctx context.Context, id int, entity entity.UserEntity) error {
+	return errors.New("not implemented")
+}
+
+func (ur userRepository) UpdatePassword(ctx context.Context, name, password string) error {
+	_, err := ur.GetByName(ctx, name)
+	if err != nil {
+		return nil
+	}
+
+	queryBuilder := sqlbuilder.NewUpdateBuilder()
+	queryBuilder.Update(usersTableName)
+	//queryBuilder.Set
+	return nil
+}
+
+func (ur userRepository) UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error) {
+	_, err := ur.GetByName(ctx, name)
+	if err != nil {
+		return 0, err
+	}
+
+	q := sqlbuilder.NewUpdateBuilder()
+	q.Update(usersTableName)
+	q.Set(
+		q.Equal("SessionToken", sessionToken),
+	)
+	q.Where(
+		q.Equal("Name", name),
+	)
+
+	query, args := q.Build()
+	rowsUpdates, err := ur.connection.ExecContext(ctx, query, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	return rowsUpdates.RowsAffected()
+}
+
+// If the hash matches what we have in the database, an error will not be returned.
+// If the user does not exist or the hash does not match, an error will be returned
+func (ur userRepository) CheckUserHash(ctx context.Context, name, password string) error {
+	record, err := ur.GetByName(ctx, name)
+	if err != nil {
+		return err
+	}
+
+	err = bcrypt.CompareHashAndPassword([]byte(record.Hash), []byte(password))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ur userRepository) UpdateScopes(ctx context.Context, name, scope string) error {
+	builder := sqlbuilder.NewUpdateBuilder()
+	builder.Update("users")
+	builder.Set(
+		builder.Assign("Scopes", scope),
+	)
+	builder.Where(
+		builder.Equal("Name", name),
+	)
+	query, args := builder.Build()
+
+	_, err := ur.connection.ExecContext(ctx, query, args...)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ur userRepository) processRows(rows *sql.Rows) []entity.UserEntity {
+	items := []entity.UserEntity{}
+
+	for rows.Next() {
+		var id int64
+		var username string
+		var hash string
+		var createdAt time.Time
+		var updatedAt time.Time
+		var deletedAt sql.NullTime
+		var scopes string
+		var sessionToken string
+		err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &hash, &scopes, &sessionToken)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		item := entity.UserEntity{
+			ID:           id,
+			UpdatedAt:    updatedAt,
+			Username:     username,
+			Hash:         hash,
+			Scopes:       scopes,
+			CreatedAt:    createdAt,
+			SessionToken: sessionToken,
+		}
+		if deletedAt.Valid {
+			item.DeletedAt = deletedAt.Time
+		}
+
+		items = append(items, item)
+	}
+
+	return items
+}

internal/repository/users_test.go

@@ -0,0 +1,88 @@
+package repository_test
+
+import (
+	"context"
+	"database/sql"
+	"log"
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+
+	_ "github.com/glebarez/go-sqlite"
+	"github.com/pressly/goose/v3"
+)
+
+func TestCanCreateNewUser(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	defer db.Close()
+
+	repo := repository.NewUserRepository(db)
+	updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
+	if err != nil {
+		log.Println(err)
+		t.FailNow()
+	}
+	log.Println(updated)
+}
+
+func TestCanFindUserInTable(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		log.Println("unable to open connection")
+		t.FailNow()
+	}
+	defer db.Close()
+
+	repo := repository.NewUserRepository(db)
+	updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	if updated != 1 {
+		t.Log("expected a row to come back")
+		t.FailNow()
+	}
+
+	user, err := repo.GetByName(context.Background(), "testing")
+	if err != nil {
+		log.Println(err)
+		t.FailNow()
+	}
+	log.Println(user)
+}
+
+func TestCheckUserHash(t *testing.T) {
+	db, err := setupInMemoryDb()
+	if err != nil {
+		log.Println("unable to open connection")
+		t.FailNow()
+	}
+	defer db.Close()
+
+	repo := repository.NewUserRepository(db)
+	repo.CheckUserHash(context.Background(), "testing", "NotSecure")
+}
+
+func setupInMemoryDb() (*sql.DB, error) {
+	db, err := sql.Open("sqlite", ":memory:")
+	if err != nil {
+		return nil, err
+	}
+
+	err = goose.SetDialect("sqlite3")
+	if err != nil {
+		return nil, err
+	}
+
+	err = goose.Up(db, "../database/migrations")
+	if err != nil {
+		return nil, err
+	}
+	return db, nil
+}

internal/repositoryServices/refreshTokens.go

@@ -0,0 +1,87 @@
+package repositoryservices
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+	"github.com/google/uuid"
+)
+
+const (
+	ErrUnexpectedAmountOfRowsUpdated = "got a unexpected of rows updated"
+)
+
+type RefreshToken interface {
+	Create(ctx context.Context, username string) (string, error)
+	GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
+	Delete(ctx context.Context, id int64) (int64, error)
+	IsRequestValid(ctx context.Context, username, refreshToken string) error
+}
+
+// A new jwt token can be made if the user has the correct refresh token for the user.
+// It will also require the old JWT token so the expire time is pulled and part of the validation
+type RefreshTokenService struct {
+	table repository.RefreshTokenRepository
+}
+
+func NewRefreshTokenService(conn *sql.DB) RefreshTokenService {
+	return RefreshTokenService{
+		table: repository.NewRefreshTokenRepository(conn),
+	}
+}
+
+func (rt RefreshTokenService) Create(ctx context.Context, username string) (string, error) {
+	//if a refresh token already exists for a user, reuse
+	existingToken, err := rt.GetByName(ctx, username)
+	if err == nil {
+		rowsRemoved, err := rt.Delete(ctx, existingToken.ID)
+		if err != nil {
+			return "", err
+		}
+
+		if rowsRemoved != 1 {
+			return "", errors.New(ErrUnexpectedAmountOfRowsUpdated)
+		}
+	}
+
+	token, err := uuid.NewV7()
+	if err != nil {
+		return "", err
+	}
+
+	rows, err := rt.table.Create(ctx, username, token.String())
+	if err != nil {
+		return "", err
+	}
+
+	if rows != 1 {
+		return "", errors.New("expected one row but got none")
+	}
+	return token.String(), nil
+}
+
+// Find the saved refresh token for a user and return it if it exists
+func (rt RefreshTokenService) GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
+	return rt.table.GetByUsername(ctx, name)
+}
+
+// This will request that a object is removed from the database
+func (rt RefreshTokenService) Delete(ctx context.Context, id int64) (int64, error) {
+	return rt.table.DeleteById(ctx, id)
+}
+
+func (rt RefreshTokenService) IsRequestValid(ctx context.Context, username, refreshToken string) error {
+	token, err := rt.GetByName(ctx, username)
+	if err != nil {
+		return err
+	}
+
+	if token.Token != refreshToken {
+		return errors.New("the refresh token given does not match")
+	}
+
+	return nil
+}

internal/repositoryServices/userService.go

@@ -0,0 +1,198 @@
+package repositoryservices
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"strings"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+	"github.com/google/uuid"
+
+	"golang.org/x/crypto/bcrypt"
+)
+
+const (
+	ErrPasswordNotLongEnough           = "password needs to be 8 character or longer"
+	ErrPasswordMissingSpecialCharacter = "password needs to contain one of the following: !, @, #"
+	ErrInvalidPassword                 = "invalid password"
+)
+
+type UserServices interface {
+	DoesUserExist(ctx context.Context, username string) error
+	DoesPasswordMatchHash(ctx context.Context, username, password string) error
+	GetUser(ctx context.Context, username string) (entity.UserEntity, error)
+	AddScopes(ctx context.Context, username string, scopes []string) error
+	RemoveScopes(ctx context.Context, username string, scopes []string) error
+	Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error)
+	NewSessionToken(ctx context.Context, name string) (string, error)
+	CheckPasswordForRequirements(password string) error
+}
+
+// This will handle operations that are user related, but one layer higher then the repository
+type UserService struct {
+	repo repository.Users
+}
+
+// This is a layer on top of the Users Repository.
+// Use this over directly talking to the table when ever possible.
+func NewUserService(conn *sql.DB) UserService {
+	return UserService{
+		repo: repository.NewUserRepository(conn),
+	}
+}
+
+func (us UserService) DoesUserExist(ctx context.Context, username string) error {
+	_, err := us.repo.GetByName(ctx, username)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (us UserService) DoesPasswordMatchHash(ctx context.Context, username, password string) error {
+	model, err := us.GetUser(ctx, username)
+	if err != nil {
+		return err
+	}
+
+	err = bcrypt.CompareHashAndPassword([]byte(model.Hash), []byte(password))
+	if err != nil {
+		return errors.New(ErrInvalidPassword)
+	}
+
+	return nil
+}
+
+func (us UserService) GetUser(ctx context.Context, username string) (entity.UserEntity, error) {
+	return us.repo.GetByName(ctx, username)
+}
+
+func (us UserService) AddScopes(ctx context.Context, username string, scopes []string) error {
+	usr, err := us.repo.GetByName(ctx, username)
+	if err != nil {
+		return err
+	}
+
+	if usr.Username != username {
+		return errors.New(repository.ErrUserNotFound)
+	}
+
+	currentScopes := strings.Split(usr.Scopes, ",")
+
+	// check the current scopes
+	for _, item := range scopes {
+		if !strings.Contains(usr.Scopes, item) {
+			currentScopes = append(currentScopes, item)
+		}
+	}
+	return us.repo.UpdateScopes(ctx, username, strings.Join(currentScopes, ","))
+}
+
+func (us UserService) RemoveScopes(ctx context.Context, username string, scopes []string) error {
+	usr, err := us.repo.GetByName(ctx, username)
+	if err != nil {
+		return err
+	}
+
+	if usr.Username != username {
+		return errors.New(repository.ErrUserNotFound)
+	}
+
+	var newScopes []string
+
+	// check all the scopes that are currently assigned
+	for _, item := range strings.Split(usr.Scopes, ",") {
+
+		// check the scopes given, if one matches skip it
+		if us.doesScopeExist(scopes, item) {
+			continue
+		}
+
+		// did not match, add it
+		newScopes = append(newScopes, item)
+	}
+
+	return us.repo.UpdateScopes(ctx, username, strings.Join(newScopes, ","))
+}
+
+func (us UserService) doesScopeExist(scopes []string, target string) bool {
+	for _, item := range scopes {
+		if item == target {
+			return true
+		}
+	}
+	return false
+}
+
+func (us UserService) Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error) {
+	err := us.CheckPasswordForRequirements(password)
+	if err != nil {
+		return entity.UserEntity{}, err
+	}
+
+	token, err := uuid.NewV7()
+	if err != nil {
+		return entity.UserEntity{}, err
+	}
+
+	us.repo.Create(ctx, name, password, token.String(), domain.ScopeArticleRead)
+	return entity.UserEntity{}, nil
+}
+
+func (us UserService) NewSessionToken(ctx context.Context, name string) (string, error) {
+	token, err := uuid.NewV7()
+	if err != nil {
+		return "", err
+	}
+
+	rows, err := us.repo.UpdateSessionToken(ctx, name, token.String())
+	if err != nil {
+		return "", err
+	}
+
+	if rows != 1 {
+		return "", fmt.Errorf("UserService.NewSessionToken %w", err)
+	}
+
+	return token.String(), nil
+}
+
+func (us UserService) CheckPasswordForRequirements(password string) error {
+	err := us.checkPasswordLength(password)
+	if err != nil {
+		return err
+	}
+
+	err = us.checkPasswordForSpecialCharacters(password)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (us UserService) checkPasswordLength(password string) error {
+	if len(password) < 8 {
+		return errors.New(ErrPasswordNotLongEnough)
+	}
+	return nil
+}
+
+func (us UserService) checkPasswordForSpecialCharacters(password string) error {
+	var chars []string
+	chars = append(chars, "!")
+	chars = append(chars, "@")
+	chars = append(chars, "#")
+
+	for _, char := range chars {
+		if strings.Contains(password, char) {
+			return nil
+		}
+	}
+
+	return errors.New(ErrPasswordMissingSpecialCharacter)
+}

internal/services/cache/cache.go

@@ -3,36 +3,40 @@ package cache
 import (
 	"time"
 
-	"github.com/jtom38/newsbot/collector/domain/model"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
 )
 
-type CacheClient struct{
-	group string
+type CacheClient struct {
+	group        string
 	DefaultTimer time.Duration
 }
 
 func NewCacheClient(group string) CacheClient {
 	return CacheClient{
-		group: group,
+		group:        group,
 		DefaultTimer: time.Hour,
 	}
 }
 
 func (cc *CacheClient) Insert(key string, value string) {
-	item := model.CacheItem{
-		Key: key,
-		Value: value,
-		Group: cc.group,
-		Expires: time.Now().Add(1 * time.Hour),
+	item := domain.CacheItem{
+		Key:       key,
+		Value:     value,
+		Group:     cc.group,
+		Expires:   time.Now().Add(1 * time.Hour),
 		IsTainted: false,
 	}
 	cacheStorage = append(cacheStorage, &item)
 }
 
-func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
+func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
 	for _, item := range cacheStorage {
-		if item.Group != cc.group { continue }
-		if item.Key != key { continue }
+		if item.Group != cc.group {
+			continue
+		}
+		if item.Key != key {
+			continue
+		}
 
 		// if it was tainted, renew the timer and remove the taint as this record was still needed
 		if item.IsTainted {
@@ -42,13 +46,17 @@ func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
 		return item, nil
 	}
 
-	return &model.CacheItem{}, ErrCacheRecordMissing
+	return &domain.CacheItem{}, ErrCacheRecordMissing
 }
 
-func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
+func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
 	for _, item := range cacheStorage {
-		if item.Group != cc.group { continue }
-		if item.Value != value { continue }
+		if item.Group != cc.group {
+			continue
+		}
+		if item.Value != value {
+			continue
+		}
 
 		// if it was tainted, renew the timer and remove the taint as this record was still needed
 		if item.IsTainted {
@@ -57,6 +65,5 @@ func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
 		}
 		return item, nil
 	}
-	return &model.CacheItem{}, ErrCacheRecordMissing
+	return &domain.CacheItem{}, ErrCacheRecordMissing
 }
-

internal/services/cache/cache_test.go

@@ -3,7 +3,7 @@ package cache_test
 import (
 	"testing"
 
-	"github.com/jtom38/newsbot/collector/services/cache"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
 )
 
 func TestNewCacheClient(t *testing.T) {
@@ -18,14 +18,18 @@ func TestInsert(t *testing.T) {
 func TestFindGroupMissing(t *testing.T) {
 	cache := cache.NewCacheClient("faker")
 	_, err := cache.FindByKey("UnitTesting")
-	if err == nil { panic("Nothing was appended with the requested group.") }
+	if err == nil {
+		panic("Nothing was appended with the requested group.")
+	}
 }
 
 func TestFindGroupExists(t *testing.T) {
 	cache := cache.NewCacheClient("Testing")
 	cache.Insert("UnitTesting", "Something")
 	_, err := cache.FindByKey("UnitTesting")
-	if err != nil { panic("") }
+	if err != nil {
+		panic("")
+	}
 }
 
 func TestCacheStorage(t *testing.T) {
@@ -35,6 +39,7 @@ func TestCacheStorage(t *testing.T) {
 
 	cache := cache.NewCacheClient("Testing")
 	_, err := cache.FindByKey("UnitTesting02")
-	if err != nil { panic("expected to find the value")}
+	if err != nil {
+		panic("expected to find the value")
+	}
 }
-

internal/services/cache/common.go

@@ -3,11 +3,11 @@ package cache
 import (
 	"errors"
 
-	"github.com/jtom38/newsbot/collector/domain/model"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
 )
 
 var (
-	cacheStorage []*model.CacheItem
+	cacheStorage []*domain.CacheItem
 
 	ErrCacheRecordMissing = errors.New("unable to find the requested record")
-)
+)

internal/services/cache/monitor.go

@@ -3,13 +3,13 @@ package cache
 import (
 	"time"
 
-	"github.com/jtom38/newsbot/collector/domain/model"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
 )
 
 // When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
 // If a record is tainted and used again, the taint will be removed and a new Expires value will be set.
 // If its not renewed, it will be dropped.
-type CacheAgeMonitor struct {}
+type CacheAgeMonitor struct{}
 
 func NewCacheAgeMonitor() CacheAgeMonitor {
 	return CacheAgeMonitor{}
@@ -20,10 +20,10 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
 	now := time.Now()
 	for index, item := range cacheStorage {
 		if now.After(item.Expires) {
-			
+
 			// the timer expired, and its not tainted, taint it
 			if !item.IsTainted {
-				item.IsTainted = true 
+				item.IsTainted = true
 				item.Expires = now.Add(1 * time.Hour)
 			}
 
@@ -36,10 +36,12 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
 }
 
 // This creates a new slice and skips over the item that needs to be dropped
-func (cam CacheAgeMonitor) removeEntry(index int) []*model.CacheItem {
-	var temp []*model.CacheItem
+func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
+	var temp []*domain.CacheItem
 	for i, item := range cacheStorage {
-		if i != index { temp = append(temp, item )}
+		if i != index {
+			temp = append(temp, item)
+		}
 	}
 	return temp
 }

internal/services/cache/monitor_test.go

@@ -3,11 +3,11 @@ package cache_test
 import (
 	"testing"
 
-	"github.com/jtom38/newsbot/collector/services/cache"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
 )
 
 func TestCacheTaintItem(t *testing.T) {
 	cc := cache.NewCacheClient("Testing")
 	cc.Insert("UnitTesting01", "test")
-	
-}
+
+}

internal/services/config.go

@@ -0,0 +1,154 @@
+package services
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/joho/godotenv"
+)
+
+const (
+	ServerAddress = "ServerAddress"
+
+	//Sql_Connection_String = "SQL_CONNECTION_STRING"
+
+	FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
+	REDDIT_PULL_TOP               = "REDDIT_PULL_TOP"
+	REDDIT_PULL_HOT               = "REDDIT_PULL_HOT"
+	REDDIT_PULL_NSFW              = "REDDIT_PULL_NSFW"
+
+	FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
+	YOUTUBE_DEBUG                  = "YOUTUBE_DEBUG"
+
+	FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
+	TWITCH_CLIENT_ID              = "TWITCH_CLIENT_ID"
+	TWITCH_CLIENT_SECRET          = "TWITCH_CLIENT_SECRET"
+	TWITCH_MONITOR_CLIPS          = "TWITCH_MONITOR_CLIPS"
+	TWITCH_MONITOR_VOD            = "TWITCH_MONITOR_VOD"
+
+	FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
+)
+
+type Configs struct {
+	ServerAddress string
+	JwtSecret     string
+	AdminSecret   string
+
+	RedditEnabled  bool
+	RedditPullTop  bool
+	RedditPullHot  bool
+	RedditPullNsfw bool
+
+	YoutubeEnabled bool
+	YoutubeDebug   bool
+
+	TwitchEnabled      bool
+	TwitchClientId     string
+	TwitchClientSecret string
+	TwitchMonitorClips bool
+	TwitchMonitorVOD   bool
+
+	FfxivEnabled bool
+}
+
+type ConfigClient struct{}
+
+func NewConfig() ConfigClient {
+	c := ConfigClient{}
+	c.RefreshEnv()
+
+	return c
+}
+
+func GetEnvConfig() Configs {
+	return Configs{
+		ServerAddress: os.Getenv(ServerAddress),
+		JwtSecret:     os.Getenv("JwtSecret"),
+		AdminSecret:   os.Getenv("AdminSecret"),
+
+		RedditEnabled:  processBoolConfig(os.Getenv(FEATURE_ENABLE_REDDIT_BACKEND)),
+		RedditPullTop:  processBoolConfig(os.Getenv(REDDIT_PULL_TOP)),
+		RedditPullHot:  processBoolConfig(os.Getenv(REDDIT_PULL_HOT)),
+		RedditPullNsfw: processBoolConfig(os.Getenv(REDDIT_PULL_NSFW)),
+
+		YoutubeEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_YOUTUBE_BACKEND)),
+		YoutubeDebug:   processBoolConfig(os.Getenv(YOUTUBE_DEBUG)),
+
+		TwitchEnabled:      processBoolConfig(os.Getenv(FEATURE_ENABLE_TWITCH_BACKEND)),
+		TwitchClientId:     os.Getenv(TWITCH_CLIENT_ID),
+		TwitchClientSecret: os.Getenv(TWITCH_CLIENT_SECRET),
+		TwitchMonitorClips: processBoolConfig(TWITCH_MONITOR_CLIPS),
+		TwitchMonitorVOD:   processBoolConfig(os.Getenv(TWITCH_MONITOR_VOD)),
+
+		FfxivEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_FFXIV_BACKEND)),
+	}
+}
+
+// This will parse a string and convert it to a bool.
+// If it runs into any errors, it will default to false
+func processBoolConfig(value string) bool {
+	b, err := strconv.ParseBool(value)
+	if err != nil {
+		return false
+	}
+	return b
+}
+
+func (cc *ConfigClient) GetConfig(key string) string {
+	res, filled := os.LookupEnv(key)
+	if !filled {
+		log.Printf("Missing the a value for '%v'.  Could generate errors.", key)
+	}
+	return res
+}
+
+// Looks for a value in the env and will panic if it does not exist.
+func (c ConfigClient) MustGetString(key string) string {
+	res, filled := os.LookupEnv(key)
+	if !filled {
+		msg := fmt.Sprintf("No value was found for '%v'", key)
+		panic(msg)
+	}
+
+	return res
+}
+
+func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
+	cc.RefreshEnv()
+
+	res, filled := os.LookupEnv(flag)
+	if !filled {
+		errorMessage := fmt.Sprintf("'%v' was not found", flag)
+		return false, errors.New(errorMessage)
+	}
+
+	b, err := strconv.ParseBool(res)
+	if err != nil {
+		return false, err
+	}
+	return b, nil
+}
+
+// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
+func (cc *ConfigClient) RefreshEnv() {
+	// Check to see if we have the env file on the system
+	_, err := os.Stat(".env")
+
+	// We have the file, load it.
+	if err == nil {
+		_, err := os.Open(".env")
+		if err == nil {
+			loadEnvFile()
+		}
+	}
+}
+
+func loadEnvFile() {
+	err := godotenv.Load()
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

internal/services/config_test.go

@@ -0,0 +1,22 @@
+package services_test
+
+import (
+	"os"
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+)
+
+func TestNewClient(t *testing.T) {
+	services.NewConfig()
+}
+
+func TestGetConfigExpectNull(t *testing.T) {
+	cc := services.NewConfig()
+	os.Setenv(services.REDDIT_PULL_HOT, "")
+	res := cc.GetConfig(services.REDDIT_PULL_HOT)
+	if res != "" {
+		panic("expected blank")
+	}
+
+}

internal/services/cron/collectors.go

@@ -0,0 +1,220 @@
+package cron
+
+import (
+	"log"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+)
+
+func (c *Cron) CollectRssPosts() {
+	log.Println("Starting ")
+	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
+	if err != nil {
+		log.Println(err)
+	}
+
+	for sourceIndex, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+
+		rssClient := input.NewRssClient(source)
+		articles, err := rssClient.GetArticles()
+		if err != nil {
+			log.Println(err)
+		}
+
+		for _, article := range articles {
+			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
+			if err == nil {
+				continue
+			}
+
+			rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
+			if err != nil {
+				log.Println(err)
+			}
+			if rowsCreated != 1 {
+				log.Println("Got back the wrong number of rows")
+			}
+		}
+
+		if sourceIndex != len(sources) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+}
+
+func (c *Cron) CollectRedditPosts() {
+	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
+	if err != nil {
+		log.Printf("[Reddit] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+
+		log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
+		rc := input.NewRedditClient(source)
+		raw, err := rc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+
+		redditArticles := rc.ConvertToArticles(raw)
+		for _, article := range redditArticles {
+			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
+			if err == nil {
+				continue
+			}
+
+			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
+			if err != nil {
+				log.Printf("Failed to add a new reddit article to the database: %s", err)
+			}
+
+			if rowsAdded != 1 {
+				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
+			}
+		}
+	}
+	log.Print("[Reddit] Done!")
+}
+
+func (c *Cron) CollectYoutubePosts() {
+	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
+	if err != nil {
+		log.Printf("[Youtube] No sources found to query - %v\r", err)
+	}
+
+	for sourceIndex, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+
+		log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
+		yc := input.NewYoutubeClient(source)
+		raw, err := yc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+
+		for _, article := range raw {
+			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
+			if err == nil {
+				continue
+			}
+
+			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
+			if err != nil {
+				log.Printf("Failed to add a new youtube article to the database: %s", err)
+			}
+
+			if rowsAdded != 1 {
+				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
+			}
+		}
+
+		if sourceIndex != len(sources) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+	log.Print("[YouTube] Done!")
+}
+
+func (c *Cron) CollectFfxivPosts() {
+	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
+	if err != nil {
+		log.Printf("[FFXIV] No sources found to query - %v\r", err)
+	}
+
+	for sourceIndex, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+
+		fc := input.NewFFXIVClient(source)
+		items, err := fc.CheckSource()
+		if err != nil {
+			log.Println(err)
+		}
+
+		for _, article := range items {
+			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
+			if err == nil {
+				continue
+			}
+
+			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
+			if err != nil {
+				log.Printf("Failed to add a new FFXIV article to the database: %s", err)
+			}
+
+			if rowsAdded != 1 {
+				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
+			}
+		}
+
+		if sourceIndex != len(sources) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+	log.Printf("[FFXIV Done!]")
+}
+
+func (c *Cron) CollectTwitchPosts() {
+	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
+	if err != nil {
+		log.Printf("[Twitch] No sources found to query - %v\r", err)
+	}
+
+	tc, err := input.NewTwitchClient()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	err = tc.Login()
+	if err != nil {
+		log.Println(err)
+	}
+
+	for sourceIndex, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+
+		log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
+		tc.ReplaceSourceRecord(source)
+		items, err := tc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+
+		for _, article := range items {
+			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
+			if err == nil {
+				continue
+			}
+
+			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
+			if err != nil {
+				log.Printf("Failed to add a new Twitch article to the database: %s", err)
+			}
+
+			if rowsAdded != 1 {
+				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
+			}
+		}
+
+		if sourceIndex != len(sources) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+
+	log.Print("[Twitch] Done!")
+}

internal/services/cron/collectors_test.go

@@ -0,0 +1,43 @@
+package cron_test
+
+import (
+	"context"
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
+)
+
+func TestRssPullsCorrectly(t *testing.T) {
+	conn, err := setupInMemoryDb()
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+	defer conn.Close()
+
+	ctx := context.Background()
+	db := services.NewRepositoryService(conn)
+	rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+
+	if rowsCreated != 1 {
+		t.Error("failed to create the source record")
+		t.FailNow()
+	}
+
+	client := cron.NewScheduler(ctx, conn)
+	client.CollectRssPosts()
+
+	articles, err := db.Articles.ListByPage(ctx, 0, 100)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+
+	t.Log(len(articles))
+}

internal/services/cron/scheduler.go

@@ -0,0 +1,131 @@
+package cron
+
+import (
+	"context"
+	"database/sql"
+
+	_ "github.com/lib/pq"
+	"github.com/robfig/cron/v3"
+
+	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+)
+
+type Cron struct {
+	//Db    *database.Queries
+	ctx   context.Context
+	timer *cron.Cron
+	repo  services.RepositoryService
+}
+
+func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
+	c := &Cron{
+		ctx:  ctx,
+		repo: services.NewRepositoryService(conn),
+	}
+	timer := cron.New()
+
+	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
+	//features := services.GetEnvConfig()
+
+	timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
+	//timer.AddFunc("10 * * * *", c.CollectRedditPosts)
+	//timer.AddFunc("15 * * * *", c.CheckYoutube)
+	//timer.AddFunc("20 * * * *", c.CheckFfxiv)
+	//timer.AddFunc("25 * * * *", c.CheckTwitch)
+	//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
+
+	c.timer = timer
+	return c
+}
+
+func (c *Cron) Start() {
+	c.timer.Start()
+}
+
+func (c *Cron) Stop() {
+	c.timer.Stop()
+}
+
+/*
+func (c *Cron) CheckDiscordQueue() {
+	// Get items from the table
+	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
+	if err != nil {
+		return err
+	}
+
+	for _, queue := range queueItems {
+		// Get the articleByID
+		article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
+		if err != nil {
+			return err
+		}
+
+		var endpoints []string
+		// List Subscription by SourceID
+		subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
+		if err != nil {
+			return err
+		}
+
+		// if no one is subscribed to it, remove it from the index.
+		if len(subs) == 0 {
+			log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
+			err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// Get the webhhooks to send to
+		for _, sub := range subs {
+			webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
+			if err != nil {
+				return err
+			}
+
+			// store them in an array
+			endpoints = append(endpoints, webhook.Url)
+		}
+
+		// Create Discord Message
+		dwh := output.NewDiscordWebHookMessage(article)
+		msg, err := dwh.GeneratePayload()
+		if err != nil {
+			return err
+		}
+
+		// Send Message(s)
+		for _, i := range endpoints {
+			err = dwh.SendPayload(msg, i)
+
+			if err != nil {
+				return err
+			}
+		}
+
+		// Remove the item from the queue, given we sent our notification.
+		err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
+		if err != nil {
+			return err
+		}
+
+		time.Sleep(10 * time.Second)
+	}
+
+	return nil
+}
+*/
+
+//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
+//	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
+//		ID:        uuid.New(),
+//		Articleid: Id,
+//	})
+//	if err != nil {
+//		return err
+//	}
+//	return nil
+//}

internal/services/cron/scheduler_test.go

@@ -0,0 +1,53 @@
+package cron_test
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose/v3"
+)
+
+/*
+func TestInvokeTwitch(t *testing.T) {
+
+}
+
+// TODO add database mocks but not sure how to do that yet.
+func TestCheckReddit(t *testing.T) {
+	ctx := context.Background()
+	c := cron.NewScheduler(ctx)
+	c.Col()
+}
+
+func TestCheckYouTube(t *testing.T) {
+	ctx := context.Background()
+	c := cron.NewScheduler(ctx)
+	c.CheckYoutube()
+}
+
+func TestCheckTwitch(t *testing.T) {
+	ctx := context.Background()
+	c := cron.NewScheduler(ctx)
+	err := c.CheckTwitch()
+	if err != nil {
+		t.Error(err)
+	}
+}
+*/
+
+func setupInMemoryDb() (*sql.DB, error) {
+	db, err := sql.Open("sqlite", ":memory:")
+	if err != nil {
+		return nil, err
+	}
+
+	err = goose.SetDialect("sqlite3")
+	if err != nil {
+		return nil, err
+	}
+
+	err = goose.Up(db, "../../database/migrations")
+	if err != nil {
+		return nil, err
+	}
+	return db, nil
+}

internal/services/database.go

@@ -0,0 +1,30 @@
+package services
+
+import (
+	"database/sql"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
+	repositoryservices "git.jamestombleson.com/jtom38/newsbot-api/internal/repositoryServices"
+)
+
+type RepositoryService struct {
+	AlertDiscord            repository.AlertDiscordRepo
+	Articles                repository.ArticlesRepo
+	DiscordWebHooks         repository.DiscordWebHookRepo
+	RefreshTokens           repositoryservices.RefreshToken
+	Sources                 repository.Sources
+	Users                   repositoryservices.UserServices
+	UserSourceSubscriptions repository.UserSourceRepo
+}
+
+func NewRepositoryService(conn *sql.DB) RepositoryService {
+	return RepositoryService{
+		AlertDiscord:            repository.NewAlertDiscordRepository(conn),
+		Articles:                repository.NewArticleRepository(conn),
+		DiscordWebHooks:         repository.NewDiscordWebHookRepository(conn),
+		RefreshTokens:           repositoryservices.NewRefreshTokenService(conn),
+		Sources:                 repository.NewSourceRepository(conn),
+		Users:                   repositoryservices.NewUserService(conn),
+		UserSourceSubscriptions: repository.NewUserSourceRepository(conn),
+	}
+}

internal/services/input/common.go

@@ -14,4 +14,4 @@ var (
 	ErrInvalidAuthorImage = errors.New("expected value looks to be wrong, something is missing")
 )
 
-const DATETIME_FORMAT string = "1/2/2006 3:4 PM"
+const DATETIME_FORMAT string = "1/2/2006 3:4 PM"

internal/services/input/ffxiv.go

@@ -1,7 +1,6 @@
 package input
 
 import (
-	"database/sql"
 	"errors"
 	"log"
 	"net/http"
@@ -13,8 +12,8 @@ import (
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/google/uuid"
 
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/cache"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
 )
 
 const (
@@ -25,7 +24,7 @@ const (
 )
 
 type FFXIVClient struct {
-	record database.Source
+	record entity.SourceEntity
 	//SourceID uint
 	//Url string
 	//Region string
@@ -33,65 +32,80 @@ type FFXIVClient struct {
 	cacheGroup string
 }
 
-func NewFFXIVClient(Record database.Source) FFXIVClient {
+func NewFFXIVClient(Record entity.SourceEntity) FFXIVClient {
 	return FFXIVClient{
-		record: Record,
+		record:     Record,
 		cacheGroup: "ffxiv",
 	}
 }
 
-func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
-	var articles []database.Article
+func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
+	var articles []entity.ArticleEntity
 
 	parser := fc.GetBrowser()
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { return articles, err }
+	if err != nil {
+		return articles, err
+	}
 
 	cache := cache.NewCacheClient(fc.cacheGroup)
 
 	for _, link := range links {
 		// Check cache/db if this link has been seen already, skip
 		_, err := cache.FindByValue(link)
-		if err == nil { continue }
-		
+		if err == nil {
+			continue
+		}
 
 		page := fc.GetPage(parser, link)
 
 		title, err := fc.ExtractTitle(page)
-		if err != nil { return articles, err }
+		if err != nil {
+			return articles, err
+		}
 
 		thumb, err := fc.ExtractThumbnail(page)
-		if err != nil { return articles, err }
+		if err != nil {
+			return articles, err
+		}
 
 		pubDate, err := fc.ExtractPubDate(page)
-		if err != nil { return articles, err }
+		if err != nil {
+			return articles, err
+		}
 
 		description, err := fc.ExtractDescription(page)
-		if err != nil { return articles, err }
+		if err != nil {
+			return articles, err
+		}
 
 		authorName, err := fc.ExtractAuthor(page)
-		if err != nil { return articles, err }
+		if err != nil {
+			return articles, err
+		}
 
 		authorImage, err := fc.ExtractAuthorImage(page)
-		if err != nil { return articles, err }
+		if err != nil {
+			return articles, err
+		}
 
 		tags, err := fc.ExtractTags(page)
-		if err != nil { return articles, err } 
+		if err != nil {
+			return articles, err
+		}
 
-		article := database.Article{
-			Sourceid: fc.record.ID,
-			Tags: tags,
-			Title: title,
-			Url: link,
-			Pubdate: pubDate,
-			Videoheight: 0,
-			Videowidth: 0,
-			Thumbnail: thumb,
-			Description: description,
-			Authorname: sql.NullString{String: authorName},
-			Authorimage: sql.NullString{String: authorImage},
+		article := entity.ArticleEntity{
+			SourceID:       fc.record.ID,
+			Tags:           tags,
+			Title:          title,
+			Url:            link,
+			PubDate:        pubDate,
+			Thumbnail:      thumb,
+			Description:    description,
+			AuthorName:     authorName,
+			AuthorImageUrl: authorImage,
 		}
 		log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
 
@@ -105,15 +119,19 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
 
 func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
 	html, err := http.Get(fc.record.Url)
-	if err != nil { return nil, err }
+	if err != nil {
+		return nil, err
+	}
 	defer html.Body.Close()
-	
+
 	doc, err := goquery.NewDocumentFromReader(html.Body)
-	if err != nil { return nil, err }
+	if err != nil {
+		return nil, err
+	}
 	return doc, nil
 }
 
-func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
+func (fc *FFXIVClient) GetBrowser() *rod.Browser {
 	var browser *rod.Browser
 	if path, exists := launcher.LookPath(); exists {
 		u := launcher.New().Bin(path).MustLaunch()
@@ -133,26 +151,26 @@ func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) {
 
 	// find all the li items
 	items := res.MustElements("li")
-	
+
 	for _, item := range items {
 		// in each li, find the a items
 		a, err := item.Element("a")
-		if err != nil { 
+		if err != nil {
 			log.Println("Unable to find the a item, skipping")
-			continue 
+			continue
 		}
 
-		// find the href behind the a 
+		// find the href behind the a
 		url, err := a.Property("href")
-		if err != nil { 
+		if err != nil {
 			log.Println("Unable to find a href link, skipping")
-			continue 
+			continue
 		}
 
 		urlString := url.String()
 		isTopic := strings.Contains(urlString, "topics")
 		if isTopic {
-			links = append(links, urlString)	
+			links = append(links, urlString)
 		}
 	}
 
@@ -166,8 +184,10 @@ func (rc *FFXIVClient) GetPage(parser *rod.Browser, url string) *rod.Page {
 
 func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
 	thumbnail := page.MustElementX("/html/body/div[3]/div[2]/div[1]/article/div[1]/img").MustProperty("src").String()
-	if thumbnail == "" { return "", errors.New("unable to find thumbnail")}
-	
+	if thumbnail == "" {
+		return "", errors.New("unable to find thumbnail")
+	}
+
 	title := page.MustElement(".news__header > h1:nth-child(2)").MustText()
 	log.Println(title)
 
@@ -176,17 +196,23 @@ func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
 
 func (fc *FFXIVClient) ExtractPubDate(page *rod.Page) (time.Time, error) {
 	stringDate := page.MustElement(".news__ic--topics").MustText()
-	if stringDate == "" { return time.Now(), errors.New("unable to locate the publish date on the post")}
+	if stringDate == "" {
+		return time.Now(), errors.New("unable to locate the publish date on the post")
+	}
 
 	PubDate, err := time.Parse(FFXIV_TIME_FORMAT, stringDate)
-	if err != nil { return time.Now(), err }
+	if err != nil {
+		return time.Now(), err
+	}
 
 	return PubDate, nil
 }
 
 func (fc *FFXIVClient) ExtractDescription(page *rod.Page) (string, error) {
 	res := page.MustElement(".news__detail__wrapper").MustText()
-	if res == "" { return "", errors.New("unable to locate the description on the post")}
+	if res == "" {
+		return "", errors.New("unable to locate the description on the post")
+	}
 
 	return res, nil
 }
@@ -195,12 +221,18 @@ func (fc *FFXIVClient) ExtractAuthor(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > meta")
 	for _, item := range meta {
 		name, err := item.Property("name")
-		if err != nil { return "", err }
+		if err != nil {
+			return "", err
+		}
 
-		if name.String() != "author" { continue }
+		if name.String() != "author" {
+			continue
+		}
 		content, err := item.Property("content")
-		if err != nil { return "", err }
-		
+		if err != nil {
+			return "", err
+		}
+
 		return content.String(), nil
 	}
 	//log.Println(meta)
@@ -211,12 +243,18 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > meta")
 	for _, item := range meta {
 		name, err := item.Property("name")
-		if err != nil { return "", err }
+		if err != nil {
+			return "", err
+		}
 
-		if name.String() != "keywords" { continue }
+		if name.String() != "keywords" {
+			continue
+		}
 		content, err := item.Property("content")
-		if err != nil { return "", err }
-		
+		if err != nil {
+			return "", err
+		}
+
 		return content.String(), nil
 	}
 	//log.Println(meta)
@@ -225,13 +263,19 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
 
 func (fc *FFXIVClient) ExtractTitle(page *rod.Page) (string, error) {
 	title, err := page.MustElement("head > title").Text()
-	if err != nil { return "", err }
+	if err != nil {
+		return "", err
+	}
 
-	if !strings.Contains(title, "|") { return "", errors.New("unable to split the title, missing | in the string")}
+	if !strings.Contains(title, "|") {
+		return "", errors.New("unable to split the title, missing | in the string")
+	}
 
 	res := strings.Split(title, "|")
-	if title != "" { return res[0], nil }
-	
+	if title != "" {
+		return res[0], nil
+	}
+
 	//log.Println(meta)
 	return "", errors.New("unable to find the author on the page")
 }
@@ -240,15 +284,20 @@ func (fc *FFXIVClient) ExtractAuthorImage(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > link")
 	for _, item := range meta {
 		name, err := item.Property("rel")
-		if err != nil { return "", err }
+		if err != nil {
+			return "", err
+		}
 
-		if name.String() != "apple-touch-icon-precomposed" { continue }
+		if name.String() != "apple-touch-icon-precomposed" {
+			continue
+		}
 		content, err := item.Property("href")
-		if err != nil { return "", err }
-		
+		if err != nil {
+			return "", err
+		}
+
 		return content.String(), nil
 	}
 	//log.Println(meta)
 	return "", errors.New("unable to find the author image on the page")
 }
-

internal/services/input/ffxiv_test.go

@@ -3,24 +3,25 @@ package input_test
 import (
 	"testing"
 
-	"github.com/google/uuid"
-	"github.com/jtom38/newsbot/collector/database"
-	ffxiv "github.com/jtom38/newsbot/collector/services/input"
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
 )
 
-var FFXIVRecord database.Source = database.Source{
-	ID: uuid.New(),
-	Site: "ffxiv",
-	Name: "Final Fantasy XIV - NA",
-	Source: "ffxiv",
-	Url: "https://na.finalfantasyxiv.com/lodestone/",
-	Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
+var FFXIVRecord entity.SourceEntity = entity.SourceEntity{
+	ID:          9999,
+	DisplayName: "Final Fantasy XIV - NA",
+	Source:      domain.SourceCollectorFfxiv,
+	Url:         "https://na.finalfantasyxiv.com/lodestone/",
+	Tags:        "ffxiv, final, fantasy, xiv, na, lodestone",
 }
 
 func TestFfxivGetParser(t *testing.T) {
 	fc := ffxiv.NewFFXIVClient(FFXIVRecord)
 	_, err := fc.GetParser()
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestFfxivPullFeed(t *testing.T) {
@@ -30,8 +31,12 @@ func TestFfxivPullFeed(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
-	if len(links) == 0 { panic("expected links to come back but got 0") }
+	if err != nil {
+		t.Error(err)
+	}
+	if len(links) == 0 {
+		t.Error("expected links to come back but got 0")
+	}
 
 }
 
@@ -42,14 +47,20 @@ func TestFfxivExtractThumbnail(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	thumb, err := fc.ExtractThumbnail(page)
-	if err != nil { panic(err) }
-	if thumb == "" { panic("expected a link but got nothing.")}
+	if err != nil {
+		t.Error(err)
+	}
+	if thumb == "" {
+		t.Error("expected a link but got nothing.")
+	}
 }
 
 func TestFfxivExtractPubDate(t *testing.T) {
@@ -59,13 +70,17 @@ func TestFfxivExtractPubDate(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	_, err = fc.ExtractPubDate(page)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestFfxivExtractDescription(t *testing.T) {
@@ -75,13 +90,17 @@ func TestFfxivExtractDescription(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	_, err = fc.ExtractDescription(page)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestFfxivExtractAuthor(t *testing.T) {
@@ -91,14 +110,20 @@ func TestFfxivExtractAuthor(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	author, err := fc.ExtractAuthor(page)
-	if err != nil { panic(err) }
-	if author == "" { panic("failed to locate the author name") }
+	if err != nil {
+		t.Error(err)
+	}
+	if author == "" {
+		t.Error("failed to locate the author name")
+	}
 }
 
 func TestFfxivExtractTags(t *testing.T) {
@@ -108,14 +133,20 @@ func TestFfxivExtractTags(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractTags(page)
-	if err != nil { panic(err) }
-	if res == "" {panic("failed to locate the tags")}
+	if err != nil {
+		t.Error(err)
+	}
+	if res == "" {
+		t.Error("failed to locate the tags")
+	}
 }
 
 func TestFfxivExtractTitle(t *testing.T) {
@@ -125,14 +156,20 @@ func TestFfxivExtractTitle(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractTitle(page)
-	if err != nil { panic(err) }
-	if res == "" { panic("failed to locate the tags") }
+	if err != nil {
+		t.Error(err)
+	}
+	if res == "" {
+		t.Error("failed to locate the tags")
+	}
 }
 
 func TestFFxivExtractAuthorIamge(t *testing.T) {
@@ -142,18 +179,24 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractAuthorImage(page)
-	if err != nil { panic(err) }
-	if res == "" { panic("failed to locate the tags") }
+	if err != nil {
+		t.Error(err)
+	}
+	if res == "" {
+		t.Error("failed to locate the tags")
+	}
 }
 
 func TestFfxivCheckSource(t *testing.T) {
 	fc := ffxiv.NewFFXIVClient(FFXIVRecord)
 	fc.CheckSource()
-	
-}
+
+}

internal/services/input/httpClient.go

@@ -2,7 +2,7 @@ package input
 
 import (
 	"crypto/tls"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 )
@@ -19,20 +19,26 @@ func getHttpContent(uri string) ([]byte, error) {
 	client := &http.Client{
 		Transport: tr,
 	}
-	
+
 	req, err := http.NewRequest("GET", uri, nil)
-	if err != nil { return nil, err }
+	if err != nil {
+		return nil, err
+	}
 
 	// set the user agent header to avoid kick backs.. as much
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
 
 	log.Printf("Requesting content from %v\n", uri)
 	resp, err := client.Do(req)
-	if err != nil { log.Fatalln(err) }
+	if err != nil {
+		log.Fatalln(err)
+	}
 	defer resp.Body.Close()
 
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil { return nil, err }
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
 
 	return body, nil
-}
+}

internal/services/input/reddit.go

@@ -1,7 +1,6 @@
 package input
 
 import (
-	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -9,16 +8,16 @@ import (
 	"strings"
 	"time"
 
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/launcher"
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/domain/model"
-	"github.com/jtom38/newsbot/collector/services/config"
 )
 
 type RedditClient struct {
 	config RedditConfig
-	record database.Source
+	record entity.SourceEntity
 }
 
 type RedditConfig struct {
@@ -27,14 +26,14 @@ type RedditConfig struct {
 	PullNSFW string
 }
 
-func NewRedditClient(Record database.Source) *RedditClient {
+func NewRedditClient(Record entity.SourceEntity) *RedditClient {
 	rc := RedditClient{
 		record: Record,
 	}
-	cc := config.New()
-	rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
-	rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
-	rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
+	cc := services.NewConfig()
+	rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
+	rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
+	rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
 
 	//rc.disableHttp2Client()
 
@@ -47,7 +46,6 @@ func NewRedditClient(Record database.Source) *RedditClient {
 //	os.Setenv("GODEBUG", "http2client=0")
 //}
 
-
 func (rc *RedditClient) GetBrowser() *rod.Browser {
 	var browser *rod.Browser
 	if path, exists := launcher.LookPath(); exists {
@@ -62,19 +60,17 @@ func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
 	return page
 }
 
-//func (rc RedditClient) 
+//func (rc RedditClient)
 
 // GetContent() reaches out to Reddit and pulls the Json data.
 // It will then convert the data to a struct and return the struct.
-func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
-	var items model.RedditJsonContent = model.RedditJsonContent{}
+func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
+	var items domain.RedditJsonContent = domain.RedditJsonContent{}
 
 	// TODO Wire this to support the config options
 	Url := fmt.Sprintf("%v.json", rc.record.Url)
 
-	log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
-
-
+	log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
 
 	content, err := getHttpContent(Url)
 	if err != nil {
@@ -91,10 +87,10 @@ func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
 	return items, nil
 }
 
-func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
-	var redditArticles []database.Article
+func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []entity.ArticleEntity {
+	var redditArticles []entity.ArticleEntity
 	for _, item := range items.Data.Children {
-		var article database.Article
+		var article entity.ArticleEntity
 		article, err := rc.convertToArticle(item.Data)
 		if err != nil {
 			log.Printf("[Reddit] %v", err)
@@ -107,8 +103,8 @@ func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []datab
 
 // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
 // This data can be passed to the database.
-func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
-	var item database.Article
+func (rc *RedditClient) convertToArticle(source domain.RedditPost) (entity.ArticleEntity, error) {
+	var item entity.ArticleEntity
 
 	if source.Content == "" && source.Url != "" {
 		item = rc.convertPicturePost(source)
@@ -134,65 +130,57 @@ func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Arti
 	return item, nil
 }
 
-func (rc *RedditClient) convertPicturePost(source model.RedditPost) database.Article {
-	var item = database.Article{
-		Sourceid:    rc.record.ID,
-		Title:       source.Title,
-		Tags:        fmt.Sprintf("%v", rc.record.Tags),
-		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		Pubdate:     time.Now(),
-		Video:       sql.NullString{String: "null"},
-		Videoheight: 0,
-		Videowidth:  0,
-		Thumbnail:   source.Thumbnail,
-		Description: source.Content,
-		Authorname:  sql.NullString{String: source.Author},
-		Authorimage: sql.NullString{String: "null"},
+func (rc *RedditClient) convertPicturePost(source domain.RedditPost) entity.ArticleEntity {
+	var item = entity.ArticleEntity{
+		SourceID:       rc.record.ID,
+		Title:          source.Title,
+		Tags:           fmt.Sprintf("%v", rc.record.Tags),
+		Url:            fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		PubDate:        time.Now(),
+		IsVideo:        false,
+		Thumbnail:      source.Thumbnail,
+		Description:    source.Content,
+		AuthorName:     source.Author,
+		AuthorImageUrl: "null",
 	}
 	return item
 }
 
-func (rc *RedditClient) convertTextPost(source model.RedditPost) database.Article {
-	var item = database.Article{
-		Sourceid:    rc.record.ID,
+func (rc *RedditClient) convertTextPost(source domain.RedditPost) entity.ArticleEntity {
+	var item = entity.ArticleEntity{
+		SourceID:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		Pubdate:     time.Now(),
-		Videoheight: 0,
-		Videowidth:  0,
+		PubDate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		Authorname:  sql.NullString{String: source.Author},
+		AuthorName:  source.Author,
 		Description: source.Content,
 	}
 	return item
 }
 
-func (rc *RedditClient) convertVideoPost(source model.RedditPost) database.Article {
-	var item = database.Article{
-		Sourceid:    rc.record.ID,
+func (rc *RedditClient) convertVideoPost(source domain.RedditPost) entity.ArticleEntity {
+	var item = entity.ArticleEntity{
+		SourceID:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		Pubdate:     time.Now(),
+		PubDate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		Videoheight: 0,
-		Videowidth:  0,
-		Authorname:  sql.NullString{String: source.Author},
+		AuthorName:  source.Author,
 		Description: source.Media.RedditVideo.FallBackUrl,
 	}
 	return item
 }
 
 // This post is nothing more then a redirect to another location.
-func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
-	var item = database.Article{
-		Sourceid:    rc.record.ID,
+func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) entity.ArticleEntity {
+	var item = entity.ArticleEntity{
+		SourceID:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		Pubdate:     time.Now(),
+		PubDate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		Videoheight: 0,
-		Videowidth:  0,
-		Authorname:  sql.NullString{String: source.Author},
+		AuthorName:  source.Author,
 		Description: source.UrlOverriddenByDest,
 	}
 	return item

internal/services/input/reddit_test.go

@@ -0,0 +1,32 @@
+package input_test
+
+import (
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+)
+
+var RedditRecord entity.SourceEntity = entity.SourceEntity{
+	ID:          9999,
+	DisplayName: "dadjokes",
+	Source:      domain.SourceCollectorRss,
+	Url:         "https://reddit.com/r/dadjokes",
+	Tags:        "reddit, dadjokes",
+}
+
+func TestGetContent(t *testing.T) {
+	//This test is flaky right now due to the http changes in 1.17
+	rc := input.NewRedditClient(RedditRecord)
+	raw, err := rc.GetContent()
+	if err != nil {
+		t.Error(err)
+	}
+	redditArticles := rc.ConvertToArticles(raw)
+	for _, posts := range redditArticles {
+		if posts.Title == "" {
+			t.Error("Title is missing")
+		}
+	}
+}

internal/services/input/rss.go

@@ -0,0 +1,77 @@
+package input
+
+import (
+	"strings"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/mmcdole/gofeed"
+)
+
+type FeedInput interface {
+	GetArticles() (entity.ArticleEntity, error)
+}
+
+type rssClient struct {
+	SourceRecord entity.SourceEntity
+}
+
+func NewRssClient(sourceRecord entity.SourceEntity) rssClient {
+	client := rssClient{
+		SourceRecord: sourceRecord,
+	}
+
+	return client
+}
+
+func (rc rssClient) GetArticles() ([]entity.ArticleEntity, error) {
+	parser := gofeed.NewParser()
+	feed, err := parser.ParseURL(rc.SourceRecord.Url)
+	if err != nil {
+		return nil, err
+	}
+
+	sourceTags := strings.Split(rc.SourceRecord.Tags, ",")
+	var articles []entity.ArticleEntity
+	for _, post := range feed.Items {
+		article := entity.ArticleEntity{
+			SourceID:    rc.SourceRecord.ID,
+			Title:       post.Title,
+			Description: post.Content,
+			Url:         post.Link,
+			PubDate:     *post.PublishedParsed,
+			//AuthorName:  post.Authors[0].Email,
+		}
+
+		if len(post.Authors) != 0 {
+			article.AuthorName = post.Authors[0].Email
+		}
+
+		var postTags []string
+		postTags = append(postTags, sourceTags...)
+		postTags = append(postTags, post.Categories...)
+		article.Tags = strings.Join(postTags, ",")
+
+		/*
+			pageContent, err := getHttpContent(article.Url)
+			if err != nil {
+				continue
+			}
+
+			htmlNode, err := html.Parse(bytes.NewReader(pageContent))
+			if err != nil {
+				continue
+			}
+			htmlNode.
+
+			fmt.Println(htmlNode)
+		*/
+
+		if post.Image == nil {
+			article.Thumbnail = ""
+		}
+
+		articles = append(articles, article)
+	}
+
+	return articles, nil
+}

internal/services/input/rss_test.go

@@ -0,0 +1,43 @@
+package input_test
+
+import (
+	"testing"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+)
+
+var rssRecord = entity.SourceEntity{
+	ID:          1,
+	DisplayName: "ArsTechnica",
+	Url:         "https://feeds.arstechnica.com/arstechnica/index",
+	Source:      domain.SourceCollectorRss,
+}
+
+func TestRssClientConstructor(t *testing.T) {
+	input.NewRssClient(rssRecord)
+}
+
+func TestRssGetFeed(t *testing.T) {
+	client := input.NewRssClient(rssRecord)
+	_, err := client.GetArticles()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestRssAgainstGita(t *testing.T) {
+	client := input.NewRssClient(entity.SourceEntity{
+		ID:          2,
+		DisplayName: "Gitea - Newsbot-api",
+		Source:      domain.SourceCollectorRss,
+		Url:         "https://git.jamestombleson.com/jtom38/newsbot-api.rss",
+		Tags:        "rss,gitea,newsbot-api",
+	})
+	_, err := client.GetArticles()
+	if err != nil {
+		t.Error(err)
+	}
+
+}

internal/services/input/twitch.go

@@ -1,19 +1,18 @@
 package input
 
 import (
-	"database/sql"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/config"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
 	"github.com/nicklaw5/helix/v2"
 )
 
 type TwitchClient struct {
-	SourceRecord database.Source
+	SourceRecord entity.SourceEntity
 
 	// config
 	monitorClips string
@@ -31,14 +30,14 @@ var (
 )
 
 func NewTwitchClient() (TwitchClient, error) {
-	c := config.New()
+	c := services.NewConfig()
 
-	id := c.GetConfig(config.TWITCH_CLIENT_ID)
+	id := c.GetConfig(services.TWITCH_CLIENT_ID)
 	if id == "" {
 		return TwitchClient{}, ErrTwitchClientIdMissing
 	}
 
-	secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
+	secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
 	if secret == "" {
 		return TwitchClient{}, ErrTwitchClientSecretMissing
 	}
@@ -50,8 +49,8 @@ func NewTwitchClient() (TwitchClient, error) {
 
 	client := TwitchClient{
 		//SourceRecord: &source,
-		monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
-		monitorVod:   c.GetConfig(config.TWITCH_MONITOR_VOD),
+		monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
+		monitorVod:   c.GetConfig(services.TWITCH_MONITOR_VOD),
 		api:          &api,
 	}
 
@@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
 }
 
 // This will let you replace the bound source record to keep the same session alive.
-func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
+func (tc *TwitchClient) ReplaceSourceRecord(source entity.SourceEntity) {
 	tc.SourceRecord = source
 }
 
@@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
 	return nil
 }
 
-func (tc *TwitchClient) GetContent() ([]database.Article, error) {
-	var items []database.Article
+func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
+	var items []entity.ArticleEntity
 
 	user, err := tc.GetUserDetails()
 	if err != nil {
@@ -101,34 +100,50 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
 	}
 
 	for _, video := range posts {
-		var article database.Article
+		var article entity.ArticleEntity
 
 		AuthorName, err := tc.ExtractAuthor(video)
-		if err != nil { return items, err }
-		article.Authorname = sql.NullString{String: AuthorName}
-		
+		if err != nil {
+			return items, err
+		}
+		article.AuthorName = AuthorName
+
 		Authorimage, err := tc.ExtractAuthorImage(user)
-		if err != nil { return items, err }
-		article.Authorimage = sql.NullString{String: Authorimage}
+		if err != nil {
+			return items, err
+		}
+		article.AuthorImageUrl = Authorimage
 
 		article.Description, err = tc.ExtractDescription(video)
-		if err != nil {return items, err }
+		if err != nil {
+			return items, err
+		}
 
-		article.Pubdate, err = tc.ExtractPubDate(video)
-		if err != nil { return items, err }
+		article.PubDate, err = tc.ExtractPubDate(video)
+		if err != nil {
+			return items, err
+		}
 
-		article.Sourceid = tc.SourceRecord.ID
+		article.SourceID = tc.SourceRecord.ID
 		article.Tags, err = tc.ExtractTags(video, user)
-		if err != nil { return items, err }
+		if err != nil {
+			return items, err
+		}
 
 		article.Thumbnail, err = tc.ExtractThumbnail(video)
-		if err != nil { return items, err }
+		if err != nil {
+			return items, err
+		}
 
 		article.Title, err = tc.ExtractTitle(video)
-		if err != nil { return items, err }
-		
+		if err != nil {
+			return items, err
+		}
+
 		article.Url, err = tc.ExtractUrl(video)
-		if err != nil { return items, err }
+		if err != nil {
+			return items, err
+		}
 
 		items = append(items, article)
 	}
@@ -140,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
 	var blank helix.User
 
 	users, err := tc.api.GetUsers(&helix.UsersParams{
-		Logins: []string{tc.SourceRecord.Name},
+		Logins: []string{tc.SourceRecord.DisplayName},
 	})
 	if err != nil {
 		return blank, err
@@ -210,8 +225,12 @@ func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
 
 // Extracts the avatar of the author with some validation.
 func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
-	if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
-	if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
+	if user.ProfileImageURL == "" {
+		return "", ErrMissingAuthorImage
+	}
+	if !strings.Contains(user.ProfileImageURL, "-profile_image-") {
+		return "", ErrInvalidAuthorImage
+	}
 	return user.ProfileImageURL, nil
 }
 
@@ -231,6 +250,8 @@ func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
 }
 
 func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
-	if post.URL == "" { return "", ErrMissingUrl }
+	if post.URL == "" {
+		return "", ErrMissingUrl
+	}
 	return post.URL, nil
-}
+}

internal/services/input/twitch_test.go

@@ -4,21 +4,21 @@ import (
 	"log"
 	"testing"
 
-	"github.com/google/uuid"
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/input"
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
 )
 
-var TwitchSourceRecord = database.Source {
-	ID:     uuid.New(),
-	Name:   "nintendo",
-	Source: "Twitch",
+var TwitchSourceRecord = entity.SourceEntity{
+	ID:          9999,
+	DisplayName: "nintendo",
+	Source:      domain.SourceCollectorTwitch,
 }
 
-var TwitchInvalidRecord = database.Source {
-	ID:     uuid.New(),
-	Name:   "EvilNintendo",
-	Source: "Twitch",
+var TwitchInvalidRecord = entity.SourceEntity{
+	ID:          9999,
+	DisplayName: "EvilNintendo",
+	Source:      domain.SourceCollectorTwitch,
 }
 
 func TestTwitchLogin(t *testing.T) {
@@ -115,40 +115,62 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
 
 func TestTwitchReturnsThumbnail(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	value, err := tc.ExtractThumbnail(posts[0])
-	if err != nil { t.Error(err) }
-	if value == "" { t.Error("uable to parse username") }
+	if err != nil {
+		t.Error(err)
+	}
+	if value == "" {
+		t.Error("uable to parse username")
+	}
 }
 
 func TestTwitchReturnsPubDate(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	date, err := tc.ExtractPubDate(posts[0])
 	log.Println(date)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestTwitchReturnsDescription(t *testing.T) {
@@ -181,17 +203,25 @@ func TestTwitchReturnsDescription(t *testing.T) {
 
 func TestTwitchReturnsAuthorImage(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil {t.Error(err) }
-	
+	if err != nil {
+		t.Error(err)
+	}
+
 	_, err = tc.ExtractAuthorImage(user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestTwitchReturnsTags(t *testing.T) {
@@ -212,10 +242,14 @@ func TestTwitchReturnsTags(t *testing.T) {
 	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	_, err = tc.ExtractTags(posts[0], user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestTwitchReturnsTitle(t *testing.T) {
@@ -236,42 +270,70 @@ func TestTwitchReturnsTitle(t *testing.T) {
 	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	res, err := tc.ExtractTitle(posts[0])
-	if err != nil { t.Error(err) }
-	if res == "" { t.Error("expected a filled string but got nil")}
+	if err != nil {
+		t.Error(err)
+	}
+	if res == "" {
+		t.Error("expected a filled string but got nil")
+	}
 }
 
 func TestTwitchReturnsUrl(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	res, err := tc.ExtractUrl(posts[0])
-	if err != nil { t.Error(err) }
-	if res == "" { t.Error("expected a filled string but got nil")}
+	if err != nil {
+		t.Error(err)
+	}
+	if res == "" {
+		t.Error("expected a filled string but got nil")
+	}
 }
 
 func TestTwitchGetContent(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil { t.Error(err) }
+	if err != nil {
+		t.Error(err)
+	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil { t.Error(err) }
-	
+	if err != nil {
+		t.Error(err)
+	}
+
 	posts, err := tc.GetContent()
-	if err != nil {t.Error(err) }
-	if len(posts) == 0 { t.Error("posts came back with 0 posts") }
-	if len(posts) != 20 { t.Error("expected 20 posts") } 
-}
+	if err != nil {
+		t.Error(err)
+	}
+	if len(posts) == 0 {
+		t.Error("posts came back with 0 posts")
+	}
+	if len(posts) != 20 {
+		t.Error("expected 20 posts")
+	}
+}

internal/services/input/youtube.go

@@ -1,24 +1,21 @@
 package input
 
 import (
-	"database/sql"
 	"errors"
 	"fmt"
 	"log"
 	"net/http"
 
-	//"strconv"
-
 	"github.com/PuerkitoBio/goquery"
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/mmcdole/gofeed"
 
-	"github.com/jtom38/newsbot/collector/database"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
 )
 
 type YoutubeClient struct {
-	record database.Source
+	record entity.SourceEntity
 
 	// internal variables at time of collection
 	channelID string
@@ -28,7 +25,7 @@ type YoutubeClient struct {
 	//debug bool
 
 	// cache config
-	cacheGroup string
+	//cacheGroup string
 }
 
 var (
@@ -39,24 +36,16 @@ var (
 
 const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
 
-func NewYoutubeClient(Record database.Source) YoutubeClient {
+func NewYoutubeClient(Record entity.SourceEntity) YoutubeClient {
 	yc := YoutubeClient{
-		record:     Record,
-		cacheGroup: "youtube",
+		record: Record,
 	}
-	/*
-		cc := NewConfigClient()
-
-		debug, err := strconv.ParseBool(cc.GetConfig(YOUTUBE_DEBUG))
-		if err != nil { panic("'YOUTUBE_DEBUG' was not a bool value")}
-		yc.Config.Debug = debug
-	*/
 	return yc
 }
 
 // CheckSource will go and run all the commands needed to process a source.
-func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
-	var items []database.Article
+func (yc *YoutubeClient) GetContent() ([]entity.ArticleEntity, error) {
+	var items []entity.ArticleEntity
 	docParser, err := yc.GetParser(yc.record.Url)
 	if err != nil {
 		return items, err
@@ -109,14 +98,18 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
 	return items, nil
 }
 
-func (yc *YoutubeClient) GetBrowser() *rod.Browser {
+func (yc *YoutubeClient) GetBrowser() (*rod.Browser, error) {
 	//browser := rod.New().MustConnect()
 	var browser *rod.Browser
 	if path, exists := launcher.LookPath(); exists {
-		u := launcher.New().Bin(path).MustLaunch()
+		u, err := launcher.New().Bin(path).Launch()
+		if err != nil {
+			return browser, err
+		}
+
 		browser = rod.New().ControlURL(u).MustConnect()
 	}
-	return browser
+	return browser, nil
 }
 
 func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page {
@@ -163,7 +156,11 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) {
 	var AvatarUri string
 
 	//browser := rod.New().MustConnect()
-	browser := yc.GetBrowser()
+	browser, err := yc.GetBrowser()
+	if err != nil {
+		return "", err
+	}
+
 	page := browser.MustPage(yc.record.Url)
 
 	res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
@@ -248,7 +245,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
 	return false
 }
 
-func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
+func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) entity.ArticleEntity {
 	parser, err := yc.GetParser(item.Link)
 	if err != nil {
 		log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
@@ -266,16 +263,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
 		log.Printf("[YouTube] %v", msg)
 	}
 
-	var article = database.Article{
-		Sourceid:    yc.record.ID,
-		Tags:        tags,
-		Title:       item.Title,
-		Url:         item.Link,
-		Pubdate:     *item.PublishedParsed,
-		Thumbnail:   thumb,
-		Description: item.Description,
-		Authorname:  sql.NullString{String: item.Author.Name},
-		Authorimage: sql.NullString{String: yc.avatarUri},
+	var article = entity.ArticleEntity{
+		SourceID:       yc.record.ID,
+		Tags:           tags,
+		Title:          item.Title,
+		Url:            item.Link,
+		PubDate:        *item.PublishedParsed,
+		Thumbnail:      thumb,
+		Description:    item.Description,
+		AuthorName:     item.Author.Name,
+		AuthorImageUrl: yc.avatarUri,
 	}
 	return article
 }

internal/services/input/youtube_test.go

@@ -3,51 +3,66 @@ package input_test
 import (
 	"testing"
 
-	"github.com/google/uuid"
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/input"
+	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
 )
 
-var YouTubeRecord database.Source = database.Source{
-	ID: uuid.New(),
-	Name: "dadjokes",
-	Source: "reddit",
-	Site: "reddit",
-	Url: "https://youtube.com/gamegrumps",
+var YouTubeRecord = entity.SourceEntity{
+	ID:          9999,
+	DisplayName: "dadjokes",
+	Source:      domain.SourceCollectorReddit,
+	Url:         "https://youtube.com/gamegrumps",
 }
 
 func TestGetPageParser(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	_, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestGetChannelId(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	parser, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	_, err = yc.GetChannelId(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestPullFeed(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	parser, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	_, err = yc.GetChannelId(parser)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	_, err = yc.PullFeed()
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestGetAvatarUri(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	res, err := yc.GetAvatarUri()
-	if err != nil { panic(err) }
-	if res == "" { panic(input.ErrMissingAuthorImage)}
+	if err != nil {
+		t.Error(err)
+	}
+	if res == "" {
+		t.Error(input.ErrMissingAuthorImage)
+	}
 }
 
 func TestGetVideoTags(t *testing.T) {
@@ -56,39 +71,59 @@ func TestGetVideoTags(t *testing.T) {
 	var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
 
 	parser, err := yc.GetParser(videoUri)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	tags, err := yc.GetTags(parser)
-	if err == nil && tags == "" { panic("err was empty but value was missing.")}
-	if err != nil { panic(err) }
+	if err == nil && tags == "" {
+		t.Error("err was empty but value was missing.")
+	}
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestGetChannelTags(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 
 	parser, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	tags, err := yc.GetTags(parser)
-	if err == nil && tags == "" { panic("no err but expected value was missing.")}
-	if err != nil { panic(err) }
+	if err == nil && tags == "" {
+		t.Error("no err but expected value was missing.")
+	}
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestGetVideoThumbnail(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
-	if err != nil {panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 
 	thumb, err := yc.GetVideoThumbnail(parser)
-	if err == nil && thumb == "" { panic("no err but expected result was missing")}
-	if err != nil { panic(err) }
+	if err == nil && thumb == "" {
+		t.Error("no err but expected result was missing")
+	}
+	if err != nil {
+		t.Error(err)
+	}
 
 }
 
 func TestCheckSource(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	_, err := yc.GetContent()
-	if err != nil { panic(err) }
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestCheckUriCache(t *testing.T) {
@@ -97,14 +132,18 @@ func TestCheckUriCache(t *testing.T) {
 
 	input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
 	res := yc.CheckUriCache(&item)
-	if res == false { panic("expected a value to come back")}
+	if res == false {
+		t.Error("expected a value to come back")
+	}
 }
 
 func TestCheckUriCacheFails(t *testing.T) {
 	yc := input.NewYoutubeClient(YouTubeRecord)
 	item := "demo1"
-	
-	res := yc.CheckUriCache(&item)
-	if res == true { panic("expected no value to come back")}
 
-}
+	res := yc.CheckUriCache(&item)
+	if res == true {
+		t.Error("expected no value to come back")
+	}
+
+}

internal/services/output/discordwebhook.go

@@ -8,7 +8,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/jtom38/newsbot/collector/database"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 )
 
 type discordField struct {
@@ -63,11 +64,11 @@ const (
 
 type Discord struct {
 	Subscriptions []string
-	article       database.Article
+	article       entity.ArticleEntity
 	Message       *DiscordMessage
 }
 
-func NewDiscordWebHookMessage(Article database.Article) Discord {
+func NewDiscordWebHookMessage(Article entity.ArticleEntity) Discord {
 	return Discord{
 		article: Article,
 	}

internal/services/output/discordwebhook_test.go

@@ -4,24 +4,20 @@ import (
 	"os"
 	"strings"
 	"testing"
-	//"time"
 
-	"github.com/google/uuid"
+	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
 	"github.com/joho/godotenv"
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/output"
 )
 
 var (
-	article database.Article = database.Article{
-		ID: uuid.New(),
-		Sourceid: uuid.New(),
-		Tags: "unit, testing",
-		Title: "Demo",
-		Url: "https://github.com/jtom38/newsbot.collector.api",
-		//Pubdate: time.Now(),
-		Videoheight: 0,
-		Videowidth: 0,
+	article entity.ArticleEntity = entity.ArticleEntity{
+		ID:       999,
+		SourceID: 1,
+		Tags:     "unit, testing",
+		Title:    "Demo",
+		Url:      "https://github.com/jtom38/newsbot.collector.api",
 		Description: "Hello World",
 	}
 	blank string = ""
@@ -33,7 +29,7 @@ func TestDiscordMessageContainsTitle(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	
+
 	for _, i := range *msg.Embeds {
 		if i.Title == &blank {
 			t.Error("title missing")
@@ -47,7 +43,7 @@ func TestDiscordMessageContainsDescription(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	
+
 	for _, i := range *msg.Embeds {
 		if i.Description == &blank {
 			t.Error("description missing")
@@ -68,7 +64,7 @@ func TestDiscordMessageFooter(t *testing.T) {
 		}
 		if i.Footer.IconUrl == &blank {
 			t.Error("missing footer url")
-		} 
+		}
 	}
 }
 
@@ -81,7 +77,7 @@ func TestDiscordMessageFields(t *testing.T) {
 	}
 	for _, embed := range *msg.Embeds {
 		for _, field := range embed.Fields {
-			var fName string 
+			var fName string
 			if field.Name != nil {
 				fName = *field.Name
 			} else {
@@ -110,7 +106,7 @@ func TestDiscordMessagePost(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-		
+
 	err = godotenv.Load()
 	if err != nil {
 		t.Error(err)
@@ -131,8 +127,8 @@ func TestDiscordMessagePost(t *testing.T) {
 		t.Error(err)
 	}
 
- 	err = d.SendPayload(msg, endpoints[0])
+	err = d.SendPayload(msg, endpoints[0])
 	if err != nil {
 		t.Error(err)
 	}
-}
+}

internal/services/output/interface.go

@@ -3,4 +3,4 @@ package output
 type Output interface {
 	GeneratePayload() error
 	SendPayload() error
-}
+}

main.go

@@ -1,36 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-
-	"github.com/jtom38/newsbot/collector/docs"
-	"github.com/jtom38/newsbot/collector/routes"
-	"github.com/jtom38/newsbot/collector/services/config"
-	"github.com/jtom38/newsbot/collector/services/cron"
-)
-
-// @title     NewsBot collector
-// @version   0.1
-// @BasePath  /api
-func main() {
-	cfg := config.New()
-	address := cfg.GetConfig(config.ServerAddress)
-	docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
-
-	ctx := context.Background()
-	c := cron.New(ctx)
-	c.Start()
-
-	server := routes.NewServer(ctx)
-
-	fmt.Println("API is online and waiting for requests.")
-	fmt.Printf("API: http://%v:8081/api\r\n", address)
-	fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", address)
-
-	err := http.ListenAndServe(":8081", server.Router)
-	if err != nil { 
-		panic(err)
-	}
-}

makefile

@@ -3,24 +3,24 @@ help: ## Shows this help command
 	@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
 build: ## builds the application with the current go runtime
-	sqlc generate
 	~/go/bin/swag f
-	~/go/bin/swag i
-	go build .
+	~/go/bin/swag init -g cmd/server.go
+	go build cmd/server.go
+	ls -lh server
 	
 docker-build: ## Generates the docker image
 	docker build -t "newsbot.collector.api" .
 	docker image ls | grep newsbot.collector.api
 
 migrate-dev: ## Apply sql migrations to dev db
-	goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up 
+	goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up 
 
 migrate-dev-down: ## revert sql migrations to dev db
-	goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down 
+	goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down 
 
 swag: ## Generates the swagger documentation with the swag tool
-	~/go/bin/swag f
-	~/go/bin/swag i
+	~/go/bin/swag f 
+	~/go/bin/swag init -g cmd/server.go
 
 gensql: ## Generates SQL code with sqlc
 	sqlc generate

routes/articles.go

@@ -1,132 +0,0 @@
-package routes
-
-import (
-	"encoding/json"
-	"net/http"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/google/uuid"
-)
-
-// ListArticles
-// @Summary  Lists the top 50 records
-// @Produce  application/json
-// @Tags     Articles
-// @Router   /articles [get]
-func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	res, err := s.Db.ListArticlesByDate(*s.ctx, 50)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-	w.Write(bres)
-}
-
-// GetArticleById
-// @Summary  Returns an article based on defined ID.
-// @Param    id  path  string  true  "uuid"
-// @Produce  application/json
-// @Tags     Articles
-// @Router   /articles/{id} [get]
-func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	id := chi.URLParam(r, "ID")
-	uuid, err := uuid.Parse(id)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	res, err := s.Db.GetArticleByID(*s.ctx, uuid)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	w.Write(bres)
-}
-
-// TODO add page support
-// GetArticlesBySourceID
-// @Summary  Finds the articles based on the SourceID provided.  Returns the top 50.
-// @Param    id  query  string  true  "Source ID UUID"
-// @Produce  application/json
-// @Tags     Articles
-// @Router   /articles/by/sourceid [get]
-func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	r.URL.Query()
-	query := r.URL.Query()
-	_id := query["id"][0]
-
-	uuid, err := uuid.Parse(_id)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	w.Write(bres)
-}
-
-// TODO add page support
-// GetArticlesByTag
-// @Summary  Finds the articles based on the SourceID provided.  Returns the top 50.
-// @Param    tag  query  string  true  "Tag name"
-// @Produce  application/json
-// @Tags     Articles
-// @Router   /articles/by/tag [get]
-func (s *Server) GetArticlesByTag(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	r.URL.Query()
-	query := r.URL.Query()
-	_id := query["tag"][0]
-
-	uuid, err := uuid.Parse(_id)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	w.Write(bres)
-}

routes/discordQueue.go

@@ -1,29 +0,0 @@
-package routes
-
-import (
-	"encoding/json"
-	"net/http"
-)
-
-// GetDiscordQueue
-// @Summary  Returns the top 100 entries from the queue to be processed.
-// @Produce  application/json
-// @Tags     Debug, Discord, Queue
-// @Router   /discord/queue [get]
-func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-	
-	res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	w.Write(bres)
-}

routes/discordwebhooks.go

@@ -1,115 +0,0 @@
-package routes
-
-import (
-	"encoding/json"
-	"log"
-	"net/http"
-	"strings"
-
-	"github.com/google/uuid"
-	"github.com/jtom38/newsbot/collector/database"
-)
-
-// GetDiscordWebHooks
-// @Summary  Returns the top 100 entries from the queue to be processed.
-// @Produce  application/json
-// @Tags     Config, Discord, Webhook
-// @Router   /discord/webhooks [get]
-func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	res, err := s.Db.ListDiscordWebhooks(*s.ctx, 100)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	w.Write(bres)
-}
-
-// GetDiscorWebHooksById
-// @Summary  Returns the top 100 entries from the queue to be processed.
-// @Produce  application/json
-// @Param    id  query  string  true  "id"
-// @Tags     Config, Discord, Webhook
-// @Router   /discord/webhooks/byId [get]
-func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	query := r.URL.Query()
-	_id := query["id"][0]
-	if _id == "" {
-		http.Error(w, "id is missing", http.StatusBadRequest)
-		return
-	}
-
-	uuid, err := uuid.Parse(_id)
-	if err != nil {
-		http.Error(w, "unable to parse id value", http.StatusBadRequest)
-		return
-	}
-	
-	res, err := s.Db.GetDiscordWebHooksByID(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, "no record found", http.StatusBadRequest)
-		return
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		http.Error(w, "unable to convert to json", http.StatusBadRequest)
-		panic(err)
-	}
-
-	w.Write(bres)
-}
-
-// NewDiscordWebHook
-// @Summary  Creates a new record for a discord web hook to post data to.
-// @Param    url      query  string  true  "url"
-// @Param    server   query  string  true  "Server name"
-// @Param    channel  query  string  true  "Channel name"
-// @Tags     Config, Discord, Webhook
-// @Router   /discord/webhooks/new [post]
-func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) {
-	query := r.URL.Query()
-	_url := query["url"][0]
-	_server := query["server"][0]
-	_channel := query["channel"][0]
-
-	if _url == "" {
-		http.Error(w, "url is missing a value", http.StatusBadRequest)
-		return
-	}
-	if !strings.Contains(_url, "discord.com/api/webhooks") {
-		http.Error(w, "invalid url", http.StatusBadRequest)
-		return
-	}
-	if _server ==  ""{
-		http.Error(w, "server is missing", http.StatusBadRequest)
-	}
-	if _channel == "" {
-		http.Error(w, "channel is missing", http.StatusBadRequest)
-	}
-	params := database.CreateDiscordWebHookParams{
-		ID:      uuid.New(),
-		Url:     _url,
-		Server:  _server,
-		Channel: _channel,
-		Enabled: true,
-	}
-	s.Db.CreateDiscordWebHook(*s.ctx, params)
-
-	bJson, err := json.Marshal(&params)
-	if err != nil {
-		log.Panicln(err)
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bJson)
-}

routes/root.go

@@ -1,50 +0,0 @@
-package routes
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/go-chi/chi/v5"
-)
-
-func RootRoutes() chi.Router {
-	app := chi.NewRouter()
-	app.Route("/", func(r chi.Router) {
-		r.Get("/helloworld", helloWorld)
-		r.Get("/ping", ping)
-		r.Route("/hello/{who}", func(r chi.Router) {
-			r.Get("/", helloWho)
-		})
-	})
-	return app
-}
-
-// HelloWorld
-// @Summary  Responds back with "Hello world!"
-// @Produce  plain
-// @Tags     Debug
-// @Router   /helloworld [get]
-func helloWorld(w http.ResponseWriter, r *http.Request) {
-	w.Write([]byte("Hello World!"))
-}
-
-// Ping
-// @Summary  Sends back "pong".  Good to test with.
-// @Produce  plain
-// @Tags     Debug
-// @Router   /ping [get]
-func ping(w http.ResponseWriter, r *http.Request) {
-	msg := "pong"
-	w.Write([]byte(msg))
-}
-
-// HelloWho
-// @Summary  Responds back with "Hello x" depending on param passed in.
-// @Param    who  path  string  true  "Who"
-// @Produce  plain
-// @Tags     Debug
-// @Router   /hello/{who} [get]
-func helloWho(w http.ResponseWriter, r *http.Request) {
-	msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who"))
-	w.Write([]byte(msg))
-}

routes/root_test.go

@@ -1,2 +0,0 @@
-package routes_test
-

routes/server.go

@@ -1,115 +0,0 @@
-package routes
-
-import (
-	"context"
-	"database/sql"
-
-	//"net/http"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/go-chi/chi/v5/middleware"
-	_ "github.com/lib/pq"
-	httpSwagger "github.com/swaggo/http-swagger"
-
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/config"
-)
-
-type Server struct {
-	Router *chi.Mux
-	Db     *database.Queries
-	ctx    *context.Context
-}
-
-var (
-	ErrIdValueMissing        string = "id value is missing"
-	ErrValueNotUuid          string = "a value given was expected to be a uuid but was not correct."
-	ErrNoRecordFound         string = "no record was found."
-	ErrUnableToConvertToJson string = "Unable to convert to json"
-)
-
-func NewServer(ctx context.Context) *Server {
-	s := &Server{
-		ctx: &ctx,
-	}
-
-	db, err := openDatabase(ctx)
-	if err != nil {
-		panic(err)
-	}
-	s.Db = db
-
-	s.Router = chi.NewRouter()
-	s.MountMiddleware()
-	s.MountRoutes()
-	return s
-}
-
-func openDatabase(ctx context.Context) (*database.Queries, error) {
-	_env := config.New()
-	connString := _env.GetConfig(config.Sql_Connection_String)
-	db, err := sql.Open("postgres", connString)
-	if err != nil {
-		panic(err)
-	}
-
-	queries := database.New(db)
-	return queries, err
-}
-
-func (s *Server) MountMiddleware() {
-	s.Router.Use(middleware.Logger)
-	s.Router.Use(middleware.Recoverer)
-	//s.Router.Use(middleware.Heartbeat())
-}
-
-func (s *Server) MountRoutes() {
-	s.Router.Get("/swagger/*", httpSwagger.Handler(
-		httpSwagger.URL("http://localhost:8081/swagger/doc.json"), //The url pointing to API definition
-	))
-
-	/* Root Routes */
-	s.Router.Get("/api/helloworld", helloWorld)
-	s.Router.Get("/api/hello/{who}", helloWho)
-	s.Router.Get("/api/ping", ping)
-
-	/* Article Routes */
-	s.Router.Get("/api/articles", s.listArticles)
-	s.Router.Route("/api/articles/{ID}", func(r chi.Router) {
-		r.Get("/", s.getArticleById)
-	})
-	s.Router.Get("/api/articles/by/sourceid", s.GetArticlesBySourceId)
-
-	/* Discord Queue */
-	s.Router.Get("/api/discord/queue", s.GetDiscordQueue)
-
-	/* Discord WebHooks */
-	s.Router.Post("/api/discord/webhooks/new", s.NewDiscordWebHook)
-	s.Router.Get("/api/discord/webhooks", s.GetDiscordWebHooks)
-	s.Router.Get("/api/discord/webhooks/byId", s.GetDiscordWebHooksById)
-
-	/* Settings */
-	s.Router.Get("/api/settings", s.getSettings)
-
-	/* Source Routes */
-	s.Router.Get("/api/config/sources", s.listSources)
-
-	/* Reddit Source Routes */
-
-	s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource)
-
-	s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource)
-	s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource)
-	s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) {
-		r.Get("/", s.getSources)
-		r.Delete("/", s.deleteSources)
-		r.Post("/disable", s.disableSource)
-		r.Post("/enable", s.enableSource)
-	})
-
-	/* Subscriptions */
-	s.Router.Get("/api/subscriptions", s.ListSubscriptions)
-	s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId)
-	s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId)
-	s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription)
-}

routes/settings.go

@@ -1,44 +0,0 @@
-package routes
-
-import (
-	"encoding/json"
-	"log"
-	"net/http"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/google/uuid"
-)
-
-// GetSettings
-// @Summary  Returns a object based on the Key that was given.
-// @Param    key  path  string  true  "Settings Key value"
-// @Produce  application/json
-// @Tags     Settings
-// @Router   /settings/{key} [get]
-func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
-	//var item model.Sources
-	id := chi.URLParam(r, "ID")
-
-	uuid, err := uuid.Parse(id)
-	if err != nil {
-		panic(err)
-	}
-
-	res, err := s.Db.GetSourceByID(*s.ctx, uuid)
-	if err != nil {
-		panic(err)
-	}
-
-	//itemId := fmt.Sprint(item.ID)
-	//if id != itemId {
-	//	log.Panicln("Unable to find the requested record.  Either unable to access SQL or the record does not exist.")
-	//}
-
-	bResult, err := json.Marshal(res)
-	if err != nil {
-		log.Panicln(err)
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bResult)
-}

routes/sources.go

@@ -1,282 +0,0 @@
-package routes
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/google/uuid"
-	"github.com/jtom38/newsbot/collector/database"
-)
-
-// ListSources
-// @Summary  Lists the top 50 records
-// @Produce  application/json
-// @Tags     Config, Source
-// @Router   /config/sources [get]
-func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
-	//TODO Add top?
-	/*
-		top := chi.URLParam(r, "top")
-		topInt, err := strconv.ParseInt(top, 0, 32)
-		if err != nil {
-			panic(err)
-		}
-		res, err := s.Db.ListSources(*s.ctx, int32(topInt))
-	*/
-
-	res, err := s.Db.ListSources(*s.ctx, 50)
-	if err != nil {
-		http.Error(w, "url is missing a value", http.StatusBadRequest)
-		return
-	}
-
-	bResult, err := json.Marshal(res)
-	if err != nil {
-		http.Error(w, "unable to convert to json", http.StatusBadRequest)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bResult)
-}
-
-// GetSource
-// @Summary  Returns a single entity by ID
-// @Param    id  path  string  true  "uuid"
-// @Produce  application/json
-// @Tags     Config, Source
-// @Router   /config/sources/{id} [get]
-func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
-	id := chi.URLParam(r, "ID")
-
-	uuid, err := uuid.Parse(id)
-	if err != nil {
-		http.Error(w, "id is not a uuid", http.StatusBadRequest)
-		return
-	}
-
-	res, err := s.Db.GetSourceByID(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, "invalid id was given", http.StatusBadRequest)
-		panic(err)
-	}
-
-	bResult, err := json.Marshal(res)
-	if err != nil {
-		log.Panicln(err)
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bResult)
-}
-
-// NewRedditSource
-// @Summary  Creates a new reddit source to monitor.
-// @Param    name  query  string  true  "name"
-// @Param    url   query  string  true  "url"
-// @Tags     Config, Source, Reddit
-// @Router   /config/sources/new/reddit [post]
-func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
-	query := r.URL.Query()
-	_name := query["name"][0]
-	_url := query["url"][0]
-	_tags := query["tags"][0]
-
-	if _url == "" {
-		http.Error(w, "url is missing a value", http.StatusBadRequest)
-		return
-	}
-	if !strings.Contains(_url, "reddit.com") {
-		http.Error(w, "invalid url", http.StatusBadRequest)
-		return
-	}
-
-	tags := fmt.Sprintf("reddit, %v, %v", _name, _tags)
-	params := database.CreateSourceParams{
-		ID:      uuid.New(),
-		Site:    "reddit",
-		Name:    _name,
-		Source:  "reddit",
-		Type:    "feed",
-		Enabled: true,
-		Url:     _url,
-		Tags:    tags,
-	}
-	s.Db.CreateSource(*s.ctx, params)
-
-	bJson, err := json.Marshal(&params)
-	if err != nil {
-		log.Panicln(err)
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bJson)
-}
-
-func (s *Server) getSourceByType(w http.ResponseWriter, r *http.Request) {
-
-}
-
-// NewYoutubeSource
-// @Summary  Creates a new youtube source to monitor.
-// @Param    name  query  string  true  "name"
-// @Param    url   query  string  true  "url"
-// @Param    tags  query  string  true  "tags"
-// @Tags     Config, Source, YouTube
-// @Router   /config/sources/new/youtube [post]
-func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
-	query := r.URL.Query()
-	_name := query["name"][0]
-	_url := query["url"][0]
-	_tags := query["tags"][0]
-
-	if _url == "" {
-		http.Error(w, "url is missing a value", http.StatusBadRequest)
-		return
-	}
-	if !strings.Contains(_url, "youtube.com") {
-		http.Error(w, "invalid url", http.StatusBadRequest)
-		return
-	}
-
-	tags := fmt.Sprintf("youtube, %v, %v", _name, _tags)
-	params := database.CreateSourceParams{
-		ID:      uuid.New(),
-		Site:    "youtube",
-		Name:    _name,
-		Source:  "youtube",
-		Type:    "feed",
-		Enabled: true,
-		Url:     _url,
-		Tags:    tags,
-	}
-	s.Db.CreateSource(*s.ctx, params)
-
-	bJson, err := json.Marshal(&params)
-	if err != nil {
-		log.Panicln(err)
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bJson)
-}
-
-// NewTwitchSource
-// @Summary  Creates a new twitch source to monitor.
-// @Param    name  query  string  true  "name"
-// @Param    url   query  string  true  "url"
-// @Param    tags  query  string  true  "tags"
-// @Tags     Config, Source, Twitch
-// @Router   /config/sources/new/twitch [post]
-func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
-	query := r.URL.Query()
-	_name := query["name"][0]
-	_url := query["url"][0]
-	_tags := query["tags"][0]
-
-	if _url == "" {
-		http.Error(w, "url is missing a value", http.StatusBadRequest)
-		return
-	}
-	if !strings.Contains(_url, "twitch.tv") {
-		http.Error(w, "invalid url", http.StatusBadRequest)
-		return
-	}
-
-	tags := fmt.Sprintf("twitch, %v, %v", _name, _tags)
-	params := database.CreateSourceParams{
-		ID:      uuid.New(),
-		Site:    "twitch",
-		Name:    _name,
-		Source:  "twitch",
-		Type:    "api",
-		Enabled: true,
-		Url:     _url,
-		Tags:    tags,
-	}
-	s.Db.CreateSource(*s.ctx, params)
-
-	bJson, err := json.Marshal(&params)
-	if err != nil {
-		log.Panicln(err)
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bJson)
-}
-
-// DeleteSource
-// @Summary  Deletes a record by ID.
-// @Param    id  path  string  true  "id"
-// @Tags     Config, Source
-// @Router   /config/sources/{id} [delete]
-func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
-	//var item model.Sources = model.Sources{}
-
-	id := chi.URLParam(r, "ID")
-	uuid, err := uuid.Parse(id)
-	if err != nil {
-		log.Panicln(err)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.Db.GetSourceByID(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-	}
-
-	// Delete the record
-	err = s.Db.DeleteSource(*s.ctx, uuid)
-	if err != nil {
-		log.Panic(err)
-	}
-}
-
-// DisableSource
-// @Summary  Disables a source from processing.
-// @Param    id  path  string  true  "id"
-// @Tags     Config, Source
-// @Router   /config/sources/{id}/disable [post]
-func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
-	id := chi.URLParam(r, "ID")
-	uuid, err := uuid.Parse(id)
-	if err != nil {
-		log.Panicln(err)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.Db.GetSourceByID(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-	}
-
-	err = s.Db.DisableSource(*s.ctx, uuid)
-	if err != nil {
-		log.Panic(err)
-	}
-}
-
-// EnableSource
-// @Summary  Enables a source to continue processing.
-// @Param    id  path  string  true  "id"
-// @Tags     Config, Source
-// @Router   /config/sources/{id}/enable [post]
-func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) {
-	id := chi.URLParam(r, "ID")
-	uuid, err := uuid.Parse(id)
-	if err != nil {
-		log.Panicln(err)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.Db.GetSourceByID(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-	}
-
-	err = s.Db.EnableSource(*s.ctx, uuid)
-	if err != nil {
-		log.Panic(err)
-	}
-}

routes/subscriptions.go

@@ -1,173 +0,0 @@
-package routes
-
-import (
-	"encoding/json"
-	"net/http"
-
-	"github.com/google/uuid"
-	"github.com/jtom38/newsbot/collector/database"
-)
-
-// GetSubscriptions
-// @Summary  Returns the top 100 entries from the queue to be processed.
-// @Produce  application/json
-// @Tags     Config, Subscription
-// @Router   /subscriptions [get]
-func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	res, err := s.Db.ListSubscriptions(*s.ctx, 100)
-	if err != nil {
-		w.Write([]byte(err.Error()))
-		panic(err)
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		http.Error(w, ErrUnableToConvertToJson, http.StatusBadRequest)
-		panic(err)
-	}
-
-	w.Write(bres)
-}
-
-// GetSubscriptionsByDiscordId
-// @Summary  Returns the top 100 entries from the queue to be processed.
-// @Produce  application/json
-// @Param    id  query  string  true  "id"
-// @Tags     Config, Subscription
-// @Router   /subscriptions/byDiscordId [get]
-func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	query := r.URL.Query()
-	_id := query["id"][0]
-	if _id == "" {
-		http.Error(w, ErrIdValueMissing, http.StatusBadRequest)
-		return
-	}
-
-	uuid, err := uuid.Parse(_id)
-	if err != nil {
-		http.Error(w, ErrValueNotUuid, http.StatusBadRequest)
-		return
-	}
-
-	res, err := s.Db.GetSubscriptionsByDiscordWebHookId(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, ErrNoRecordFound, http.StatusBadRequest)
-		return
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		http.Error(w, ErrUnableToConvertToJson, http.StatusBadRequest)
-		return
-	}
-
-	w.Write(bres)
-}
-
-// GetSubscriptionsBySourceId
-// @Summary  Returns the top 100 entries from the queue to be processed.
-// @Produce  application/json
-// @Param    id  query  string  true  "id"
-// @Tags     Config, Subscription
-// @Router   /subscriptions/bySourceId [get]
-func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	query := r.URL.Query()
-	_id := query["id"][0]
-	if _id == "" {
-		http.Error(w, ErrIdValueMissing, http.StatusBadRequest)
-		return
-	}
-
-	uuid, err := uuid.Parse(_id)
-	if err != nil {
-		http.Error(w, ErrValueNotUuid, http.StatusBadRequest)
-		return
-	}
-
-	res, err := s.Db.GetSubscriptionsByDiscordWebHookId(*s.ctx, uuid)
-	if err != nil {
-		http.Error(w, ErrNoRecordFound, http.StatusBadRequest)
-		return
-	}
-
-	bres, err := json.Marshal(res)
-	if err != nil {
-		http.Error(w, ErrUnableToConvertToJson, http.StatusBadRequest)
-		return
-	}
-
-	w.Write(bres)
-}
-
-// NewDiscordWebHookSubscription
-// @Summary  Creates a new subscription to link a post from a Source to a DiscordWebHook.
-// @Param    discordWebHookId  query  string  true  "discordWebHookId"
-// @Param    sourceId          query  string  true  "sourceId"
-// @Tags     Config, Source, Discord, Subscription
-// @Router   /subscriptions/new/discordwebhook [post]
-func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
-	// Extract the values given
-	query := r.URL.Query()
-	discordWebHookId := query["discordWebHookId"][0]
-	sourceId := query["sourceId"][0]
-
-	// Check to make we didnt get a null
-	if discordWebHookId == "" {
-		http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest )
-		return
-	}
-	if sourceId == "" {
-		http.Error(w, "invalid sourceID given", http.StatusBadRequest )
-		return
-	}
-
-	// Valide they are UUID values
-	uHook, err := uuid.Parse(discordWebHookId)
-	if err != nil {
-		http.Error(w, "DiscordWebHooksID was not a uuid value.", http.StatusBadRequest)
-		return
-	}
-	uSource, err := uuid.Parse(sourceId)
-	if err != nil {
-		http.Error(w, "SourceId was not a uuid value", http.StatusBadRequest)
-		return
-	}
-
-	// Check if the sub already exists
-	item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{
-		Discordwebhookid:  uHook,
-		Sourceid: uSource,
-	})
-	if err == nil {
-		bJson, err := json.Marshal(&item)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.Write(bJson)
-		return
-	}
-	
-	// Does not exist, so make it.
-	params := database.CreateSubscriptionParams{
-		ID: uuid.New(),
-		Discordwebhookid: uHook,
-		Sourceid: uSource,
-	}
-	s.Db.CreateSubscription(*s.ctx, params)
-	
-	bJson, err := json.Marshal(&params)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(bJson)
-}

services/config/config.go

@@ -1,87 +0,0 @@
-package config
-
-import (
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"strconv"
-
-	"github.com/joho/godotenv"
-)
-
-const (
-	ServerAddress = "SERVER_ADDRESS"
-
-	Sql_Connection_String = "SQL_CONNECTION_STRING"
-
-	FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
-	REDDIT_PULL_TOP               = "REDDIT_PULL_TOP"
-	REDDIT_PULL_HOT               = "REDDIT_PULL_HOT"
-	REDDIT_PULL_NSFW              = "REDDIT_PULL_NSFW"
-
-	FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
-	YOUTUBE_DEBUG                  = "YOUTUBE_DEBUG"
-
-	FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
-	TWITCH_CLIENT_ID              = "TWITCH_CLIENT_ID"
-	TWITCH_CLIENT_SECRET          = "TWITCH_CLIENT_SECRET"
-	TWITCH_MONITOR_CLIPS          = "TWITCH_MONITOR_CLIPS"
-	TWITCH_MONITOR_VOD            = "TWITCH_MONITOR_VOD"
-
-	FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
-)
-
-type ConfigClient struct{}
-
-func New() ConfigClient {
-	c := ConfigClient{}
-	c.RefreshEnv()
-
-	return c
-}
-
-func (cc *ConfigClient) GetConfig(key string) string {
-	res, filled := os.LookupEnv(key)
-	if !filled {
-		log.Printf("Missing the a value for '%v'.  Could generate errors.", key)
-	}
-	return res
-}
-
-func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
-	cc.RefreshEnv()
-
-	res, filled := os.LookupEnv(flag)
-	if !filled {
-		errorMessage := fmt.Sprintf("'%v' was not found", flag)
-		return false, errors.New(errorMessage)
-	}
-
-	b, err := strconv.ParseBool(res)
-	if err != nil {
-		return false, err
-	}
-	return b, nil
-}
-
-// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
-func (cc *ConfigClient) RefreshEnv() {
-	// Check to see if we have the env file on the system
-	_, err := os.Stat(".env")
-
-	// We have the file, load it.
-	if err == nil {
-		_, err := os.Open(".env")
-		if err == nil {
-			loadEnvFile()
-		}
-	}
-}
-
-func loadEnvFile() {
-	err := godotenv.Load()
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

services/config/config_test.go

@@ -1,20 +0,0 @@
-package config_test
-
-import (
-	"testing"
-	"os"
-
-	"github.com/jtom38/newsbot/collector/services/config"
-)
-
-func TestNewClient(t *testing.T) {
-	config.New()
-}
-
-func TestGetConfigExpectNull(t *testing.T) {
-	cc := config.New()
-	os.Setenv(config.REDDIT_PULL_HOT, "")
-	res := cc.GetConfig(config.REDDIT_PULL_HOT)
-	if res != "" { panic("expected blank")}
-
-}

services/cron/scheduler.go

@@ -1,314 +0,0 @@
-package cron
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"log"
-	"time"
-
-	"github.com/google/uuid"
-	_ "github.com/lib/pq"
-	"github.com/robfig/cron/v3"
-
-	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services/config"
-	"github.com/jtom38/newsbot/collector/services/input"
-	"github.com/jtom38/newsbot/collector/services/output"
-)
-
-type Cron struct {
-	Db    *database.Queries
-	ctx   *context.Context
-	timer *cron.Cron
-}
-
-func openDatabase() (*database.Queries, error) {
-	_env := config.New()
-	connString := _env.GetConfig(config.Sql_Connection_String)
-	if connString == "" {
-		panic("Connection String is null!")
-	}
-	db, err := sql.Open("postgres", connString)
-	if err != nil {
-		panic(err)
-	}
-
-	queries := database.New(db)
-	return queries, err
-}
-
-func New(ctx context.Context) *Cron {
-	c := &Cron{
-		ctx: &ctx,
-	}
-
-	timer := cron.New()
-	queries, err := openDatabase()
-	if err != nil {
-		panic(err)
-	}
-	c.Db = queries
-
-	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
-	features := config.New()
-
-	res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
-	if res {
-		timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
-		log.Print("[Input] Reddit backend was enabled")
-		//go c.CheckReddit()
-	}
-
-	res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
-	if res {
-		timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
-		log.Print("[Input] YouTube backend was enabled")
-	}
-
-	res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
-	if res {
-		timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
-		log.Print("[Input] FFXIV backend was enabled")
-	}
-
-	res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
-	if res {
-		timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
-		log.Print("[Input] Twitch backend was enabled")
-	}
-
-	timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
-	log.Print("[Output] Discord Output was enabled")
-
-	c.timer = timer
-	return c
-}
-
-func (c *Cron) Start() {
-	c.timer.Start()
-}
-
-func (c *Cron) Stop() {
-	c.timer.Stop()
-}
-
-// This is the main entry point to query all the reddit services
-func (c *Cron) CheckReddit() {
-	sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
-	if err != nil {
-		log.Printf("[Reddit] No sources found to query - %v\r", err)
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-		log.Printf("[Reddit] Checking '%v'...", source.Name)
-		rc := input.NewRedditClient(source)
-		raw, err := rc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-		redditArticles := rc.ConvertToArticles(raw)
-		c.checkPosts(redditArticles, "Reddit")
-	}
-	log.Print("[Reddit] Done!")
-}
-
-func (c *Cron) CheckYoutube() {
-	// Add call to the db to request youtube sources.
-	sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
-	if err != nil {
-		log.Printf("[Youtube] No sources found to query - %v\r", err)
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-		log.Printf("[YouTube] Checking '%v'...", source.Name)
-		yc := input.NewYoutubeClient(source)
-		raw, err := yc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-		c.checkPosts(raw, "YouTube")
-	}
-	log.Print("[YouTube] Done!")
-}
-
-func (c *Cron) CheckFfxiv() {
-	sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
-	if err != nil {
-		log.Printf("[FFXIV] No sources found to query - %v\r", err)
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-		fc := input.NewFFXIVClient(source)
-		items, err := fc.CheckSource()
-		if err != nil {
-			log.Println(err)
-		}
-		c.checkPosts(items, "FFXIV")
-	}
-	log.Printf("[FFXIV Done!]")
-}
-
-func (c *Cron) CheckTwitch() error {
-	sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
-	if err != nil {
-		log.Printf("[Twitch] No sources found to query - %v\r", err)
-	}
-
-	tc, err := input.NewTwitchClient()
-	if err != nil {
-		return err
-	}
-
-	err = tc.Login()
-	if err != nil {
-		return err
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-		log.Printf("[Twitch] Checking '%v'...", source.Name)
-		tc.ReplaceSourceRecord(source)
-		items, err := tc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-		c.checkPosts(items, "Twitch")
-	}
-
-	log.Print("[Twitch] Done!")
-	return nil
-}
-
-func (c *Cron) CheckDiscordQueue() error {
-	// Get items from the table
-	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
-	if err != nil {
-		return err
-	}
-
-	for _, queue := range queueItems {
-		// Get the articleByID
-		article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
-		if err != nil {
-			return err
-		}
-
-		var endpoints []string
-		// List Subscription by SourceID
-		subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
-		if err != nil {
-			return err
-		}
-
-		// if no one is subscribed to it, remove it from the index.
-		if len(subs) == 0 {
-			log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
-			err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		// Get the webhhooks to send to
-		for _, sub := range subs {
-			webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
-			if err != nil {
-				return err
-			}
-
-			// store them in an array
-			endpoints = append(endpoints, webhook.Url)
-		}
-
-		// Create Discord Message
-		dwh := output.NewDiscordWebHookMessage(article)
-		msg, err := dwh.GeneratePayload()
-		if err != nil {
-			return err
-		}
-
-		// Send Message(s)
-		for _, i := range endpoints {
-			err = dwh.SendPayload(msg, i)
-
-			if err != nil {
-				return err
-			}
-		}
-
-		// Remove the item from the queue, given we sent our notification.
-		err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
-		if err != nil {
-			return err
-		}
-
-		time.Sleep(10 * time.Second)
-	}
-
-	return nil
-}
-
-func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
-	for _, item := range posts {
-		_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
-		if err != nil {
-			id := uuid.New()
-
-			err := c.postArticle(id, item)
-			if err != nil {
-				return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
-			}
-
-			err = c.addToDiscordQueue(id)
-			if err != nil {
-				return err
-			}
-
-		}
-	}
-	time.Sleep(30 * time.Second)
-	return nil
-}
-
-func (c *Cron) postArticle(id uuid.UUID,item database.Article) error {
-	err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
-		ID:          id,
-		Sourceid:    item.Sourceid,
-		Tags:        item.Tags,
-		Title:       item.Title,
-		Url:         item.Url,
-		Pubdate:     item.Pubdate,
-		Video:       item.Video,
-		Videoheight: item.Videoheight,
-		Videowidth:  item.Videowidth,
-		Thumbnail:   item.Thumbnail,
-		Description: item.Description,
-		Authorname:  item.Authorname,
-		Authorimage: item.Authorimage,
-	})
-	return err
-}
-
-func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
-	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
-		ID: uuid.New(),
-		Articleid: Id,
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}

services/cron/scheduler_test.go

@@ -1,34 +0,0 @@
-package cron_test
-
-import (
-	"context"
-	"testing"
-
-	"github.com/jtom38/newsbot/collector/services/cron"
-)
-
-func TestInvokeTwitch(t *testing.T) {
-
-}
-
-// TODO add database mocks but not sure how to do that yet.
-func TestCheckReddit(t *testing.T) {
-	ctx := context.Background()
-	c := cron.New(ctx)
-	c.CheckReddit()
-}
-
-func TestCheckYouTube(t *testing.T) {
-	ctx := context.Background()
-	c := cron.New(ctx)
-	c.CheckYoutube()
-}
-
-func TestCheckTwitch(t *testing.T) {
-	ctx := context.Background()
-	c := cron.New(ctx)
-	err := c.CheckTwitch()
-	if err != nil {
-		t.Error(err)
-	}
-}