added a new endpoint to get a list of sources by the source value

.drone.yaml

@@ -1,60 +0,0 @@
----
-kind: pipeline
-type: docker
-name: buildLatestImage
-
-steps:
-  - name: buildLatestImage
-    image: plugins/docker
-    settings:
-      repo: jtom38/newsbot-collector
-      username: jtom38
-      password:
-        from_secret: DockerPushPat
-trigger:
-  branch:
-    include:
-      - main
-
-  event:
-    exclude:
-      - pull_request
----
-kind: pipeline
-type: docker
-name: buildReleaseImage
-
-steps:
-  - name: buildReleaseImage
-    image: plugins/docker
-    settings:
-      repo: jtom38/newsbot-collector
-      username: jtom38
-      password:
-        from_secret: DockerPushPat
-trigger:
-  branch:
-    include:
-      - releases/*
-  ref:
-    include:
-      - refs/tags/**
-  event:
-    exclude:
-      - pull_request
-
-
----
-kind: pipeline
-type: docker
-name: PullRequestCompileTest
-steps:
-  - name: Compile project
-    image: golang:1.22
-    commands:
-      - go test ./internal/repository
-      - go build ./cmd/server.go
-      - 
-trigger:
-  event:
-    - pull_request

.github/workflows/docker.build.yaml

@@ -0,0 +1,64 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  #schedule:
+  #  - cron: '21 19 * * *'
+  push:
+    branches: [ master ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  #pull_request:
+  #  branches: [ master ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          #images: ${{ env.REGISTRY }}/newsbot.worker
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/go-build.yml

@@ -0,0 +1,25 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.18
+
+    - name: Build
+      run: go build -v ./...
+
+    #- name: Test
+    #  run: go test -v ./...

.gitignore

@@ -1,9 +1,7 @@
 .env
 dev.session.sql
-__debug_bin
+
-server
 .vscode
-openapi.json
 
 # Binaries for programs and plugins
 *.exe
@@ -12,13 +10,10 @@ openapi.json
 *.so
 *.dylib
 collector
-newsbot.db
-tmp/
 
 # Test binary, built with `go test -c`
 *.test
 
-
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "."
+        }
+    ]
+}

Dockerfile

@@ -1,14 +1,9 @@
-FROM golang:1.22 as build
+FROM golang:1.18.4 as build
 
 COPY . /app
 WORKDIR /app
-
+RUN go build .
-# Always make sure that swagger docs are updated
+RUN go install github.com/pressly/goose/v3/cmd/goose@latest
-RUN go install github.com/swaggo/swag/cmd/swag@latest
-RUN /go/bin/swag init -g cmd/server.go
-
-#RUN go build .
-#RUN go install github.com/pressly/goose/v3/cmd/goose@latest
 
 FROM alpine:latest as app
 
@@ -17,7 +12,8 @@ RUN apk --no-cache add libc6-compat
 RUN apk --no-cache add chromium
 
 RUN mkdir /app && mkdir /app/migrations
-COPY --from=build /app/server /app
+COPY --from=build /app/collector /app
-COPY ./internal/database/migrations/ /app/migrations
+COPY --from=build /go/bin/goose /app
+COPY ./database/migrations/ /app/migrations
 
 CMD [ "/app/collector" ]

api.http

@@ -1,6 +0,0 @@
-### Select Sources fro mthe top
-GET http://localhost:8081/api/v1/sources/
-
-
-### Select Sources by type
-GET http://localhost:8081/api/v1/sources/by/source?source=rss

api/client.yaml

@@ -1,6 +0,0 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/deepmap/oapi-codegen/HEAD/configuration-schema.json
-package: api
-output: api.gen.go
-generate:
-  models: true
-  client: true

cmd/server.go

@@ -1,90 +0,0 @@
-package main
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"os"
-
-	_ "github.com/glebarez/go-sqlite"
-	"github.com/pressly/goose/v3"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/docs"
-	v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
-)
-
-//  @title                       NewsBot collector
-//  @version                     0.1
-//  @BasePath                    /api
-//  @securityDefinitions.apikey  Bearer
-//  @in                          header
-//  @name                        Authorization
-//  @description                 Type "Bearer" followed by a space and JWT token.
-func main() {
-	ctx := context.Background()
-
-	cfg := services.NewConfig()
-	configs := services.GetEnvConfig()
-	address := cfg.GetConfig(services.ServerAddress)
-	docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
-
-	db, err := sql.Open("sqlite", "newsbot.db")
-	if err != nil {
-		panic(err)
-	}
-
-	err = migrateDatabase(db)
-	if err != nil {
-		fmt.Print(err)
-	}
-
-	c := cron.NewScheduler(ctx, db)
-	c.Start()
-
-	server := v1.NewServer(ctx, configs, db)
-
-	fmt.Println("API is online and waiting for requests.")
-	fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
-	fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress)
-
-	server.Router.Start(":8081")
-}
-
-func migrateDatabase(db *sql.DB) error {
-	err := goose.SetDialect("sqlite3")
-	if err != nil {
-		panic(err)
-	}
-
-	err = goose.Up(db, "../internal/database/migrations")
-	if err != nil {
-		panic(err)
-	}
-
-	_, err = os.Stat("./migrations")
-	if err == nil {
-
-		err = goose.Up(db, "../internal/database/migrations")
-		if err != nil {
-			panic(err)
-		}
-		
-		return nil
-	}
-
-	_, err = os.Stat("../internal/database/migrations")
-	if err == nil {
-
-		err = goose.Up(db, "../internal/database/migrations")
-		if err != nil {
-			panic(err)
-		}
-
-		return nil
-	}
-
-	return errors.New("failed to find the migration files")
-}

database/db.go

@@ -0,0 +1,31 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.13.0
+
+package database
+
+import (
+	"context"
+	"database/sql"
+)
+
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+func New(db DBTX) *Queries {
+	return &Queries{db: db}
+}
+
+type Queries struct {
+	db DBTX
+}
+
+func (q *Queries) WithTx(tx *sql.Tx) *Queries {
+	return &Queries{
+		db: tx,
+	}
+}

database/migrations/20220522083756_init.sql

@@ -0,0 +1,72 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+CREATE TABLE Articles (
+	ID 			uuid PRIMARY KEY,
+	SourceId 	uuid NOT null,
+	Tags 		TEXT NOT NULL,
+	Title 		TEXT NOT NULL,
+	Url 		TEXT NOT NULL,
+	PubDate 	timestamp NOT NULL,
+	Video 		TEXT,
+	VideoHeight int NOT NULL,
+	VideoWidth 	int NOT NULL,
+	Thumbnail 	TEXT NOT NULL,
+	Description TEXT NOT NULL,
+	AuthorName 	TEXT,
+	AuthorImage TEXT
+);
+
+CREATE Table DiscordQueue ( 
+    ID 			uuid PRIMARY KEY,
+    ArticleId 	uuid NOT NULL
+);
+
+CREATE Table DiscordWebHooks (
+	ID 		uuid PRIMARY KEY,
+	Name 	TEXT NOT NULL, -- Defines webhook purpose
+	Key 	TEXT,
+	Url 	TEXT NOT NULL, -- Webhook Url
+	Server 	TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
+	Channel TEXT NOT NULL, -- Defines the channel its bound to.  Used for refrence
+	Enabled BOOLEAN NOT NULL
+);
+
+CREATE Table Icons (
+	ID 			uuid PRIMARY Key,
+	FileName 	TEXT NOT NULL,
+	Site 		TEXT NOT NULL
+);
+
+Create Table Settings (
+	ID 		uuid PRIMARY Key,
+	Key 	TEXT NOT NULL, -- How you search for a entry
+	Value 	TEXT NOT NULL, -- The value for one
+	Options TEXT -- any notes about the entry
+);
+
+Create Table Sources (
+	ID 		uuid PRIMARY Key,
+	Site 	TEXT NOT NULL, -- Vanity name
+	Name 	TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
+	Source 	TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
+	Type 	TEXT NOT NULL, -- Defines what kind of feed this is.  feed, user, tag
+	Value 	TEXT,
+	Enabled BOOLEAN NOT NULL,
+	Url 	TEXT NOT NULL,
+	Tags 	TEXT NOT NULL
+);
+
+
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+Drop Table Articles;
+Drop Table DiscordQueue;
+Drop Table DiscordWebHooks;
+Drop Table Icons;
+Drop Table Settings;
+Drop Table Sources;
+-- +goose StatementEnd

database/migrations/20220529082459_seed.sql

@@ -0,0 +1,50 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+
+-- Enable UUID's
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Final Fantasy XIV Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
+
+-- Reddit Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
+
+-- Youtube Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
+
+-- RSS Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
+
+-- Twitch Entries 
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+--SELECT 'down SQL query';
+
+DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
+DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
+DELETE FROM sources where source = 'ffxiv';
+DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
+DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
+DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
+-- +goose StatementEnd

database/migrations/20220619085634_subscriptions.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+Create TABLE Subscriptions (
+	ID 			      uuid Primary Key,
+	DiscordWebHookID  uuid Not Null,
+	SourceID         uuid Not Null
+);
+
+ALTER TABLE discordwebhooks drop COLUMN Name;
+ALTER TABLE discordwebhooks drop COLUMN Key;
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query';
+Drop Table Subscriptions;
+ALTER TABLE discordwebhooks Add COLUMN Name TEXT;
+--ALTER TABLE discordwebhooks Add COLUMN Key TEXT;
+-- +goose StatementEnd

database/models.go

@@ -0,0 +1,72 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.13.0
+
+package database
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type Article struct {
+	ID          uuid.UUID
+	Sourceid    uuid.UUID
+	Tags        string
+	Title       string
+	Url         string
+	Pubdate     time.Time
+	Video       sql.NullString
+	Videoheight int32
+	Videowidth  int32
+	Thumbnail   string
+	Description string
+	Authorname  sql.NullString
+	Authorimage sql.NullString
+}
+
+type Discordqueue struct {
+	ID        uuid.UUID
+	Articleid uuid.UUID
+}
+
+type Discordwebhook struct {
+	ID      uuid.UUID
+	Url     string
+	Server  string
+	Channel string
+	Enabled bool
+}
+
+type Icon struct {
+	ID       uuid.UUID
+	Filename string
+	Site     string
+}
+
+type Setting struct {
+	ID      uuid.UUID
+	Key     string
+	Value   string
+	Options sql.NullString
+}
+
+type Source struct {
+	ID      uuid.UUID
+	Site    string
+	Name    string
+	Source  string
+	Type    string
+	Value   sql.NullString
+	Enabled bool
+	Url     string
+	Tags    string
+}
+
+type Subscription struct {
+	ID               uuid.UUID
+	Discordwebhookid uuid.UUID
+	Sourceid         uuid.UUID
+}

database/schema/query.sql

@@ -0,0 +1,176 @@
+/* Articles */
+-- name: GetArticleByID :one
+Select * from Articles
+WHERE ID = $1 LIMIT 1;
+
+-- name: GetArticleByUrl :one
+Select * from Articles
+Where Url = $1 LIMIT 1;
+
+-- name: ListArticles :many
+Select * From articles Limit $1;
+
+-- name: ListArticlesByDate :many
+Select * From articles ORDER BY pubdate desc Limit $1;
+
+-- name: GetArticlesBySource :many
+select * from articles
+INNER join sources on articles.sourceid=Sources.ID
+where site = $1;
+
+-- name: GetArticlesBySourceId :many
+Select * From articles
+Where sourceid = $1 Limit 50;
+
+-- name: GetArticlesBySourceName :many
+select 
+articles.ID, articles.SourceId, articles.Tags, articles.Title, articles.Url, articles.PubDate, articles.Video, articles.VideoHeight, articles.VideoWidth, articles.Thumbnail, articles.Description, articles.AuthorName, articles.AuthorImage, sources.source, sources.name
+From articles
+Left Join sources
+On articles.sourceid = sources.id
+Where name = $1;
+
+-- name: CreateArticle :exec
+INSERT INTO Articles 
+(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
+Values
+($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
+
+
+/* DiscordQueue */
+-- name: CreateDiscordQueue :exec
+Insert into DiscordQueue
+(ID, ArticleId)
+Values
+($1, $2);
+
+-- name: GetDiscordQueueByID :one
+Select * from DiscordQueue
+Where ID = $1 LIMIT 1;
+
+-- name: DeleteDiscordQueueItem :exec
+Delete From DiscordQueue Where ID = $1;
+
+-- name: ListDiscordQueueItems :many
+Select * from DiscordQueue LIMIT $1;
+
+/* DiscordWebHooks */
+-- name: CreateDiscordWebHook :exec
+Insert Into DiscordWebHooks
+(ID, Url, Server, Channel, Enabled)
+Values
+($1, $2, $3, $4, $5);
+
+-- name: GetDiscordWebHooksByID :one
+Select * from DiscordWebHooks
+Where ID = $1 LIMIT 1;
+
+-- name: ListDiscordWebHooksByServer :many
+Select * From DiscordWebHooks
+Where Server = $1;
+
+-- name: GetDiscordWebHookByUrl :one
+Select * From DiscordWebHooks Where url = $1;
+
+-- name: ListDiscordWebhooks :many
+Select * From discordwebhooks LIMIT $1;
+
+-- name: DeleteDiscordWebHooks :exec
+Delete From discordwebhooks Where ID = $1;
+
+
+/* Icons */
+
+-- name: CreateIcon :exec
+INSERT INTO Icons
+(ID, FileName, Site)
+VALUES
+($1,$2,$3);
+
+-- name: GetIconByID :one
+Select * FROM Icons
+Where ID = $1 Limit 1;
+
+-- name: GetIconBySite :one
+Select * FROM Icons
+Where Site = $1 Limit 1;
+
+-- name: DeleteIcon :exec
+Delete From Icons where ID = $1;
+
+/* Settings */
+
+-- name: CreateSettings :one
+Insert Into settings
+(ID, Key, Value, OPTIONS)
+Values
+($1,$2,$3,$4)
+RETURNING *;
+
+-- name: GetSettingByID :one
+Select * From settings
+Where ID = $1 Limit 1;
+
+-- name: GetSettingByKey :one
+Select * From settings Where 
+Key = $1 Limit 1;
+
+-- name: GetSettingByValue :one
+Select * From settings Where 
+Value = $1 Limit 1;
+
+-- name: DeleteSetting :exec
+Delete From settings Where ID = $1;
+
+/* Sources */
+
+-- name: CreateSource :exec
+Insert Into Sources
+(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
+Values
+($1,$2,$3,$4,$5,$6,$7,$8,$9);
+
+-- name: GetSourceByID :one
+Select * From Sources where ID = $1 Limit 1;
+
+-- name: GetSourceByName :one
+Select * from Sources where name = $1 Limit 1;
+
+-- name: ListSources :many
+Select * From Sources Limit $1;
+
+-- name: ListSourcesBySource :many
+Select * From Sources where Source = $1;
+
+-- name: DeleteSource :exec
+DELETE From sources where id = $1;
+
+-- name: DisableSource :exec
+Update Sources Set Enabled = FALSE where ID = $1;
+
+-- name: EnableSource :exec
+Update Sources Set Enabled = TRUE where ID = $1;
+
+
+/* Subscriptions */
+
+-- name: CreateSubscription :exec
+Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3);
+
+-- name: ListSubscriptions :many
+Select * From subscriptions Limit $1;
+
+-- name: ListSubscriptionsBySourceId :many
+Select * From subscriptions where sourceid = $1;
+
+-- name: QuerySubscriptions :one
+Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1;
+
+-- name: GetSubscriptionsBySourceID :many
+Select * From subscriptions Where sourceid = $1;
+
+-- name: GetSubscriptionsByDiscordWebHookId :many
+Select * from subscriptions Where discordwebhookid = $1;
+
+-- name: DeleteSubscription :exec
+Delete From subscriptions Where id = $1;

database/schema/schema.sql

@@ -0,0 +1,60 @@
+CREATE TABLE Articles (
+	ID 			uuid PRIMARY KEY,
+	SourceId 	uuid NOT null,
+	Tags 		TEXT NOT NULL,
+	Title 		TEXT NOT NULL,
+	Url 		TEXT NOT NULL,
+	PubDate 	timestamp NOT NULL,
+	Video 		TEXT,
+	VideoHeight int NOT NULL,
+	VideoWidth 	int NOT NULL,
+	Thumbnail 	TEXT NOT NULL,
+	Description TEXT NOT NULL,
+	AuthorName 	TEXT,
+	AuthorImage TEXT
+);
+
+CREATE Table DiscordQueue ( 
+    ID 			uuid PRIMARY KEY,
+    ArticleId 	uuid NOT NULL
+);
+
+CREATE Table DiscordWebHooks (
+	ID 		uuid PRIMARY KEY,
+	Url 	TEXT NOT NULL, -- Webhook Url
+	Server 	TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
+	Channel TEXT NOT NULL, -- Defines the channel its bound to.  Used for refrence
+	Enabled BOOLEAN NOT NULL
+);
+
+CREATE Table Icons (
+	ID 			uuid PRIMARY Key,
+	FileName 	TEXT NOT NULL,
+	Site 		TEXT NOT NULL
+);
+
+Create Table Settings (
+	ID 		uuid PRIMARY Key,
+	Key 	TEXT NOT NULL,
+	Value 	TEXT NOT NULL,
+	Options TEXT
+);
+
+Create Table Sources (
+	ID 		uuid PRIMARY Key,
+	Site 	TEXT NOT NULL, -- Vanity name
+	Name 	TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
+	Source 	TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
+	Type 	TEXT NOT NULL, -- Defines what kind of feed this is.  feed, user, tag
+	Value 	TEXT,
+	Enabled BOOLEAN NOT NULL,
+	Url 	TEXT NOT NULL,
+	Tags 	TEXT NOT NULL
+);
+
+/* This table is used to track what the Web Hook wants to have sent by Source */;
+Create TABLE Subscriptions (
+	ID 					uuid Primary Key,
+	DiscordWebHookID  	uuid Not Null,
+	SourceID         	uuid Not Null
+);

domain/const.go

@@ -1,9 +0,0 @@
-package domain
-
-const (
-	SourceCollectorRss     = "rss"
-	SourceCollectorFfxiv   = "ffxiv"
-	SourceCollectorTwitch  = "twitch"
-	SourceCollectorYoutube = "youtube"
-	SourceCollectorReddit  = "reddit"
-)

domain/dto.go

@@ -1,64 +0,0 @@
-package domain
-
-import "time"
-
-type ArticleDto struct {
-	ID             int64     `json:"id"`
-	SourceID       int64     `json:"sourceId"`
-	Tags           string    `json:"tags"`
-	Title          string    `json:"title"`
-	Url            string    `json:"url"`
-	PubDate        time.Time `json:"pubDate"`
-	IsVideo        bool      `json:"isVideo"`
-	Thumbnail      string    `json:"thumbnail"`
-	Description    string    `json:"description"`
-	AuthorName     string    `json:"authorName"`
-	AuthorImageUrl string    `json:"authorImage"`
-}
-
-type DiscordQueueDto struct {
-	ID        int64 `json:"id"`
-	ArticleId int64 `json:"articleId"`
-	SourceId  int64 `json:"sourceId"`
-}
-
-type DiscordWebHookDto struct {
-	ID      int64   `json:"id"`
-	//Name    string `json:"name"`
-	//Key     string `json:"key"`
-	Url     string `json:"url"`
-	Server  string `json:"server"`
-	Channel string `json:"channel"`
-	Enabled bool   `json:"enabled"`
-}
-
-type IconDto struct {
-	ID       int64  `json:"id"`
-	FileName string `json:"fileName"`
-	Site     string `json:"site"`
-}
-
-type SettingDto struct {
-	ID      int64  `json:"id"`
-	Key     string `json:"key"`
-	Value   string `json:"value"`
-	Options string `json:"options"`
-}
-
-type SubscriptionDto struct {
-	ID          int64  `json:"id"`
-	SourceID    int64  `json:"sourceId"`
-	SourceType  string `json:"sourceType"`
-	SourceName  string `json:"sourceName"`
-	DiscordID   int64  `json:"discordId"`
-	DiscordName string `json:"discordName"`
-}
-
-type SourceDto struct {
-	ID          int64  `json:"id"`
-	Source      string `json:"source"`
-	DisplayName string `json:"name"`
-	Url         string `json:"url"`
-	Tags        string `json:"tags"`
-	Enabled     bool   `json:"enabled"`
-}

domain/interfaces/source.go

@@ -0,0 +1,23 @@
+package interfaces
+
+import (
+	"github.com/go-rod/rod"
+	"github.com/mmcdole/gofeed"
+)
+
+type Sources interface {
+	CheckSource() error
+	PullFeed() (*gofeed.Feed, error)
+
+	GetBrowser() *rod.Browser
+	GetPage(parser *rod.Browser, url string) *rod.Page
+
+	ExtractThumbnail(page *rod.Page) (string, error)
+	ExtractPubDate(page *rod.Page) (string, error)
+	ExtractDescription(page *rod.Page) (string, error)
+	ExtractAuthor(page *rod.Page) (string, error)
+	ExtractAuthorImage(page *rod.Page) (string, error)
+	ExtractTags(page *rod.Page) (string, error)
+	ExtractTitle(page *rod.Page) (string, error)
+}
+

domain/model/cache.go

@@ -1,6 +1,8 @@
-package domain
+package model
 
-import "time"
+import (
+	"time"
+)
 
 type CacheItem struct {
 	Key   string

domain/model/database.go

@@ -0,0 +1,90 @@
+package model
+
+import (
+	"time"
+)
+
+// Articles represents the model for an Article
+type Articles struct {
+	ID uint `json:"ID"`
+	SourceID uint `json:"sourceId"`
+	Tags string	`json:"tags"`
+	Title string `json:"title"`
+	Url string `json:"url"`
+	PubDate time.Time `json:"pubdate"`
+	Video string `json:"video"`
+	VideoHeight uint16 `json:"videoHeight"`
+	VideoWidth uint16 `json:"videoWidth"`
+	Thumbnail string `json:"thumbnail"`
+	Description string `json:"description"`
+	AuthorName string `json:"authorName"`
+	AuthorImage string `json:"authorImage"`
+}
+
+type DiscordQueue struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+	ArticleId string `json:"articleId"`
+}
+
+type DiscordWebHooks struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+
+	Name string 	`json:"name"`
+	Key string 		`json:"key"`
+	Url string 		`json:"url"`
+	Server string 	`json:"server"`
+	Channel string	`json:"channel"`
+	Enabled bool	`json:"enabled"`
+}
+
+type Icons struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+
+	FileName string	`json:"fileName"`
+	Site string		`json:"site"`
+}
+
+type Settings struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+
+	Key string		`json:"key"`
+	Value string	`json:"value"`
+	Options string	`json:"options"`
+}
+
+type Sources struct {
+	ID uint 		`json:"ID"`
+	Site string		`json:"site"`
+	Name string		`json:"name"`
+	Source string	`json:"source"`
+	Type string		`json:"type"`
+	Value string	`json:"value"`
+	Enabled bool	`json:"enabled"`
+	Url string		`json:"url"`
+	Tags string		`json:"tags"`
+}
+
+type SourceLinks struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+	
+	SourceID uint		`json:"sourceId"`
+	SourceType string	`json:"sourceType"`
+	SourceName string	`json:"sourceName"`
+	DiscordID uint	`json:"discordId"`
+	DiscordName string	`json:"discordName"`
+}

domain/model/reddit.go

@@ -0,0 +1,54 @@
+package model
+
+// This is the root Json object.  It does not contain data that we care about though.
+type RedditJsonContent struct {
+	Kind string `json:"kind"`
+	Data RedditJsonContentData `json:"data"`
+}
+
+type RedditJsonContentData struct {
+	After string `json:"after"`
+	Dist int `json:"dist"`
+	Modhash string `json:"modhash"`
+	Children []RedditJsonContentChildren `json:"children"`
+}
+
+type RedditJsonContentChildren struct {
+	Kind string `json:"kind"`
+	Data RedditPost `json:"data"`
+}
+
+// RedditPost contains the information that was posted by a user.
+type RedditPost struct {
+	Subreddit string `json:"subreddit"`
+	Title string `json:"title"`
+	Content string `json:"selftext"`
+	ContentHtml string `json:"selftext_html"`
+	Author string `json:"author"`
+	Permalink string `json:"permalink"`
+	IsVideo bool `json:"is_video"`
+	Media RedditPostMedia `json:"media"`
+	Url string `json:"url"`
+	UrlOverriddenByDest string `json:"url_overridden_by_dest"`
+
+	Thumbnail string `json:"thumbnail"`
+}
+
+// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
+type RedditPostMedia struct {
+	RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
+}
+
+// RedditVideo contains information about the video in the post.
+type RedditPostMediaRedditVideo struct {
+	BitrateKbps int `json:"bitrate_kpbs"`
+	FallBackUrl string `json:"fallback_url"`
+	Height int `json:"height"`
+	Width int `json:"width"`
+	ScrubberMediaUrl string `json:"scrubber_media_url"`
+	DashUrl string `json:"dash_url"`
+	Duration int `json:"duration"`
+	HlsUrl string `json:"hls_url"`
+	IsGif bool `json:"is_gif"`
+	TranscodingStatus string `json:"transcoding_status"`
+}

domain/requests.go

@@ -1,27 +0,0 @@
-package domain
-
-type LoginFormRequest struct {
-	Username string `form:"username"`
-	Password string `form:"password"`
-}
-
-type GetSourceBySourceAndNameParamRequest struct {
-	Name   string `query:"name"`
-	Source string `query:"source"`
-}
-
-type NewSourceParamRequest struct {
-	Name string `json:"name"`
-	Url  string `json:"url"`
-	Tags string `json:"tags"`
-}
-
-type RefreshTokenRequest struct {
-	Username     string `json:"username"`
-	RefreshToken string `json:"refreshToken"`
-}
-
-type UpdateScopesRequest struct {
-	Username string   `json:"username"`
-	Scopes   []string `json:"scopes" validate:"required"`
-}

domain/responses.go

@@ -1,38 +0,0 @@
-package domain
-
-type BaseResponse struct {
-	Message string `json:"message"`
-	IsError bool   `json:"isError"`
-}
-
-type LoginResponse struct {
-	BaseResponse
-	Token        string `json:"token"`
-	Type         string `json:"type"`
-	RefreshToken string `json:"refreshToken"`
-}
-
-type ArticleResponse struct {
-	BaseResponse
-	Payload []ArticleDto `json:"payload"`
-}
-
-type ArticleAndSourceModel struct {
-	Article ArticleDto `json:"article"`
-	Source  SourceDto  `json:"source"`
-}
-
-type ArticleDetailedResponse struct {
-	BaseResponse
-	Payload ArticleAndSourceModel `json:"payload"`
-}
-
-type DiscordWebhookResponse struct {
-	BaseResponse
-	Payload []DiscordWebHookDto `json:"payload"`
-}
-
-type SourcesResponse struct {
-	BaseResponse
-	Payload []SourceDto `json:"payload"`
-}

domain/scopes.go

@@ -1,14 +0,0 @@
-package domain
-
-const (
-	ScopeAll = "newsbot:all"
-
-	ScopeArticleRead    = "newsbot:article:read"
-	ScopeArticleDisable = "newsbot:article:disable"
-
-	ScopeSourceRead   = "newsbot:source:read"
-	ScopeSourceCreate = "newsbot:source:create"
-
-	ScopeDiscordWebHookCreate = "newsbot:discordwebhook:create"
-	ScopeDiscordWebhookRead   = "newsbot:discordwebhook:read"
-)

go.mod

@@ -1,48 +1,18 @@
-module git.jamestombleson.com/jtom38/newsbot-api
+module github.com/jtom38/newsbot/collector
 
-go 1.22
+go 1.18
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
-	github.com/glebarez/go-sqlite v1.22.0
+	github.com/go-chi/chi/v5 v5.0.7
 	github.com/go-rod/rod v0.107.1
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/google/uuid v1.3.0
-	github.com/google/uuid v1.6.0
-	github.com/huandu/go-sqlbuilder v1.27.1
 	github.com/joho/godotenv v1.4.0
-	github.com/labstack/echo-jwt/v4 v4.2.0
-	github.com/labstack/echo/v4 v4.12.0
 	github.com/mmcdole/gofeed v1.1.3
 	github.com/nicklaw5/helix/v2 v2.4.0
-	github.com/pressly/goose/v3 v3.20.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/swaggo/echo-swagger v1.4.1
+	github.com/swaggo/http-swagger v1.3.0
-	github.com/swaggo/swag v1.8.12
+	github.com/swaggo/swag v1.8.2
-	golang.org/x/crypto v0.22.0
-)
-
-require (
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/labstack/gommon v0.4.2 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mfridman/interpolate v0.0.2 // indirect
-	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/sethvargo/go-retry v0.2.4 // indirect
-	github.com/swaggo/files/v2 v2.0.0 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasttemplate v1.2.2 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	modernc.org/libc v1.41.0 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.7.2 // indirect
-	modernc.org/sqlite v1.29.6 // indirect
 )
 
 require (
@@ -52,7 +22,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/spec v0.20.6 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/lib/pq v1.10.6
@@ -60,12 +30,13 @@ require (
 	github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
 	github.com/ysmood/gson v0.7.2 // indirect
 	github.com/ysmood/leakless v0.7.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/tools v0.1.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -12,12 +12,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
-github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -31,25 +27,11 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK
 github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY=
 github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
-github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
-github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
-github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
-github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
-github.com/huandu/go-sqlbuilder v1.27.1 h1:7UU/3EMIQYYX8wn+L7BNcGVz1aEs5TPNOVFd7ryrPos=
-github.com/huandu/go-sqlbuilder v1.27.1/go.mod h1:nUVmMitjOmn/zacMLXT0d3Yd3RHoO2K+vy906JzqxMI=
-github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
-github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -62,12 +44,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo-jwt/v4 v4.2.0 h1:odSISV9JgcSCuhgQSV/6Io3i7nUmfM/QkBeR5GVJj5c=
-github.com/labstack/echo-jwt/v4 v4.2.0/go.mod h1:MA2RqdXdEn4/uEglx0HcUOgQSyBaTh5JcaHIan3biwU=
-github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
-github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
-github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
-github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
 github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
 github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -75,13 +51,6 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
-github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
 github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
 github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
 github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
@@ -93,41 +62,27 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
-github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nicklaw5/helix/v2 v2.4.0 h1:ZvqCKVqza1eJYyqgTRrZ/xjDq0w/EQVFNkN067Utls0=
 github.com/nicklaw5/helix/v2 v2.4.0/go.mod h1:0ONzvVi1cH+k3a7EDIFNNqxfW0podhf+CqlmFvuexq8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0=
-github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
-github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc=
-github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk=
+github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
-github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc=
+github.com/swaggo/http-swagger v1.3.0 h1:1+6M4qRorIbdyTWTsGrwnb0r9jGK5dcWN82O6oY/yHQ=
-github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
+github.com/swaggo/http-swagger v1.3.0/go.mod h1:9glekdg40lwclrrKNRGgj/IMDxpNPZ3kzab4oPcF8EM=
-github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
+github.com/swaggo/swag v1.8.2 h1:D4aBiVS2a65zhyk3WFqOUz7Rz0sOaUcgeErcid5uGL4=
-github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w=
+github.com/swaggo/swag v1.8.2/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
-github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its=
 github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
-github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
 github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
 github.com/ysmood/got v0.29.5 h1:+wMnm8UjoyYFMfeAsr57a1bahWTkloysc0Hxsu2gmnM=
@@ -139,39 +94,29 @@ github.com/ysmood/gson v0.7.2 h1:1iWUvpi5DPvd2j59W7ifRPR9DiAZ3Ga+fmMl1mJrRbM=
 github.com/ysmood/gson v0.7.2/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
 github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
 github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
-go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 h1:PgOr27OhUx2IRqGJ2RxAWI4dJQ7bi9cSrB82uzFzfUA=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
@@ -180,20 +125,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
-modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
-modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
-modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
-modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
-modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
-modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
-modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4=
-modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
-modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
-modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
-modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/database/migrations/20240425083756_init.sql

@@ -1,115 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-CREATE TABLE Articles (
-	ID 				INTEGER 	PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 		DATETIME	NOT NULL,
-	UpdatedAt 		DATETIME 	NOT NULL,
-	DeletedAt 		DATETIME	NOT NULL,
-	SourceId 		NUMBER 		NOT NULL,
-	Tags 			TEXT 		NOT NULL,
-	Title 			TEXT 		NOT NULL,
-	Url 			TEXT 		NOT NULL,
-	PubDate 		DATETIME 	NOT NULL,
-	IsVideo 		TEXT		NOT NULL,
-	ThumbnailUrl 	TEXT 		NOT NULL,
-	Description 	TEXT 		NOT NULL,
-	AuthorName 		TEXT		NOT NULL,
-	AuthorImageUrl 	TEXT		NOT NULL
-);
-
-CREATE Table DiscordWebHooks (
-	ID 			INTEGER 	PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-	UserID 		INTEGER 	NOT NULL,
-	Url 		TEXT 		NOT NULL, -- Webhook Url
-	Server 		TEXT 		NOT NULL, -- Defines the server its bound it. Used for reference
-	Channel 	TEXT 		NOT NULL, -- Defines the channel its bound to.  Used for reference
-	Enabled 	BOOLEAN 	NOT NULL
-);
-
-CREATE Table Icons (
-	ID 			INTEGER	 	PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME,
-	FileName 	TEXT 		NOT NULL,
-	Site 		TEXT 		NOT NULL
-);
-
-Create Table Settings (
-	ID 			INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME,
-	Key 		TEXT 		NOT NULL, -- How you search for a entry
-	Value 		TEXT 		NOT NULL, -- The value for one
-	Options 	TEXT -- any notes about the entry
-);
-
-CREATE Table Sources (
-	ID 			INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-	DisplayName	TEXT 		NOT NULL, -- Vanity name
-	Source 		TEXT 		NOT NULL, -- Defines the service that will use this record. IE reddit or youtube
-	Enabled 	BOOLEAN 	NOT NULL,
-	Url 		TEXT 		NOT NULL,
-	Tags 		TEXT 		NOT NULL
-);
-
-CREATE TABLE UserSourceSubscriptions (
-	ID 					INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 			DATETIME 	NOT NULL,
-	UpdatedAt 			DATETIME 	NOT NULL,
-	DeletedAt 			DATETIME 	NOT NULL,
-	UserID				NUMBER		NOT NULL,
-	SourceID 			NUMBER 		NOT NULL
-);
-
-CREATE TABLE AlertDiscord (
-	ID 					INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 			DATETIME 	NOT NULL,
-	UpdatedAt 			DATETIME 	NOT NULL,
-	DeletedAt 			DATETIME 	NOT NULL,
-	UserID				NUMBER 		NOT NULL,
-	SourceID 			NUMBER 		NOT NULL,
-	DiscordWebHookID 	NUMBER 		NOT NULL
-);
-
-CREATE TABLE Users (
-    ID          INTEGER     PRIMARY KEY AUTOINCREMENT,
-    CreatedAt   DATETIME    NOT NULL,
-    UpdatedAt	DATETIME    NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-    Name        TEXT        NOT NULL,
-    Hash        TEXT        NOT NULL,
-	Scopes 		TEXT		NOT NULL
-); 
-
-CREATE TABLE RefreshTokens (
-    ID          INTEGER     PRIMARY KEY AUTOINCREMENT,
-    CreatedAt   DATETIME    NOT NULL,
-    UpdatedAt 	DATETIME    NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-    Username    TEXT        NOT NULL,
-    Token       TEXT        NOT NULL
-);
-
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-DROP TABLE AlertDiscord;
-Drop Table Articles;
-Drop Table DiscordWebHooks;
-Drop Table Icons;
-DROP TABLE RefreshTokens;
-Drop Table Sources;
-DROP TABLE Users;
-DROP TABLE UserSourceSubscriptions;
--- +goose StatementEnd

internal/database/migrations/20240425092459_seed.sql

@@ -1,50 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-
--- Enable UUID's
---CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-
--- Final Fantasy XIV Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES  
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES  
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-
--- Reddit Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-
--- Youtube Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-
--- RSS Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-
--- Twitch Entries 
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
---SELECT 'down SQL query';
-
-DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes';
-DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck';
-DELETE FROM sources where Source = 'ffxiv';
-DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo';
-DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps';
-DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck';
--- +goose StatementEnd

internal/database/migrations/20240521194914_session_token.sql

@@ -1,9 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-ALTER TABLE Users ADD SessionToken TEXT; 
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-ALTER TABLE Users DROP SessionToken;
--- +goose StatementEnd

internal/domain/modelsReddit.go

@@ -1,54 +0,0 @@
-package domain
-
-// This is the root Json object.  It does not contain data that we care about though.
-type RedditJsonContent struct {
-	Kind string                `json:"kind"`
-	Data RedditJsonContentData `json:"data"`
-}
-
-type RedditJsonContentData struct {
-	After    string                      `json:"after"`
-	Dist     int                         `json:"dist"`
-	Modhash  string                      `json:"modhash"`
-	Children []RedditJsonContentChildren `json:"children"`
-}
-
-type RedditJsonContentChildren struct {
-	Kind string     `json:"kind"`
-	Data RedditPost `json:"data"`
-}
-
-// RedditPost contains the information that was posted by a user.
-type RedditPost struct {
-	Subreddit           string          `json:"subreddit"`
-	Title               string          `json:"title"`
-	Content             string          `json:"selftext"`
-	ContentHtml         string          `json:"selftext_html"`
-	Author              string          `json:"author"`
-	Permalink           string          `json:"permalink"`
-	IsVideo             bool            `json:"is_video"`
-	Media               RedditPostMedia `json:"media"`
-	Url                 string          `json:"url"`
-	UrlOverriddenByDest string          `json:"url_overridden_by_dest"`
-
-	Thumbnail string `json:"thumbnail"`
-}
-
-// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
-type RedditPostMedia struct {
-	RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
-}
-
-// RedditVideo contains information about the video in the post.
-type RedditPostMediaRedditVideo struct {
-	BitrateKbps       int    `json:"bitrate_kpbs"`
-	FallBackUrl       string `json:"fallback_url"`
-	Height            int    `json:"height"`
-	Width             int    `json:"width"`
-	ScrubberMediaUrl  string `json:"scrubber_media_url"`
-	DashUrl           string `json:"dash_url"`
-	Duration          int    `json:"duration"`
-	HlsUrl            string `json:"hls_url"`
-	IsGif             bool   `json:"is_gif"`
-	TranscodingStatus string `json:"transcoding_status"`
-}

internal/dtoconv/dtoconv.go

@@ -1,67 +0,0 @@
-package dtoconv
-
-import (
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-)
-
-func ArticlesToDto(items []entity.ArticleEntity) []domain.ArticleDto {
-	var dtos []domain.ArticleDto
-	for _, item := range items {
-		dtos = append(dtos, ArticleToDto(item))
-	}
-	return dtos
-}
-
-func ArticleToDto(item entity.ArticleEntity) domain.ArticleDto {
-	return domain.ArticleDto{
-		ID:             item.ID,
-		SourceID:       item.SourceID,
-		Tags:           item.Tags,
-		Title:          item.Title,
-		Url:            item.Url,
-		PubDate:        item.PubDate,
-		IsVideo:        item.IsVideo,
-		Thumbnail:      item.Thumbnail,
-		Description:    item.Description,
-		AuthorName:     item.AuthorName,
-		AuthorImageUrl: item.AuthorImageUrl,
-	}
-}
-
-func DiscordWebhooksToDto(items []entity.DiscordWebHookEntity) []domain.DiscordWebHookDto{
-	var dtos []domain.DiscordWebHookDto
-	for _, item := range items {
-		dtos = append(dtos, DiscordWebhookToDto(item))
-	}
-	return dtos
-}
-
-func DiscordWebhookToDto(item entity.DiscordWebHookEntity) domain.DiscordWebHookDto {
-	return domain.DiscordWebHookDto{
-		ID:      item.ID,
-		Server:  item.Server,
-		Channel: item.Channel,
-		Url:     item.Url,
-		Enabled: item.Enabled,
-	}
-}
-
-func SourcesToDto(items []entity.SourceEntity) []domain.SourceDto {
-	var dtos []domain.SourceDto
-	for _, item := range items {
-		dtos = append(dtos, SourceToDto(item))
-	}
-	return dtos
-}
-
-func SourceToDto(item entity.SourceEntity) domain.SourceDto {
-	return domain.SourceDto{
-		ID:          item.ID,
-		Source:      item.Source,
-		DisplayName: item.DisplayName,
-		Url:         item.Url,
-		Tags:        item.Tags,
-		Enabled:     item.Enabled,
-	}
-}

internal/entity/entity.go

@@ -1,141 +0,0 @@
-package entity
-
-import (
-	"time"
-)
-
-// This links a source to a discord webhook.
-// It is owned by a user so they can remove the link
-type AlertDiscordEntity struct {
-	ID               int64
-	CreatedAt        time.Time
-	UpdatedAt        time.Time
-	DeletedAt        time.Time
-	UserID           int64
-	SourceID         int64
-	DiscordWebHookId int64
-}
-
-type ArticleEntity struct {
-	ID             int64
-	CreatedAt      time.Time
-	UpdatedAt      time.Time
-	DeletedAt      time.Time
-	SourceID       int64
-	Tags           string
-	Title          string
-	Url            string
-	PubDate        time.Time
-	IsVideo        bool
-	Thumbnail      string
-	Description    string
-	AuthorName     string
-	AuthorImageUrl string
-}
-
-type DiscordQueueEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	ArticleId int64
-	SourceId  int64
-}
-
-type DiscordWebHookEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	UserID    int64
-	Url       string
-	Server    string
-	Channel   string
-	Enabled   bool
-}
-
-type IconEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	FileName  string
-	Site      string
-}
-
-type SettingEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	Key       string
-	Value     string
-	Options   string
-}
-
-type SourceEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-
-	// Who will collect from it.  Used
-	// domain.SourceCollector...
-	Source string
-
-	// Human Readable value to state what is getting collected
-	DisplayName string
-
-	// Tells the parser where to look for data
-	Url string
-
-	// Static tags for this defined record
-	Tags string
-
-	// If the record is disabled, then it will be skipped on processing
-	Enabled bool
-}
-
-//type SubscriptionEntity struct {
-//	ID        int64
-//	CreatedAt time.Time
-//	UpdatedAt time.Time
-//	DeletedAt time.Time
-//	UserID    int64
-//	SourceID  int64
-//	//SourceType  string
-//	//SourceName  string
-//	DiscordID int64
-//	//DiscordName string
-//}
-
-// This defines what sources a user wants to follow.
-// These will show up for the user as a front page
-type UserSourceSubscriptionEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	UserID    int64
-	SourceID  int64
-}
-
-type UserEntity struct {
-	ID           int64
-	CreatedAt    time.Time
-	UpdatedAt    time.Time
-	DeletedAt    time.Time
-	Username     string
-	Hash         string
-	Scopes       string
-	SessionToken string
-}
-
-type RefreshTokenEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	Username  string
-	Token     string
-}

internal/handler/v1/articles.go

@@ -1,181 +0,0 @@
-package v1
-
-import (
-	"net/http"
-	"strconv"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
-	"github.com/labstack/echo/v4"
-)
-
-// ListArticles
-//  @Summary   Lists the top 25 records ordering from newest to oldest.
-//  @Produce   application/json
-//  @Param     page  query  int  true  "page number"  test
-//  @Tags      Articles
-//  @Router    /v1/articles [get]
-//  @Success   200  {object}  domain.ArticleResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) listArticles(c echo.Context) error {
-	resp := domain.ArticleResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, resp)
-	}
-
-	page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		page = 0
-	}
-
-	res, err := s.repo.Articles.ListByPage(c.Request().Context(), page, 25)
-	if err != nil {
-		return c.JSON(http.StatusInternalServerError, resp)
-	}
-
-	resp.Payload = dtoconv.ArticlesToDto(res)
-	resp.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, resp)
-}
-
-// GetArticle
-//  @Summary   Returns an article based on defined ID.
-//  @Param     id  path  string  true  "int"
-//  @Produce   application/json
-//  @Tags      Articles
-//  @Router    /v1/articles/{id} [get]
-//  @Success   200  {object}  domain.ArticleResponse  "OK"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) getArticle(c echo.Context) error {
-	p := domain.ArticleResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id := c.Param("ID")
-	idNumber, err := strconv.Atoi(id)
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Articles.GetById(c.Request().Context(), int64(idNumber))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	var dtos []domain.ArticleDto
-	dtos = append(dtos, dtoconv.ArticleToDto(item))
-	p.Payload = dtos
-
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetArticleDetails
-//  @Summary   Returns an article and source based on defined ID.
-//  @Param     id  path  string  true  "int"
-//  @Produce   application/json
-//  @Tags      Articles
-//  @Router    /v1/articles/{id}/details [get]
-//  @Success   200  {object}  domain.ArticleDetailedResponse  "OK"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) getArticleDetails(c echo.Context) error {
-	p := domain.ArticleDetailedResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-		Payload: domain.ArticleAndSourceModel{},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	article, err := s.repo.Articles.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	source, err := s.repo.Sources.GetById(c.Request().Context(), article.SourceID)
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	p.Payload.Article = dtoconv.ArticleToDto(article)
-	p.Payload.Source = dtoconv.SourceToDto(source)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// ListArticlesBySourceID
-//  @Summary   Finds the articles based on the SourceID provided.  Returns the top 25.
-//  @Param     id    path   string  true   "source id"
-//  @Param     page  query  int     false  "Page to query"
-//  @Produce   application/json
-//  @Tags      Articles
-//  @Router    /v1/articles/by/source/{id} [get]
-//  @Success   200  {object}  domain.ArticleResponse  "OK"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) ListArticlesBySourceId(c echo.Context) error {
-	p := domain.ArticleResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.QueryParam("id"))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// if the page number is missing, default to 0
-	_page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		_page = 0
-	}
-
-	items, err := s.repo.Articles.ListBySource(c.Request().Context(), _page, 25, id, "")
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	p.Payload = dtoconv.ArticlesToDto(items)
-	p.BaseResponse.IsError = false
-
-	return c.JSON(http.StatusOK, p)
-}

internal/handler/v1/discordwebhooks.go

@@ -1,443 +0,0 @@
-package v1
-
-import (
-	"net/http"
-	"strconv"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
-	"github.com/labstack/echo/v4"
-)
-
-// ListDiscordWebhooks
-//  @Summary   Returns the top 100
-//  @Produce   application/json
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks [get]
-//  @Success   200  {object}  domain.DiscordWebhookResponse
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) ListDiscordWebHooks(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	res, err := s.repo.DiscordWebHooks.ListByServerName(c.Request().Context(), "")
-	if err != nil {
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.DiscordWebhooksToDto(res)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetDiscordWebHook
-//  @Summary   Returns the top 100 entries from the queue to be processed.
-//  @Produce   application/json
-//  @Param     id  path  int  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id} [get]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) GetDiscordWebHooksById(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	res, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(res))
-	p.Payload = dtos
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetDiscordWebHookByServerAndChannel
-//  @Summary   Returns all the known web hooks based on the Server and Channel given.
-//  @Produce   application/json
-//  @Param     server   query  string  true  "Fancy Server"
-//  @Param     channel  query  string  true  "memes"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/by/serverAndChannel [get]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	_server := c.QueryParam("server")
-	if _server == "" {
-		p.BaseResponse.Message = "server was not defined"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	_channel := c.QueryParam("channel")
-	if _channel == "" {
-		p.BaseResponse.Message = "channel was not defined"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	res, err := s.repo.DiscordWebHooks.ListByServerAndChannel(c.Request().Context(), _server, _channel)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.DiscordWebhooksToDto(res)
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewDiscordWebHook
-//  @Summary   Creates a new record for a discord web hook to post data to.
-//  @Param     url      query  string  true  "url"
-//  @Param     server   query  string  true  "Server name"
-//  @Param     channel  query  string  true  "Channel name"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/new [post]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) NewDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	token, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	_url := c.QueryParam("url")
-	_server := c.QueryParam("server")
-	_channel := c.QueryParam("channel")
-
-	if _url == "" {
-		p.Message = "url is missing a value"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if !strings.Contains(_url, "discord.com/api/webhooks") {
-		p.Message = "invalid url"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if _server == "" {
-		p.Message = "server is missing"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if _channel == "" {
-		p.Message = "channel is missing"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	rows, err := s.repo.DiscordWebHooks.Create(c.Request().Context(), user.ID, _url, _server, _channel, true)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.Message = "data was not written to database"
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetByUrl(c.Request().Context(), _url)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DisableDiscordWebHooks
-//  @Summary   Disables a Webhook from being used.
-//  @Param     id  path  int  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id}/disable [post]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) disableDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if record.UserID != s.GetUserIdFromJwtToken(c) {
-		p.BaseResponse.Message = ErrYouDontOwnTheRecord
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// flip the it
-	updated, err := s.repo.DiscordWebHooks.Disable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// make sure we got a row updated
-	if updated != 1 {
-		p.BaseResponse.Message = "unexpected number of updates found"
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// EnableDiscordWebHook
-//  @Summary   Enables a source to continue processing.
-//  @Param     id  path  int  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id}/enable [post]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) enableDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if record.UserID != s.GetUserIdFromJwtToken(c) {
-		p.BaseResponse.Message = ErrYouDontOwnTheRecord
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	updated, err := s.repo.DiscordWebHooks.Enable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if updated != 1 {
-		p.BaseResponse.Message = ErrFailedToUpdateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DeleteDiscordWebHook
-//  @Summary   Deletes a record by ID.
-//  @Param     id  path  string  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id} [delete]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) deleteDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if record.UserID != s.GetUserIdFromJwtToken(c) {
-		p.BaseResponse.Message = ErrYouDontOwnTheRecord
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Soft delete the record
-	updated, err := s.repo.DiscordWebHooks.SoftDelete(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if updated != 1 {
-		p.BaseResponse.Message = ErrFailedToUpdateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// UpdateDiscordWebHook
-//	@Summary	Updates a valid discord webhook ID based on the body given.
-//	@Param		id	path	string	true	"id"
-//	@Tags		DiscordWebhook
-//	@Router		/v1/discord/webhooks/{id} [patch]
-//	@Success	200	{object}	domain.DiscordWebhookResponse	"OK"
-//	@Failure	400	{object}	domain.BaseResponse
-//	@Failure	500	{object}	domain.BaseResponse
-//func (s *Handler) UpdateDiscordWebHook(c echo.Context) error {
-//	id, err := strconv.Atoi(c.Param("ID"))
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	// Check to make sure we can find the record
-//	_, err = s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	// Soft delete the record
-//	updated, err := s.repo.DiscordWebHooks(c.Request().Context(), int64(id))
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	_, err = s.Db.GetDiscordQueueByID(c.Request().Context(), uuid)
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	return nil
-//}

internal/handler/v1/handler.go

@@ -1,197 +0,0 @@
-package v1
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-
-	"github.com/golang-jwt/jwt/v5"
-	echojwt "github.com/labstack/echo-jwt/v4"
-	"github.com/labstack/echo/v4"
-	"github.com/labstack/echo/v4/middleware"
-	swagger "github.com/swaggo/echo-swagger"
-
-	_ "git.jamestombleson.com/jtom38/newsbot-api/docs"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-)
-
-type Handler struct {
-	Router *echo.Echo
-	//Db     *database.Queries
-	config services.Configs
-	repo   services.RepositoryService
-}
-
-const (
-	ErrParameterIdMissing = "The requested parameter ID was not found."
-	ErrParameterMissing   = "The requested parameter was not found found:"
-	ErrUnableToParseId    = "Unable to parse the requested ID"
-
-	ErrRecordMissing        = "The requested record was not found"
-	ErrFailedToCreateRecord = "The record was not created due to a database problem"
-	ErrFailedToUpdateRecord = "The requested record was not updated due to a database problem"
-
-	ErrUserUnknown         = "User is unknown"
-	ErrYouDontOwnTheRecord = "The record requested does not belong to you"
-
-	ResponseMessageSuccess = "Success"
-)
-
-var (
-	ErrIdValueMissing        string = "id value is missing"
-	ErrValueNotUuid          string = "a value given was expected to be a uuid but was not correct."
-	ErrNoRecordFound         string = "no record was found."
-	ErrUnableToConvertToJson string = "Unable to convert to json"
-)
-
-func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
-	s := &Handler{
-		config: configs,
-		repo:   services.NewRepositoryService(conn),
-	}
-
-	jwtConfig := echojwt.Config{
-		NewClaimsFunc: func(c echo.Context) jwt.Claims {
-			return new(JwtToken)
-		},
-		SigningKey: []byte(configs.JwtSecret),
-	}
-
-	router := echo.New()
-	router.Pre(middleware.RemoveTrailingSlash())
-	router.Pre(middleware.Logger())
-	router.Pre(middleware.Recover())
-	router.GET("/swagger/*", swagger.WrapHandler)
-
-	v1 := router.Group("/api/v1")
-	articles := v1.Group("/articles")
-	articles.Use(echojwt.WithConfig(jwtConfig))
-	articles.GET("", s.listArticles)
-	articles.GET(":id", s.getArticle)
-	articles.GET(":id/details", s.getArticleDetails)
-	articles.GET("by/source/:id", s.ListArticlesBySourceId)
-
-	//dwh := v1.Group("/discord/webhooks")
-	//dwh.GET("/", s.ListDiscordWebHooks)
-	//dwh.POST("/new", s.NewDiscordWebHook)
-	//dwh.GET("/by/serverAndChannel", s.GetDiscordWebHooksByServerAndChannel)
-	//dwh.GET("/:ID", s.GetDiscordWebHooksById)
-	//dwh.DELETE("/:ID", s.deleteDiscordWebHook)
-	//dwh.POST("/:ID/disable", s.disableDiscordWebHook)
-	//dwh.POST("/:ID/enable", s.enableDiscordWebHook)
-
-	//queue := v1.Group("/queue")
-	//queue.GET("/discord/webhooks", s.ListDiscordWebhookQueue) // TODO this needs to be reworked
-
-	//settings := v1.Group("/settings")
-	//settings.GET("/", s.getSettings)
-
-	sources := v1.Group("/sources")
-	sources.Use(echojwt.WithConfig(jwtConfig))
-	sources.GET("", s.listSources)
-	sources.GET("/by/source", s.listSourcesBySource)
-	sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName)
-	//sources.POST("/new/reddit", s.newRedditSource)
-	//sources.POST("/new/youtube", s.newYoutubeSource)
-	//sources.POST("/new/twitch", s.newTwitchSource)
-	sources.POST("/new/rss", s.newRssSource)
-	sources.GET("/:id", s.getSource)
-	sources.DELETE("/:id", s.deleteSources)
-	sources.POST("/:id/disable", s.disableSource)
-	sources.POST("/:id/enable", s.enableSource)
-
-	users := v1.Group("/users")
-	users.POST("/login", s.AuthLogin)
-	users.POST("/register", s.AuthRegister)
-	users.Use(echojwt.WithConfig(jwtConfig))
-	users.POST("/scopes/add", s.AddScopes)
-	users.POST("/scopes/remove", s.RemoveScopes)
-	users.POST("/refresh/token", s.RefreshJwtToken)
-	users.POST("/refresh/sessionToken", s.NewSessionToken)
-
-	s.Router = router
-	return s
-}
-
-//type ApiStatusModel struct {
-//	StatusCode int    `json:"status"`
-//	Message    string `json:"message"`
-//}
-
-//type ApiError struct {
-//	*ApiStatusModel
-//}
-
-//func (s *Handler) WriteError(c echo.Context, errMessage error, HttpStatusCode int) error {
-//	return c.JSON(HttpStatusCode, domain.BaseResponse{
-//		Message: errMessage.Error(),
-//		IsError: true,
-//	})
-//}
-
-//func (s *Handler) WriteMessage(c echo.Context, msg string, HttpStatusCode int) error {
-//	return c.JSON(HttpStatusCode, domain.BaseResponse{
-//		Message: msg,
-//	})
-//}
-
-//func (s *Handler) InternalServerErrorResponse(c echo.Context, msg string) error {
-//	return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
-//		Message: msg,
-//	})
-//}
-
-//func (s *Handler) UnauthorizedResponse(c echo.Context, msg string) error {
-//	return c.JSON(http.StatusUnauthorized, domain.BaseResponse{
-//		Message: msg,
-//	})
-//}
-
-// If the token is not valid then an json error will be returned.
-// If the token has the wrong scope, a json error will be returned.
-// If the token passes all the checks, it is valid and is returned back to the caller.
-func (s *Handler) ValidateJwtToken(c echo.Context, requiredScope string) (JwtToken, error) {
-	token, err := s.getJwtTokenFromContext(c)
-	if err != nil {
-		return JwtToken{}, errors.New(ErrJwtMissing)
-	}
-
-	err = token.hasExpired()
-	if err != nil {
-		return JwtToken{}, errors.New(ErrJwtExpired)
-	}
-
-	err = token.hasScope(requiredScope)
-	if err != nil {
-		return JwtToken{}, errors.New(ErrJwtScopeMissing)
-	}
-
-	if token.Iss != s.config.ServerAddress {
-		return JwtToken{}, errors.New(ErrJwtInvalidIssuer)
-	}
-
-	// If you are the built in admin account, skip the username and session token check
-	if token.UserName == "admin" {
-		return token, nil
-	}
-
-	user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
-	if err != nil {
-		return JwtToken{}, errors.New("user record not found")
-	}
-
-	if user.SessionToken != token.SessionToken {
-		return JwtToken{}, errors.New("invalid session token")
-	}
-
-	return token, nil
-}
-
-func (s *Handler) GetUserIdFromJwtToken(c echo.Context) int64 {
-	token, err := s.getJwtTokenFromContext(c)
-	if err != nil {
-		return -1
-	}
-
-	return token.GetUserId()
-}

internal/handler/v1/jwt.go

@@ -1,130 +0,0 @@
-package v1
-
-import (
-	"errors"
-	"strings"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/labstack/echo/v4"
-)
-
-const (
-	ErrJwtMissing       = "auth token is missing"
-	ErrJwtClaimsMissing = "claims missing on token"
-	ErrJwtExpired       = "auth token has expired"
-	ErrJwtScopeMissing  = "required scope is missing"
-	ErrJwtInvalidIssuer = "incorrect server issued the token"
-)
-
-type JwtToken struct {
-	Exp          time.Time `json:"exp"`
-	Iss          string    `json:"iss"`
-	Authorized   bool      `json:"authorized"`
-	UserName     string    `json:"username"`
-	UserId       int64     `json:"userId"`
-	Scopes       []string  `json:"scopes"`
-	SessionToken string    `json:"sessionToken"`
-	jwt.RegisteredClaims
-}
-
-func (j JwtToken) IsValid(scope string) error {
-	err := j.hasExpired()
-	if err != nil {
-		return err
-	}
-
-	// Check to see if they have the scope to do anything
-	// if they do, let them pass
-	err = j.hasScope(domain.ScopeAll)
-	if err == nil {
-		return nil
-	}
-
-	err = j.hasScope(scope)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (j JwtToken) GetUsername() string {
-	return j.UserName
-}
-
-func (j JwtToken) GetUserId() int64 {
-	return j.UserId
-}
-
-func (j JwtToken) hasExpired() error {
-	// Check to see if the token has expired
-	//hasExpired := j.Exp.Compare(time.Now())
-	hasExpired := time.Now().Compare(j.Exp)
-	if hasExpired == 1 {
-		return errors.New(ErrJwtExpired)
-	}
-	return nil
-}
-
-// This will check the users token to make sure they have the correct scope to access the handler.
-// It will evaluate if you have the admin scope or the required scope for the handler.
-func (j JwtToken) hasScope(scope string) error {
-	// they have the scope to access everything, so let them pass.
-	userScopes := strings.Join(j.Scopes, "")
-	if strings.Contains(domain.ScopeAll, userScopes) {
-		return nil
-	}
-
-	if strings.Contains(userScopes, scope) {
-		return nil
-	}
-
-	return errors.New(ErrJwtScopeMissing)
-}
-
-func (h *Handler) generateJwt(username, issuer, sessionToken string, userScopes []string, userId int64) (string, error) {
-	return h.generateJwtWithExp(username, issuer, sessionToken, userScopes, userId, time.Now().Add(10*time.Minute))
-}
-
-func (h *Handler) generateJwtWithExp(username, issuer, sessionToken string, userScopes []string, userId int64, expiresAt time.Time) (string, error) {
-	secret := []byte(h.config.JwtSecret)
-
-	// Anyone who wants to decrypt the key needs to use the same method
-	token := jwt.New(jwt.SigningMethodHS256)
-	claims := token.Claims.(jwt.MapClaims)
-	claims["exp"] = expiresAt
-	claims["authorized"] = true
-	claims["username"] = username
-	claims["iss"] = issuer
-	claims["userId"] = userId
-	claims["sessionToken"] = sessionToken
-
-	var scopes []string
-	scopes = append(scopes, userScopes...)
-	claims["scopes"] = scopes
-
-	tokenString, err := token.SignedString(secret)
-	if err != nil {
-		return "", err
-	}
-
-	return tokenString, nil
-}
-
-func (h *Handler) getJwtTokenFromContext(c echo.Context) (JwtToken, error) {
-	// Make sure that the request came with a jwtToken
-	token, ok := c.Get("user").(*jwt.Token)
-	if !ok {
-		return JwtToken{}, errors.New(ErrJwtMissing)
-	}
-
-	// Generate the claims from the token
-	claims, ok := token.Claims.(*JwtToken)
-	if !ok {
-		return JwtToken{}, errors.New(ErrJwtClaimsMissing)
-	}
-
-	return *claims, nil
-}

internal/handler/v1/sources.go

@@ -1,653 +0,0 @@
-package v1
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/labstack/echo/v4"
-)
-
-// ListSources
-//  @Summary   Lists the top 50 records
-//  @Param     page  query  string  false  "page number"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse  "Unable to reach SQL or Data problems"
-//  @Security  Bearer
-func (s *Handler) listSources(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		page = 0
-	}
-
-	// Default way of showing all sources
-	items, err := s.repo.Sources.List(c.Request().Context(), page, 25)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.SourcesToDto(items)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// ListSourcesBySource
-//  @Summary   Lists the top 50 records based on the name given. Example: reddit
-//  @Param     source  query  string  true   "Source Name"
-//  @Param     page    query  string  false  "page number"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources/by/source [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) listSourcesBySource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	source := c.QueryParam("source")
-	if source == "" {
-		p.BaseResponse.Message = fmt.Sprintf("%s source", ErrParameterMissing)
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		page = 0
-	}
-
-	// Shows the list by Sources.source
-	items, err := s.repo.Sources.ListBySource(c.Request().Context(), page, 25, source)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.SourcesToDto(items)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetSource
-//  @Summary   Returns a single entity by ID
-//  @Param     id  path  int  true  "id"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources/{id} [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) getSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("id"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetSourceByNameAndSource
-//  @Summary   Returns a single entity by ID
-//  @Param     name    query  string  true  "dadjokes"
-//  @Param     source  query  string  true  "reddit"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources/by/sourceAndName [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) GetSourceBySourceAndName(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.GetSourceBySourceAndNameParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), param.Source, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewRedditSource
-//  @Summary   Creates a new reddit source to monitor.
-//  @Param     name  query  string  true  "name"
-//  @Param     url   query  string  true  "url"
-//  @Tags      Source
-//  @Router    /v1/sources/new/reddit [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newRedditSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.NewSourceParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if param.Url == "" {
-		p.BaseResponse.Message = "url is missing"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if !strings.Contains(param.Url, "reddit.com") {
-		p.BaseResponse.Message = "invalid url"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to see if we already have this record, if we do, return it.
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
-	if err == nil {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	tags := fmt.Sprintf("twitch, %v, %s", param.Name, param.Tags)
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorReddit, param.Name, param.Url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewYoutubeSource
-//  @Summary   Creates a new youtube source to monitor.
-//  @Param     name  query  string  true  "name"
-//  @Param     url   query  string  true  "url"
-//  @Tags      Source
-//  @Router    /v1/sources/new/youtube [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newYoutubeSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-		},
-	}
-
-	// Validate the jwt
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.NewSourceParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if param.Url == "" {
-		p.BaseResponse.Message = "url is missing a value"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if !strings.Contains(param.Url, "youtube.com") {
-		p.BaseResponse.Message = "invalid url"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
-	if err == nil {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	tags := fmt.Sprintf("twitch, %v", param.Name)
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorYoutube, param.Name, param.Url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewTwitchSource
-//  @Summary   Creates a new twitch source to monitor.
-//  @Param     name  query  string  true  "name"
-//  @Tags      Source
-//  @Router    /v1/sources/new/twitch [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newTwitchSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.NewSourceParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	tags := fmt.Sprintf("twitch, %v", param.Name)
-	url := fmt.Sprintf("https://twitch.tv/%v", param.Name)
-
-	// Check if the record already exists
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
-	if err == nil {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorTwitch, param.Name, url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, _ = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewRssSource
-//  @Summary   Creates a new rss source to monitor.
-//  @Param     request  body  domain.NewSourceParamRequest  true  "body"
-//  @Tags      Source
-//  @Router    /v1/sources/new/rss [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newRssSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	param := domain.NewSourceParamRequest{}
-	err = c.Bind(&param)
-	//err = (&echo.DefaultBinder{}).BindBody(c, &param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if param.Url == "" {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check if the record already exists
-	blankRecord := entity.SourceEntity{}
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
-	if err != nil {
-		p.IsError = true
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// No record was found, but no error returned
-	if item != blankRecord {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	//if err == nil {
-	//	var dto []domain.SourceDto
-	//	dto = append(dto, dtoconv.SourceToDto(item))
-	//	p.Payload = dto
-	//	p.BaseResponse.IsError = false
-	//	return c.JSON(http.StatusOK, p)
-	//}
-
-	tags := fmt.Sprintf("rss, %v, %s", param.Name, param.Tags)
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorRss, param.Name, param.Url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DeleteSource
-//  @Summary   Marks a source as deleted based on its ID value.
-//  @Param     id  path  int  true  "id"
-//  @Tags      Source
-//  @Router    /v1/sources/{id} [POST]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) deleteSources(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// Delete the record
-	rows, err := s.repo.Sources.SoftDelete(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToUpdateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// pull the record with its updated value
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var items []domain.SourceDto
-	items = append(items, dtoconv.SourceToDto(item))
-	p.Payload = items
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DisableSource
-//  @Summary   Disables a source from processing.
-//  @Param     id  path  int  true  "id"
-//  @Tags      Source
-//  @Router    /v1/sources/{id}/disable [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) disableSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	_, err = s.repo.Sources.Disable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// EnableSource
-//  @Summary   Enables a source to continue processing.
-//  @Param     id  path  string  true  "id"
-//  @Tags      Source
-//  @Router    /v1/sources/{id}/enable [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) enableSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	_, err = s.repo.Sources.Enable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}

internal/handler/v1/users.go

@@ -1,351 +0,0 @@
-package v1
-
-import (
-	"net/http"
-	"strings"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-
-	"github.com/google/uuid"
-	"github.com/labstack/echo/v4"
-)
-
-const (
-	ErrUserNotFound          = "requested user does not exist"
-	ErrUsernameAlreadyExists = "the requested username already exists"
-)
-
-//  @Summary  Creates a new user
-//  @Router   /v1/users/register [post]
-//  @Param    request  formData  domain.LoginFormRequest  true  "form"
-//  @Accepts  x-www-form-urlencoded
-//  @Produce  json
-//  @Tags     Users
-//  @Success  201  {object}  domain.BaseResponse
-//  @Failure  400  {object}  domain.BaseResponse
-//  @Failure  500  {object}  domain.BaseResponse
-func (h *Handler) AuthRegister(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	username := c.FormValue("username")
-	password := c.FormValue("password")
-
-	//username := c.QueryParam("username")
-	exists, err := h.repo.Users.GetUser(c.Request().Context(), username)
-	if err != nil {
-		// if we have an err, validate that if its not user not found.
-		// if the user is not found, we can use that name
-		if err.Error() != repository.ErrUserNotFound {
-			p.Message = err.Error()
-			return c.JSON(http.StatusBadRequest, p)
-		}
-	}
-	if exists.Username == username {
-		p.Message = ErrUsernameAlreadyExists
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	//password := c.QueryParam("password")
-	err = h.repo.Users.CheckPasswordForRequirements(password)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	_, err = h.repo.Users.Create(c.Request().Context(), username, password, domain.ScopeArticleRead)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	return c.JSON(http.StatusCreated, p)
-}
-
-//  @Summary  Logs into the API and returns a bearer token if successful
-//  @Router   /v1/users/login [post]
-//  @Param    request  formData  domain.LoginFormRequest  true  "form"
-//  @Accepts  x-www-form-urlencoded
-//  @Produce  json
-//  @Tags     Users
-//  @Success  200  {object}  domain.LoginResponse
-//  @Failure  400  {object}  domain.LoginResponse
-//  @Failure  500  {object}  domain.LoginResponse
-func (h *Handler) AuthLogin(c echo.Context) error {
-	p := domain.LoginResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-		//Token:        jwt,
-		Type: "Bearer",
-		//RefreshToken: refresh,
-	}
-
-	username := c.FormValue("username")
-	password := c.FormValue("password")
-
-	// Check to see if they are trying to login with the admin token
-	if username == "" {
-		return h.createAdminToken(c, password)
-	}
-
-	// check if the user exists
-	user, err := h.repo.Users.GetUser(c.Request().Context(), username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// make sure the hash matches
-	err = h.repo.Users.DoesPasswordMatchHash(c.Request().Context(), username, password)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// TODO think about moving this down some?
-	expiresAt := time.Now().Add(time.Hour * 48)
-	userScopes := strings.Split(user.Scopes, ",")
-
-	jwt, err := h.generateJwtWithExp(username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, expiresAt)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	refresh, err := h.repo.RefreshTokens.Create(c.Request().Context(), username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Token = jwt
-	p.RefreshToken = refresh
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-func (h *Handler) createAdminToken(c echo.Context, password string) error {
-	p := domain.LoginResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-		//Token: token,
-		Type: "Bearer",
-	}
-
-	// if the admin token is blank, then the admin wanted this disabled.
-	// this will fail right away and not progress.
-	if h.config.AdminSecret == "" {
-		p.BaseResponse.Message = ErrUserNotFound
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if h.config.AdminSecret != password {
-		p.BaseResponse.Message = ErrUserNotFound
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	var userScopes []string
-	userScopes = append(userScopes, domain.ScopeAll)
-	sessionToken, err := uuid.NewV7()
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	token, err := h.generateJwt("admin", h.config.ServerAddress, sessionToken.String(), userScopes, -1)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Token = token
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// This will take collect some information about the requested refresh, validate and then return a new jwt token if approved.
-// Register
-//  @Summary   Generates a new token
-//  @Router    /v1/users/refresh/token [post]
-//  @Param     request  body  domain.RefreshTokenRequest  true  "body"
-//  @Tags      Users
-//  @Success   200  {object}  domain.LoginResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) RefreshJwtToken(c echo.Context) error {
-	p := domain.LoginResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: "OK",
-		},
-		//Token:        jwt,
-		Type: "Bearer",
-		//RefreshToken: newRefreshToken,
-	}
-
-	_, err := h.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	// Check the context for the refresh token
-	var request domain.RefreshTokenRequest
-	err = (&echo.DefaultBinder{}).BindBody(c, &request)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	err = h.repo.RefreshTokens.IsRequestValid(c.Request().Context(), request.Username, request.RefreshToken)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	user, err := h.repo.Users.GetUser(c.Request().Context(), request.Username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-	userScopes := strings.Split(user.Scopes, ",")
-
-	jwt, err := h.generateJwtWithExp(request.Username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, time.Now().Add(time.Hour*48))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	newRefreshToken, err := h.repo.RefreshTokens.Create(c.Request().Context(), request.Username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Token = jwt
-	p.RefreshToken = newRefreshToken
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-//  @Summary   Adds a new scope to a user account
-//  @Router    /v1/users/scopes/add [post]
-//  @Param     request  body  domain.UpdateScopesRequest  true  "body"
-//  @Tags      Users
-//  @Accept    json
-//  @Produce   json
-//  @Success   200  {object}  domain.BaseResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) AddScopes(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	_, err := h.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	request := domain.UpdateScopesRequest{}
-	err = (&echo.DefaultBinder{}).BindBody(c, &request)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	err = h.repo.Users.AddScopes(c.Request().Context(), request.Username, request.Scopes)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-//  @Summary   Adds a new scope to a user account
-//  @Router    /v1/users/scopes/remove [post]
-//  @Param     request  body  domain.UpdateScopesRequest  true  "body"
-//  @Tags      Users
-//  @Accept    json
-//  @Produce   json
-//  @Success   200  {object}  domain.BaseResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) RemoveScopes(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	token, err := h.getJwtTokenFromContext(c)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	err = token.IsValid(domain.ScopeAll)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	request := domain.UpdateScopesRequest{}
-	err = (&echo.DefaultBinder{}).BindBody(c, &request)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	err = h.repo.Users.RemoveScopes(c.Request().Context(), request.Username, request.Scopes)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-//  @Summary   Revokes the current session token and replaces it with a new one.
-//  @Router    /v1/users/refresh/sessionToken [post]
-//  @Tags      Users
-//  @Accept    json
-//  @Produce   json
-//  @Success   200  {object}  domain.BaseResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) NewSessionToken(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	token, err := h.getJwtTokenFromContext(c)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	_, err = h.repo.Users.NewSessionToken(c.Request().Context(), token.UserName)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.IsError = false
-	return c.JSON(http.StatusInternalServerError, p)
-}

internal/repository/alertDiscord.go

@@ -1,122 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type AlertDiscordRepo interface {
-	Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error)
-}
-
-type alertDiscordRepository struct {
-	conn          *sql.DB
-	defaultLimit  int
-	defaultOffset int
-}
-
-func NewAlertDiscordRepository(conn *sql.DB) alertDiscordRepository {
-	return alertDiscordRepository{
-		conn:          conn,
-		defaultLimit:  50,
-		defaultOffset: 50,
-	}
-}
-
-func (r alertDiscordRepository) Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("AlertDiscord")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID", "DiscordWebHookID")
-	queryBuilder.Values(dt, dt, timeZero, userId, sourceId, webhookId)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r alertDiscordRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "AlertDiscord", id)
-}
-
-func (r alertDiscordRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "AlertDiscord", id)
-}
-
-func (r alertDiscordRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "AlertDiscord", id)
-}
-
-func (r alertDiscordRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("AlertDiscord")
-	builder.Where(
-		builder.Equal("UserID", userId),
-	)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.AlertDiscordEntity{}, err
-	}
-
-	data := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.AlertDiscordEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ur alertDiscordRepository) processRows(rows *sql.Rows) []entity.AlertDiscordEntity {
-	items := []entity.AlertDiscordEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var userId int64
-		var sourceId int64
-		var webhookId int64
-
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt, &deletedAt,
-			&userId, &sourceId, &webhookId,
-		)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.AlertDiscordEntity{
-			ID:               id,
-			CreatedAt:        createdAt,
-			UpdatedAt:        updatedAt,
-			DeletedAt:        deletedAt,
-			UserID:           userId,
-			SourceID:         sourceId,
-			DiscordWebHookId: webhookId,
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/alertDiscord_test.go

@@ -1,63 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestAlertDiscordCreate(t *testing.T) {
-	t.Log(time.Time{})
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	r := repository.NewAlertDiscordRepository(db)
-	created, err := r.Create(context.Background(), 1, 1, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-}
-
-func TestAlertDiscordCreateAndValidate(t *testing.T) {
-	t.Log(time.Time{})
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	source := repository.NewSourceRepository(db)
-	source.Create(context.Background(), domain.SourceCollectorRss, "Unit Testing", "www.fake.com", "testing,units", true)
-	sourceRecord, _ := source.GetBySourceAndName(context.Background(), domain.SourceCollectorRss, "Unit Testing")
-
-	webhookRepo := repository.NewDiscordWebHookRepository(db)
-	webhookRepo.Create(context.Background(), 999, "discord.com", "Unit Testing", "memes", true)
-	webhook, _ := webhookRepo.GetByUrl(context.Background(), "discord.com")
-	
-	r := repository.NewAlertDiscordRepository(db)
-	r.Create(context.Background(), 999, sourceRecord.ID, webhook.ID)
-	alert, err := r.ListByUser(context.Background(), 0, 10, 999)
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	if len(alert) != 1 {
-		t.Error("got the incorrect number of rows back")
-		t.FailNow()
-	}
-}

internal/repository/article.go

@@ -1,261 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-const (
-	ArticleOrderByPublishDateDesc = "pubDate desc"
-	ArticleOrderByPublishDateAsc  = "pubDate asc"
-)
-
-type ArticlesRepo interface {
-	GetById(ctx context.Context, id int64) (entity.ArticleEntity, error)
-	GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error)
-	ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error)
-	ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error)
-	ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error)
-	ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error)
-	Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error)
-	CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error)
-}
-
-type ArticleRepository struct {
-	conn          *sql.DB
-	defaultLimit  int
-	defaultOffset int
-}
-
-func NewArticleRepository(conn *sql.DB) ArticleRepository {
-	return ArticleRepository{
-		conn:          conn,
-		defaultLimit:  50,
-		defaultOffset: 50,
-	}
-}
-
-func (ar ArticleRepository) GetById(ctx context.Context, id int64) (entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles").Where(
-		builder.E("id", id),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data[0], nil
-}
-
-func (ar ArticleRepository) GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles").Where(
-		builder.E("url", url),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data[0], nil
-}
-
-func (ar ArticleRepository) ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ar ArticleRepository) ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	builder.OrderBy(ArticleOrderByPublishDateDesc)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ar ArticleRepository) ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	if orderBy != "" {
-		builder.OrderBy(orderBy)
-	}
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-	return data, nil
-}
-
-func (ar ArticleRepository) ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	builder.JoinWithOption("InnerJoin", "sources", "articles.sourceId=sources.Id")
-	if orderBy != "" {
-		builder.OrderBy(orderBy)
-	}
-	builder.Where(
-		builder.Equal("SourceId", sourceId),
-	)
-	builder.Offset(50)
-	builder.Limit(page * limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-	return data, nil
-}
-
-func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("articles")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
-	queryBuilder.Values(dt, dt, timeZero, sourceId, tags, title, url, pubDate, isVideo, thumbnailUrl, description, authorName, authorImageUrl)
-	query, args := queryBuilder.Build()
-
-	_, err := ar.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("articles")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
-	queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl)
-	query, args := queryBuilder.Build()
-
-	_, err := ar.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (ur ArticleRepository) processRows(rows *sql.Rows) []entity.ArticleEntity {
-	items := []entity.ArticleEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var sourceId int64
-		var tags string
-		var title string
-		var url string
-		var pubDate time.Time
-		var isVideo bool
-		var thumbnail string
-		var description string
-		var authorName string
-		var authorImageUrl string
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt,
-			&deletedAt, &sourceId, &tags,
-			&title, &url, &pubDate,
-			&isVideo, &thumbnail, &description,
-			&authorName, &authorImageUrl)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.ArticleEntity{
-			ID:             id,
-			CreatedAt:      createdAt,
-			UpdatedAt:      updatedAt,
-			DeletedAt:      deletedAt,
-			SourceID:       sourceId,
-			Tags:           tags,
-			Title:          title,
-			Url:            url,
-			PubDate:        pubDate,
-			IsVideo:        isVideo,
-			Thumbnail:      thumbnail,
-			Description:    description,
-			AuthorName:     authorName,
-			AuthorImageUrl: authorImageUrl,
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/article_test.go

@@ -1,156 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-const (
-	articleFakeDotCom = "www.fake.com"
-)
-
-func TestCreateArticle(t *testing.T) {
-	t.Log(time.Time{})
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	r := repository.NewArticleRepository(db)
-	created, err := r.Create(context.Background(), 1, "", "unit test", articleFakeDotCom, "", "testing", "", "", time.Now(), false)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-}
-
-func TestArticleByUrl(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-
-	err = insertFakeArticles(r, "u1", 0)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	article, err := r.GetByUrl(context.Background(), articleFakeDotCom)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if article.Url != "www.fake.com" {
-		t.Log("failed to find the requested record")
-		t.FailNow()
-	}
-}
-
-func TestPullingMultipleArticlesWithLimit(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-	insertFakeArticles(r, "u1", 0)
-	insertFakeArticles(r, "u2", 0)
-	insertFakeArticles(r, "u3", 0)
-	insertFakeArticles(r, "u4", 0)
-
-	items, err := r.ListTop(context.Background(), 3)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if len(items) != 3 {
-		t.Log("expected 3 rows back")
-		t.FailNow()
-	}
-}
-
-func TestPullingMultipleArticlesWithPaging(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-	insertFakeArticles(r, "u1", 0)
-	insertFakeArticles(r, "u2", 0)
-	insertFakeArticles(r, "u3", 0)
-	insertFakeArticles(r, "u4", 0)
-
-	items, err := r.ListByPage(context.Background(), 2, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if items[0].Title != "u2" {
-		t.Log("pulled the wrong record with paging")
-		t.FailNow()
-	}
-}
-
-func TestPullingByPublishDate(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-
-	today := time.Now()
-
-	insertFakeArticles(r, "u1", 0)
-	insertFakeArticles(r, "u1", -1)
-	insertFakeArticles(r, "u1", -2)
-
-	items, err := r.ListByPublishDate(context.Background(), 0, 2, repository.ArticleOrderByPublishDateDesc)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if len(items) != 2 {
-		t.Log("expected two items back")
-		t.FailNow()
-	}
-	pubDate := items[1].PubDate.Day()
-	expectedDay := today.Day() - 1
-	if pubDate != expectedDay {
-		t.Log("expected a record that was 2 days old")
-		t.FailNow()
-	}
-}
-
-//func TestArticleBySource
-
-func insertFakeArticles(r repository.ArticleRepository, title string, daysOld int) error {
-	pubDate := time.Now().AddDate(0,0, daysOld)
-	_, err := r.Create(context.Background(), 1, "", title, articleFakeDotCom, "", "testing", "", "", pubDate, false)
-	if err != nil {
-		return err
-	}
-	return nil
-}

internal/repository/common.go

@@ -1,71 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"github.com/huandu/go-sqlbuilder"
-)
-
-var (
-	timeZero = time.Time{}
-)
-
-func deleteFromTable(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
-	b := sqlbuilder.NewDeleteBuilder()
-	b.DeleteFrom(tableName)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func restoreRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
-	timeZero := time.Time{}
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update(tableName)
-	b.Set(
-		b.Assign("UpdatedAt", time.Now()),
-		b.Assign("DeletedAt", timeZero),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func softDeleteRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
-	now := time.Now()
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update(tableName)
-	b.Set(
-		b.Assign("UpdatedAt", now),
-		b.Assign("DeletedAt", now),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}

internal/repository/discordWebHooks.go

@@ -1,228 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type DiscordWebHookRepo interface {
-	Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error)
-	Enable(ctx context.Context, id int64) (int64, error)
-	Disable(ctx context.Context, id int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error)
-	GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error)
-	ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error)
-	ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error)
-}
-
-type discordWebHookRepository struct {
-	conn *sql.DB
-}
-
-func NewDiscordWebHookRepository(conn *sql.DB) discordWebHookRepository {
-	return discordWebHookRepository{
-		conn: conn,
-	}
-}
-
-func (r discordWebHookRepository) Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("DiscordWebHooks")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "Url", "Server", "Channel", "Enabled")
-	queryBuilder.Values(dt, dt, timeZero, userId, url, server, channel, enabled)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r discordWebHookRepository) Enable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("DiscordWebHooks")
-	b.Set(
-		b.Assign("Enabled", true),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r discordWebHookRepository) Disable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("DiscordWebHooks")
-	b.Set(
-		b.Assign("Enabled", false),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r discordWebHookRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "DiscordWebHooks", id)
-}
-
-func (r discordWebHookRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "DiscordWebHooks", id)
-}
-
-func (r discordWebHookRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "DiscordWebHooks", id)
-}
-
-func (r discordWebHookRepository) GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.E("id", id),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r discordWebHookRepository) GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.E("Url", url),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r discordWebHookRepository) ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.E("Server", name),
-	)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r discordWebHookRepository) ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.Equal("Server", server),
-		builder.Equal("Channel", channel),
-	)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r discordWebHookRepository) processRows(rows *sql.Rows) ([]entity.DiscordWebHookEntity, error) {
-	items := []entity.DiscordWebHookEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var userId int64
-		var url string
-		var server string
-		var channel string
-		var enabled bool
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt,
-			&deletedAt, &userId, &url, &server,
-			&channel, &enabled,
-		)
-		if err != nil {
-			return items, err
-		}
-
-		item := entity.DiscordWebHookEntity{
-			ID:        id,
-			CreatedAt: createdAt,
-			UpdatedAt: updatedAt,
-			DeletedAt: deletedAt,
-			UserID:    userId,
-			Url:       url,
-			Server:    server,
-			Channel:   channel,
-			Enabled:   enabled,
-		}
-
-		items = append(items, item)
-	}
-
-	return items, nil
-}

internal/repository/discordWebHooks_test.go

@@ -1,287 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestCreateDiscordWebHookRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	r := repository.NewDiscordWebHookRepository(db)
-	created, err := r.Create(context.Background(), 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookGetById(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewDiscordWebHookRepository(db)
-	created, err := r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.ID != 1 {
-		t.Log("got the wrong record back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookGetByUrl(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
-	item, err := r.GetByUrl(ctx, "www.discord.com/bad/webhook")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Url != "www.discord.com/bad/webhook" {
-		t.Log("got the wrong record back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookListByServerName(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, "memes", true)
-
-	item, err := r.ListByServerName(ctx, serverName)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item[0].Server != serverName {
-		t.Log("got the wrong record back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookListByServerAndChannel(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-
-	item, err := r.ListByServerAndChannel(ctx, serverName, channel)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item[0].Server != serverName {
-		t.Log("got the wrong wrong server back")
-		t.FailNow()
-	}
-
-	if item[0].Channel != channel {
-		t.Log("got the wrong channel back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookEnableRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, false)
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != false {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Enable(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookDisableRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != true {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Disable(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookSoftDelete(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-	_, err = r.SoftDelete(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, _ := r.GetById(ctx, 1)
-	t.Log(updated.DeletedAt)
-}
-
-func TestDiscordWebHookRestore(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	timeZero := time.Time{}
-
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-	item, _ := r.GetById(ctx, 1)
-	if item.DeletedAt != timeZero {
-		t.Log("DeletedAt was not zero")
-		t.FailNow()
-	}
-	_, _ = r.SoftDelete(ctx, 1)
-	softDeleted, _ := r.GetById(ctx, 1)
-	if softDeleted.ID != 1 {
-		t.Log("record went boom")
-		t.FailNow()
-	}
-
-	_, err = r.Restore(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, _ := r.GetById(ctx, 1)
-	t.Log(updated.DeletedAt)
-}

internal/repository/refreshTokens.go

@@ -1,120 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-const (
-	refreshTokenTableName = "RefreshTokens"
-)
-
-type RefreshToken interface {
-	Create(ctx context.Context, username string, token string) (int64, error)
-	GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
-	DeleteById(ctx context.Context, id int64) (int64, error)
-}
-
-type RefreshTokenRepository struct {
-	connection *sql.DB
-}
-
-func NewRefreshTokenRepository(conn *sql.DB) RefreshTokenRepository {
-	return RefreshTokenRepository{
-		connection: conn,
-	}
-}
-
-func (rt RefreshTokenRepository) Create(ctx context.Context, username string, token string) (int64, error) {
-	dt := time.Now()
-	builder := sqlbuilder.NewInsertBuilder()
-	builder.InsertInto(refreshTokenTableName)
-	builder.Cols("Username", "Token", "CreatedAt", "UpdatedAt", "DeletedAt")
-	builder.Values(username, token, dt, dt, time.Time{})
-	query, args := builder.Build()
-
-	_, err := rt.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (rt RefreshTokenRepository) GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*").From(refreshTokenTableName).Where(
-		builder.E("Username", name),
-	)
-
-	query, args := builder.Build()
-	rows, err := rt.connection.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.RefreshTokenEntity{}, err
-	}
-
-	data := rt.processRows(rows)
-	if len(data) == 0 {
-		return entity.RefreshTokenEntity{}, errors.New("no token found for user")
-	}
-
-	return data[0], nil
-}
-
-func (rt RefreshTokenRepository) DeleteById(ctx context.Context, id int64) (int64, error) {
-	builder := sqlbuilder.NewDeleteBuilder()
-	builder.DeleteFrom(refreshTokenTableName)
-	builder.Where(
-		builder.EQ("Id", id),
-	)
-
-	query, args := builder.Build()
-	rows, err := rt.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return -1, err
-	}
-
-	return rows.RowsAffected()
-}
-
-func (rd RefreshTokenRepository) processRows(rows *sql.Rows) []entity.RefreshTokenEntity {
-	items := []entity.RefreshTokenEntity{}
-
-	for rows.Next() {
-		var id int64
-		var username string
-		var token string
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt sql.NullTime
-
-		err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &token)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.RefreshTokenEntity{
-			ID:        id,
-			Username:  username,
-			Token:     token,
-			CreatedAt: createdAt,
-			UpdatedAt: updatedAt,
-		}
-
-		if deletedAt.Valid {
-			item.DeletedAt = deletedAt.Time
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}
-
-//func (rt RefreshTokenRepository) Delete()

internal/repository/refreshTokens_test.go

@@ -1,93 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestRefreshTokenCreate(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	client := repository.NewRefreshTokenRepository(conn)
-	rows, err := client.Create(context.Background(), "tester", "BadTokenDontUse")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if rows == 0 {
-		t.Log("expected one row to come back but got 0")
-	}
-}
-
-func TestRefreshTokenGetByUsername(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	client := repository.NewRefreshTokenRepository(conn)
-	rows, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if rows != 1 {
-		t.Log("expected a row to be added but not the wrong value back")
-		t.FailNow()
-	}
-
-	model, err := client.GetByUsername(context.Background(), "tester")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if model.Username != "tester" {
-		t.Log("got the wrong user back")
-		t.FailNow()
-	}
-}
-
-func TestRefreshTokenDeleteById(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	client := repository.NewRefreshTokenRepository(conn)
-	created, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if created != 1 {
-		t.Log("Unexpected number back for rows created")
-	}
-
-	model, err := client.GetByUsername(context.Background(), "tester")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := client.DeleteById(context.Background(), model.ID)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if updated != 1 {
-		t.Log("deleted the wrong number of records")
-		t.FailNow()
-	}
-}

internal/repository/source.go

@@ -1,277 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type Sources interface {
-	Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error)
-	GetById(ctx context.Context, id int64) (entity.SourceEntity, error)
-	GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error)
-	GetBySource(ctx context.Context, source string) (entity.SourceEntity, error)
-	GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error)
-	List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error)
-	ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error)
-	Enable(ctx context.Context, id int64) (int64, error)
-	Disable(ctx context.Context, id int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-}
-
-type sourceRepository struct {
-	conn *sql.DB
-}
-
-func NewSourceRepository(conn *sql.DB) sourceRepository {
-	return sourceRepository{
-		conn: conn,
-	}
-}
-
-func (r sourceRepository) Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("Sources")
-	queryBuilder.Cols("CreatedAt", "UpdatedAt", "DeletedAt", "DisplayName", "Source", "Url", "Tags", "Enabled")
-	queryBuilder.Values(dt, dt, timeZero, displayName, source, url, tags, enabled)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r sourceRepository) GetById(ctx context.Context, id int64) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("Id", id),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("DisplayName", displayName),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) GetBySource(ctx context.Context, source string) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("Source", source),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("Source", source),
-		b.Equal("DisplayName", name),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("Sources")
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.SourceEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r sourceRepository) ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("Sources")
-	builder.Where(
-		builder.Equal("Source", source),
-	)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.SourceEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r sourceRepository) Enable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("Sources")
-	b.Set(
-		b.Assign("Enabled", true),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r sourceRepository) Disable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("Sources")
-	b.Set(
-		b.Assign("Enabled", false),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r sourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "Sources", id)
-}
-
-func (r sourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "Sources", id)
-}
-
-func (r sourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "Sources", id)
-}
-
-func (r sourceRepository) processRows(rows *sql.Rows) ([]entity.SourceEntity, error) {
-	items := []entity.SourceEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var displayName string
-		var source string
-		var enabled bool
-		var url string
-		var tags string
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt,
-			&deletedAt, &displayName, &source,
-			&enabled, &url, &tags,
-		)
-		if err != nil {
-			return items, err
-		}
-
-		item := entity.SourceEntity{
-			ID:          id,
-			CreatedAt:   createdAt,
-			UpdatedAt:   updatedAt,
-			DeletedAt:   deletedAt,
-			DisplayName: displayName,
-			Source:      source,
-			Enabled:     enabled,
-			Url:         url,
-			Tags:        tags,
-		}
-
-		items = append(items, item)
-	}
-
-	return items, nil
-}

internal/repository/source_test.go

@@ -1,246 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestSourceCreate(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	rows, err := r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if rows != 1 {
-		t.Log("failed to create a record, why")
-		t.FailNow()
-	}
-}
-
-func TestSourceGetById(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if item.ID != 1 {
-		t.Log("got no record or the wrong one")
-		t.FailNow()
-	}
-}
-
-func TestSourceGetByDisplayName(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	item, err := r.GetByDisplayName(ctx, "Test")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if item.DisplayName != "Test" {
-		t.Log("got no record or the wrong one")
-		t.FailNow()
-	}
-}
-
-func TestSourceGetBySource(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	item, err := r.GetBySource(ctx, domain.SourceCollectorRss)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if item.Source != domain.SourceCollectorRss {
-		t.Log("got no record or the wrong one")
-		t.FailNow()
-	}
-}
-
-func TestSourceList(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-
-	items, err := r.List(ctx, 0, 4)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if len(items ) != 4 {
-		t.Log("something bad happened here")
-		t.FailNow()
-	}
-}
-
-func TestSourceListBySource(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-
-	items, err := r.ListBySource(ctx, 0, 4, domain.SourceCollectorRss)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if len(items ) != 4 {
-		t.Log("something bad happened here")
-		t.FailNow()
-	}
-}
-
-func TestSourcesEnableRecord(t *testing.T) {
-	// This depends on the seed migration
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-
-	r := repository.NewSourceRepository(db)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", false)
-	item, err := r.GetByDisplayName(ctx, "Test")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != false {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Enable(ctx, item.ID)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, item.ID)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}
-
-func TestSourcesDisableRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	item, err := r.GetByDisplayName(ctx, "Test")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != true {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Disable(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}

internal/repository/userSourceSubscription.go

@@ -1,120 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type UserSourceRepo interface {
-	Create(ctx context.Context, userId, sourceId int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error)
-}
-
-type userSourceRepository struct {
-	conn          *sql.DB
-	defaultLimit  int
-	defaultOffset int
-}
-
-func NewUserSourceRepository(conn *sql.DB) userSourceRepository {
-	return userSourceRepository{
-		conn:          conn,
-		defaultLimit:  50,
-		defaultOffset: 50,
-	}
-}
-
-func (r userSourceRepository) Create(ctx context.Context, userId, sourceId int64) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("UserSourceSubscriptions")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID")
-	queryBuilder.Values(dt, dt, timeZero, userId, sourceId)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r userSourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "UserSourceSubscriptions", id)
-}
-
-func (r userSourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "UserSourceSubscriptions", id)
-}
-
-func (r userSourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "UserSourceSubscriptions", id)
-}
-
-func (r userSourceRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("UserSourceSubscriptions")
-	builder.Where(
-		builder.Equal("UserID", userId),
-	)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.UserSourceSubscriptionEntity{}, err
-	}
-
-	data := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.UserSourceSubscriptionEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ur userSourceRepository) processRows(rows *sql.Rows) []entity.UserSourceSubscriptionEntity {
-	items := []entity.UserSourceSubscriptionEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var userId int64
-		var sourceId int64
-
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt, &deletedAt,
-			&userId, &sourceId,
-		)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.UserSourceSubscriptionEntity{
-			ID:        id,
-			CreatedAt: createdAt,
-			UpdatedAt: updatedAt,
-			DeletedAt: deletedAt,
-			UserID:    userId,
-			SourceID:  sourceId,
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/users.go

@@ -1,192 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-
-	"github.com/huandu/go-sqlbuilder"
-	"golang.org/x/crypto/bcrypt"
-)
-
-const (
-	usersTableName  string = "users"
-	ErrUserNotFound string = "requested user was not found"
-)
-
-type Users interface {
-	GetByName(ctx context.Context, name string) (entity.UserEntity, error)
-	Create(ctx context.Context, name, password, sessionTOken, scope string) (int64, error)
-	Update(ctx context.Context, id int, entity entity.UserEntity) error
-	UpdatePassword(ctx context.Context, name, password string) error
-	CheckUserHash(ctx context.Context, name, password string) error
-	UpdateScopes(ctx context.Context, name, scope string) error
-	UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error)
-}
-
-// Creates a new instance of UserRepository with the bound sql
-func NewUserRepository(conn *sql.DB) userRepository {
-	return userRepository{
-		connection: conn,
-	}
-}
-
-type userRepository struct {
-	connection *sql.DB
-}
-
-func (ur userRepository) GetByName(ctx context.Context, name string) (entity.UserEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*").From("users").Where(
-		builder.E("Name", name),
-	)
-	query, args := builder.Build()
-
-	rows, err := ur.connection.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.UserEntity{}, err
-	}
-
-	data := ur.processRows(rows)
-	if len(data) == 0 {
-		return entity.UserEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data[0], nil
-}
-
-func (ur userRepository) Create(ctx context.Context, name, password, sessionToken, scope string) (int64, error) {
-	passwordBytes := []byte(password)
-	hash, err := bcrypt.GenerateFromPassword(passwordBytes, bcrypt.DefaultCost)
-	if err != nil {
-		return 0, err
-	}
-
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("users")
-	queryBuilder.Cols("Name", "Hash", "UpdatedAt", "CreatedAt", "DeletedAt", "Scopes", "SessionToken")
-	queryBuilder.Values(name, string(hash), dt, dt, time.Time{}, scope, sessionToken)
-	query, args := queryBuilder.Build()
-
-	_, err = ur.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (ur userRepository) Update(ctx context.Context, id int, entity entity.UserEntity) error {
-	return errors.New("not implemented")
-}
-
-func (ur userRepository) UpdatePassword(ctx context.Context, name, password string) error {
-	_, err := ur.GetByName(ctx, name)
-	if err != nil {
-		return nil
-	}
-
-	queryBuilder := sqlbuilder.NewUpdateBuilder()
-	queryBuilder.Update(usersTableName)
-	//queryBuilder.Set
-	return nil
-}
-
-func (ur userRepository) UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error) {
-	_, err := ur.GetByName(ctx, name)
-	if err != nil {
-		return 0, err
-	}
-
-	q := sqlbuilder.NewUpdateBuilder()
-	q.Update(usersTableName)
-	q.Set(
-		q.Equal("SessionToken", sessionToken),
-	)
-	q.Where(
-		q.Equal("Name", name),
-	)
-
-	query, args := q.Build()
-	rowsUpdates, err := ur.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return rowsUpdates.RowsAffected()
-}
-
-// If the hash matches what we have in the database, an error will not be returned.
-// If the user does not exist or the hash does not match, an error will be returned
-func (ur userRepository) CheckUserHash(ctx context.Context, name, password string) error {
-	record, err := ur.GetByName(ctx, name)
-	if err != nil {
-		return err
-	}
-
-	err = bcrypt.CompareHashAndPassword([]byte(record.Hash), []byte(password))
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (ur userRepository) UpdateScopes(ctx context.Context, name, scope string) error {
-	builder := sqlbuilder.NewUpdateBuilder()
-	builder.Update("users")
-	builder.Set(
-		builder.Assign("Scopes", scope),
-	)
-	builder.Where(
-		builder.Equal("Name", name),
-	)
-	query, args := builder.Build()
-
-	_, err := ur.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (ur userRepository) processRows(rows *sql.Rows) []entity.UserEntity {
-	items := []entity.UserEntity{}
-
-	for rows.Next() {
-		var id int64
-		var username string
-		var hash string
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt sql.NullTime
-		var scopes string
-		var sessionToken string
-		err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &hash, &scopes, &sessionToken)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.UserEntity{
-			ID:           id,
-			UpdatedAt:    updatedAt,
-			Username:     username,
-			Hash:         hash,
-			Scopes:       scopes,
-			CreatedAt:    createdAt,
-			SessionToken: sessionToken,
-		}
-		if deletedAt.Valid {
-			item.DeletedAt = deletedAt.Time
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/users_test.go

@@ -1,88 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"database/sql"
-	"log"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-
-	_ "github.com/glebarez/go-sqlite"
-	"github.com/pressly/goose/v3"
-)
-
-func TestCanCreateNewUser(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	repo := repository.NewUserRepository(db)
-	updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
-	if err != nil {
-		log.Println(err)
-		t.FailNow()
-	}
-	log.Println(updated)
-}
-
-func TestCanFindUserInTable(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		log.Println("unable to open connection")
-		t.FailNow()
-	}
-	defer db.Close()
-
-	repo := repository.NewUserRepository(db)
-	updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if updated != 1 {
-		t.Log("expected a row to come back")
-		t.FailNow()
-	}
-
-	user, err := repo.GetByName(context.Background(), "testing")
-	if err != nil {
-		log.Println(err)
-		t.FailNow()
-	}
-	log.Println(user)
-}
-
-func TestCheckUserHash(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		log.Println("unable to open connection")
-		t.FailNow()
-	}
-	defer db.Close()
-
-	repo := repository.NewUserRepository(db)
-	repo.CheckUserHash(context.Background(), "testing", "NotSecure")
-}
-
-func setupInMemoryDb() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", ":memory:")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.SetDialect("sqlite3")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.Up(db, "../database/migrations")
-	if err != nil {
-		return nil, err
-	}
-	return db, nil
-}

internal/repositoryServices/refreshTokens.go

@@ -1,87 +0,0 @@
-package repositoryservices
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-	"github.com/google/uuid"
-)
-
-const (
-	ErrUnexpectedAmountOfRowsUpdated = "got a unexpected of rows updated"
-)
-
-type RefreshToken interface {
-	Create(ctx context.Context, username string) (string, error)
-	GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	IsRequestValid(ctx context.Context, username, refreshToken string) error
-}
-
-// A new jwt token can be made if the user has the correct refresh token for the user.
-// It will also require the old JWT token so the expire time is pulled and part of the validation
-type RefreshTokenService struct {
-	table repository.RefreshTokenRepository
-}
-
-func NewRefreshTokenService(conn *sql.DB) RefreshTokenService {
-	return RefreshTokenService{
-		table: repository.NewRefreshTokenRepository(conn),
-	}
-}
-
-func (rt RefreshTokenService) Create(ctx context.Context, username string) (string, error) {
-	//if a refresh token already exists for a user, reuse
-	existingToken, err := rt.GetByName(ctx, username)
-	if err == nil {
-		rowsRemoved, err := rt.Delete(ctx, existingToken.ID)
-		if err != nil {
-			return "", err
-		}
-
-		if rowsRemoved != 1 {
-			return "", errors.New(ErrUnexpectedAmountOfRowsUpdated)
-		}
-	}
-
-	token, err := uuid.NewV7()
-	if err != nil {
-		return "", err
-	}
-
-	rows, err := rt.table.Create(ctx, username, token.String())
-	if err != nil {
-		return "", err
-	}
-
-	if rows != 1 {
-		return "", errors.New("expected one row but got none")
-	}
-	return token.String(), nil
-}
-
-// Find the saved refresh token for a user and return it if it exists
-func (rt RefreshTokenService) GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
-	return rt.table.GetByUsername(ctx, name)
-}
-
-// This will request that a object is removed from the database
-func (rt RefreshTokenService) Delete(ctx context.Context, id int64) (int64, error) {
-	return rt.table.DeleteById(ctx, id)
-}
-
-func (rt RefreshTokenService) IsRequestValid(ctx context.Context, username, refreshToken string) error {
-	token, err := rt.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	if token.Token != refreshToken {
-		return errors.New("the refresh token given does not match")
-	}
-
-	return nil
-}

internal/repositoryServices/userService.go

@@ -1,198 +0,0 @@
-package repositoryservices
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-	"github.com/google/uuid"
-
-	"golang.org/x/crypto/bcrypt"
-)
-
-const (
-	ErrPasswordNotLongEnough           = "password needs to be 8 character or longer"
-	ErrPasswordMissingSpecialCharacter = "password needs to contain one of the following: !, @, #"
-	ErrInvalidPassword                 = "invalid password"
-)
-
-type UserServices interface {
-	DoesUserExist(ctx context.Context, username string) error
-	DoesPasswordMatchHash(ctx context.Context, username, password string) error
-	GetUser(ctx context.Context, username string) (entity.UserEntity, error)
-	AddScopes(ctx context.Context, username string, scopes []string) error
-	RemoveScopes(ctx context.Context, username string, scopes []string) error
-	Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error)
-	NewSessionToken(ctx context.Context, name string) (string, error)
-	CheckPasswordForRequirements(password string) error
-}
-
-// This will handle operations that are user related, but one layer higher then the repository
-type UserService struct {
-	repo repository.Users
-}
-
-// This is a layer on top of the Users Repository.
-// Use this over directly talking to the table when ever possible.
-func NewUserService(conn *sql.DB) UserService {
-	return UserService{
-		repo: repository.NewUserRepository(conn),
-	}
-}
-
-func (us UserService) DoesUserExist(ctx context.Context, username string) error {
-	_, err := us.repo.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (us UserService) DoesPasswordMatchHash(ctx context.Context, username, password string) error {
-	model, err := us.GetUser(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	err = bcrypt.CompareHashAndPassword([]byte(model.Hash), []byte(password))
-	if err != nil {
-		return errors.New(ErrInvalidPassword)
-	}
-
-	return nil
-}
-
-func (us UserService) GetUser(ctx context.Context, username string) (entity.UserEntity, error) {
-	return us.repo.GetByName(ctx, username)
-}
-
-func (us UserService) AddScopes(ctx context.Context, username string, scopes []string) error {
-	usr, err := us.repo.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	if usr.Username != username {
-		return errors.New(repository.ErrUserNotFound)
-	}
-
-	currentScopes := strings.Split(usr.Scopes, ",")
-
-	// check the current scopes
-	for _, item := range scopes {
-		if !strings.Contains(usr.Scopes, item) {
-			currentScopes = append(currentScopes, item)
-		}
-	}
-	return us.repo.UpdateScopes(ctx, username, strings.Join(currentScopes, ","))
-}
-
-func (us UserService) RemoveScopes(ctx context.Context, username string, scopes []string) error {
-	usr, err := us.repo.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	if usr.Username != username {
-		return errors.New(repository.ErrUserNotFound)
-	}
-
-	var newScopes []string
-
-	// check all the scopes that are currently assigned
-	for _, item := range strings.Split(usr.Scopes, ",") {
-
-		// check the scopes given, if one matches skip it
-		if us.doesScopeExist(scopes, item) {
-			continue
-		}
-
-		// did not match, add it
-		newScopes = append(newScopes, item)
-	}
-
-	return us.repo.UpdateScopes(ctx, username, strings.Join(newScopes, ","))
-}
-
-func (us UserService) doesScopeExist(scopes []string, target string) bool {
-	for _, item := range scopes {
-		if item == target {
-			return true
-		}
-	}
-	return false
-}
-
-func (us UserService) Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error) {
-	err := us.CheckPasswordForRequirements(password)
-	if err != nil {
-		return entity.UserEntity{}, err
-	}
-
-	token, err := uuid.NewV7()
-	if err != nil {
-		return entity.UserEntity{}, err
-	}
-
-	us.repo.Create(ctx, name, password, token.String(), domain.ScopeArticleRead)
-	return entity.UserEntity{}, nil
-}
-
-func (us UserService) NewSessionToken(ctx context.Context, name string) (string, error) {
-	token, err := uuid.NewV7()
-	if err != nil {
-		return "", err
-	}
-
-	rows, err := us.repo.UpdateSessionToken(ctx, name, token.String())
-	if err != nil {
-		return "", err
-	}
-
-	if rows != 1 {
-		return "", fmt.Errorf("UserService.NewSessionToken %w", err)
-	}
-
-	return token.String(), nil
-}
-
-func (us UserService) CheckPasswordForRequirements(password string) error {
-	err := us.checkPasswordLength(password)
-	if err != nil {
-		return err
-	}
-
-	err = us.checkPasswordForSpecialCharacters(password)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (us UserService) checkPasswordLength(password string) error {
-	if len(password) < 8 {
-		return errors.New(ErrPasswordNotLongEnough)
-	}
-	return nil
-}
-
-func (us UserService) checkPasswordForSpecialCharacters(password string) error {
-	var chars []string
-	chars = append(chars, "!")
-	chars = append(chars, "@")
-	chars = append(chars, "#")
-
-	for _, char := range chars {
-		if strings.Contains(password, char) {
-			return nil
-		}
-	}
-
-	return errors.New(ErrPasswordMissingSpecialCharacter)
-}

internal/services/config.go

@@ -1,154 +0,0 @@
-package services
-
-import (
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"strconv"
-
-	"github.com/joho/godotenv"
-)
-
-const (
-	ServerAddress = "ServerAddress"
-
-	//Sql_Connection_String = "SQL_CONNECTION_STRING"
-
-	FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
-	REDDIT_PULL_TOP               = "REDDIT_PULL_TOP"
-	REDDIT_PULL_HOT               = "REDDIT_PULL_HOT"
-	REDDIT_PULL_NSFW              = "REDDIT_PULL_NSFW"
-
-	FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
-	YOUTUBE_DEBUG                  = "YOUTUBE_DEBUG"
-
-	FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
-	TWITCH_CLIENT_ID              = "TWITCH_CLIENT_ID"
-	TWITCH_CLIENT_SECRET          = "TWITCH_CLIENT_SECRET"
-	TWITCH_MONITOR_CLIPS          = "TWITCH_MONITOR_CLIPS"
-	TWITCH_MONITOR_VOD            = "TWITCH_MONITOR_VOD"
-
-	FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
-)
-
-type Configs struct {
-	ServerAddress string
-	JwtSecret     string
-	AdminSecret   string
-
-	RedditEnabled  bool
-	RedditPullTop  bool
-	RedditPullHot  bool
-	RedditPullNsfw bool
-
-	YoutubeEnabled bool
-	YoutubeDebug   bool
-
-	TwitchEnabled      bool
-	TwitchClientId     string
-	TwitchClientSecret string
-	TwitchMonitorClips bool
-	TwitchMonitorVOD   bool
-
-	FfxivEnabled bool
-}
-
-type ConfigClient struct{}
-
-func NewConfig() ConfigClient {
-	c := ConfigClient{}
-	c.RefreshEnv()
-
-	return c
-}
-
-func GetEnvConfig() Configs {
-	return Configs{
-		ServerAddress: os.Getenv(ServerAddress),
-		JwtSecret:     os.Getenv("JwtSecret"),
-		AdminSecret:   os.Getenv("AdminSecret"),
-
-		RedditEnabled:  processBoolConfig(os.Getenv(FEATURE_ENABLE_REDDIT_BACKEND)),
-		RedditPullTop:  processBoolConfig(os.Getenv(REDDIT_PULL_TOP)),
-		RedditPullHot:  processBoolConfig(os.Getenv(REDDIT_PULL_HOT)),
-		RedditPullNsfw: processBoolConfig(os.Getenv(REDDIT_PULL_NSFW)),
-
-		YoutubeEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_YOUTUBE_BACKEND)),
-		YoutubeDebug:   processBoolConfig(os.Getenv(YOUTUBE_DEBUG)),
-
-		TwitchEnabled:      processBoolConfig(os.Getenv(FEATURE_ENABLE_TWITCH_BACKEND)),
-		TwitchClientId:     os.Getenv(TWITCH_CLIENT_ID),
-		TwitchClientSecret: os.Getenv(TWITCH_CLIENT_SECRET),
-		TwitchMonitorClips: processBoolConfig(TWITCH_MONITOR_CLIPS),
-		TwitchMonitorVOD:   processBoolConfig(os.Getenv(TWITCH_MONITOR_VOD)),
-
-		FfxivEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_FFXIV_BACKEND)),
-	}
-}
-
-// This will parse a string and convert it to a bool.
-// If it runs into any errors, it will default to false
-func processBoolConfig(value string) bool {
-	b, err := strconv.ParseBool(value)
-	if err != nil {
-		return false
-	}
-	return b
-}
-
-func (cc *ConfigClient) GetConfig(key string) string {
-	res, filled := os.LookupEnv(key)
-	if !filled {
-		log.Printf("Missing the a value for '%v'.  Could generate errors.", key)
-	}
-	return res
-}
-
-// Looks for a value in the env and will panic if it does not exist.
-func (c ConfigClient) MustGetString(key string) string {
-	res, filled := os.LookupEnv(key)
-	if !filled {
-		msg := fmt.Sprintf("No value was found for '%v'", key)
-		panic(msg)
-	}
-
-	return res
-}
-
-func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
-	cc.RefreshEnv()
-
-	res, filled := os.LookupEnv(flag)
-	if !filled {
-		errorMessage := fmt.Sprintf("'%v' was not found", flag)
-		return false, errors.New(errorMessage)
-	}
-
-	b, err := strconv.ParseBool(res)
-	if err != nil {
-		return false, err
-	}
-	return b, nil
-}
-
-// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
-func (cc *ConfigClient) RefreshEnv() {
-	// Check to see if we have the env file on the system
-	_, err := os.Stat(".env")
-
-	// We have the file, load it.
-	if err == nil {
-		_, err := os.Open(".env")
-		if err == nil {
-			loadEnvFile()
-		}
-	}
-}
-
-func loadEnvFile() {
-	err := godotenv.Load()
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

internal/services/config_test.go

@@ -1,22 +0,0 @@
-package services_test
-
-import (
-	"os"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-)
-
-func TestNewClient(t *testing.T) {
-	services.NewConfig()
-}
-
-func TestGetConfigExpectNull(t *testing.T) {
-	cc := services.NewConfig()
-	os.Setenv(services.REDDIT_PULL_HOT, "")
-	res := cc.GetConfig(services.REDDIT_PULL_HOT)
-	if res != "" {
-		panic("expected blank")
-	}
-
-}

internal/services/cron/collectors.go

@@ -1,220 +0,0 @@
-package cron
-
-import (
-	"log"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-func (c *Cron) CollectRssPosts() {
-	log.Println("Starting ")
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
-	if err != nil {
-		log.Println(err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		rssClient := input.NewRssClient(source)
-		articles, err := rssClient.GetArticles()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range articles {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Println(err)
-			}
-			if rowsCreated != 1 {
-				log.Println("Got back the wrong number of rows")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-}
-
-func (c *Cron) CollectRedditPosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
-	if err != nil {
-		log.Printf("[Reddit] No sources found to query - %v\r", err)
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
-		rc := input.NewRedditClient(source)
-		raw, err := rc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		redditArticles := rc.ConvertToArticles(raw)
-		for _, article := range redditArticles {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new reddit article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-	}
-	log.Print("[Reddit] Done!")
-}
-
-func (c *Cron) CollectYoutubePosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
-	if err != nil {
-		log.Printf("[Youtube] No sources found to query - %v\r", err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
-		yc := input.NewYoutubeClient(source)
-		raw, err := yc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range raw {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new youtube article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-	log.Print("[YouTube] Done!")
-}
-
-func (c *Cron) CollectFfxivPosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
-	if err != nil {
-		log.Printf("[FFXIV] No sources found to query - %v\r", err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		fc := input.NewFFXIVClient(source)
-		items, err := fc.CheckSource()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range items {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new FFXIV article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-	log.Printf("[FFXIV Done!]")
-}
-
-func (c *Cron) CollectTwitchPosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
-	if err != nil {
-		log.Printf("[Twitch] No sources found to query - %v\r", err)
-	}
-
-	tc, err := input.NewTwitchClient()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	err = tc.Login()
-	if err != nil {
-		log.Println(err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
-		tc.ReplaceSourceRecord(source)
-		items, err := tc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range items {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new Twitch article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-
-	log.Print("[Twitch] Done!")
-}

internal/services/cron/collectors_test.go

@@ -1,43 +0,0 @@
-package cron_test
-
-import (
-	"context"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
-)
-
-func TestRssPullsCorrectly(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-	defer conn.Close()
-
-	ctx := context.Background()
-	db := services.NewRepositoryService(conn)
-	rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	if rowsCreated != 1 {
-		t.Error("failed to create the source record")
-		t.FailNow()
-	}
-
-	client := cron.NewScheduler(ctx, conn)
-	client.CollectRssPosts()
-
-	articles, err := db.Articles.ListByPage(ctx, 0, 100)
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	t.Log(len(articles))
-}

internal/services/cron/scheduler.go

@@ -1,131 +0,0 @@
-package cron
-
-import (
-	"context"
-	"database/sql"
-
-	_ "github.com/lib/pq"
-	"github.com/robfig/cron/v3"
-
-	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-)
-
-type Cron struct {
-	//Db    *database.Queries
-	ctx   context.Context
-	timer *cron.Cron
-	repo  services.RepositoryService
-}
-
-func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
-	c := &Cron{
-		ctx:  ctx,
-		repo: services.NewRepositoryService(conn),
-	}
-	timer := cron.New()
-
-	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
-	//features := services.GetEnvConfig()
-
-	timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
-	//timer.AddFunc("10 * * * *", c.CollectRedditPosts)
-	//timer.AddFunc("15 * * * *", c.CheckYoutube)
-	//timer.AddFunc("20 * * * *", c.CheckFfxiv)
-	//timer.AddFunc("25 * * * *", c.CheckTwitch)
-	//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
-
-	c.timer = timer
-	return c
-}
-
-func (c *Cron) Start() {
-	c.timer.Start()
-}
-
-func (c *Cron) Stop() {
-	c.timer.Stop()
-}
-
-/*
-func (c *Cron) CheckDiscordQueue() {
-	// Get items from the table
-	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
-	if err != nil {
-		return err
-	}
-
-	for _, queue := range queueItems {
-		// Get the articleByID
-		article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
-		if err != nil {
-			return err
-		}
-
-		var endpoints []string
-		// List Subscription by SourceID
-		subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
-		if err != nil {
-			return err
-		}
-
-		// if no one is subscribed to it, remove it from the index.
-		if len(subs) == 0 {
-			log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
-			err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		// Get the webhhooks to send to
-		for _, sub := range subs {
-			webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
-			if err != nil {
-				return err
-			}
-
-			// store them in an array
-			endpoints = append(endpoints, webhook.Url)
-		}
-
-		// Create Discord Message
-		dwh := output.NewDiscordWebHookMessage(article)
-		msg, err := dwh.GeneratePayload()
-		if err != nil {
-			return err
-		}
-
-		// Send Message(s)
-		for _, i := range endpoints {
-			err = dwh.SendPayload(msg, i)
-
-			if err != nil {
-				return err
-			}
-		}
-
-		// Remove the item from the queue, given we sent our notification.
-		err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
-		if err != nil {
-			return err
-		}
-
-		time.Sleep(10 * time.Second)
-	}
-
-	return nil
-}
-*/
-
-//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
-//	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
-//		ID:        uuid.New(),
-//		Articleid: Id,
-//	})
-//	if err != nil {
-//		return err
-//	}
-//	return nil
-//}

internal/services/cron/scheduler_test.go

@@ -1,53 +0,0 @@
-package cron_test
-
-import (
-	"database/sql"
-
-	"github.com/pressly/goose/v3"
-)
-
-/*
-func TestInvokeTwitch(t *testing.T) {
-
-}
-
-// TODO add database mocks but not sure how to do that yet.
-func TestCheckReddit(t *testing.T) {
-	ctx := context.Background()
-	c := cron.NewScheduler(ctx)
-	c.Col()
-}
-
-func TestCheckYouTube(t *testing.T) {
-	ctx := context.Background()
-	c := cron.NewScheduler(ctx)
-	c.CheckYoutube()
-}
-
-func TestCheckTwitch(t *testing.T) {
-	ctx := context.Background()
-	c := cron.NewScheduler(ctx)
-	err := c.CheckTwitch()
-	if err != nil {
-		t.Error(err)
-	}
-}
-*/
-
-func setupInMemoryDb() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", ":memory:")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.SetDialect("sqlite3")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.Up(db, "../../database/migrations")
-	if err != nil {
-		return nil, err
-	}
-	return db, nil
-}

internal/services/database.go

@@ -1,30 +0,0 @@
-package services
-
-import (
-	"database/sql"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-	repositoryservices "git.jamestombleson.com/jtom38/newsbot-api/internal/repositoryServices"
-)
-
-type RepositoryService struct {
-	AlertDiscord            repository.AlertDiscordRepo
-	Articles                repository.ArticlesRepo
-	DiscordWebHooks         repository.DiscordWebHookRepo
-	RefreshTokens           repositoryservices.RefreshToken
-	Sources                 repository.Sources
-	Users                   repositoryservices.UserServices
-	UserSourceSubscriptions repository.UserSourceRepo
-}
-
-func NewRepositoryService(conn *sql.DB) RepositoryService {
-	return RepositoryService{
-		AlertDiscord:            repository.NewAlertDiscordRepository(conn),
-		Articles:                repository.NewArticleRepository(conn),
-		DiscordWebHooks:         repository.NewDiscordWebHookRepository(conn),
-		RefreshTokens:           repositoryservices.NewRefreshTokenService(conn),
-		Sources:                 repository.NewSourceRepository(conn),
-		Users:                   repositoryservices.NewUserService(conn),
-		UserSourceSubscriptions: repository.NewUserSourceRepository(conn),
-	}
-}

internal/services/input/reddit_test.go

@@ -1,32 +0,0 @@
-package input_test
-
-import (
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-var RedditRecord entity.SourceEntity = entity.SourceEntity{
-	ID:          9999,
-	DisplayName: "dadjokes",
-	Source:      domain.SourceCollectorRss,
-	Url:         "https://reddit.com/r/dadjokes",
-	Tags:        "reddit, dadjokes",
-}
-
-func TestGetContent(t *testing.T) {
-	//This test is flaky right now due to the http changes in 1.17
-	rc := input.NewRedditClient(RedditRecord)
-	raw, err := rc.GetContent()
-	if err != nil {
-		t.Error(err)
-	}
-	redditArticles := rc.ConvertToArticles(raw)
-	for _, posts := range redditArticles {
-		if posts.Title == "" {
-			t.Error("Title is missing")
-		}
-	}
-}

internal/services/input/rss.go

@@ -1,77 +0,0 @@
-package input
-
-import (
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/mmcdole/gofeed"
-)
-
-type FeedInput interface {
-	GetArticles() (entity.ArticleEntity, error)
-}
-
-type rssClient struct {
-	SourceRecord entity.SourceEntity
-}
-
-func NewRssClient(sourceRecord entity.SourceEntity) rssClient {
-	client := rssClient{
-		SourceRecord: sourceRecord,
-	}
-
-	return client
-}
-
-func (rc rssClient) GetArticles() ([]entity.ArticleEntity, error) {
-	parser := gofeed.NewParser()
-	feed, err := parser.ParseURL(rc.SourceRecord.Url)
-	if err != nil {
-		return nil, err
-	}
-
-	sourceTags := strings.Split(rc.SourceRecord.Tags, ",")
-	var articles []entity.ArticleEntity
-	for _, post := range feed.Items {
-		article := entity.ArticleEntity{
-			SourceID:    rc.SourceRecord.ID,
-			Title:       post.Title,
-			Description: post.Content,
-			Url:         post.Link,
-			PubDate:     *post.PublishedParsed,
-			//AuthorName:  post.Authors[0].Email,
-		}
-
-		if len(post.Authors) != 0 {
-			article.AuthorName = post.Authors[0].Email
-		}
-
-		var postTags []string
-		postTags = append(postTags, sourceTags...)
-		postTags = append(postTags, post.Categories...)
-		article.Tags = strings.Join(postTags, ",")
-
-		/*
-			pageContent, err := getHttpContent(article.Url)
-			if err != nil {
-				continue
-			}
-
-			htmlNode, err := html.Parse(bytes.NewReader(pageContent))
-			if err != nil {
-				continue
-			}
-			htmlNode.
-
-			fmt.Println(htmlNode)
-		*/
-
-		if post.Image == nil {
-			article.Thumbnail = ""
-		}
-
-		articles = append(articles, article)
-	}
-
-	return articles, nil
-}

internal/services/input/rss_test.go

@@ -1,43 +0,0 @@
-package input_test
-
-import (
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-var rssRecord = entity.SourceEntity{
-	ID:          1,
-	DisplayName: "ArsTechnica",
-	Url:         "https://feeds.arstechnica.com/arstechnica/index",
-	Source:      domain.SourceCollectorRss,
-}
-
-func TestRssClientConstructor(t *testing.T) {
-	input.NewRssClient(rssRecord)
-}
-
-func TestRssGetFeed(t *testing.T) {
-	client := input.NewRssClient(rssRecord)
-	_, err := client.GetArticles()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestRssAgainstGita(t *testing.T) {
-	client := input.NewRssClient(entity.SourceEntity{
-		ID:          2,
-		DisplayName: "Gitea - Newsbot-api",
-		Source:      domain.SourceCollectorRss,
-		Url:         "https://git.jamestombleson.com/jtom38/newsbot-api.rss",
-		Tags:        "rss,gitea,newsbot-api",
-	})
-	_, err := client.GetArticles()
-	if err != nil {
-		t.Error(err)
-	}
-
-}

main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/jtom38/newsbot/collector/docs"
+	"github.com/jtom38/newsbot/collector/routes"
+	"github.com/jtom38/newsbot/collector/services/config"
+	"github.com/jtom38/newsbot/collector/services/cron"
+)
+
+// @title     NewsBot collector
+// @version   0.1
+// @BasePath  /api
+func main() {
+	cfg := config.New()
+	address := cfg.GetConfig(config.ServerAddress)
+	docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
+
+	ctx := context.Background()
+	c := cron.New(ctx)
+	c.Start()
+
+	server := routes.NewServer(ctx)
+
+	fmt.Println("API is online and waiting for requests.")
+	fmt.Printf("API: http://%v:8081/api\r\n", address)
+	fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", address)
+
+	err := http.ListenAndServe(":8081", server.Router)
+	if err != nil { 
+		panic(err)
+	}
+}

makefile

@@ -3,29 +3,24 @@ help: ## Shows this help command
 	@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
 build: ## builds the application with the current go runtime
+	sqlc generate
 	~/go/bin/swag f
-	~/go/bin/swag init -g cmd/server.go
+	~/go/bin/swag i
-	go build cmd/server.go
+	go build .
-	ls -lh server
 	
 docker-build: ## Generates the docker image
 	docker build -t "newsbot.collector.api" .
 	docker image ls | grep newsbot.collector.api
 
 migrate-dev: ## Apply sql migrations to dev db
-	goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up 
+	goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up 
 
 migrate-dev-down: ## revert sql migrations to dev db
-	goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down 
+	goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down 
 
 swag: ## Generates the swagger documentation with the swag tool
-	~/go/bin/swag f -g cmd/server.go
+	~/go/bin/swag f
-	~/go/bin/swag init -g cmd/server.go
+	~/go/bin/swag i
-	go run tools/swaggertoopenapi/main.go
-	oapi-codegen -config api/client.yaml docs/openapi.json
-	mv api.gen.go api/
 
-install-tools: ## Installs the required tools for this project
+gensql: ## Generates SQL code with sqlc
-	go install github.com/swaggo/swag/cmd/swag@v1.8.1
+	sqlc generate
-	go install github.com/pressly/goose/v3/cmd/goose@latest
-	go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.3.0

routes/articles.go

@@ -0,0 +1,132 @@
+package routes
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/google/uuid"
+)
+
+// ListArticles
+// @Summary  Lists the top 50 records
+// @Produce  application/json
+// @Tags     Articles
+// @Router   /articles [get]
+func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	res, err := s.Db.ListArticlesByDate(*s.ctx, 50)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+	w.Write(bres)
+}
+
+// GetArticleById
+// @Summary  Returns an article based on defined ID.
+// @Param    id  path  string  true  "uuid"
+// @Produce  application/json
+// @Tags     Articles
+// @Router   /articles/{id} [get]
+func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	id := chi.URLParam(r, "ID")
+	uuid, err := uuid.Parse(id)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	res, err := s.Db.GetArticleByID(*s.ctx, uuid)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	w.Write(bres)
+}
+
+// TODO add page support
+// GetArticlesBySourceID
+// @Summary  Finds the articles based on the SourceID provided.  Returns the top 50.
+// @Param    id  query  string  true  "Source ID UUID"
+// @Produce  application/json
+// @Tags     Articles
+// @Router   /articles/by/sourceid [get]
+func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	r.URL.Query()
+	query := r.URL.Query()
+	_id := query["id"][0]
+
+	uuid, err := uuid.Parse(_id)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	w.Write(bres)
+}
+
+// TODO add page support
+// GetArticlesByTag
+// @Summary  Finds the articles based on the SourceID provided.  Returns the top 50.
+// @Param    tag  query  string  true  "Tag name"
+// @Produce  application/json
+// @Tags     Articles
+// @Router   /articles/by/tag [get]
+func (s *Server) GetArticlesByTag(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	r.URL.Query()
+	query := r.URL.Query()
+	_id := query["tag"][0]
+
+	uuid, err := uuid.Parse(_id)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	w.Write(bres)
+}

routes/discordQueue.go

@@ -0,0 +1,29 @@
+package routes
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+// GetDiscordQueue
+// @Summary  Returns the top 100 entries from the queue to be processed.
+// @Produce  application/json
+// @Tags     Debug, Discord, Queue
+// @Router   /discord/queue [get]
+func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	
+	res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	w.Write(bres)
+}

routes/discordwebhooks.go

@@ -0,0 +1,115 @@
+package routes
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/jtom38/newsbot/collector/database"
+)
+
+// GetDiscordWebHooks
+// @Summary  Returns the top 100 entries from the queue to be processed.
+// @Produce  application/json
+// @Tags     Config, Discord, Webhook
+// @Router   /discord/webhooks [get]
+func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	res, err := s.Db.ListDiscordWebhooks(*s.ctx, 100)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	w.Write(bres)
+}
+
+// GetDiscorWebHooksById
+// @Summary  Returns the top 100 entries from the queue to be processed.
+// @Produce  application/json
+// @Param    id  query  string  true  "id"
+// @Tags     Config, Discord, Webhook
+// @Router   /discord/webhooks/byId [get]
+func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	query := r.URL.Query()
+	_id := query["id"][0]
+	if _id == "" {
+		http.Error(w, "id is missing", http.StatusBadRequest)
+		return
+	}
+
+	uuid, err := uuid.Parse(_id)
+	if err != nil {
+		http.Error(w, "unable to parse id value", http.StatusBadRequest)
+		return
+	}
+	
+	res, err := s.Db.GetDiscordWebHooksByID(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, "no record found", http.StatusBadRequest)
+		return
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		http.Error(w, "unable to convert to json", http.StatusBadRequest)
+		panic(err)
+	}
+
+	w.Write(bres)
+}
+
+// NewDiscordWebHook
+// @Summary  Creates a new record for a discord web hook to post data to.
+// @Param    url      query  string  true  "url"
+// @Param    server   query  string  true  "Server name"
+// @Param    channel  query  string  true  "Channel name"
+// @Tags     Config, Discord, Webhook
+// @Router   /discord/webhooks/new [post]
+func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+	_url := query["url"][0]
+	_server := query["server"][0]
+	_channel := query["channel"][0]
+
+	if _url == "" {
+		http.Error(w, "url is missing a value", http.StatusBadRequest)
+		return
+	}
+	if !strings.Contains(_url, "discord.com/api/webhooks") {
+		http.Error(w, "invalid url", http.StatusBadRequest)
+		return
+	}
+	if _server ==  ""{
+		http.Error(w, "server is missing", http.StatusBadRequest)
+	}
+	if _channel == "" {
+		http.Error(w, "channel is missing", http.StatusBadRequest)
+	}
+	params := database.CreateDiscordWebHookParams{
+		ID:      uuid.New(),
+		Url:     _url,
+		Server:  _server,
+		Channel: _channel,
+		Enabled: true,
+	}
+	s.Db.CreateDiscordWebHook(*s.ctx, params)
+
+	bJson, err := json.Marshal(&params)
+	if err != nil {
+		log.Panicln(err)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bJson)
+}

routes/root.go

@@ -0,0 +1,50 @@
+package routes
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+)
+
+func RootRoutes() chi.Router {
+	app := chi.NewRouter()
+	app.Route("/", func(r chi.Router) {
+		r.Get("/helloworld", helloWorld)
+		r.Get("/ping", ping)
+		r.Route("/hello/{who}", func(r chi.Router) {
+			r.Get("/", helloWho)
+		})
+	})
+	return app
+}
+
+// HelloWorld
+// @Summary  Responds back with "Hello world!"
+// @Produce  plain
+// @Tags     Debug
+// @Router   /helloworld [get]
+func helloWorld(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte("Hello World!"))
+}
+
+// Ping
+// @Summary  Sends back "pong".  Good to test with.
+// @Produce  plain
+// @Tags     Debug
+// @Router   /ping [get]
+func ping(w http.ResponseWriter, r *http.Request) {
+	msg := "pong"
+	w.Write([]byte(msg))
+}
+
+// HelloWho
+// @Summary  Responds back with "Hello x" depending on param passed in.
+// @Param    who  path  string  true  "Who"
+// @Produce  plain
+// @Tags     Debug
+// @Router   /hello/{who} [get]
+func helloWho(w http.ResponseWriter, r *http.Request) {
+	msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who"))
+	w.Write([]byte(msg))
+}

routes/root_test.go

@@ -0,0 +1,2 @@
+package routes_test
+

routes/server.go

@@ -0,0 +1,116 @@
+package routes
+
+import (
+	"context"
+	"database/sql"
+
+	//"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
+	_ "github.com/lib/pq"
+	httpSwagger "github.com/swaggo/http-swagger"
+
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services/config"
+)
+
+type Server struct {
+	Router *chi.Mux
+	Db     *database.Queries
+	ctx    *context.Context
+}
+
+var (
+	ErrIdValueMissing        string = "id value is missing"
+	ErrValueNotUuid          string = "a value given was expected to be a uuid but was not correct."
+	ErrNoRecordFound         string = "no record was found."
+	ErrUnableToConvertToJson string = "Unable to convert to json"
+)
+
+func NewServer(ctx context.Context) *Server {
+	s := &Server{
+		ctx: &ctx,
+	}
+
+	db, err := openDatabase(ctx)
+	if err != nil {
+		panic(err)
+	}
+	s.Db = db
+
+	s.Router = chi.NewRouter()
+	s.MountMiddleware()
+	s.MountRoutes()
+	return s
+}
+
+func openDatabase(ctx context.Context) (*database.Queries, error) {
+	_env := config.New()
+	connString := _env.GetConfig(config.Sql_Connection_String)
+	db, err := sql.Open("postgres", connString)
+	if err != nil {
+		panic(err)
+	}
+
+	queries := database.New(db)
+	return queries, err
+}
+
+func (s *Server) MountMiddleware() {
+	s.Router.Use(middleware.Logger)
+	s.Router.Use(middleware.Recoverer)
+	//s.Router.Use(middleware.Heartbeat())
+}
+
+func (s *Server) MountRoutes() {
+	s.Router.Get("/swagger/*", httpSwagger.Handler(
+		httpSwagger.URL("http://localhost:8081/swagger/doc.json"), //The url pointing to API definition
+	))
+
+	/* Root Routes */
+	s.Router.Get("/api/helloworld", helloWorld)
+	s.Router.Get("/api/hello/{who}", helloWho)
+	s.Router.Get("/api/ping", ping)
+
+	/* Article Routes */
+	s.Router.Get("/api/articles", s.listArticles)
+	s.Router.Route("/api/articles/{ID}", func(r chi.Router) {
+		r.Get("/", s.getArticleById)
+	})
+	s.Router.Get("/api/articles/by/sourceid", s.GetArticlesBySourceId)
+
+	/* Discord Queue */
+	s.Router.Get("/api/discord/queue", s.GetDiscordQueue)
+
+	/* Discord WebHooks */
+	s.Router.Post("/api/discord/webhooks/new", s.NewDiscordWebHook)
+	s.Router.Get("/api/discord/webhooks", s.GetDiscordWebHooks)
+	s.Router.Get("/api/discord/webhooks/byId", s.GetDiscordWebHooksById)
+
+	/* Settings */
+	s.Router.Get("/api/settings", s.getSettings)
+
+	/* Source Routes */
+	s.Router.Get("/api/config/sources", s.listSources)
+	s.Router.Get("/api/config/sources/by/source", s.listSourcesBySource)
+
+	/* Reddit Source Routes */
+
+	s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource)
+
+	s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource)
+	s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource)
+	s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) {
+		r.Get("/", s.getSources)
+		r.Delete("/", s.deleteSources)
+		r.Post("/disable", s.disableSource)
+		r.Post("/enable", s.enableSource)
+	})
+
+	/* Subscriptions */
+	s.Router.Get("/api/subscriptions", s.ListSubscriptions)
+	s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId)
+	s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId)
+	s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription)
+}

routes/settings.go

@@ -0,0 +1,44 @@
+package routes
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/google/uuid"
+)
+
+// GetSettings
+// @Summary  Returns a object based on the Key that was given.
+// @Param    key  path  string  true  "Settings Key value"
+// @Produce  application/json
+// @Tags     Settings
+// @Router   /settings/{key} [get]
+func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
+	//var item model.Sources
+	id := chi.URLParam(r, "ID")
+
+	uuid, err := uuid.Parse(id)
+	if err != nil {
+		panic(err)
+	}
+
+	res, err := s.Db.GetSourceByID(*s.ctx, uuid)
+	if err != nil {
+		panic(err)
+	}
+
+	//itemId := fmt.Sprint(item.ID)
+	//if id != itemId {
+	//	log.Panicln("Unable to find the requested record.  Either unable to access SQL or the record does not exist.")
+	//}
+
+	bResult, err := json.Marshal(res)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bResult)
+}

routes/sources.go

@@ -0,0 +1,315 @@
+package routes
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/google/uuid"
+	"github.com/jtom38/newsbot/collector/database"
+)
+
+// ListSources
+// @Summary  Lists the top 50 records
+// @Produce  application/json
+// @Tags     Config, Source
+// @Router   /config/sources [get]
+func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
+	//TODO Add top?
+	/*
+		top := chi.URLParam(r, "top")
+		topInt, err := strconv.ParseInt(top, 0, 32)
+		if err != nil {
+			panic(err)
+		}
+		res, err := s.Db.ListSources(*s.ctx, int32(topInt))
+	*/
+
+	// Default way of showing all sources
+	res, err := s.Db.ListSources(*s.ctx, 50)
+	if err != nil {
+		http.Error(w, "url is missing a value", http.StatusBadRequest)
+		return
+	}
+	bResult, err := json.Marshal(res)
+	if err != nil {
+		http.Error(w, "unable to convert to json", http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bResult)
+
+}
+
+// ListSourcesBySource
+// @Summary  Lists the top 50 records based on the name given. Example: reddit
+// @Param    source  query  string  true  "Source Name"
+// @Produce  application/json
+// @Tags     Config, Source
+// @Router   /config/sources/by/source [get]
+func (s *Server) listSourcesBySource(w http.ResponseWriter, r *http.Request) {
+	//TODO Add top?
+	/*
+		top := chi.URLParam(r, "top")
+		topInt, err := strconv.ParseInt(top, 0, 32)
+		if err != nil {
+			panic(err)
+		}
+		res, err := s.Db.ListSources(*s.ctx, int32(topInt))
+	*/
+
+	query := r.URL.Query()
+	_source := query["source"][0]
+
+	// Shows the list by Sources.source
+	res, err := s.Db.ListSourcesBySource(*s.ctx, strings.ToLower(_source))
+	if err != nil {
+		http.Error(w, "invalid source is missing a value", http.StatusBadRequest)
+		return
+	}
+	bResult, err := json.Marshal(res)
+	if err != nil {
+		http.Error(w, "unable to convert to json", http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bResult)
+}
+
+// GetSource
+// @Summary  Returns a single entity by ID
+// @Param    id  path  string  true  "uuid"
+// @Produce  application/json
+// @Tags     Config, Source
+// @Router   /config/sources/{id} [get]
+func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
+	id := chi.URLParam(r, "ID")
+
+	uuid, err := uuid.Parse(id)
+	if err != nil {
+		http.Error(w, "id is not a uuid", http.StatusBadRequest)
+		return
+	}
+
+	res, err := s.Db.GetSourceByID(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, "invalid id was given", http.StatusBadRequest)
+		panic(err)
+	}
+
+	bResult, err := json.Marshal(res)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bResult)
+}
+
+// NewRedditSource
+// @Summary  Creates a new reddit source to monitor.
+// @Param    name  query  string  true  "name"
+// @Param    url   query  string  true  "url"
+// @Tags     Config, Source, Reddit
+// @Router   /config/sources/new/reddit [post]
+func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+	_name := query["name"][0]
+	_url := query["url"][0]
+	_tags := query["tags"][0]
+
+	if _url == "" {
+		http.Error(w, "url is missing a value", http.StatusBadRequest)
+		return
+	}
+	if !strings.Contains(_url, "reddit.com") {
+		http.Error(w, "invalid url", http.StatusBadRequest)
+		return
+	}
+
+	tags := fmt.Sprintf("reddit, %v, %v", _name, _tags)
+	params := database.CreateSourceParams{
+		ID:      uuid.New(),
+		Site:    "reddit",
+		Name:    _name,
+		Source:  "reddit",
+		Type:    "feed",
+		Enabled: true,
+		Url:     _url,
+		Tags:    tags,
+	}
+	s.Db.CreateSource(*s.ctx, params)
+
+	bJson, err := json.Marshal(&params)
+	if err != nil {
+		log.Panicln(err)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bJson)
+}
+
+// NewYoutubeSource
+// @Summary  Creates a new youtube source to monitor.
+// @Param    name  query  string  true  "name"
+// @Param    url   query  string  true  "url"
+// @Param    tags  query  string  true  "tags"
+// @Tags     Config, Source, YouTube
+// @Router   /config/sources/new/youtube [post]
+func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+	_name := query["name"][0]
+	_url := query["url"][0]
+	_tags := query["tags"][0]
+
+	if _url == "" {
+		http.Error(w, "url is missing a value", http.StatusBadRequest)
+		return
+	}
+	if !strings.Contains(_url, "youtube.com") {
+		http.Error(w, "invalid url", http.StatusBadRequest)
+		return
+	}
+
+	tags := fmt.Sprintf("youtube, %v, %v", _name, _tags)
+	params := database.CreateSourceParams{
+		ID:      uuid.New(),
+		Site:    "youtube",
+		Name:    _name,
+		Source:  "youtube",
+		Type:    "feed",
+		Enabled: true,
+		Url:     _url,
+		Tags:    tags,
+	}
+	s.Db.CreateSource(*s.ctx, params)
+
+	bJson, err := json.Marshal(&params)
+	if err != nil {
+		log.Panicln(err)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bJson)
+}
+
+// NewTwitchSource
+// @Summary  Creates a new twitch source to monitor.
+// @Param    name  query  string  true  "name"
+// @Param    url   query  string  true  "url"
+// @Param    tags  query  string  true  "tags"
+// @Tags     Config, Source, Twitch
+// @Router   /config/sources/new/twitch [post]
+func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
+	query := r.URL.Query()
+	_name := query["name"][0]
+	_url := query["url"][0]
+	_tags := query["tags"][0]
+
+	if _url == "" {
+		http.Error(w, "url is missing a value", http.StatusBadRequest)
+		return
+	}
+	if !strings.Contains(_url, "twitch.tv") {
+		http.Error(w, "invalid url", http.StatusBadRequest)
+		return
+	}
+
+	tags := fmt.Sprintf("twitch, %v, %v", _name, _tags)
+	params := database.CreateSourceParams{
+		ID:      uuid.New(),
+		Site:    "twitch",
+		Name:    _name,
+		Source:  "twitch",
+		Type:    "api",
+		Enabled: true,
+		Url:     _url,
+		Tags:    tags,
+	}
+	s.Db.CreateSource(*s.ctx, params)
+
+	bJson, err := json.Marshal(&params)
+	if err != nil {
+		log.Panicln(err)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bJson)
+}
+
+// DeleteSource
+// @Summary  Deletes a record by ID.
+// @Param    id  path  string  true  "id"
+// @Tags     Config, Source
+// @Router   /config/sources/{id} [delete]
+func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
+	//var item model.Sources = model.Sources{}
+
+	id := chi.URLParam(r, "ID")
+	uuid, err := uuid.Parse(id)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	// Check to make sure we can find the record
+	_, err = s.Db.GetSourceByID(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	// Delete the record
+	err = s.Db.DeleteSource(*s.ctx, uuid)
+	if err != nil {
+		log.Panic(err)
+	}
+}
+
+// DisableSource
+// @Summary  Disables a source from processing.
+// @Param    id  path  string  true  "id"
+// @Tags     Config, Source
+// @Router   /config/sources/{id}/disable [post]
+func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
+	id := chi.URLParam(r, "ID")
+	uuid, err := uuid.Parse(id)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	// Check to make sure we can find the record
+	_, err = s.Db.GetSourceByID(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	err = s.Db.DisableSource(*s.ctx, uuid)
+	if err != nil {
+		log.Panic(err)
+	}
+}
+
+// EnableSource
+// @Summary  Enables a source to continue processing.
+// @Param    id  path  string  true  "id"
+// @Tags     Config, Source
+// @Router   /config/sources/{id}/enable [post]
+func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) {
+	id := chi.URLParam(r, "ID")
+	uuid, err := uuid.Parse(id)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	// Check to make sure we can find the record
+	_, err = s.Db.GetSourceByID(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	err = s.Db.EnableSource(*s.ctx, uuid)
+	if err != nil {
+		log.Panic(err)
+	}
+}

routes/subscriptions.go

@@ -0,0 +1,173 @@
+package routes
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/google/uuid"
+	"github.com/jtom38/newsbot/collector/database"
+)
+
+// GetSubscriptions
+// @Summary  Returns the top 100 entries from the queue to be processed.
+// @Produce  application/json
+// @Tags     Config, Subscription
+// @Router   /subscriptions [get]
+func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	res, err := s.Db.ListSubscriptions(*s.ctx, 100)
+	if err != nil {
+		w.Write([]byte(err.Error()))
+		panic(err)
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		http.Error(w, ErrUnableToConvertToJson, http.StatusBadRequest)
+		panic(err)
+	}
+
+	w.Write(bres)
+}
+
+// GetSubscriptionsByDiscordId
+// @Summary  Returns the top 100 entries from the queue to be processed.
+// @Produce  application/json
+// @Param    id  query  string  true  "id"
+// @Tags     Config, Subscription
+// @Router   /subscriptions/byDiscordId [get]
+func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	query := r.URL.Query()
+	_id := query["id"][0]
+	if _id == "" {
+		http.Error(w, ErrIdValueMissing, http.StatusBadRequest)
+		return
+	}
+
+	uuid, err := uuid.Parse(_id)
+	if err != nil {
+		http.Error(w, ErrValueNotUuid, http.StatusBadRequest)
+		return
+	}
+
+	res, err := s.Db.GetSubscriptionsByDiscordWebHookId(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, ErrNoRecordFound, http.StatusBadRequest)
+		return
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		http.Error(w, ErrUnableToConvertToJson, http.StatusBadRequest)
+		return
+	}
+
+	w.Write(bres)
+}
+
+// GetSubscriptionsBySourceId
+// @Summary  Returns the top 100 entries from the queue to be processed.
+// @Produce  application/json
+// @Param    id  query  string  true  "id"
+// @Tags     Config, Subscription
+// @Router   /subscriptions/bySourceId [get]
+func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	query := r.URL.Query()
+	_id := query["id"][0]
+	if _id == "" {
+		http.Error(w, ErrIdValueMissing, http.StatusBadRequest)
+		return
+	}
+
+	uuid, err := uuid.Parse(_id)
+	if err != nil {
+		http.Error(w, ErrValueNotUuid, http.StatusBadRequest)
+		return
+	}
+
+	res, err := s.Db.GetSubscriptionsByDiscordWebHookId(*s.ctx, uuid)
+	if err != nil {
+		http.Error(w, ErrNoRecordFound, http.StatusBadRequest)
+		return
+	}
+
+	bres, err := json.Marshal(res)
+	if err != nil {
+		http.Error(w, ErrUnableToConvertToJson, http.StatusBadRequest)
+		return
+	}
+
+	w.Write(bres)
+}
+
+// NewDiscordWebHookSubscription
+// @Summary  Creates a new subscription to link a post from a Source to a DiscordWebHook.
+// @Param    discordWebHookId  query  string  true  "discordWebHookId"
+// @Param    sourceId          query  string  true  "sourceId"
+// @Tags     Config, Source, Discord, Subscription
+// @Router   /subscriptions/new/discordwebhook [post]
+func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
+	// Extract the values given
+	query := r.URL.Query()
+	discordWebHookId := query["discordWebHookId"][0]
+	sourceId := query["sourceId"][0]
+
+	// Check to make we didnt get a null
+	if discordWebHookId == "" {
+		http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest )
+		return
+	}
+	if sourceId == "" {
+		http.Error(w, "invalid sourceID given", http.StatusBadRequest )
+		return
+	}
+
+	// Valide they are UUID values
+	uHook, err := uuid.Parse(discordWebHookId)
+	if err != nil {
+		http.Error(w, "DiscordWebHooksID was not a uuid value.", http.StatusBadRequest)
+		return
+	}
+	uSource, err := uuid.Parse(sourceId)
+	if err != nil {
+		http.Error(w, "SourceId was not a uuid value", http.StatusBadRequest)
+		return
+	}
+
+	// Check if the sub already exists
+	item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{
+		Discordwebhookid:  uHook,
+		Sourceid: uSource,
+	})
+	if err == nil {
+		bJson, err := json.Marshal(&item)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(bJson)
+		return
+	}
+	
+	// Does not exist, so make it.
+	params := database.CreateSubscriptionParams{
+		ID: uuid.New(),
+		Discordwebhookid: uHook,
+		Sourceid: uSource,
+	}
+	s.Db.CreateSubscription(*s.ctx, params)
+	
+	bJson, err := json.Marshal(&params)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bJson)
+}

services/cache/cache.go

@@ -3,10 +3,10 @@ package cache
 import (
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"github.com/jtom38/newsbot/collector/domain/model"
 )
 
-type CacheClient struct {
+type CacheClient struct{
 	group string
 	DefaultTimer time.Duration
 }
@@ -19,7 +19,7 @@ func NewCacheClient(group string) CacheClient {
 }
 
 func (cc *CacheClient) Insert(key string, value string) {
-	item := domain.CacheItem{
+	item := model.CacheItem{
 		Key: key,
 		Value: value,
 		Group: cc.group,
@@ -29,14 +29,10 @@ func (cc *CacheClient) Insert(key string, value string) {
 	cacheStorage = append(cacheStorage, &item)
 }
 
-func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
+func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
 	for _, item := range cacheStorage {
-		if item.Group != cc.group {
+		if item.Group != cc.group { continue }
-			continue
+		if item.Key != key { continue }
-		}
-		if item.Key != key {
-			continue
-		}
 
 		// if it was tainted, renew the timer and remove the taint as this record was still needed
 		if item.IsTainted {
@@ -46,17 +42,13 @@ func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
 		return item, nil
 	}
 
-	return &domain.CacheItem{}, ErrCacheRecordMissing
+	return &model.CacheItem{}, ErrCacheRecordMissing
 }
 
-func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
+func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
 	for _, item := range cacheStorage {
-		if item.Group != cc.group {
+		if item.Group != cc.group { continue }
-			continue
+		if item.Value != value { continue }
-		}
-		if item.Value != value {
-			continue
-		}
 
 		// if it was tainted, renew the timer and remove the taint as this record was still needed
 		if item.IsTainted {
@@ -65,5 +57,6 @@ func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
 		}
 		return item, nil
 	}
-	return &domain.CacheItem{}, ErrCacheRecordMissing
+	return &model.CacheItem{}, ErrCacheRecordMissing
 }
+

services/cache/cache_test.go

@@ -3,7 +3,7 @@ package cache_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
+	"github.com/jtom38/newsbot/collector/services/cache"
 )
 
 func TestNewCacheClient(t *testing.T) {
@@ -18,18 +18,14 @@ func TestInsert(t *testing.T) {
 func TestFindGroupMissing(t *testing.T) {
 	cache := cache.NewCacheClient("faker")
 	_, err := cache.FindByKey("UnitTesting")
-	if err == nil {
+	if err == nil { panic("Nothing was appended with the requested group.") }
-		panic("Nothing was appended with the requested group.")
-	}
 }
 
 func TestFindGroupExists(t *testing.T) {
 	cache := cache.NewCacheClient("Testing")
 	cache.Insert("UnitTesting", "Something")
 	_, err := cache.FindByKey("UnitTesting")
-	if err != nil {
+	if err != nil { panic("") }
-		panic("")
-	}
 }
 
 func TestCacheStorage(t *testing.T) {
@@ -39,7 +35,6 @@ func TestCacheStorage(t *testing.T) {
 
 	cache := cache.NewCacheClient("Testing")
 	_, err := cache.FindByKey("UnitTesting02")
-	if err != nil {
+	if err != nil { panic("expected to find the value")}
-		panic("expected to find the value")
-	}
 }
+

services/cache/common.go

@@ -3,11 +3,11 @@ package cache
 import (
 	"errors"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"github.com/jtom38/newsbot/collector/domain/model"
 )
 
 var (
-	cacheStorage []*domain.CacheItem
+	cacheStorage []*model.CacheItem
 
 	ErrCacheRecordMissing = errors.New("unable to find the requested record")
 )

services/cache/monitor.go

@@ -3,13 +3,13 @@ package cache
 import (
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"github.com/jtom38/newsbot/collector/domain/model"
 )
 
 // When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
 // If a record is tainted and used again, the taint will be removed and a new Expires value will be set.
 // If its not renewed, it will be dropped.
-type CacheAgeMonitor struct{}
+type CacheAgeMonitor struct {}
 
 func NewCacheAgeMonitor() CacheAgeMonitor {
 	return CacheAgeMonitor{}
@@ -36,12 +36,10 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
 }
 
 // This creates a new slice and skips over the item that needs to be dropped
-func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
+func (cam CacheAgeMonitor) removeEntry(index int) []*model.CacheItem {
-	var temp []*domain.CacheItem
+	var temp []*model.CacheItem
 	for i, item := range cacheStorage {
-		if i != index {
+		if i != index { temp = append(temp, item )}
-			temp = append(temp, item)
-		}
 	}
 	return temp
 }

services/cache/monitor_test.go

@@ -3,7 +3,7 @@ package cache_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
+	"github.com/jtom38/newsbot/collector/services/cache"
 )
 
 func TestCacheTaintItem(t *testing.T) {

services/config/config.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/joho/godotenv"
+)
+
+const (
+	ServerAddress = "SERVER_ADDRESS"
+
+	Sql_Connection_String = "SQL_CONNECTION_STRING"
+
+	FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
+	REDDIT_PULL_TOP               = "REDDIT_PULL_TOP"
+	REDDIT_PULL_HOT               = "REDDIT_PULL_HOT"
+	REDDIT_PULL_NSFW              = "REDDIT_PULL_NSFW"
+
+	FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
+	YOUTUBE_DEBUG                  = "YOUTUBE_DEBUG"
+
+	FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
+	TWITCH_CLIENT_ID              = "TWITCH_CLIENT_ID"
+	TWITCH_CLIENT_SECRET          = "TWITCH_CLIENT_SECRET"
+	TWITCH_MONITOR_CLIPS          = "TWITCH_MONITOR_CLIPS"
+	TWITCH_MONITOR_VOD            = "TWITCH_MONITOR_VOD"
+
+	FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
+)
+
+type ConfigClient struct{}
+
+func New() ConfigClient {
+	c := ConfigClient{}
+	c.RefreshEnv()
+
+	return c
+}
+
+func (cc *ConfigClient) GetConfig(key string) string {
+	res, filled := os.LookupEnv(key)
+	if !filled {
+		log.Printf("Missing the a value for '%v'.  Could generate errors.", key)
+	}
+	return res
+}
+
+func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
+	cc.RefreshEnv()
+
+	res, filled := os.LookupEnv(flag)
+	if !filled {
+		errorMessage := fmt.Sprintf("'%v' was not found", flag)
+		return false, errors.New(errorMessage)
+	}
+
+	b, err := strconv.ParseBool(res)
+	if err != nil {
+		return false, err
+	}
+	return b, nil
+}
+
+// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
+func (cc *ConfigClient) RefreshEnv() {
+	// Check to see if we have the env file on the system
+	_, err := os.Stat(".env")
+
+	// We have the file, load it.
+	if err == nil {
+		_, err := os.Open(".env")
+		if err == nil {
+			loadEnvFile()
+		}
+	}
+}
+
+func loadEnvFile() {
+	err := godotenv.Load()
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

services/config/config_test.go

@@ -0,0 +1,20 @@
+package config_test
+
+import (
+	"testing"
+	"os"
+
+	"github.com/jtom38/newsbot/collector/services/config"
+)
+
+func TestNewClient(t *testing.T) {
+	config.New()
+}
+
+func TestGetConfigExpectNull(t *testing.T) {
+	cc := config.New()
+	os.Setenv(config.REDDIT_PULL_HOT, "")
+	res := cc.GetConfig(config.REDDIT_PULL_HOT)
+	if res != "" { panic("expected blank")}
+
+}

services/cron/scheduler.go

@@ -0,0 +1,314 @@
+package cron
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/google/uuid"
+	_ "github.com/lib/pq"
+	"github.com/robfig/cron/v3"
+
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services/config"
+	"github.com/jtom38/newsbot/collector/services/input"
+	"github.com/jtom38/newsbot/collector/services/output"
+)
+
+type Cron struct {
+	Db    *database.Queries
+	ctx   *context.Context
+	timer *cron.Cron
+}
+
+func openDatabase() (*database.Queries, error) {
+	_env := config.New()
+	connString := _env.GetConfig(config.Sql_Connection_String)
+	if connString == "" {
+		panic("Connection String is null!")
+	}
+	db, err := sql.Open("postgres", connString)
+	if err != nil {
+		panic(err)
+	}
+
+	queries := database.New(db)
+	return queries, err
+}
+
+func New(ctx context.Context) *Cron {
+	c := &Cron{
+		ctx: &ctx,
+	}
+
+	timer := cron.New()
+	queries, err := openDatabase()
+	if err != nil {
+		panic(err)
+	}
+	c.Db = queries
+
+	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
+	features := config.New()
+
+	res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
+	if res {
+		timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
+		log.Print("[Input] Reddit backend was enabled")
+		//go c.CheckReddit()
+	}
+
+	res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
+	if res {
+		timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
+		log.Print("[Input] YouTube backend was enabled")
+	}
+
+	res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
+	if res {
+		timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
+		log.Print("[Input] FFXIV backend was enabled")
+	}
+
+	res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
+	if res {
+		timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
+		log.Print("[Input] Twitch backend was enabled")
+	}
+
+	timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
+	log.Print("[Output] Discord Output was enabled")
+
+	c.timer = timer
+	return c
+}
+
+func (c *Cron) Start() {
+	c.timer.Start()
+}
+
+func (c *Cron) Stop() {
+	c.timer.Stop()
+}
+
+// This is the main entry point to query all the reddit services
+func (c *Cron) CheckReddit() {
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
+	if err != nil {
+		log.Printf("[Reddit] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		log.Printf("[Reddit] Checking '%v'...", source.Name)
+		rc := input.NewRedditClient(source)
+		raw, err := rc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		redditArticles := rc.ConvertToArticles(raw)
+		c.checkPosts(redditArticles, "Reddit")
+	}
+	log.Print("[Reddit] Done!")
+}
+
+func (c *Cron) CheckYoutube() {
+	// Add call to the db to request youtube sources.
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
+	if err != nil {
+		log.Printf("[Youtube] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		log.Printf("[YouTube] Checking '%v'...", source.Name)
+		yc := input.NewYoutubeClient(source)
+		raw, err := yc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		c.checkPosts(raw, "YouTube")
+	}
+	log.Print("[YouTube] Done!")
+}
+
+func (c *Cron) CheckFfxiv() {
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
+	if err != nil {
+		log.Printf("[FFXIV] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		fc := input.NewFFXIVClient(source)
+		items, err := fc.CheckSource()
+		if err != nil {
+			log.Println(err)
+		}
+		c.checkPosts(items, "FFXIV")
+	}
+	log.Printf("[FFXIV Done!]")
+}
+
+func (c *Cron) CheckTwitch() error {
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
+	if err != nil {
+		log.Printf("[Twitch] No sources found to query - %v\r", err)
+	}
+
+	tc, err := input.NewTwitchClient()
+	if err != nil {
+		return err
+	}
+
+	err = tc.Login()
+	if err != nil {
+		return err
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		log.Printf("[Twitch] Checking '%v'...", source.Name)
+		tc.ReplaceSourceRecord(source)
+		items, err := tc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		c.checkPosts(items, "Twitch")
+	}
+
+	log.Print("[Twitch] Done!")
+	return nil
+}
+
+func (c *Cron) CheckDiscordQueue() error {
+	// Get items from the table
+	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
+	if err != nil {
+		return err
+	}
+
+	for _, queue := range queueItems {
+		// Get the articleByID
+		article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
+		if err != nil {
+			return err
+		}
+
+		var endpoints []string
+		// List Subscription by SourceID
+		subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
+		if err != nil {
+			return err
+		}
+
+		// if no one is subscribed to it, remove it from the index.
+		if len(subs) == 0 {
+			log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
+			err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// Get the webhhooks to send to
+		for _, sub := range subs {
+			webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
+			if err != nil {
+				return err
+			}
+
+			// store them in an array
+			endpoints = append(endpoints, webhook.Url)
+		}
+
+		// Create Discord Message
+		dwh := output.NewDiscordWebHookMessage(article)
+		msg, err := dwh.GeneratePayload()
+		if err != nil {
+			return err
+		}
+
+		// Send Message(s)
+		for _, i := range endpoints {
+			err = dwh.SendPayload(msg, i)
+
+			if err != nil {
+				return err
+			}
+		}
+
+		// Remove the item from the queue, given we sent our notification.
+		err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
+		if err != nil {
+			return err
+		}
+
+		time.Sleep(10 * time.Second)
+	}
+
+	return nil
+}
+
+func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
+	for _, item := range posts {
+		_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
+		if err != nil {
+			id := uuid.New()
+
+			err := c.postArticle(id, item)
+			if err != nil {
+				return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
+			}
+
+			err = c.addToDiscordQueue(id)
+			if err != nil {
+				return err
+			}
+
+		}
+	}
+	time.Sleep(30 * time.Second)
+	return nil
+}
+
+func (c *Cron) postArticle(id uuid.UUID,item database.Article) error {
+	err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
+		ID:          id,
+		Sourceid:    item.Sourceid,
+		Tags:        item.Tags,
+		Title:       item.Title,
+		Url:         item.Url,
+		Pubdate:     item.Pubdate,
+		Video:       item.Video,
+		Videoheight: item.Videoheight,
+		Videowidth:  item.Videowidth,
+		Thumbnail:   item.Thumbnail,
+		Description: item.Description,
+		Authorname:  item.Authorname,
+		Authorimage: item.Authorimage,
+	})
+	return err
+}
+
+func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
+	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
+		ID: uuid.New(),
+		Articleid: Id,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}

services/cron/scheduler_test.go

@@ -0,0 +1,34 @@
+package cron_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/jtom38/newsbot/collector/services/cron"
+)
+
+func TestInvokeTwitch(t *testing.T) {
+
+}
+
+// TODO add database mocks but not sure how to do that yet.
+func TestCheckReddit(t *testing.T) {
+	ctx := context.Background()
+	c := cron.New(ctx)
+	c.CheckReddit()
+}
+
+func TestCheckYouTube(t *testing.T) {
+	ctx := context.Background()
+	c := cron.New(ctx)
+	c.CheckYoutube()
+}
+
+func TestCheckTwitch(t *testing.T) {
+	ctx := context.Background()
+	c := cron.New(ctx)
+	err := c.CheckTwitch()
+	if err != nil {
+		t.Error(err)
+	}
+}

services/input/ffxiv.go

@@ -1,6 +1,7 @@
 package input
 
 import (
+	"database/sql"
 	"errors"
 	"log"
 	"net/http"
@@ -12,8 +13,8 @@ import (
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/google/uuid"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/jtom38/newsbot/collector/database"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
+	"github.com/jtom38/newsbot/collector/services/cache"
 )
 
 const (
@@ -24,7 +25,7 @@ const (
 )
 
 type FFXIVClient struct {
-	record entity.SourceEntity
+	record database.Source
 	//SourceID uint
 	//Url string
 	//Region string
@@ -32,80 +33,65 @@ type FFXIVClient struct {
 	cacheGroup string
 }
 
-func NewFFXIVClient(Record entity.SourceEntity) FFXIVClient {
+func NewFFXIVClient(Record database.Source) FFXIVClient {
 	return FFXIVClient{
 		record: Record,
 		cacheGroup: "ffxiv",
 	}
 }
 
-func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
+func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
-	var articles []entity.ArticleEntity
+	var articles []database.Article
 
 	parser := fc.GetBrowser()
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { return articles, err }
-		return articles, err
-	}
 
 	cache := cache.NewCacheClient(fc.cacheGroup)
 
 	for _, link := range links {
 		// Check cache/db if this link has been seen already, skip
 		_, err := cache.FindByValue(link)
-		if err == nil {
+		if err == nil { continue }
-			continue
+		
-		}
 
 		page := fc.GetPage(parser, link)
 
 		title, err := fc.ExtractTitle(page)
-		if err != nil {
+		if err != nil { return articles, err }
-			return articles, err
-		}
 
 		thumb, err := fc.ExtractThumbnail(page)
-		if err != nil {
+		if err != nil { return articles, err }
-			return articles, err
-		}
 
 		pubDate, err := fc.ExtractPubDate(page)
-		if err != nil {
+		if err != nil { return articles, err }
-			return articles, err
-		}
 
 		description, err := fc.ExtractDescription(page)
-		if err != nil {
+		if err != nil { return articles, err }
-			return articles, err
-		}
 
 		authorName, err := fc.ExtractAuthor(page)
-		if err != nil {
+		if err != nil { return articles, err }
-			return articles, err
-		}
 
 		authorImage, err := fc.ExtractAuthorImage(page)
-		if err != nil {
+		if err != nil { return articles, err }
-			return articles, err
-		}
 
 		tags, err := fc.ExtractTags(page)
-		if err != nil {
+		if err != nil { return articles, err } 
-			return articles, err
-		}
 
-		article := entity.ArticleEntity{
+		article := database.Article{
-			SourceID:       fc.record.ID,
+			Sourceid: fc.record.ID,
 			Tags: tags,
 			Title: title,
 			Url: link,
-			PubDate:        pubDate,
+			Pubdate: pubDate,
+			Videoheight: 0,
+			Videowidth: 0,
 			Thumbnail: thumb,
 			Description: description,
-			AuthorName:     authorName,
+			Authorname: sql.NullString{String: authorName},
-			AuthorImageUrl: authorImage,
+			Authorimage: sql.NullString{String: authorImage},
 		}
 		log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
 
@@ -119,19 +105,15 @@ func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
 
 func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
 	html, err := http.Get(fc.record.Url)
-	if err != nil {
+	if err != nil { return nil, err }
-		return nil, err
-	}
 	defer html.Body.Close()
 	
 	doc, err := goquery.NewDocumentFromReader(html.Body)
-	if err != nil {
+	if err != nil { return nil, err }
-		return nil, err
-	}
 	return doc, nil
 }
 
-func (fc *FFXIVClient) GetBrowser() *rod.Browser {
+func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
 	var browser *rod.Browser
 	if path, exists := launcher.LookPath(); exists {
 		u := launcher.New().Bin(path).MustLaunch()
@@ -184,9 +166,7 @@ func (rc *FFXIVClient) GetPage(parser *rod.Browser, url string) *rod.Page {
 
 func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
 	thumbnail := page.MustElementX("/html/body/div[3]/div[2]/div[1]/article/div[1]/img").MustProperty("src").String()
-	if thumbnail == "" {
+	if thumbnail == "" { return "", errors.New("unable to find thumbnail")}
-		return "", errors.New("unable to find thumbnail")
-	}
 	
 	title := page.MustElement(".news__header > h1:nth-child(2)").MustText()
 	log.Println(title)
@@ -196,23 +176,17 @@ func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
 
 func (fc *FFXIVClient) ExtractPubDate(page *rod.Page) (time.Time, error) {
 	stringDate := page.MustElement(".news__ic--topics").MustText()
-	if stringDate == "" {
+	if stringDate == "" { return time.Now(), errors.New("unable to locate the publish date on the post")}
-		return time.Now(), errors.New("unable to locate the publish date on the post")
-	}
 
 	PubDate, err := time.Parse(FFXIV_TIME_FORMAT, stringDate)
-	if err != nil {
+	if err != nil { return time.Now(), err }
-		return time.Now(), err
-	}
 
 	return PubDate, nil
 }
 
 func (fc *FFXIVClient) ExtractDescription(page *rod.Page) (string, error) {
 	res := page.MustElement(".news__detail__wrapper").MustText()
-	if res == "" {
+	if res == "" { return "", errors.New("unable to locate the description on the post")}
-		return "", errors.New("unable to locate the description on the post")
-	}
 
 	return res, nil
 }
@@ -221,17 +195,11 @@ func (fc *FFXIVClient) ExtractAuthor(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > meta")
 	for _, item := range meta {
 		name, err := item.Property("name")
-		if err != nil {
+		if err != nil { return "", err }
-			return "", err
-		}
 
-		if name.String() != "author" {
+		if name.String() != "author" { continue }
-			continue
-		}
 		content, err := item.Property("content")
-		if err != nil {
+		if err != nil { return "", err }
-			return "", err
-		}
 		
 		return content.String(), nil
 	}
@@ -243,17 +211,11 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > meta")
 	for _, item := range meta {
 		name, err := item.Property("name")
-		if err != nil {
+		if err != nil { return "", err }
-			return "", err
-		}
 
-		if name.String() != "keywords" {
+		if name.String() != "keywords" { continue }
-			continue
-		}
 		content, err := item.Property("content")
-		if err != nil {
+		if err != nil { return "", err }
-			return "", err
-		}
 		
 		return content.String(), nil
 	}
@@ -263,18 +225,12 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
 
 func (fc *FFXIVClient) ExtractTitle(page *rod.Page) (string, error) {
 	title, err := page.MustElement("head > title").Text()
-	if err != nil {
+	if err != nil { return "", err }
-		return "", err
-	}
 
-	if !strings.Contains(title, "|") {
+	if !strings.Contains(title, "|") { return "", errors.New("unable to split the title, missing | in the string")}
-		return "", errors.New("unable to split the title, missing | in the string")
-	}
 
 	res := strings.Split(title, "|")
-	if title != "" {
+	if title != "" { return res[0], nil }
-		return res[0], nil
-	}
 	
 	//log.Println(meta)
 	return "", errors.New("unable to find the author on the page")
@@ -284,20 +240,15 @@ func (fc *FFXIVClient) ExtractAuthorImage(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > link")
 	for _, item := range meta {
 		name, err := item.Property("rel")
-		if err != nil {
+		if err != nil { return "", err }
-			return "", err
-		}
 
-		if name.String() != "apple-touch-icon-precomposed" {
+		if name.String() != "apple-touch-icon-precomposed" { continue }
-			continue
-		}
 		content, err := item.Property("href")
-		if err != nil {
+		if err != nil { return "", err }
-			return "", err
-		}
 		
 		return content.String(), nil
 	}
 	//log.Println(meta)
 	return "", errors.New("unable to find the author image on the page")
 }
+

services/input/ffxiv_test.go

@@ -3,15 +3,16 @@ package input_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"github.com/google/uuid"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/jtom38/newsbot/collector/database"
-	ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	ffxiv "github.com/jtom38/newsbot/collector/services/input"
 )
 
-var FFXIVRecord entity.SourceEntity = entity.SourceEntity{
+var FFXIVRecord database.Source = database.Source{
-	ID:          9999,
+	ID: uuid.New(),
-	DisplayName: "Final Fantasy XIV - NA",
+	Site: "ffxiv",
-	Source:      domain.SourceCollectorFfxiv,
+	Name: "Final Fantasy XIV - NA",
+	Source: "ffxiv",
 	Url: "https://na.finalfantasyxiv.com/lodestone/",
 	Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
 }
@@ -19,9 +20,7 @@ var FFXIVRecord entity.SourceEntity = entity.SourceEntity{
 func TestFfxivGetParser(t *testing.T) {
 	fc := ffxiv.NewFFXIVClient(FFXIVRecord)
 	_, err := fc.GetParser()
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 }
 
 func TestFfxivPullFeed(t *testing.T) {
@@ -31,12 +30,8 @@ func TestFfxivPullFeed(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
+	if len(links) == 0 { panic("expected links to come back but got 0") }
-	}
-	if len(links) == 0 {
-		t.Error("expected links to come back but got 0")
-	}
 
 }
 
@@ -47,20 +42,14 @@ func TestFfxivExtractThumbnail(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	thumb, err := fc.ExtractThumbnail(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
+	if thumb == "" { panic("expected a link but got nothing.")}
-	}
-	if thumb == "" {
-		t.Error("expected a link but got nothing.")
-	}
 }
 
 func TestFfxivExtractPubDate(t *testing.T) {
@@ -70,17 +59,13 @@ func TestFfxivExtractPubDate(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	_, err = fc.ExtractPubDate(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 }
 
 func TestFfxivExtractDescription(t *testing.T) {
@@ -90,17 +75,13 @@ func TestFfxivExtractDescription(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	_, err = fc.ExtractDescription(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 }
 
 func TestFfxivExtractAuthor(t *testing.T) {
@@ -110,20 +91,14 @@ func TestFfxivExtractAuthor(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	author, err := fc.ExtractAuthor(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
+	if author == "" { panic("failed to locate the author name") }
-	}
-	if author == "" {
-		t.Error("failed to locate the author name")
-	}
 }
 
 func TestFfxivExtractTags(t *testing.T) {
@@ -133,20 +108,14 @@ func TestFfxivExtractTags(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractTags(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
+	if res == "" {panic("failed to locate the tags")}
-	}
-	if res == "" {
-		t.Error("failed to locate the tags")
-	}
 }
 
 func TestFfxivExtractTitle(t *testing.T) {
@@ -156,20 +125,14 @@ func TestFfxivExtractTitle(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractTitle(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
+	if res == "" { panic("failed to locate the tags") }
-	}
-	if res == "" {
-		t.Error("failed to locate the tags")
-	}
 }
 
 func TestFFxivExtractAuthorIamge(t *testing.T) {
@@ -179,20 +142,14 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
-	}
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractAuthorImage(page)
-	if err != nil {
+	if err != nil { panic(err) }
-		t.Error(err)
+	if res == "" { panic("failed to locate the tags") }
-	}
-	if res == "" {
-		t.Error("failed to locate the tags")
-	}
 }
 
 func TestFfxivCheckSource(t *testing.T) {

services/input/httpClient.go

@@ -2,7 +2,7 @@ package input
 
 import (
 	"crypto/tls"
-	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 )
@@ -21,24 +21,18 @@ func getHttpContent(uri string) ([]byte, error) {
 	}
 	
 	req, err := http.NewRequest("GET", uri, nil)
-	if err != nil {
+	if err != nil { return nil, err }
-		return nil, err
-	}
 
 	// set the user agent header to avoid kick backs.. as much
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
 
 	log.Printf("Requesting content from %v\n", uri)
 	resp, err := client.Do(req)
-	if err != nil {
+	if err != nil { log.Fatalln(err) }
-		log.Fatalln(err)
-	}
 	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp.Body)
+	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
+	if err != nil { return nil, err }
-		return nil, err
-	}
 
 	return body, nil
 }

services/input/reddit.go

@@ -1,6 +1,7 @@
 package input
 
 import (
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -8,16 +9,16 @@ import (
 	"strings"
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/launcher"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/domain/model"
+	"github.com/jtom38/newsbot/collector/services/config"
 )
 
 type RedditClient struct {
 	config RedditConfig
-	record entity.SourceEntity
+	record database.Source
 }
 
 type RedditConfig struct {
@@ -26,14 +27,14 @@ type RedditConfig struct {
 	PullNSFW string
 }
 
-func NewRedditClient(Record entity.SourceEntity) *RedditClient {
+func NewRedditClient(Record database.Source) *RedditClient {
 	rc := RedditClient{
 		record: Record,
 	}
-	cc := services.NewConfig()
+	cc := config.New()
-	rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
+	rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
-	rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
+	rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
-	rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
+	rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
 
 	//rc.disableHttp2Client()
 
@@ -46,6 +47,7 @@ func NewRedditClient(Record entity.SourceEntity) *RedditClient {
 //	os.Setenv("GODEBUG", "http2client=0")
 //}
 
+
 func (rc *RedditClient) GetBrowser() *rod.Browser {
 	var browser *rod.Browser
 	if path, exists := launcher.LookPath(); exists {
@@ -64,13 +66,15 @@ func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
 
 // GetContent() reaches out to Reddit and pulls the Json data.
 // It will then convert the data to a struct and return the struct.
-func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
+func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
-	var items domain.RedditJsonContent = domain.RedditJsonContent{}
+	var items model.RedditJsonContent = model.RedditJsonContent{}
 
 	// TODO Wire this to support the config options
 	Url := fmt.Sprintf("%v.json", rc.record.Url)
 
-	log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
+	log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
+
+
 
 	content, err := getHttpContent(Url)
 	if err != nil {
@@ -87,10 +91,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
 	return items, nil
 }
 
-func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []entity.ArticleEntity {
+func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
-	var redditArticles []entity.ArticleEntity
+	var redditArticles []database.Article
 	for _, item := range items.Data.Children {
-		var article entity.ArticleEntity
+		var article database.Article
 		article, err := rc.convertToArticle(item.Data)
 		if err != nil {
 			log.Printf("[Reddit] %v", err)
@@ -103,8 +107,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []enti
 
 // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
 // This data can be passed to the database.
-func (rc *RedditClient) convertToArticle(source domain.RedditPost) (entity.ArticleEntity, error) {
+func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
-	var item entity.ArticleEntity
+	var item database.Article
 
 	if source.Content == "" && source.Url != "" {
 		item = rc.convertPicturePost(source)
@@ -130,57 +134,65 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (entity.Artic
 	return item, nil
 }
 
-func (rc *RedditClient) convertPicturePost(source domain.RedditPost) entity.ArticleEntity {
+func (rc *RedditClient) convertPicturePost(source model.RedditPost) database.Article {
-	var item = entity.ArticleEntity{
+	var item = database.Article{
-		SourceID:       rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Title:       source.Title,
 		Tags:        fmt.Sprintf("%v", rc.record.Tags),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		PubDate:        time.Now(),
+		Pubdate:     time.Now(),
-		IsVideo:        false,
+		Video:       sql.NullString{String: "null"},
+		Videoheight: 0,
+		Videowidth:  0,
 		Thumbnail:   source.Thumbnail,
 		Description: source.Content,
-		AuthorName:     source.Author,
+		Authorname:  sql.NullString{String: source.Author},
-		AuthorImageUrl: "null",
+		Authorimage: sql.NullString{String: "null"},
 	}
 	return item
 }
 
-func (rc *RedditClient) convertTextPost(source domain.RedditPost) entity.ArticleEntity {
+func (rc *RedditClient) convertTextPost(source model.RedditPost) database.Article {
-	var item = entity.ArticleEntity{
+	var item = database.Article{
-		SourceID:    rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		PubDate:     time.Now(),
+		Pubdate:     time.Now(),
+		Videoheight: 0,
+		Videowidth:  0,
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
+		Authorname:  sql.NullString{String: source.Author},
 		Description: source.Content,
 	}
 	return item
 }
 
-func (rc *RedditClient) convertVideoPost(source domain.RedditPost) entity.ArticleEntity {
+func (rc *RedditClient) convertVideoPost(source model.RedditPost) database.Article {
-	var item = entity.ArticleEntity{
+	var item = database.Article{
-		SourceID:    rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		PubDate:     time.Now(),
+		Pubdate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
+		Videoheight: 0,
+		Videowidth:  0,
+		Authorname:  sql.NullString{String: source.Author},
 		Description: source.Media.RedditVideo.FallBackUrl,
 	}
 	return item
 }
 
 // This post is nothing more then a redirect to another location.
-func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) entity.ArticleEntity {
+func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
-	var item = entity.ArticleEntity{
+	var item = database.Article{
-		SourceID:    rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		PubDate:     time.Now(),
+		Pubdate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
+		Videoheight: 0,
+		Videowidth:  0,
+		Authorname:  sql.NullString{String: source.Author},
 		Description: source.UrlOverriddenByDest,
 	}
 	return item

services/input/reddit_test.go

@@ -0,0 +1,33 @@
+package input_test
+
+import (
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services/input"
+)
+
+var RedditRecord database.Source = database.Source{
+	ID: uuid.New(),
+	Name: "dadjokes",
+	Source: "reddit",
+	Site: "reddit",
+	Url: "https://reddit.com/r/dadjokes",
+	Tags: "reddit, dadjokes",
+}
+
+func TestGetContent(t *testing.T) {
+	//This test is flaky right now due to the http changes in 1.17
+	rc := input.NewRedditClient(RedditRecord)
+	raw, err := rc.GetContent()
+	if err != nil {
+		t.Error(err)
+	}
+	redditArticles := rc.ConvertToArticles(raw)
+	for _, posts := range redditArticles {
+		if posts.Title == "" {
+			t.Error("Title is missing")
+		}
+	}
+}

services/input/rss.go

@@ -0,0 +1,55 @@
+package input
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/jtom38/newsbot/collector/domain/model"
+	"github.com/jtom38/newsbot/collector/services/cache"
+	"github.com/mmcdole/gofeed"
+)
+
+type rssClient struct {
+	SourceRecord model.Sources
+}
+
+func NewRssClient(sourceRecord model.Sources) rssClient {
+	client := rssClient{
+		SourceRecord: sourceRecord,
+	}
+
+	return client
+}
+
+//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
+	//rc.SourceRecord = source
+//}
+
+func (rc rssClient) getCacheGroup() string {
+	return fmt.Sprintf("rss-%v", rc.SourceRecord.Name)
+}
+
+func (rc rssClient) GetContent() error {
+	feed, err := rc.PullFeed()
+	if err != nil { return err }
+
+	cacheClient := cache.NewCacheClient(rc.getCacheGroup())
+
+	for _, item := range feed.Items {
+		log.Println(item)
+
+		cacheClient.FindByValue(item.Link)
+
+	}
+
+	return nil
+}
+
+func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
+	feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
+	fp := gofeed.NewParser()
+	feed, err := fp.ParseURL(feedUri)
+	if err != nil { return nil, err }
+
+	return feed, nil
+}

services/input/rss_test.go

@@ -0,0 +1,26 @@
+package input_test
+
+import (
+	"testing"
+
+	"github.com/jtom38/newsbot/collector/domain/model"
+	"github.com/jtom38/newsbot/collector/services/input"
+)
+
+var rssRecord = model.Sources {
+	ID: 1,
+	Name: "ArsTechnica",
+	Url: "https://feeds.arstechnica.com/arstechnica/index",
+}
+
+func TestRssClientConstructor(t *testing.T) {
+	input.NewRssClient(rssRecord)
+}
+
+func TestRssGetFeed(t *testing.T) {
+	client := input.NewRssClient(rssRecord)
+	feed, err := client.PullFeed()
+	if err != nil { t.Error(err) }
+	if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")}
+
+}

services/input/twitch.go

@@ -1,18 +1,19 @@
 package input
 
 import (
+	"database/sql"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/jtom38/newsbot/collector/database"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+	"github.com/jtom38/newsbot/collector/services/config"
 	"github.com/nicklaw5/helix/v2"
 )
 
 type TwitchClient struct {
-	SourceRecord entity.SourceEntity
+	SourceRecord database.Source
 
 	// config
 	monitorClips string
@@ -30,14 +31,14 @@ var (
 )
 
 func NewTwitchClient() (TwitchClient, error) {
-	c := services.NewConfig()
+	c := config.New()
 
-	id := c.GetConfig(services.TWITCH_CLIENT_ID)
+	id := c.GetConfig(config.TWITCH_CLIENT_ID)
 	if id == "" {
 		return TwitchClient{}, ErrTwitchClientIdMissing
 	}
 
-	secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
+	secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
 	if secret == "" {
 		return TwitchClient{}, ErrTwitchClientSecretMissing
 	}
@@ -49,8 +50,8 @@ func NewTwitchClient() (TwitchClient, error) {
 
 	client := TwitchClient{
 		//SourceRecord: &source,
-		monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
+		monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
-		monitorVod:   c.GetConfig(services.TWITCH_MONITOR_VOD),
+		monitorVod:   c.GetConfig(config.TWITCH_MONITOR_VOD),
 		api:          &api,
 	}
 
@@ -71,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
 }
 
 // This will let you replace the bound source record to keep the same session alive.
-func (tc *TwitchClient) ReplaceSourceRecord(source entity.SourceEntity) {
+func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
 	tc.SourceRecord = source
 }
 
@@ -86,8 +87,8 @@ func (tc *TwitchClient) Login() error {
 	return nil
 }
 
-func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
+func (tc *TwitchClient) GetContent() ([]database.Article, error) {
-	var items []entity.ArticleEntity
+	var items []database.Article
 
 	user, err := tc.GetUserDetails()
 	if err != nil {
@@ -100,50 +101,34 @@ func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
 	}
 
 	for _, video := range posts {
-		var article entity.ArticleEntity
+		var article database.Article
 
 		AuthorName, err := tc.ExtractAuthor(video)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
+		article.Authorname = sql.NullString{String: AuthorName}
-		}
-		article.AuthorName = AuthorName
 		
 		Authorimage, err := tc.ExtractAuthorImage(user)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
+		article.Authorimage = sql.NullString{String: Authorimage}
-		}
-		article.AuthorImageUrl = Authorimage
 
 		article.Description, err = tc.ExtractDescription(video)
-		if err != nil {
+		if err != nil {return items, err }
-			return items, err
-		}
 
-		article.PubDate, err = tc.ExtractPubDate(video)
+		article.Pubdate, err = tc.ExtractPubDate(video)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
-		}
 
-		article.SourceID = tc.SourceRecord.ID
+		article.Sourceid = tc.SourceRecord.ID
 		article.Tags, err = tc.ExtractTags(video, user)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
-		}
 
 		article.Thumbnail, err = tc.ExtractThumbnail(video)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
-		}
 
 		article.Title, err = tc.ExtractTitle(video)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
-		}
 		
 		article.Url, err = tc.ExtractUrl(video)
-		if err != nil {
+		if err != nil { return items, err }
-			return items, err
-		}
 
 		items = append(items, article)
 	}
@@ -155,7 +140,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
 	var blank helix.User
 
 	users, err := tc.api.GetUsers(&helix.UsersParams{
-		Logins: []string{tc.SourceRecord.DisplayName},
+		Logins: []string{tc.SourceRecord.Name},
 	})
 	if err != nil {
 		return blank, err
@@ -225,12 +210,8 @@ func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
 
 // Extracts the avatar of the author with some validation.
 func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
-	if user.ProfileImageURL == "" {
+	if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
-		return "", ErrMissingAuthorImage
+	if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
-	}
-	if !strings.Contains(user.ProfileImageURL, "-profile_image-") {
-		return "", ErrInvalidAuthorImage
-	}
 	return user.ProfileImageURL, nil
 }
 
@@ -250,8 +231,6 @@ func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
 }
 
 func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
-	if post.URL == "" {
+	if post.URL == "" { return "", ErrMissingUrl }
-		return "", ErrMissingUrl
-	}
 	return post.URL, nil
 }

services/input/twitch_test.go

@@ -4,21 +4,21 @@ import (
 	"log"
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
+	"github.com/google/uuid"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
+	"github.com/jtom38/newsbot/collector/database"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"github.com/jtom38/newsbot/collector/services/input"
 )
 
-var TwitchSourceRecord = entity.SourceEntity{
+var TwitchSourceRecord = database.Source {
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "nintendo",
+	Name:   "nintendo",
-	Source:      domain.SourceCollectorTwitch,
+	Source: "Twitch",
 }
 
-var TwitchInvalidRecord = entity.SourceEntity{
+var TwitchInvalidRecord = database.Source {
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "EvilNintendo",
+	Name:   "EvilNintendo",
-	Source:      domain.SourceCollectorTwitch,
+	Source: "Twitch",
 }
 
 func TestTwitchLogin(t *testing.T) {
@@ -115,62 +115,40 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
 
 func TestTwitchReturnsThumbnail(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {
+	if err != nil {t.Error(err) }
-		t.Error(err)
-	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	value, err := tc.ExtractThumbnail(posts[0])
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
+	if value == "" { t.Error("uable to parse username") }
-	}
-	if value == "" {
-		t.Error("uable to parse username")
-	}
 }
 
 func TestTwitchReturnsPubDate(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	date, err := tc.ExtractPubDate(posts[0])
 	log.Println(date)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 }
 
 func TestTwitchReturnsDescription(t *testing.T) {
@@ -203,25 +181,17 @@ func TestTwitchReturnsDescription(t *testing.T) {
 
 func TestTwitchReturnsAuthorImage(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {
+	if err != nil {t.Error(err) }
-		t.Error(err)
-	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil {
+	if err != nil {t.Error(err) }
-		t.Error(err)
-	}
 	
 	_, err = tc.ExtractAuthorImage(user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 }
 
 func TestTwitchReturnsTags(t *testing.T) {
@@ -242,14 +212,10 @@ func TestTwitchReturnsTags(t *testing.T) {
 	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	_, err = tc.ExtractTags(posts[0], user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 }
 
 func TestTwitchReturnsTitle(t *testing.T) {
@@ -270,70 +236,42 @@ func TestTwitchReturnsTitle(t *testing.T) {
 	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	res, err := tc.ExtractTitle(posts[0])
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
+	if res == "" { t.Error("expected a filled string but got nil")}
-	}
-	if res == "" {
-		t.Error("expected a filled string but got nil")
-	}
 }
 
 func TestTwitchReturnsUrl(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	user, err := tc.GetUserDetails()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	posts, err := tc.GetPosts(user)
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 
 	res, err := tc.ExtractUrl(posts[0])
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
+	if res == "" { t.Error("expected a filled string but got nil")}
-	}
-	if res == "" {
-		t.Error("expected a filled string but got nil")
-	}
 }
 
 func TestTwitchGetContent(t *testing.T) {
 	tc, err := input.NewTwitchClient()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
 	err = tc.Login()
-	if err != nil {
+	if err != nil { t.Error(err) }
-		t.Error(err)
-	}
 	
 	posts, err := tc.GetContent()
-	if err != nil {
+	if err != nil {t.Error(err) }
-		t.Error(err)
+	if len(posts) == 0 { t.Error("posts came back with 0 posts") }
-	}
+	if len(posts) != 20 { t.Error("expected 20 posts") } 
-	if len(posts) == 0 {
-		t.Error("posts came back with 0 posts")
-	}
-	if len(posts) != 20 {
-		t.Error("expected 20 posts")
-	}
 }