version bump

.drone.yaml

@@ -1,60 +0,0 @@
----
-kind: pipeline
-type: docker
-name: buildLatestImage
-
-steps:
-  - name: buildLatestImage
-    image: plugins/docker
-    settings:
-      repo: jtom38/newsbot-collector
-      username: jtom38
-      password:
-        from_secret: DockerPushPat
-trigger:
-  branch:
-    include:
-      - main
-
-  event:
-    exclude:
-      - pull_request
----
-kind: pipeline
-type: docker
-name: buildReleaseImage
-
-steps:
-  - name: buildReleaseImage
-    image: plugins/docker
-    settings:
-      repo: jtom38/newsbot-collector
-      username: jtom38
-      password:
-        from_secret: DockerPushPat
-trigger:
-  branch:
-    include:
-      - releases/*
-  ref:
-    include:
-      - refs/tags/**
-  event:
-    exclude:
-      - pull_request
-
-
----
-kind: pipeline
-type: docker
-name: PullRequestCompileTest
-steps:
-  - name: Compile project
-    image: golang:1.22
-    commands:
-      - go test ./internal/repository
-      - go build ./cmd/server.go
-      - 
-trigger:
-  event:
-    - pull_request

.github/workflows/docker.build.yaml

@@ -0,0 +1,64 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  #schedule:
+  #  - cron: '21 19 * * *'
+  push:
+    branches: [ master ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  #pull_request:
+  #  branches: [ master ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          #images: ${{ env.REGISTRY }}/newsbot.worker
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/go-build.yml

@@ -0,0 +1,25 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.18
+
+    - name: Build
+      run: go build -v ./...
+
+    #- name: Test
+    #  run: go test -v ./...

.gitignore

@@ -1,9 +1,5 @@
 .env
 dev.session.sql
-__debug_bin
-server
-.vscode
-openapi.json
 
 # Binaries for programs and plugins
 *.exe
@@ -12,13 +8,10 @@ openapi.json
 *.so
 *.dylib
 collector
-newsbot.db
-tmp/
 
 # Test binary, built with `go test -c`
 *.test
 
-
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "."
+        }
+    ]
+}

Dockerfile

@@ -1,23 +1,16 @@
-FROM golang:1.22 as build
+FROM golang:1.18.3 as build
 
 COPY . /app
 WORKDIR /app
+RUN go build .
+RUN go install github.com/pressly/goose/v3/cmd/goose@latest
 
-# Always make sure that swagger docs are updated
-RUN go install github.com/swaggo/swag/cmd/swag@latest
-RUN /go/bin/swag init -g cmd/server.go
+FROM alpine
 
-#RUN go build .
-#RUN go install github.com/pressly/goose/v3/cmd/goose@latest
+RUN mkdir /app && \
+    mkdir /app/migrations
+COPY --from=build /app/collector /app
+COPY --from=build /go/bin/goose /app
+COPY ./database/migrations/ /app/migrations
 
-FROM alpine:latest as app
-
-RUN apk --no-cache add bash 
-RUN apk --no-cache add libc6-compat
-RUN apk --no-cache add chromium
-
-RUN mkdir /app && mkdir /app/migrations
-COPY --from=build /app/server /app
-COPY ./internal/database/migrations/ /app/migrations
-
-CMD [ "/app/collector" ]
+ENTRYPOINT [ "/app/collector" ]

README.md

@@ -1,25 +1,2 @@
 # newsbot.collector.api
-
 This is the collection service of newsbot to pull articles from the web.
-
-## Deployment
-
-1. Create a copy of the docker compose file and make it local
-2. Update the `docker-compose.yaml` with your secrets
-3. Run migrations
-   2. `docker compose run api /app/goose -dir "/app/migrations" up`
-4. Run app
-   1. `docker compose up -d`
-5. Once the app is running go to the swagger page and validate that you see the seeded sources.
-   1. `http://localhost:8081/swagger/index.html#/Source/get_config_sources`
-   2. `curl -X 'GET' 'http://localhost:8081/api/config/sources' -H 'accept: application/json'`
-6. Add any new sources
-7. Add a Discord Web Hook
-   1. `curl -X 'POST' 'http://localhost:8081/api/discord/webhooks/new?url=WEBHOOKURL&server=SERVERNAME&channel=CHANNELNAME' -H 'accept: application/json' -d ''`
-8. Create your subscription links
-   1. This is a link between a source and a discord web hook.  Without this, the app will not send a notification about new posts.
-
-### Errors
-
-- pq: permission denied to create extension "uuid-ossp"
-  - Might need to grant your account `ALTER USER root WITH SUPERUSER;` to create the 'uuid-ossp' for uuid creations

api.http

@@ -1,6 +0,0 @@
-### Select Sources fro mthe top
-GET http://localhost:8081/api/v1/sources/
-
-
-### Select Sources by type
-GET http://localhost:8081/api/v1/sources/by/source?source=rss

api/client.yaml

@@ -1,6 +0,0 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/deepmap/oapi-codegen/HEAD/configuration-schema.json
-package: api
-output: api.gen.go
-generate:
-  models: true
-  client: true

cmd/server.go

@@ -1,90 +0,0 @@
-package main
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"os"
-
-	_ "github.com/glebarez/go-sqlite"
-	"github.com/pressly/goose/v3"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/docs"
-	v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
-)
-
-//  @title                       NewsBot collector
-//  @version                     0.1
-//  @BasePath                    /api
-//  @securityDefinitions.apikey  Bearer
-//  @in                          header
-//  @name                        Authorization
-//  @description                 Type "Bearer" followed by a space and JWT token.
-func main() {
-	ctx := context.Background()
-
-	cfg := services.NewConfig()
-	configs := services.GetEnvConfig()
-	address := cfg.GetConfig(services.ServerAddress)
-	docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
-
-	db, err := sql.Open("sqlite", "newsbot.db")
-	if err != nil {
-		panic(err)
-	}
-
-	err = migrateDatabase(db)
-	if err != nil {
-		fmt.Print(err)
-	}
-
-	c := cron.NewScheduler(ctx, db)
-	c.Start()
-
-	server := v1.NewServer(ctx, configs, db)
-
-	fmt.Println("API is online and waiting for requests.")
-	fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
-	fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress)
-
-	server.Router.Start(":8081")
-}
-
-func migrateDatabase(db *sql.DB) error {
-	err := goose.SetDialect("sqlite3")
-	if err != nil {
-		panic(err)
-	}
-
-	err = goose.Up(db, "../internal/database/migrations")
-	if err != nil {
-		panic(err)
-	}
-
-	_, err = os.Stat("./migrations")
-	if err == nil {
-
-		err = goose.Up(db, "../internal/database/migrations")
-		if err != nil {
-			panic(err)
-		}
-		
-		return nil
-	}
-
-	_, err = os.Stat("../internal/database/migrations")
-	if err == nil {
-
-		err = goose.Up(db, "../internal/database/migrations")
-		if err != nil {
-			panic(err)
-		}
-
-		return nil
-	}
-
-	return errors.New("failed to find the migration files")
-}

database/db.go

@@ -0,0 +1,31 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.13.0
+
+package database
+
+import (
+	"context"
+	"database/sql"
+)
+
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+func New(db DBTX) *Queries {
+	return &Queries{db: db}
+}
+
+type Queries struct {
+	db DBTX
+}
+
+func (q *Queries) WithTx(tx *sql.Tx) *Queries {
+	return &Queries{
+		db: tx,
+	}
+}

database/migrations/20220522083756_init.sql

@@ -0,0 +1,72 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+CREATE TABLE Articles (
+	ID 			uuid PRIMARY KEY,
+	SourceId 	uuid NOT null,
+	Tags 		TEXT NOT NULL,
+	Title 		TEXT NOT NULL,
+	Url 		TEXT NOT NULL,
+	PubDate 	timestamp NOT NULL,
+	Video 		TEXT,
+	VideoHeight int NOT NULL,
+	VideoWidth 	int NOT NULL,
+	Thumbnail 	TEXT NOT NULL,
+	Description TEXT NOT NULL,
+	AuthorName 	TEXT,
+	AuthorImage TEXT
+);
+
+CREATE Table DiscordQueue ( 
+    ID 			uuid PRIMARY KEY,
+    ArticleId 	uuid NOT NULL
+);
+
+CREATE Table DiscordWebHooks (
+	ID 		uuid PRIMARY KEY,
+	Name 	TEXT NOT NULL, -- Defines webhook purpose
+	Key 	TEXT,
+	Url 	TEXT NOT NULL, -- Webhook Url
+	Server 	TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
+	Channel TEXT NOT NULL, -- Defines the channel its bound to.  Used for refrence
+	Enabled BOOLEAN NOT NULL
+);
+
+CREATE Table Icons (
+	ID 			uuid PRIMARY Key,
+	FileName 	TEXT NOT NULL,
+	Site 		TEXT NOT NULL
+);
+
+Create Table Settings (
+	ID 		uuid PRIMARY Key,
+	Key 	TEXT NOT NULL, -- How you search for a entry
+	Value 	TEXT NOT NULL, -- The value for one
+	Options TEXT -- any notes about the entry
+);
+
+Create Table Sources (
+	ID 		uuid PRIMARY Key,
+	Site 	TEXT NOT NULL, -- Vanity name
+	Name 	TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
+	Source 	TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
+	Type 	TEXT NOT NULL, -- Defines what kind of feed this is.  feed, user, tag
+	Value 	TEXT,
+	Enabled BOOLEAN NOT NULL,
+	Url 	TEXT NOT NULL,
+	Tags 	TEXT NOT NULL
+);
+
+
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+Drop Table Articles;
+Drop Table DiscordQueue;
+Drop Table DiscordWebHooks;
+Drop Table Icons;
+Drop Table Settings;
+Drop Table Sources;
+-- +goose StatementEnd

database/migrations/20220529082459_seed.sql

@@ -0,0 +1,50 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query';
+
+-- Enable UUID's
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Final Fantasy XIV Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
+
+-- Reddit Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
+
+-- Youtube Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
+
+-- RSS Entries
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
+
+-- Twitch Entries 
+INSERT INTO sources VALUES 
+(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+--SELECT 'down SQL query';
+
+DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
+DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
+DELETE FROM sources where source = 'ffxiv';
+DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
+DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
+DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
+-- +goose StatementEnd

database/models.go

@@ -0,0 +1,68 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.13.0
+
+package database
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type Article struct {
+	ID          uuid.UUID
+	Sourceid    uuid.UUID
+	Tags        string
+	Title       string
+	Url         string
+	Pubdate     time.Time
+	Video       sql.NullString
+	Videoheight int32
+	Videowidth  int32
+	Thumbnail   string
+	Description string
+	Authorname  sql.NullString
+	Authorimage sql.NullString
+}
+
+type Discordqueue struct {
+	ID        uuid.UUID
+	Articleid uuid.UUID
+}
+
+type Discordwebhook struct {
+	ID      uuid.UUID
+	Name    string
+	Key     sql.NullString
+	Url     string
+	Server  string
+	Channel string
+	Enabled bool
+}
+
+type Icon struct {
+	ID       uuid.UUID
+	Filename string
+	Site     string
+}
+
+type Setting struct {
+	ID      uuid.UUID
+	Key     string
+	Value   string
+	Options sql.NullString
+}
+
+type Source struct {
+	ID      uuid.UUID
+	Site    string
+	Name    string
+	Source  string
+	Type    string
+	Value   sql.NullString
+	Enabled bool
+	Url     string
+	Tags    string
+}

database/query.sql.go

@@ -0,0 +1,521 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.13.0
+// source: query.sql
+
+package database
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const createArticle = `-- name: CreateArticle :exec
+INSERT INTO Articles 
+(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
+Values
+($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+`
+
+type CreateArticleParams struct {
+	ID          uuid.UUID
+	Sourceid    uuid.UUID
+	Tags        string
+	Title       string
+	Url         string
+	Pubdate     time.Time
+	Video       sql.NullString
+	Videoheight int32
+	Videowidth  int32
+	Thumbnail   string
+	Description string
+	Authorname  sql.NullString
+	Authorimage sql.NullString
+}
+
+func (q *Queries) CreateArticle(ctx context.Context, arg CreateArticleParams) error {
+	_, err := q.db.ExecContext(ctx, createArticle,
+		arg.ID,
+		arg.Sourceid,
+		arg.Tags,
+		arg.Title,
+		arg.Url,
+		arg.Pubdate,
+		arg.Video,
+		arg.Videoheight,
+		arg.Videowidth,
+		arg.Thumbnail,
+		arg.Description,
+		arg.Authorname,
+		arg.Authorimage,
+	)
+	return err
+}
+
+const createDiscordQueue = `-- name: CreateDiscordQueue :exec
+Insert into DiscordQueue
+(ID, ArticleId)
+Values
+($1, $2)
+`
+
+type CreateDiscordQueueParams struct {
+	ID        uuid.UUID
+	Articleid uuid.UUID
+}
+
+// DiscordQueue
+func (q *Queries) CreateDiscordQueue(ctx context.Context, arg CreateDiscordQueueParams) error {
+	_, err := q.db.ExecContext(ctx, createDiscordQueue, arg.ID, arg.Articleid)
+	return err
+}
+
+const createDiscordWebHook = `-- name: CreateDiscordWebHook :exec
+Insert Into DiscordWebHooks
+(ID, Name, Key, Url, Server, Channel, Enabled)
+Values
+($1, $2, $3, $4, $5, $6, $7)
+`
+
+type CreateDiscordWebHookParams struct {
+	ID      uuid.UUID
+	Name    string
+	Key     sql.NullString
+	Url     string
+	Server  string
+	Channel string
+	Enabled bool
+}
+
+// DiscordWebHooks
+func (q *Queries) CreateDiscordWebHook(ctx context.Context, arg CreateDiscordWebHookParams) error {
+	_, err := q.db.ExecContext(ctx, createDiscordWebHook,
+		arg.ID,
+		arg.Name,
+		arg.Key,
+		arg.Url,
+		arg.Server,
+		arg.Channel,
+		arg.Enabled,
+	)
+	return err
+}
+
+const createIcon = `-- name: CreateIcon :exec
+
+INSERT INTO Icons
+(ID, FileName, Site)
+VALUES
+($1,$2,$3)
+`
+
+type CreateIconParams struct {
+	ID       uuid.UUID
+	Filename string
+	Site     string
+}
+
+// Icons
+func (q *Queries) CreateIcon(ctx context.Context, arg CreateIconParams) error {
+	_, err := q.db.ExecContext(ctx, createIcon, arg.ID, arg.Filename, arg.Site)
+	return err
+}
+
+const createSettings = `-- name: CreateSettings :one
+
+Insert Into settings
+(ID, Key, Value, OPTIONS)
+Values
+($1,$2,$3,$4)
+RETURNING id, key, value, options
+`
+
+type CreateSettingsParams struct {
+	ID      uuid.UUID
+	Key     string
+	Value   string
+	Options sql.NullString
+}
+
+// Settings
+func (q *Queries) CreateSettings(ctx context.Context, arg CreateSettingsParams) (Setting, error) {
+	row := q.db.QueryRowContext(ctx, createSettings,
+		arg.ID,
+		arg.Key,
+		arg.Value,
+		arg.Options,
+	)
+	var i Setting
+	err := row.Scan(
+		&i.ID,
+		&i.Key,
+		&i.Value,
+		&i.Options,
+	)
+	return i, err
+}
+
+const createSource = `-- name: CreateSource :exec
+
+Insert Into Sources
+(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
+Values
+($1,$2,$3,$4,$5,$6,$7,$8,$9)
+`
+
+type CreateSourceParams struct {
+	ID      uuid.UUID
+	Site    string
+	Name    string
+	Source  string
+	Type    string
+	Value   sql.NullString
+	Enabled bool
+	Url     string
+	Tags    string
+}
+
+// Sources
+func (q *Queries) CreateSource(ctx context.Context, arg CreateSourceParams) error {
+	_, err := q.db.ExecContext(ctx, createSource,
+		arg.ID,
+		arg.Site,
+		arg.Name,
+		arg.Source,
+		arg.Type,
+		arg.Value,
+		arg.Enabled,
+		arg.Url,
+		arg.Tags,
+	)
+	return err
+}
+
+const deleteDiscordQueueItem = `-- name: DeleteDiscordQueueItem :exec
+Delete From DiscordQueue Where ID = $1
+`
+
+func (q *Queries) DeleteDiscordQueueItem(ctx context.Context, id uuid.UUID) error {
+	_, err := q.db.ExecContext(ctx, deleteDiscordQueueItem, id)
+	return err
+}
+
+const deleteDiscordWebHooks = `-- name: DeleteDiscordWebHooks :exec
+Delete From discordwebhooks Where ID = $1
+`
+
+func (q *Queries) DeleteDiscordWebHooks(ctx context.Context, id uuid.UUID) error {
+	_, err := q.db.ExecContext(ctx, deleteDiscordWebHooks, id)
+	return err
+}
+
+const deleteIcon = `-- name: DeleteIcon :exec
+Delete From Icons where ID = $1
+`
+
+func (q *Queries) DeleteIcon(ctx context.Context, id uuid.UUID) error {
+	_, err := q.db.ExecContext(ctx, deleteIcon, id)
+	return err
+}
+
+const deleteSetting = `-- name: DeleteSetting :exec
+Delete From settings Where ID = $1
+`
+
+func (q *Queries) DeleteSetting(ctx context.Context, id uuid.UUID) error {
+	_, err := q.db.ExecContext(ctx, deleteSetting, id)
+	return err
+}
+
+const deleteSource = `-- name: DeleteSource :exec
+DELETE From sources where id = $1
+`
+
+func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
+	_, err := q.db.ExecContext(ctx, deleteSource, id)
+	return err
+}
+
+const getArticleByID = `-- name: GetArticleByID :one
+Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
+WHERE ID = $1 LIMIT 1
+`
+
+// Articles
+func (q *Queries) GetArticleByID(ctx context.Context, id uuid.UUID) (Article, error) {
+	row := q.db.QueryRowContext(ctx, getArticleByID, id)
+	var i Article
+	err := row.Scan(
+		&i.ID,
+		&i.Sourceid,
+		&i.Tags,
+		&i.Title,
+		&i.Url,
+		&i.Pubdate,
+		&i.Video,
+		&i.Videoheight,
+		&i.Videowidth,
+		&i.Thumbnail,
+		&i.Description,
+		&i.Authorname,
+		&i.Authorimage,
+	)
+	return i, err
+}
+
+const getArticleByUrl = `-- name: GetArticleByUrl :one
+Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
+Where Url = $1 LIMIT 1
+`
+
+func (q *Queries) GetArticleByUrl(ctx context.Context, url string) (Article, error) {
+	row := q.db.QueryRowContext(ctx, getArticleByUrl, url)
+	var i Article
+	err := row.Scan(
+		&i.ID,
+		&i.Sourceid,
+		&i.Tags,
+		&i.Title,
+		&i.Url,
+		&i.Pubdate,
+		&i.Video,
+		&i.Videoheight,
+		&i.Videowidth,
+		&i.Thumbnail,
+		&i.Description,
+		&i.Authorname,
+		&i.Authorimage,
+	)
+	return i, err
+}
+
+const getDiscordQueueByID = `-- name: GetDiscordQueueByID :one
+Select id, articleid from DiscordQueue
+Where ID = $1 LIMIT 1
+`
+
+func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discordqueue, error) {
+	row := q.db.QueryRowContext(ctx, getDiscordQueueByID, id)
+	var i Discordqueue
+	err := row.Scan(&i.ID, &i.Articleid)
+	return i, err
+}
+
+const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
+Select id, articleid from DiscordQueue LIMIT $1
+`
+
+func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
+	rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []Discordqueue
+	for rows.Next() {
+		var i Discordqueue
+		if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
+Select id, name, key, url, server, channel, enabled from DiscordWebHooks
+Where ID = $1 LIMIT 1
+`
+
+func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Discordwebhook, error) {
+	row := q.db.QueryRowContext(ctx, getDiscordWebHooksByID, id)
+	var i Discordwebhook
+	err := row.Scan(
+		&i.ID,
+		&i.Name,
+		&i.Key,
+		&i.Url,
+		&i.Server,
+		&i.Channel,
+		&i.Enabled,
+	)
+	return i, err
+}
+
+const getIconByID = `-- name: GetIconByID :one
+Select id, filename, site FROM Icons
+Where ID = $1 Limit 1
+`
+
+func (q *Queries) GetIconByID(ctx context.Context, id uuid.UUID) (Icon, error) {
+	row := q.db.QueryRowContext(ctx, getIconByID, id)
+	var i Icon
+	err := row.Scan(&i.ID, &i.Filename, &i.Site)
+	return i, err
+}
+
+const getIconBySite = `-- name: GetIconBySite :one
+Select id, filename, site FROM Icons
+Where Site = $1 Limit 1
+`
+
+func (q *Queries) GetIconBySite(ctx context.Context, site string) (Icon, error) {
+	row := q.db.QueryRowContext(ctx, getIconBySite, site)
+	var i Icon
+	err := row.Scan(&i.ID, &i.Filename, &i.Site)
+	return i, err
+}
+
+const getSettingByID = `-- name: GetSettingByID :one
+Select id, key, value, options From settings
+Where ID = $1 Limit 1
+`
+
+func (q *Queries) GetSettingByID(ctx context.Context, id uuid.UUID) (Setting, error) {
+	row := q.db.QueryRowContext(ctx, getSettingByID, id)
+	var i Setting
+	err := row.Scan(
+		&i.ID,
+		&i.Key,
+		&i.Value,
+		&i.Options,
+	)
+	return i, err
+}
+
+const getSettingByKey = `-- name: GetSettingByKey :one
+Select id, key, value, options From settings Where 
+Key = $1 Limit 1
+`
+
+func (q *Queries) GetSettingByKey(ctx context.Context, key string) (Setting, error) {
+	row := q.db.QueryRowContext(ctx, getSettingByKey, key)
+	var i Setting
+	err := row.Scan(
+		&i.ID,
+		&i.Key,
+		&i.Value,
+		&i.Options,
+	)
+	return i, err
+}
+
+const getSettingByValue = `-- name: GetSettingByValue :one
+Select id, key, value, options From settings Where 
+Value = $1 Limit 1
+`
+
+func (q *Queries) GetSettingByValue(ctx context.Context, value string) (Setting, error) {
+	row := q.db.QueryRowContext(ctx, getSettingByValue, value)
+	var i Setting
+	err := row.Scan(
+		&i.ID,
+		&i.Key,
+		&i.Value,
+		&i.Options,
+	)
+	return i, err
+}
+
+const getSourceByID = `-- name: GetSourceByID :one
+Select id, site, name, source, type, value, enabled, url, tags From Sources where ID = $1 Limit 1
+`
+
+func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, error) {
+	row := q.db.QueryRowContext(ctx, getSourceByID, id)
+	var i Source
+	err := row.Scan(
+		&i.ID,
+		&i.Site,
+		&i.Name,
+		&i.Source,
+		&i.Type,
+		&i.Value,
+		&i.Enabled,
+		&i.Url,
+		&i.Tags,
+	)
+	return i, err
+}
+
+const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
+Select id, name, key, url, server, channel, enabled From DiscordWebHooks
+Where Server = $1
+`
+
+func (q *Queries) ListDiscordWebHooksByServer(ctx context.Context, server string) ([]Discordwebhook, error) {
+	rows, err := q.db.QueryContext(ctx, listDiscordWebHooksByServer, server)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []Discordwebhook
+	for rows.Next() {
+		var i Discordwebhook
+		if err := rows.Scan(
+			&i.ID,
+			&i.Name,
+			&i.Key,
+			&i.Url,
+			&i.Server,
+			&i.Channel,
+			&i.Enabled,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listSourcesBySource = `-- name: ListSourcesBySource :many
+Select id, site, name, source, type, value, enabled, url, tags From Sources where Source = $1
+`
+
+func (q *Queries) ListSourcesBySource(ctx context.Context, source string) ([]Source, error) {
+	rows, err := q.db.QueryContext(ctx, listSourcesBySource, source)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []Source
+	for rows.Next() {
+		var i Source
+		if err := rows.Scan(
+			&i.ID,
+			&i.Site,
+			&i.Name,
+			&i.Source,
+			&i.Type,
+			&i.Value,
+			&i.Enabled,
+			&i.Url,
+			&i.Tags,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

database/schema/query.sql

@@ -0,0 +1,112 @@
+/* Articles */
+-- name: GetArticleByID :one
+Select * from Articles
+WHERE ID = $1 LIMIT 1;
+
+-- name: GetArticleByUrl :one
+Select * from Articles
+Where Url = $1 LIMIT 1;
+
+-- name: CreateArticle :exec
+INSERT INTO Articles 
+(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
+Values
+($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
+
+
+/* DiscordQueue */
+-- name: CreateDiscordQueue :exec
+Insert into DiscordQueue
+(ID, ArticleId)
+Values
+($1, $2);
+
+-- name: GetDiscordQueueByID :one
+Select * from DiscordQueue
+Where ID = $1 LIMIT 1;
+
+-- name: DeleteDiscordQueueItem :exec
+Delete From DiscordQueue Where ID = $1;
+
+-- name: GetDiscordQueueItems :many
+Select * from DiscordQueue LIMIT $1;
+
+
+/* DiscordWebHooks */
+-- name: CreateDiscordWebHook :exec
+Insert Into DiscordWebHooks
+(ID, Name, Key, Url, Server, Channel, Enabled)
+Values
+($1, $2, $3, $4, $5, $6, $7);
+
+-- name: GetDiscordWebHooksByID :one
+Select * from DiscordWebHooks
+Where ID = $1 LIMIT 1;
+
+-- name: ListDiscordWebHooksByServer :many
+Select * From DiscordWebHooks
+Where Server = $1;
+
+-- name: DeleteDiscordWebHooks :exec
+Delete From discordwebhooks Where ID = $1;
+
+
+/* Icons */
+
+-- name: CreateIcon :exec
+INSERT INTO Icons
+(ID, FileName, Site)
+VALUES
+($1,$2,$3);
+
+-- name: GetIconByID :one
+Select * FROM Icons
+Where ID = $1 Limit 1;
+
+-- name: GetIconBySite :one
+Select * FROM Icons
+Where Site = $1 Limit 1;
+
+-- name: DeleteIcon :exec
+Delete From Icons where ID = $1;
+
+/* Settings */
+
+-- name: CreateSettings :one
+Insert Into settings
+(ID, Key, Value, OPTIONS)
+Values
+($1,$2,$3,$4)
+RETURNING *;
+
+-- name: GetSettingByID :one
+Select * From settings
+Where ID = $1 Limit 1;
+
+-- name: GetSettingByKey :one
+Select * From settings Where 
+Key = $1 Limit 1;
+
+-- name: GetSettingByValue :one
+Select * From settings Where 
+Value = $1 Limit 1;
+
+-- name: DeleteSetting :exec
+Delete From settings Where ID = $1;
+
+/* Sources */
+
+-- name: CreateSource :exec
+Insert Into Sources
+(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
+Values
+($1,$2,$3,$4,$5,$6,$7,$8,$9);
+
+-- name: GetSourceByID :one
+Select * From Sources where ID = $1 Limit 1;
+
+-- name: ListSourcesBySource :many
+Select * From Sources where Source = $1;
+
+-- name: DeleteSource :exec
+DELETE From sources where id = $1;

database/schema/schema.sql

@@ -0,0 +1,56 @@
+CREATE TABLE Articles (
+	ID 			uuid PRIMARY KEY,
+	SourceId 	uuid NOT null,
+	Tags 		TEXT NOT NULL,
+	Title 		TEXT NOT NULL,
+	Url 		TEXT NOT NULL,
+	PubDate 	timestamp NOT NULL,
+	Video 		TEXT,
+	VideoHeight int NOT NULL,
+	VideoWidth 	int NOT NULL,
+	Thumbnail 	TEXT NOT NULL,
+	Description TEXT NOT NULL,
+	AuthorName 	TEXT,
+	AuthorImage TEXT
+);
+
+CREATE Table DiscordQueue ( 
+    ID 			uuid PRIMARY KEY,
+    ArticleId 	uuid NOT NULL
+);
+
+CREATE Table DiscordWebHooks (
+	ID 		uuid PRIMARY KEY,
+	Name 	TEXT NOT NULL, -- Defines webhook purpose
+	Key 	TEXT,
+	Url 	TEXT NOT NULL, -- Webhook Url
+	Server 	TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
+	Channel TEXT NOT NULL, -- Defines the channel its bound to.  Used for refrence
+	Enabled BOOLEAN NOT NULL
+);
+
+CREATE Table Icons (
+	ID 			uuid PRIMARY Key,
+	FileName 	TEXT NOT NULL,
+	Site 		TEXT NOT NULL
+);
+
+Create Table Settings (
+	ID 		uuid PRIMARY Key,
+	Key 	TEXT NOT NULL,
+	Value 	TEXT NOT NULL,
+	Options TEXT
+);
+
+Create Table Sources (
+	ID 		uuid PRIMARY Key,
+	Site 	TEXT NOT NULL, -- Vanity name
+	Name 	TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
+	Source 	TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
+	Type 	TEXT NOT NULL, -- Defines what kind of feed this is.  feed, user, tag
+	Value 	TEXT,
+	Enabled BOOLEAN NOT NULL,
+	Url 	TEXT NOT NULL,
+	Tags 	TEXT NOT NULL
+);
+

databaseRest/articles.go

@@ -0,0 +1,100 @@
+package databaseRest
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/jtom38/newsbot/collector/domain/model"
+)
+
+// Generate this struct fr
+type ArticlesClient struct {
+	rootUri string
+}
+
+func (ac *ArticlesClient) List() ([]model.Articles, error) {
+	var items []model.Articles
+	url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
+	resp, err := getContent(url)
+	if err != nil {
+		return items, err
+	}
+
+	err = json.Unmarshal(resp, &items)
+	if err != nil {
+		return []model.Articles{}, err
+	}
+
+	return items, nil
+}
+
+func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
+	var items model.Articles
+	url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
+	resp, err := getContent(url)
+	if err != nil {
+		return items, err
+	}
+
+	err = json.Unmarshal(resp, &items)
+	if err != nil {
+		return items, err
+	}
+
+	return items, nil
+}
+
+func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
+	var item model.Articles
+	get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
+	resp, err := getContent(get)
+	if err != nil {
+		return item, err
+	}
+
+	err = json.Unmarshal(resp, &item)
+	if err != nil {
+		return item, err
+	}
+
+	return item, nil
+}
+
+func (ac *ArticlesClient) Delete(id int32) error {
+	return errors.New("not implemented")
+}
+
+func (ac *ArticlesClient) Add(item model.Articles) error {
+	//return errors.New("not implemented")
+	url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
+
+	bItem, err := json.Marshal(item)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	
+	if resp.StatusCode != 200 {
+		return errors.New("failed to post to the DB")
+	}
+
+	return nil
+	//body, err := ioutil.ReadAll(resp.Body)
+	//if err != nil { return err }
+
+}

databaseRest/common.go

@@ -0,0 +1,74 @@
+package databaseRest
+
+import (
+	"errors"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/jtom38/newsbot/collector/services/config"
+)
+
+type DatabaseClient struct {
+	Diagnosis DiagnosisClient
+
+	Articles ArticlesClient
+	Sources SourcesClient
+}
+
+// This will generate a new client to interface with the API Database.
+func NewDatabaseClient() DatabaseClient {
+	cc := config.New()
+	dbUri := cc.GetConfig(config.DB_URI)
+
+	var client = DatabaseClient{}
+	client.Diagnosis.rootUri = dbUri
+	client.Sources.rootUri = dbUri
+	client.Articles.rootUri = dbUri
+
+	return client
+}
+
+func getContent(url string) ([]byte, error) {
+	client := &http.Client{}
+	var blank []byte
+	
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil { return blank, err }
+
+	// set the user agent header to avoid kick backs.. as much
+	req.Header.Set("User-Agent", getUserAgent())
+
+	log.Printf("Requesting content from %v\n", url)
+	resp, err := client.Do(req)
+	if err != nil { return blank, err }
+	if resp.StatusCode == 404 { 
+		err = errors.New("404 not found")
+		return blank, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil { return blank, err }
+
+	//log.Println(string(body))
+	return body, nil
+}
+
+func httpDelete(url string) error {
+	client := &http.Client{}
+	req, err := http.NewRequest("DELETE", url, nil)
+	if err != nil { return err }
+
+	//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
+	req.Header.Set("User-Agent", getUserAgent())
+
+	_, err = client.Do(req)
+	if err != nil { return err }
+
+	return nil
+}
+
+func getUserAgent() string {
+	return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
+}

databaseRest/diagnosis.go

@@ -0,0 +1,24 @@
+package databaseRest
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	//"github.com/jtom38/newsbot/collector/services"
+)
+
+type DiagnosisClient struct {
+	rootUri string
+}
+
+func (dc *DiagnosisClient) Ping() error {
+	dbPing := fmt.Sprintf("%v/ping", dc.rootUri)
+	resp, err := http.Get(dbPing)
+	if err != nil { return err }
+
+	_, err = ioutil.ReadAll(resp.Body)
+	if err != nil { return err }
+	return nil
+}
+

databaseRest/sources.go

@@ -0,0 +1,38 @@
+package databaseRest
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/jtom38/newsbot/collector/domain/model"
+)
+
+type SourcesClient struct {
+	rootUri string
+}
+
+func (sb *SourcesClient) List() ([]model.Sources, error) {
+	var items []model.Sources
+	url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
+	resp, err := getContent(url)
+	if err != nil { return items, err }
+
+	err = json.Unmarshal(resp, &items)
+	if err != nil { return []model.Sources{}, err }
+	
+	return items, nil
+}
+
+func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) {
+	items, err := sb.List()
+	if err != nil { log.Panicln(err) }
+
+	var res []model.Sources
+	for _, item := range(items) {
+		if item.Source == SourceType {
+			res = append(res, item)
+		}
+	}
+	return res, nil
+}

docker-compose.yaml

@@ -1,36 +0,0 @@
-version: "3"
-
-networks:
-  newsbot:
-    
-services:
-  api:
-    image: ghcr.io/jtom38/newsbot.collector.api:master
-    environment:
-      SERVER_ADDRESS: "localhost"
-
-      SQL_CONNECTION_STRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
-      
-      # Used for database migrations
-      GOOSE_DRIVER: "postgres"
-
-      # Connection String to Postgresql
-      GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
-
-      # Enable/Disable Reddit monitoring
-      FEATURE_ENABLE_REDDIT_BACKEND: true
-
-      # Enable/Disable YouTube monitoring
-      FEATURE_ENABLE_YOUTUBE_BACKEND: false
-
-      FEATURE_ENABLE_TWITCH_BACKEND: false
-      # Set your Twitch Developer ID and Secrets here and they will be used to collect updates.
-      TWITCH_CLIENT_ID: ""
-      TWITCH_CLIENT_SECRET: ""
-
-      # If you want to collect news on Final Fantasy XIV, set this to true
-      FEATURE_ENABLE_FFXIV_BACKEND: false
-    ports:
-      - 8081:8081
-    networks:
-      - newsbot

docs/swagger.yaml

@@ -1,945 +0,0 @@
-definitions:
-  domain.ArticleAndSourceModel:
-    properties:
-      article:
-        $ref: '#/definitions/domain.ArticleDto'
-      source:
-        $ref: '#/definitions/domain.SourceDto'
-    type: object
-  domain.ArticleDetailedResponse:
-    properties:
-      isError:
-        type: boolean
-      message:
-        type: string
-      payload:
-        $ref: '#/definitions/domain.ArticleAndSourceModel'
-    type: object
-  domain.ArticleDto:
-    properties:
-      authorImage:
-        type: string
-      authorName:
-        type: string
-      description:
-        type: string
-      id:
-        type: integer
-      isVideo:
-        type: boolean
-      pubDate:
-        type: string
-      sourceId:
-        type: integer
-      tags:
-        type: string
-      thumbnail:
-        type: string
-      title:
-        type: string
-      url:
-        type: string
-    type: object
-  domain.ArticleResponse:
-    properties:
-      isError:
-        type: boolean
-      message:
-        type: string
-      payload:
-        items:
-          $ref: '#/definitions/domain.ArticleDto'
-        type: array
-    type: object
-  domain.BaseResponse:
-    properties:
-      isError:
-        type: boolean
-      message:
-        type: string
-    type: object
-  domain.DiscordWebHookDto:
-    properties:
-      channel:
-        type: string
-      enabled:
-        type: boolean
-      id:
-        type: integer
-      server:
-        type: string
-      url:
-        description: |-
-          Name    string `json:"name"`
-          Key     string `json:"key"`
-        type: string
-    type: object
-  domain.DiscordWebhookResponse:
-    properties:
-      isError:
-        type: boolean
-      message:
-        type: string
-      payload:
-        items:
-          $ref: '#/definitions/domain.DiscordWebHookDto'
-        type: array
-    type: object
-  domain.LoginResponse:
-    properties:
-      isError:
-        type: boolean
-      message:
-        type: string
-      refreshToken:
-        type: string
-      token:
-        type: string
-      type:
-        type: string
-    type: object
-  domain.NewSourceParamRequest:
-    properties:
-      name:
-        type: string
-      tags:
-        type: string
-      url:
-        type: string
-    type: object
-  domain.RefreshTokenRequest:
-    properties:
-      refreshToken:
-        type: string
-      username:
-        type: string
-    type: object
-  domain.SourceDto:
-    properties:
-      enabled:
-        type: boolean
-      id:
-        type: integer
-      name:
-        type: string
-      source:
-        type: string
-      tags:
-        type: string
-      url:
-        type: string
-    type: object
-  domain.SourcesResponse:
-    properties:
-      isError:
-        type: boolean
-      message:
-        type: string
-      payload:
-        items:
-          $ref: '#/definitions/domain.SourceDto'
-        type: array
-    type: object
-  domain.UpdateScopesRequest:
-    properties:
-      scopes:
-        items:
-          type: string
-        type: array
-      username:
-        type: string
-    required:
-    - scopes
-    type: object
-info:
-  contact: {}
-paths:
-  /v1/articles:
-    get:
-      parameters:
-      - description: page number
-        in: query
-        name: page
-        required: true
-        type: integer
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.ArticleResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Lists the top 25 records ordering from newest to oldest.
-      tags:
-      - Articles
-  /v1/articles/{id}:
-    get:
-      parameters:
-      - description: int
-        in: path
-        name: id
-        required: true
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.ArticleResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Returns an article based on defined ID.
-      tags:
-      - Articles
-  /v1/articles/{id}/details:
-    get:
-      parameters:
-      - description: int
-        in: path
-        name: id
-        required: true
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.ArticleDetailedResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Returns an article and source based on defined ID.
-      tags:
-      - Articles
-  /v1/articles/by/source/{id}:
-    get:
-      parameters:
-      - description: source id
-        in: path
-        name: id
-        required: true
-        type: string
-      - description: Page to query
-        in: query
-        name: page
-        type: integer
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.ArticleResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Finds the articles based on the SourceID provided.  Returns the top
-        25.
-      tags:
-      - Articles
-  /v1/discord/webhooks:
-    get:
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Returns the top 100
-      tags:
-      - DiscordWebhook
-  /v1/discord/webhooks/{id}:
-    delete:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: string
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Deletes a record by ID.
-      tags:
-      - DiscordWebhook
-    get:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: integer
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Returns the top 100 entries from the queue to be processed.
-      tags:
-      - DiscordWebhook
-  /v1/discord/webhooks/{id}/disable:
-    post:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: integer
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Disables a Webhook from being used.
-      tags:
-      - DiscordWebhook
-  /v1/discord/webhooks/{id}/enable:
-    post:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: integer
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Enables a source to continue processing.
-      tags:
-      - DiscordWebhook
-  /v1/discord/webhooks/by/serverAndChannel:
-    get:
-      parameters:
-      - description: Fancy Server
-        in: query
-        name: server
-        required: true
-        type: string
-      - description: memes
-        in: query
-        name: channel
-        required: true
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Returns all the known web hooks based on the Server and Channel given.
-      tags:
-      - DiscordWebhook
-  /v1/discord/webhooks/new:
-    post:
-      parameters:
-      - description: url
-        in: query
-        name: url
-        required: true
-        type: string
-      - description: Server name
-        in: query
-        name: server
-        required: true
-        type: string
-      - description: Channel name
-        in: query
-        name: channel
-        required: true
-        type: string
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.DiscordWebhookResponse'
-      security:
-      - Bearer: []
-      summary: Creates a new record for a discord web hook to post data to.
-      tags:
-      - DiscordWebhook
-  /v1/sources:
-    get:
-      parameters:
-      - description: page number
-        in: query
-        name: page
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Unable to reach SQL or Data problems
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Lists the top 50 records
-      tags:
-      - Source
-  /v1/sources/{id}:
-    get:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: integer
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Returns a single entity by ID
-      tags:
-      - Source
-    post:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: integer
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Marks a source as deleted based on its ID value.
-      tags:
-      - Source
-  /v1/sources/{id}/disable:
-    post:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: integer
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Disables a source from processing.
-      tags:
-      - Source
-  /v1/sources/{id}/enable:
-    post:
-      parameters:
-      - description: id
-        in: path
-        name: id
-        required: true
-        type: string
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Enables a source to continue processing.
-      tags:
-      - Source
-  /v1/sources/by/source:
-    get:
-      parameters:
-      - description: Source Name
-        in: query
-        name: source
-        required: true
-        type: string
-      - description: page number
-        in: query
-        name: page
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: 'Lists the top 50 records based on the name given. Example: reddit'
-      tags:
-      - Source
-  /v1/sources/by/sourceAndName:
-    get:
-      parameters:
-      - description: dadjokes
-        in: query
-        name: name
-        required: true
-        type: string
-      - description: reddit
-        in: query
-        name: source
-        required: true
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Returns a single entity by ID
-      tags:
-      - Source
-  /v1/sources/new/reddit:
-    post:
-      parameters:
-      - description: name
-        in: query
-        name: name
-        required: true
-        type: string
-      - description: url
-        in: query
-        name: url
-        required: true
-        type: string
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Creates a new reddit source to monitor.
-      tags:
-      - Source
-  /v1/sources/new/rss:
-    post:
-      parameters:
-      - description: body
-        in: body
-        name: request
-        required: true
-        schema:
-          $ref: '#/definitions/domain.NewSourceParamRequest'
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Creates a new rss source to monitor.
-      tags:
-      - Source
-  /v1/sources/new/twitch:
-    post:
-      parameters:
-      - description: name
-        in: query
-        name: name
-        required: true
-        type: string
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Creates a new twitch source to monitor.
-      tags:
-      - Source
-  /v1/sources/new/youtube:
-    post:
-      parameters:
-      - description: name
-        in: query
-        name: name
-        required: true
-        type: string
-      - description: url
-        in: query
-        name: url
-        required: true
-        type: string
-      responses:
-        "200":
-          description: ok
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.SourcesResponse'
-      security:
-      - Bearer: []
-      summary: Creates a new youtube source to monitor.
-      tags:
-      - Source
-  /v1/users/login:
-    post:
-      parameters:
-      - in: formData
-        name: password
-        type: string
-      - in: formData
-        name: username
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.LoginResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.LoginResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.LoginResponse'
-      summary: Logs into the API and returns a bearer token if successful
-      tags:
-      - Users
-  /v1/users/refresh/sessionToken:
-    post:
-      consumes:
-      - application/json
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Revokes the current session token and replaces it with a new one.
-      tags:
-      - Users
-  /v1/users/refresh/token:
-    post:
-      parameters:
-      - description: body
-        in: body
-        name: request
-        required: true
-        schema:
-          $ref: '#/definitions/domain.RefreshTokenRequest'
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.LoginResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Generates a new token
-      tags:
-      - Users
-  /v1/users/register:
-    post:
-      parameters:
-      - in: formData
-        name: password
-        type: string
-      - in: formData
-        name: username
-        type: string
-      produces:
-      - application/json
-      responses:
-        "201":
-          description: Created
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      summary: Creates a new user
-      tags:
-      - Users
-  /v1/users/scopes/add:
-    post:
-      consumes:
-      - application/json
-      parameters:
-      - description: body
-        in: body
-        name: request
-        required: true
-        schema:
-          $ref: '#/definitions/domain.UpdateScopesRequest'
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Adds a new scope to a user account
-      tags:
-      - Users
-  /v1/users/scopes/remove:
-    post:
-      consumes:
-      - application/json
-      parameters:
-      - description: body
-        in: body
-        name: request
-        required: true
-        schema:
-          $ref: '#/definitions/domain.UpdateScopesRequest'
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/domain.BaseResponse'
-      security:
-      - Bearer: []
-      summary: Adds a new scope to a user account
-      tags:
-      - Users
-swagger: "2.0"

domain/const.go

@@ -1,9 +0,0 @@
-package domain
-
-const (
-	SourceCollectorRss     = "rss"
-	SourceCollectorFfxiv   = "ffxiv"
-	SourceCollectorTwitch  = "twitch"
-	SourceCollectorYoutube = "youtube"
-	SourceCollectorReddit  = "reddit"
-)

domain/dto.go

@@ -1,64 +0,0 @@
-package domain
-
-import "time"
-
-type ArticleDto struct {
-	ID             int64     `json:"id"`
-	SourceID       int64     `json:"sourceId"`
-	Tags           string    `json:"tags"`
-	Title          string    `json:"title"`
-	Url            string    `json:"url"`
-	PubDate        time.Time `json:"pubDate"`
-	IsVideo        bool      `json:"isVideo"`
-	Thumbnail      string    `json:"thumbnail"`
-	Description    string    `json:"description"`
-	AuthorName     string    `json:"authorName"`
-	AuthorImageUrl string    `json:"authorImage"`
-}
-
-type DiscordQueueDto struct {
-	ID        int64 `json:"id"`
-	ArticleId int64 `json:"articleId"`
-	SourceId  int64 `json:"sourceId"`
-}
-
-type DiscordWebHookDto struct {
-	ID      int64   `json:"id"`
-	//Name    string `json:"name"`
-	//Key     string `json:"key"`
-	Url     string `json:"url"`
-	Server  string `json:"server"`
-	Channel string `json:"channel"`
-	Enabled bool   `json:"enabled"`
-}
-
-type IconDto struct {
-	ID       int64  `json:"id"`
-	FileName string `json:"fileName"`
-	Site     string `json:"site"`
-}
-
-type SettingDto struct {
-	ID      int64  `json:"id"`
-	Key     string `json:"key"`
-	Value   string `json:"value"`
-	Options string `json:"options"`
-}
-
-type SubscriptionDto struct {
-	ID          int64  `json:"id"`
-	SourceID    int64  `json:"sourceId"`
-	SourceType  string `json:"sourceType"`
-	SourceName  string `json:"sourceName"`
-	DiscordID   int64  `json:"discordId"`
-	DiscordName string `json:"discordName"`
-}
-
-type SourceDto struct {
-	ID          int64  `json:"id"`
-	Source      string `json:"source"`
-	DisplayName string `json:"name"`
-	Url         string `json:"url"`
-	Tags        string `json:"tags"`
-	Enabled     bool   `json:"enabled"`
-}

domain/interfaces/source.go

@@ -0,0 +1,23 @@
+package interfaces
+
+import (
+	"github.com/go-rod/rod"
+	"github.com/mmcdole/gofeed"
+)
+
+type Sources interface {
+	CheckSource() error
+	PullFeed() (*gofeed.Feed, error)
+
+	GetBrowser() *rod.Browser
+	GetPage(parser *rod.Browser, url string) *rod.Page
+
+	ExtractThumbnail(page *rod.Page) (string, error)
+	ExtractPubDate(page *rod.Page) (string, error)
+	ExtractDescription(page *rod.Page) (string, error)
+	ExtractAuthor(page *rod.Page) (string, error)
+	ExtractAuthorImage(page *rod.Page) (string, error)
+	ExtractTags(page *rod.Page) (string, error)
+	ExtractTitle(page *rod.Page) (string, error)
+}
+

domain/model/cache.go

@@ -1,6 +1,8 @@
-package domain
+package model
 
-import "time"
+import (
+	"time"
+)
 
 type CacheItem struct {
 	Key   string

domain/model/database.go

@@ -0,0 +1,90 @@
+package model
+
+import (
+	"time"
+)
+
+// Articles represents the model for an Article
+type Articles struct {
+	ID uint `json:"ID"`
+	SourceID uint `json:"sourceId"`
+	Tags string	`json:"tags"`
+	Title string `json:"title"`
+	Url string `json:"url"`
+	PubDate time.Time `json:"pubdate"`
+	Video string `json:"video"`
+	VideoHeight uint16 `json:"videoHeight"`
+	VideoWidth uint16 `json:"videoWidth"`
+	Thumbnail string `json:"thumbnail"`
+	Description string `json:"description"`
+	AuthorName string `json:"authorName"`
+	AuthorImage string `json:"authorImage"`
+}
+
+type DiscordQueue struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+	ArticleId string `json:"articleId"`
+}
+
+type DiscordWebHooks struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+
+	Name string 	`json:"name"`
+	Key string 		`json:"key"`
+	Url string 		`json:"url"`
+	Server string 	`json:"server"`
+	Channel string	`json:"channel"`
+	Enabled bool	`json:"enabled"`
+}
+
+type Icons struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+
+	FileName string	`json:"fileName"`
+	Site string		`json:"site"`
+}
+
+type Settings struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+
+	Key string		`json:"key"`
+	Value string	`json:"value"`
+	Options string	`json:"options"`
+}
+
+type Sources struct {
+	ID uint 		`json:"ID"`
+	Site string		`json:"site"`
+	Name string		`json:"name"`
+	Source string	`json:"source"`
+	Type string		`json:"type"`
+	Value string	`json:"value"`
+	Enabled bool	`json:"enabled"`
+	Url string		`json:"url"`
+	Tags string		`json:"tags"`
+}
+
+type SourceLinks struct {
+	ID uint `json:"ID"`
+	CreatedAt time.Time `json:"CreatedAt"`
+	UpdatedAt time.Time `json:"UpdatedAt"`
+	DeletedAt time.Time `json:"DeletedAt"`
+	
+	SourceID uint		`json:"sourceId"`
+	SourceType string	`json:"sourceType"`
+	SourceName string	`json:"sourceName"`
+	DiscordID uint	`json:"discordId"`
+	DiscordName string	`json:"discordName"`
+}

domain/model/reddit.go

@@ -0,0 +1,54 @@
+package model
+
+// This is the root Json object.  It does not contain data that we care about though.
+type RedditJsonContent struct {
+	Kind string `json:"kind"`
+	Data RedditJsonContentData `json:"data"`
+}
+
+type RedditJsonContentData struct {
+	After string `json:"after"`
+	Dist int `json:"dist"`
+	Modhash string `json:"modhash"`
+	Children []RedditJsonContentChildren `json:"children"`
+}
+
+type RedditJsonContentChildren struct {
+	Kind string `json:"kind"`
+	Data RedditPost `json:"data"`
+}
+
+// RedditPost contains the information that was posted by a user.
+type RedditPost struct {
+	Subreddit string `json:"subreddit"`
+	Title string `json:"title"`
+	Content string `json:"selftext"`
+	ContentHtml string `json:"selftext_html"`
+	Author string `json:"author"`
+	Permalink string `json:"permalink"`
+	IsVideo bool `json:"is_video"`
+	Media RedditPostMedia `json:"media"`
+	Url string `json:"url"`
+	UrlOverriddenByDest string `json:"url_overridden_by_dest"`
+
+	Thumbnail string `json:"thumbnail"`
+}
+
+// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
+type RedditPostMedia struct {
+	RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
+}
+
+// RedditVideo contains information about the video in the post.
+type RedditPostMediaRedditVideo struct {
+	BitrateKbps int `json:"bitrate_kpbs"`
+	FallBackUrl string `json:"fallback_url"`
+	Height int `json:"height"`
+	Width int `json:"width"`
+	ScrubberMediaUrl string `json:"scrubber_media_url"`
+	DashUrl string `json:"dash_url"`
+	Duration int `json:"duration"`
+	HlsUrl string `json:"hls_url"`
+	IsGif bool `json:"is_gif"`
+	TranscodingStatus string `json:"transcoding_status"`
+}

domain/requests.go

@@ -1,27 +0,0 @@
-package domain
-
-type LoginFormRequest struct {
-	Username string `form:"username"`
-	Password string `form:"password"`
-}
-
-type GetSourceBySourceAndNameParamRequest struct {
-	Name   string `query:"name"`
-	Source string `query:"source"`
-}
-
-type NewSourceParamRequest struct {
-	Name string `json:"name"`
-	Url  string `json:"url"`
-	Tags string `json:"tags"`
-}
-
-type RefreshTokenRequest struct {
-	Username     string `json:"username"`
-	RefreshToken string `json:"refreshToken"`
-}
-
-type UpdateScopesRequest struct {
-	Username string   `json:"username"`
-	Scopes   []string `json:"scopes" validate:"required"`
-}

domain/responses.go

@@ -1,38 +0,0 @@
-package domain
-
-type BaseResponse struct {
-	Message string `json:"message"`
-	IsError bool   `json:"isError"`
-}
-
-type LoginResponse struct {
-	BaseResponse
-	Token        string `json:"token"`
-	Type         string `json:"type"`
-	RefreshToken string `json:"refreshToken"`
-}
-
-type ArticleResponse struct {
-	BaseResponse
-	Payload []ArticleDto `json:"payload"`
-}
-
-type ArticleAndSourceModel struct {
-	Article ArticleDto `json:"article"`
-	Source  SourceDto  `json:"source"`
-}
-
-type ArticleDetailedResponse struct {
-	BaseResponse
-	Payload ArticleAndSourceModel `json:"payload"`
-}
-
-type DiscordWebhookResponse struct {
-	BaseResponse
-	Payload []DiscordWebHookDto `json:"payload"`
-}
-
-type SourcesResponse struct {
-	BaseResponse
-	Payload []SourceDto `json:"payload"`
-}

domain/scopes.go

@@ -1,14 +0,0 @@
-package domain
-
-const (
-	ScopeAll = "newsbot:all"
-
-	ScopeArticleRead    = "newsbot:article:read"
-	ScopeArticleDisable = "newsbot:article:disable"
-
-	ScopeSourceRead   = "newsbot:source:read"
-	ScopeSourceCreate = "newsbot:source:create"
-
-	ScopeDiscordWebHookCreate = "newsbot:discordwebhook:create"
-	ScopeDiscordWebhookRead   = "newsbot:discordwebhook:read"
-)

go.mod

@@ -1,48 +1,19 @@
-module git.jamestombleson.com/jtom38/newsbot-api
+module github.com/jtom38/newsbot/collector
 
-go 1.22
+go 1.18
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
-	github.com/glebarez/go-sqlite v1.22.0
+	github.com/go-chi/chi/v5 v5.0.7
 	github.com/go-rod/rod v0.107.1
-	github.com/golang-jwt/jwt/v5 v5.2.1
-	github.com/google/uuid v1.6.0
-	github.com/huandu/go-sqlbuilder v1.27.1
+	github.com/google/uuid v1.3.0
 	github.com/joho/godotenv v1.4.0
-	github.com/labstack/echo-jwt/v4 v4.2.0
-	github.com/labstack/echo/v4 v4.12.0
+	github.com/lib/pq v1.10.6
 	github.com/mmcdole/gofeed v1.1.3
 	github.com/nicklaw5/helix/v2 v2.4.0
-	github.com/pressly/goose/v3 v3.20.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/swaggo/echo-swagger v1.4.1
-	github.com/swaggo/swag v1.8.12
-	golang.org/x/crypto v0.22.0
-)
-
-require (
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/labstack/gommon v0.4.2 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mfridman/interpolate v0.0.2 // indirect
-	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/sethvargo/go-retry v0.2.4 // indirect
-	github.com/swaggo/files/v2 v2.0.0 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasttemplate v1.2.2 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	modernc.org/libc v1.41.0 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.7.2 // indirect
-	modernc.org/sqlite v1.29.6 // indirect
+	github.com/swaggo/http-swagger v1.3.0
+	github.com/swaggo/swag v1.8.2
 )
 
 require (
@@ -52,20 +23,20 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/spec v0.20.6 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/lib/pq v1.10.6
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
 	github.com/ysmood/gson v0.7.2 // indirect
 	github.com/ysmood/leakless v0.7.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -12,12 +12,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
-github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
+github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
+github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -29,31 +25,20 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU=
 github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-rod/rod v0.105.1 h1:r0bNmO9siOe13lG6Vbkaak11u48rYmWGl/Hk4MJdOiE=
+github.com/go-rod/rod v0.105.1/go.mod h1:Wrnn6HokFHskwaIVke3ML1y/NBVp7XPIeB8eDzR9vuw=
 github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY=
 github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
-github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
-github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
-github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
+github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
-github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
-github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
-github.com/huandu/go-sqlbuilder v1.27.1 h1:7UU/3EMIQYYX8wn+L7BNcGVz1aEs5TPNOVFd7ryrPos=
-github.com/huandu/go-sqlbuilder v1.27.1/go.mod h1:nUVmMitjOmn/zacMLXT0d3Yd3RHoO2K+vy906JzqxMI=
-github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
-github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -62,12 +47,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo-jwt/v4 v4.2.0 h1:odSISV9JgcSCuhgQSV/6Io3i7nUmfM/QkBeR5GVJj5c=
-github.com/labstack/echo-jwt/v4 v4.2.0/go.mod h1:MA2RqdXdEn4/uEglx0HcUOgQSyBaTh5JcaHIan3biwU=
-github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
-github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
-github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
-github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
 github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
 github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -75,103 +54,87 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
-github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
 github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
 github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
 github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
 github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 h1:Z6i7ND25ixRtXFBylIUggqpvLMV1I15yprcqMVB7WZA=
 github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
-github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nicklaw5/helix/v2 v2.4.0 h1:ZvqCKVqza1eJYyqgTRrZ/xjDq0w/EQVFNkN067Utls0=
 github.com/nicklaw5/helix/v2 v2.4.0/go.mod h1:0ONzvVi1cH+k3a7EDIFNNqxfW0podhf+CqlmFvuexq8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0=
-github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
-github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk=
-github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc=
-github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
-github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
-github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w=
-github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM=
+github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
+github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc=
+github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
+github.com/swaggo/http-swagger v1.2.8 h1:TVjxLU7qoqofJ9qynJazmpTGs/p4Kx9FTp7YYwOkJb0=
+github.com/swaggo/http-swagger v1.2.8/go.mod h1:FrQwV7rx+A5t11PIX8d+tFJa2GKx11RdAXQptllPQHg=
+github.com/swaggo/http-swagger v1.3.0 h1:1+6M4qRorIbdyTWTsGrwnb0r9jGK5dcWN82O6oY/yHQ=
+github.com/swaggo/http-swagger v1.3.0/go.mod h1:9glekdg40lwclrrKNRGgj/IMDxpNPZ3kzab4oPcF8EM=
+github.com/swaggo/swag v1.8.2 h1:D4aBiVS2a65zhyk3WFqOUz7Rz0sOaUcgeErcid5uGL4=
+github.com/swaggo/swag v1.8.2/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
 github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
-github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
 github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
+github.com/ysmood/got v0.23.2 h1:U2U0vyQ/gDaawkKJZK/hyza8UUXbWCurbmazK7AcAfY=
+github.com/ysmood/got v0.23.2/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
 github.com/ysmood/got v0.29.5 h1:+wMnm8UjoyYFMfeAsr57a1bahWTkloysc0Hxsu2gmnM=
 github.com/ysmood/got v0.29.5/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
 github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
 github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
 github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
 github.com/ysmood/gson v0.7.2 h1:1iWUvpi5DPvd2j59W7ifRPR9DiAZ3Ga+fmMl1mJrRbM=
 github.com/ysmood/gson v0.7.2/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
 github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
 github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
-go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 h1:z8Hj/bl9cOV2grsOpEaQFUaly0JWN3i97mo3jXKJNp0=
+golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 h1:PgOr27OhUx2IRqGJ2RxAWI4dJQ7bi9cSrB82uzFzfUA=
+golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
+golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
@@ -180,20 +143,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
-modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
-modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
-modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
-modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
-modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
-modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
-modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4=
-modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
-modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
-modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
-modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/database/migrations/20240425083756_init.sql

@@ -1,115 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-CREATE TABLE Articles (
-	ID 				INTEGER 	PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 		DATETIME	NOT NULL,
-	UpdatedAt 		DATETIME 	NOT NULL,
-	DeletedAt 		DATETIME	NOT NULL,
-	SourceId 		NUMBER 		NOT NULL,
-	Tags 			TEXT 		NOT NULL,
-	Title 			TEXT 		NOT NULL,
-	Url 			TEXT 		NOT NULL,
-	PubDate 		DATETIME 	NOT NULL,
-	IsVideo 		TEXT		NOT NULL,
-	ThumbnailUrl 	TEXT 		NOT NULL,
-	Description 	TEXT 		NOT NULL,
-	AuthorName 		TEXT		NOT NULL,
-	AuthorImageUrl 	TEXT		NOT NULL
-);
-
-CREATE Table DiscordWebHooks (
-	ID 			INTEGER 	PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-	UserID 		INTEGER 	NOT NULL,
-	Url 		TEXT 		NOT NULL, -- Webhook Url
-	Server 		TEXT 		NOT NULL, -- Defines the server its bound it. Used for reference
-	Channel 	TEXT 		NOT NULL, -- Defines the channel its bound to.  Used for reference
-	Enabled 	BOOLEAN 	NOT NULL
-);
-
-CREATE Table Icons (
-	ID 			INTEGER	 	PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME,
-	FileName 	TEXT 		NOT NULL,
-	Site 		TEXT 		NOT NULL
-);
-
-Create Table Settings (
-	ID 			INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME,
-	Key 		TEXT 		NOT NULL, -- How you search for a entry
-	Value 		TEXT 		NOT NULL, -- The value for one
-	Options 	TEXT -- any notes about the entry
-);
-
-CREATE Table Sources (
-	ID 			INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 	DATETIME 	NOT NULL,
-	UpdatedAt 	DATETIME 	NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-	DisplayName	TEXT 		NOT NULL, -- Vanity name
-	Source 		TEXT 		NOT NULL, -- Defines the service that will use this record. IE reddit or youtube
-	Enabled 	BOOLEAN 	NOT NULL,
-	Url 		TEXT 		NOT NULL,
-	Tags 		TEXT 		NOT NULL
-);
-
-CREATE TABLE UserSourceSubscriptions (
-	ID 					INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 			DATETIME 	NOT NULL,
-	UpdatedAt 			DATETIME 	NOT NULL,
-	DeletedAt 			DATETIME 	NOT NULL,
-	UserID				NUMBER		NOT NULL,
-	SourceID 			NUMBER 		NOT NULL
-);
-
-CREATE TABLE AlertDiscord (
-	ID 					INTEGER		PRIMARY KEY AUTOINCREMENT,
-	CreatedAt 			DATETIME 	NOT NULL,
-	UpdatedAt 			DATETIME 	NOT NULL,
-	DeletedAt 			DATETIME 	NOT NULL,
-	UserID				NUMBER 		NOT NULL,
-	SourceID 			NUMBER 		NOT NULL,
-	DiscordWebHookID 	NUMBER 		NOT NULL
-);
-
-CREATE TABLE Users (
-    ID          INTEGER     PRIMARY KEY AUTOINCREMENT,
-    CreatedAt   DATETIME    NOT NULL,
-    UpdatedAt	DATETIME    NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-    Name        TEXT        NOT NULL,
-    Hash        TEXT        NOT NULL,
-	Scopes 		TEXT		NOT NULL
-); 
-
-CREATE TABLE RefreshTokens (
-    ID          INTEGER     PRIMARY KEY AUTOINCREMENT,
-    CreatedAt   DATETIME    NOT NULL,
-    UpdatedAt 	DATETIME    NOT NULL,
-	DeletedAt 	DATETIME	NOT NULL,
-    Username    TEXT        NOT NULL,
-    Token       TEXT        NOT NULL
-);
-
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-DROP TABLE AlertDiscord;
-Drop Table Articles;
-Drop Table DiscordWebHooks;
-Drop Table Icons;
-DROP TABLE RefreshTokens;
-Drop Table Sources;
-DROP TABLE Users;
-DROP TABLE UserSourceSubscriptions;
--- +goose StatementEnd

internal/database/migrations/20240425092459_seed.sql

@@ -1,50 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
-
--- Enable UUID's
---CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-
--- Final Fantasy XIV Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES  
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES  
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-
--- Reddit Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-
--- Youtube Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES 
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-
--- RSS Entries
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-
--- Twitch Entries 
-INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES   
-("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
---SELECT 'down SQL query';
-
-DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes';
-DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck';
-DELETE FROM sources where Source = 'ffxiv';
-DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo';
-DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps';
-DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck';
--- +goose StatementEnd

internal/database/migrations/20240521194914_session_token.sql

@@ -1,9 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-ALTER TABLE Users ADD SessionToken TEXT; 
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-ALTER TABLE Users DROP SessionToken;
--- +goose StatementEnd

internal/domain/modelsReddit.go

@@ -1,54 +0,0 @@
-package domain
-
-// This is the root Json object.  It does not contain data that we care about though.
-type RedditJsonContent struct {
-	Kind string                `json:"kind"`
-	Data RedditJsonContentData `json:"data"`
-}
-
-type RedditJsonContentData struct {
-	After    string                      `json:"after"`
-	Dist     int                         `json:"dist"`
-	Modhash  string                      `json:"modhash"`
-	Children []RedditJsonContentChildren `json:"children"`
-}
-
-type RedditJsonContentChildren struct {
-	Kind string     `json:"kind"`
-	Data RedditPost `json:"data"`
-}
-
-// RedditPost contains the information that was posted by a user.
-type RedditPost struct {
-	Subreddit           string          `json:"subreddit"`
-	Title               string          `json:"title"`
-	Content             string          `json:"selftext"`
-	ContentHtml         string          `json:"selftext_html"`
-	Author              string          `json:"author"`
-	Permalink           string          `json:"permalink"`
-	IsVideo             bool            `json:"is_video"`
-	Media               RedditPostMedia `json:"media"`
-	Url                 string          `json:"url"`
-	UrlOverriddenByDest string          `json:"url_overridden_by_dest"`
-
-	Thumbnail string `json:"thumbnail"`
-}
-
-// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
-type RedditPostMedia struct {
-	RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
-}
-
-// RedditVideo contains information about the video in the post.
-type RedditPostMediaRedditVideo struct {
-	BitrateKbps       int    `json:"bitrate_kpbs"`
-	FallBackUrl       string `json:"fallback_url"`
-	Height            int    `json:"height"`
-	Width             int    `json:"width"`
-	ScrubberMediaUrl  string `json:"scrubber_media_url"`
-	DashUrl           string `json:"dash_url"`
-	Duration          int    `json:"duration"`
-	HlsUrl            string `json:"hls_url"`
-	IsGif             bool   `json:"is_gif"`
-	TranscodingStatus string `json:"transcoding_status"`
-}

internal/dtoconv/dtoconv.go

@@ -1,67 +0,0 @@
-package dtoconv
-
-import (
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-)
-
-func ArticlesToDto(items []entity.ArticleEntity) []domain.ArticleDto {
-	var dtos []domain.ArticleDto
-	for _, item := range items {
-		dtos = append(dtos, ArticleToDto(item))
-	}
-	return dtos
-}
-
-func ArticleToDto(item entity.ArticleEntity) domain.ArticleDto {
-	return domain.ArticleDto{
-		ID:             item.ID,
-		SourceID:       item.SourceID,
-		Tags:           item.Tags,
-		Title:          item.Title,
-		Url:            item.Url,
-		PubDate:        item.PubDate,
-		IsVideo:        item.IsVideo,
-		Thumbnail:      item.Thumbnail,
-		Description:    item.Description,
-		AuthorName:     item.AuthorName,
-		AuthorImageUrl: item.AuthorImageUrl,
-	}
-}
-
-func DiscordWebhooksToDto(items []entity.DiscordWebHookEntity) []domain.DiscordWebHookDto{
-	var dtos []domain.DiscordWebHookDto
-	for _, item := range items {
-		dtos = append(dtos, DiscordWebhookToDto(item))
-	}
-	return dtos
-}
-
-func DiscordWebhookToDto(item entity.DiscordWebHookEntity) domain.DiscordWebHookDto {
-	return domain.DiscordWebHookDto{
-		ID:      item.ID,
-		Server:  item.Server,
-		Channel: item.Channel,
-		Url:     item.Url,
-		Enabled: item.Enabled,
-	}
-}
-
-func SourcesToDto(items []entity.SourceEntity) []domain.SourceDto {
-	var dtos []domain.SourceDto
-	for _, item := range items {
-		dtos = append(dtos, SourceToDto(item))
-	}
-	return dtos
-}
-
-func SourceToDto(item entity.SourceEntity) domain.SourceDto {
-	return domain.SourceDto{
-		ID:          item.ID,
-		Source:      item.Source,
-		DisplayName: item.DisplayName,
-		Url:         item.Url,
-		Tags:        item.Tags,
-		Enabled:     item.Enabled,
-	}
-}

internal/entity/entity.go

@@ -1,141 +0,0 @@
-package entity
-
-import (
-	"time"
-)
-
-// This links a source to a discord webhook.
-// It is owned by a user so they can remove the link
-type AlertDiscordEntity struct {
-	ID               int64
-	CreatedAt        time.Time
-	UpdatedAt        time.Time
-	DeletedAt        time.Time
-	UserID           int64
-	SourceID         int64
-	DiscordWebHookId int64
-}
-
-type ArticleEntity struct {
-	ID             int64
-	CreatedAt      time.Time
-	UpdatedAt      time.Time
-	DeletedAt      time.Time
-	SourceID       int64
-	Tags           string
-	Title          string
-	Url            string
-	PubDate        time.Time
-	IsVideo        bool
-	Thumbnail      string
-	Description    string
-	AuthorName     string
-	AuthorImageUrl string
-}
-
-type DiscordQueueEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	ArticleId int64
-	SourceId  int64
-}
-
-type DiscordWebHookEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	UserID    int64
-	Url       string
-	Server    string
-	Channel   string
-	Enabled   bool
-}
-
-type IconEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	FileName  string
-	Site      string
-}
-
-type SettingEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	Key       string
-	Value     string
-	Options   string
-}
-
-type SourceEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-
-	// Who will collect from it.  Used
-	// domain.SourceCollector...
-	Source string
-
-	// Human Readable value to state what is getting collected
-	DisplayName string
-
-	// Tells the parser where to look for data
-	Url string
-
-	// Static tags for this defined record
-	Tags string
-
-	// If the record is disabled, then it will be skipped on processing
-	Enabled bool
-}
-
-//type SubscriptionEntity struct {
-//	ID        int64
-//	CreatedAt time.Time
-//	UpdatedAt time.Time
-//	DeletedAt time.Time
-//	UserID    int64
-//	SourceID  int64
-//	//SourceType  string
-//	//SourceName  string
-//	DiscordID int64
-//	//DiscordName string
-//}
-
-// This defines what sources a user wants to follow.
-// These will show up for the user as a front page
-type UserSourceSubscriptionEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	UserID    int64
-	SourceID  int64
-}
-
-type UserEntity struct {
-	ID           int64
-	CreatedAt    time.Time
-	UpdatedAt    time.Time
-	DeletedAt    time.Time
-	Username     string
-	Hash         string
-	Scopes       string
-	SessionToken string
-}
-
-type RefreshTokenEntity struct {
-	ID        int64
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt time.Time
-	Username  string
-	Token     string
-}

internal/handler/v1/articles.go

@@ -1,181 +0,0 @@
-package v1
-
-import (
-	"net/http"
-	"strconv"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
-	"github.com/labstack/echo/v4"
-)
-
-// ListArticles
-//  @Summary   Lists the top 25 records ordering from newest to oldest.
-//  @Produce   application/json
-//  @Param     page  query  int  true  "page number"  test
-//  @Tags      Articles
-//  @Router    /v1/articles [get]
-//  @Success   200  {object}  domain.ArticleResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) listArticles(c echo.Context) error {
-	resp := domain.ArticleResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, resp)
-	}
-
-	page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		page = 0
-	}
-
-	res, err := s.repo.Articles.ListByPage(c.Request().Context(), page, 25)
-	if err != nil {
-		return c.JSON(http.StatusInternalServerError, resp)
-	}
-
-	resp.Payload = dtoconv.ArticlesToDto(res)
-	resp.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, resp)
-}
-
-// GetArticle
-//  @Summary   Returns an article based on defined ID.
-//  @Param     id  path  string  true  "int"
-//  @Produce   application/json
-//  @Tags      Articles
-//  @Router    /v1/articles/{id} [get]
-//  @Success   200  {object}  domain.ArticleResponse  "OK"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) getArticle(c echo.Context) error {
-	p := domain.ArticleResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id := c.Param("ID")
-	idNumber, err := strconv.Atoi(id)
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Articles.GetById(c.Request().Context(), int64(idNumber))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	var dtos []domain.ArticleDto
-	dtos = append(dtos, dtoconv.ArticleToDto(item))
-	p.Payload = dtos
-
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetArticleDetails
-//  @Summary   Returns an article and source based on defined ID.
-//  @Param     id  path  string  true  "int"
-//  @Produce   application/json
-//  @Tags      Articles
-//  @Router    /v1/articles/{id}/details [get]
-//  @Success   200  {object}  domain.ArticleDetailedResponse  "OK"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) getArticleDetails(c echo.Context) error {
-	p := domain.ArticleDetailedResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-		Payload: domain.ArticleAndSourceModel{},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	article, err := s.repo.Articles.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	source, err := s.repo.Sources.GetById(c.Request().Context(), article.SourceID)
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	p.Payload.Article = dtoconv.ArticleToDto(article)
-	p.Payload.Source = dtoconv.SourceToDto(source)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// ListArticlesBySourceID
-//  @Summary   Finds the articles based on the SourceID provided.  Returns the top 25.
-//  @Param     id    path   string  true   "source id"
-//  @Param     page  query  int     false  "Page to query"
-//  @Produce   application/json
-//  @Tags      Articles
-//  @Router    /v1/articles/by/source/{id} [get]
-//  @Success   200  {object}  domain.ArticleResponse  "OK"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) ListArticlesBySourceId(c echo.Context) error {
-	p := domain.ArticleResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.QueryParam("id"))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// if the page number is missing, default to 0
-	_page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		_page = 0
-	}
-
-	items, err := s.repo.Articles.ListBySource(c.Request().Context(), _page, 25, id, "")
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	p.Payload = dtoconv.ArticlesToDto(items)
-	p.BaseResponse.IsError = false
-
-	return c.JSON(http.StatusOK, p)
-}

internal/handler/v1/discordwebhooks.go

@@ -1,443 +0,0 @@
-package v1
-
-import (
-	"net/http"
-	"strconv"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
-	"github.com/labstack/echo/v4"
-)
-
-// ListDiscordWebhooks
-//  @Summary   Returns the top 100
-//  @Produce   application/json
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks [get]
-//  @Success   200  {object}  domain.DiscordWebhookResponse
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) ListDiscordWebHooks(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	res, err := s.repo.DiscordWebHooks.ListByServerName(c.Request().Context(), "")
-	if err != nil {
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.DiscordWebhooksToDto(res)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetDiscordWebHook
-//  @Summary   Returns the top 100 entries from the queue to be processed.
-//  @Produce   application/json
-//  @Param     id  path  int  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id} [get]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) GetDiscordWebHooksById(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
-	if err != nil {
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	res, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(res))
-	p.Payload = dtos
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetDiscordWebHookByServerAndChannel
-//  @Summary   Returns all the known web hooks based on the Server and Channel given.
-//  @Produce   application/json
-//  @Param     server   query  string  true  "Fancy Server"
-//  @Param     channel  query  string  true  "memes"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/by/serverAndChannel [get]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	_server := c.QueryParam("server")
-	if _server == "" {
-		p.BaseResponse.Message = "server was not defined"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	_channel := c.QueryParam("channel")
-	if _channel == "" {
-		p.BaseResponse.Message = "channel was not defined"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	res, err := s.repo.DiscordWebHooks.ListByServerAndChannel(c.Request().Context(), _server, _channel)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.DiscordWebhooksToDto(res)
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewDiscordWebHook
-//  @Summary   Creates a new record for a discord web hook to post data to.
-//  @Param     url      query  string  true  "url"
-//  @Param     server   query  string  true  "Server name"
-//  @Param     channel  query  string  true  "Channel name"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/new [post]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) NewDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	token, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	_url := c.QueryParam("url")
-	_server := c.QueryParam("server")
-	_channel := c.QueryParam("channel")
-
-	if _url == "" {
-		p.Message = "url is missing a value"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if !strings.Contains(_url, "discord.com/api/webhooks") {
-		p.Message = "invalid url"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if _server == "" {
-		p.Message = "server is missing"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if _channel == "" {
-		p.Message = "channel is missing"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	rows, err := s.repo.DiscordWebHooks.Create(c.Request().Context(), user.ID, _url, _server, _channel, true)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.Message = "data was not written to database"
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetByUrl(c.Request().Context(), _url)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DisableDiscordWebHooks
-//  @Summary   Disables a Webhook from being used.
-//  @Param     id  path  int  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id}/disable [post]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) disableDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if record.UserID != s.GetUserIdFromJwtToken(c) {
-		p.BaseResponse.Message = ErrYouDontOwnTheRecord
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// flip the it
-	updated, err := s.repo.DiscordWebHooks.Disable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// make sure we got a row updated
-	if updated != 1 {
-		p.BaseResponse.Message = "unexpected number of updates found"
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// EnableDiscordWebHook
-//  @Summary   Enables a source to continue processing.
-//  @Param     id  path  int  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id}/enable [post]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) enableDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if record.UserID != s.GetUserIdFromJwtToken(c) {
-		p.BaseResponse.Message = ErrYouDontOwnTheRecord
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	updated, err := s.repo.DiscordWebHooks.Enable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if updated != 1 {
-		p.BaseResponse.Message = ErrFailedToUpdateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DeleteDiscordWebHook
-//  @Summary   Deletes a record by ID.
-//  @Param     id  path  string  true  "id"
-//  @Tags      DiscordWebhook
-//  @Router    /v1/discord/webhooks/{id} [delete]
-//  @Success   200  {object}  domain.DiscordWebhookResponse  "OK"
-//  @Failure   400  {object}  domain.DiscordWebhookResponse
-//  @Failure   500  {object}  domain.DiscordWebhookResponse
-//  @Security  Bearer
-func (s *Handler) deleteDiscordWebHook(c echo.Context) error {
-	p := domain.DiscordWebhookResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if record.UserID != s.GetUserIdFromJwtToken(c) {
-		p.BaseResponse.Message = ErrYouDontOwnTheRecord
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Soft delete the record
-	updated, err := s.repo.DiscordWebHooks.SoftDelete(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if updated != 1 {
-		p.BaseResponse.Message = ErrFailedToUpdateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dtos []domain.DiscordWebHookDto
-	dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
-
-	p.Payload = dtos
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// UpdateDiscordWebHook
-//	@Summary	Updates a valid discord webhook ID based on the body given.
-//	@Param		id	path	string	true	"id"
-//	@Tags		DiscordWebhook
-//	@Router		/v1/discord/webhooks/{id} [patch]
-//	@Success	200	{object}	domain.DiscordWebhookResponse	"OK"
-//	@Failure	400	{object}	domain.BaseResponse
-//	@Failure	500	{object}	domain.BaseResponse
-//func (s *Handler) UpdateDiscordWebHook(c echo.Context) error {
-//	id, err := strconv.Atoi(c.Param("ID"))
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	// Check to make sure we can find the record
-//	_, err = s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	// Soft delete the record
-//	updated, err := s.repo.DiscordWebHooks(c.Request().Context(), int64(id))
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	_, err = s.Db.GetDiscordQueueByID(c.Request().Context(), uuid)
-//	if err != nil {
-//		return c.JSON(http.StatusInternalServerError, err.Error())
-//	}
-//
-//	return nil
-//}

internal/handler/v1/handler.go

@@ -1,197 +0,0 @@
-package v1
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-
-	"github.com/golang-jwt/jwt/v5"
-	echojwt "github.com/labstack/echo-jwt/v4"
-	"github.com/labstack/echo/v4"
-	"github.com/labstack/echo/v4/middleware"
-	swagger "github.com/swaggo/echo-swagger"
-
-	_ "git.jamestombleson.com/jtom38/newsbot-api/docs"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-)
-
-type Handler struct {
-	Router *echo.Echo
-	//Db     *database.Queries
-	config services.Configs
-	repo   services.RepositoryService
-}
-
-const (
-	ErrParameterIdMissing = "The requested parameter ID was not found."
-	ErrParameterMissing   = "The requested parameter was not found found:"
-	ErrUnableToParseId    = "Unable to parse the requested ID"
-
-	ErrRecordMissing        = "The requested record was not found"
-	ErrFailedToCreateRecord = "The record was not created due to a database problem"
-	ErrFailedToUpdateRecord = "The requested record was not updated due to a database problem"
-
-	ErrUserUnknown         = "User is unknown"
-	ErrYouDontOwnTheRecord = "The record requested does not belong to you"
-
-	ResponseMessageSuccess = "Success"
-)
-
-var (
-	ErrIdValueMissing        string = "id value is missing"
-	ErrValueNotUuid          string = "a value given was expected to be a uuid but was not correct."
-	ErrNoRecordFound         string = "no record was found."
-	ErrUnableToConvertToJson string = "Unable to convert to json"
-)
-
-func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
-	s := &Handler{
-		config: configs,
-		repo:   services.NewRepositoryService(conn),
-	}
-
-	jwtConfig := echojwt.Config{
-		NewClaimsFunc: func(c echo.Context) jwt.Claims {
-			return new(JwtToken)
-		},
-		SigningKey: []byte(configs.JwtSecret),
-	}
-
-	router := echo.New()
-	router.Pre(middleware.RemoveTrailingSlash())
-	router.Pre(middleware.Logger())
-	router.Pre(middleware.Recover())
-	router.GET("/swagger/*", swagger.WrapHandler)
-
-	v1 := router.Group("/api/v1")
-	articles := v1.Group("/articles")
-	articles.Use(echojwt.WithConfig(jwtConfig))
-	articles.GET("", s.listArticles)
-	articles.GET(":id", s.getArticle)
-	articles.GET(":id/details", s.getArticleDetails)
-	articles.GET("by/source/:id", s.ListArticlesBySourceId)
-
-	//dwh := v1.Group("/discord/webhooks")
-	//dwh.GET("/", s.ListDiscordWebHooks)
-	//dwh.POST("/new", s.NewDiscordWebHook)
-	//dwh.GET("/by/serverAndChannel", s.GetDiscordWebHooksByServerAndChannel)
-	//dwh.GET("/:ID", s.GetDiscordWebHooksById)
-	//dwh.DELETE("/:ID", s.deleteDiscordWebHook)
-	//dwh.POST("/:ID/disable", s.disableDiscordWebHook)
-	//dwh.POST("/:ID/enable", s.enableDiscordWebHook)
-
-	//queue := v1.Group("/queue")
-	//queue.GET("/discord/webhooks", s.ListDiscordWebhookQueue) // TODO this needs to be reworked
-
-	//settings := v1.Group("/settings")
-	//settings.GET("/", s.getSettings)
-
-	sources := v1.Group("/sources")
-	sources.Use(echojwt.WithConfig(jwtConfig))
-	sources.GET("", s.listSources)
-	sources.GET("/by/source", s.listSourcesBySource)
-	sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName)
-	//sources.POST("/new/reddit", s.newRedditSource)
-	//sources.POST("/new/youtube", s.newYoutubeSource)
-	//sources.POST("/new/twitch", s.newTwitchSource)
-	sources.POST("/new/rss", s.newRssSource)
-	sources.GET("/:id", s.getSource)
-	sources.DELETE("/:id", s.deleteSources)
-	sources.POST("/:id/disable", s.disableSource)
-	sources.POST("/:id/enable", s.enableSource)
-
-	users := v1.Group("/users")
-	users.POST("/login", s.AuthLogin)
-	users.POST("/register", s.AuthRegister)
-	users.Use(echojwt.WithConfig(jwtConfig))
-	users.POST("/scopes/add", s.AddScopes)
-	users.POST("/scopes/remove", s.RemoveScopes)
-	users.POST("/refresh/token", s.RefreshJwtToken)
-	users.POST("/refresh/sessionToken", s.NewSessionToken)
-
-	s.Router = router
-	return s
-}
-
-//type ApiStatusModel struct {
-//	StatusCode int    `json:"status"`
-//	Message    string `json:"message"`
-//}
-
-//type ApiError struct {
-//	*ApiStatusModel
-//}
-
-//func (s *Handler) WriteError(c echo.Context, errMessage error, HttpStatusCode int) error {
-//	return c.JSON(HttpStatusCode, domain.BaseResponse{
-//		Message: errMessage.Error(),
-//		IsError: true,
-//	})
-//}
-
-//func (s *Handler) WriteMessage(c echo.Context, msg string, HttpStatusCode int) error {
-//	return c.JSON(HttpStatusCode, domain.BaseResponse{
-//		Message: msg,
-//	})
-//}
-
-//func (s *Handler) InternalServerErrorResponse(c echo.Context, msg string) error {
-//	return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
-//		Message: msg,
-//	})
-//}
-
-//func (s *Handler) UnauthorizedResponse(c echo.Context, msg string) error {
-//	return c.JSON(http.StatusUnauthorized, domain.BaseResponse{
-//		Message: msg,
-//	})
-//}
-
-// If the token is not valid then an json error will be returned.
-// If the token has the wrong scope, a json error will be returned.
-// If the token passes all the checks, it is valid and is returned back to the caller.
-func (s *Handler) ValidateJwtToken(c echo.Context, requiredScope string) (JwtToken, error) {
-	token, err := s.getJwtTokenFromContext(c)
-	if err != nil {
-		return JwtToken{}, errors.New(ErrJwtMissing)
-	}
-
-	err = token.hasExpired()
-	if err != nil {
-		return JwtToken{}, errors.New(ErrJwtExpired)
-	}
-
-	err = token.hasScope(requiredScope)
-	if err != nil {
-		return JwtToken{}, errors.New(ErrJwtScopeMissing)
-	}
-
-	if token.Iss != s.config.ServerAddress {
-		return JwtToken{}, errors.New(ErrJwtInvalidIssuer)
-	}
-
-	// If you are the built in admin account, skip the username and session token check
-	if token.UserName == "admin" {
-		return token, nil
-	}
-
-	user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
-	if err != nil {
-		return JwtToken{}, errors.New("user record not found")
-	}
-
-	if user.SessionToken != token.SessionToken {
-		return JwtToken{}, errors.New("invalid session token")
-	}
-
-	return token, nil
-}
-
-func (s *Handler) GetUserIdFromJwtToken(c echo.Context) int64 {
-	token, err := s.getJwtTokenFromContext(c)
-	if err != nil {
-		return -1
-	}
-
-	return token.GetUserId()
-}

internal/handler/v1/jwt.go

@@ -1,130 +0,0 @@
-package v1
-
-import (
-	"errors"
-	"strings"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/labstack/echo/v4"
-)
-
-const (
-	ErrJwtMissing       = "auth token is missing"
-	ErrJwtClaimsMissing = "claims missing on token"
-	ErrJwtExpired       = "auth token has expired"
-	ErrJwtScopeMissing  = "required scope is missing"
-	ErrJwtInvalidIssuer = "incorrect server issued the token"
-)
-
-type JwtToken struct {
-	Exp          time.Time `json:"exp"`
-	Iss          string    `json:"iss"`
-	Authorized   bool      `json:"authorized"`
-	UserName     string    `json:"username"`
-	UserId       int64     `json:"userId"`
-	Scopes       []string  `json:"scopes"`
-	SessionToken string    `json:"sessionToken"`
-	jwt.RegisteredClaims
-}
-
-func (j JwtToken) IsValid(scope string) error {
-	err := j.hasExpired()
-	if err != nil {
-		return err
-	}
-
-	// Check to see if they have the scope to do anything
-	// if they do, let them pass
-	err = j.hasScope(domain.ScopeAll)
-	if err == nil {
-		return nil
-	}
-
-	err = j.hasScope(scope)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (j JwtToken) GetUsername() string {
-	return j.UserName
-}
-
-func (j JwtToken) GetUserId() int64 {
-	return j.UserId
-}
-
-func (j JwtToken) hasExpired() error {
-	// Check to see if the token has expired
-	//hasExpired := j.Exp.Compare(time.Now())
-	hasExpired := time.Now().Compare(j.Exp)
-	if hasExpired == 1 {
-		return errors.New(ErrJwtExpired)
-	}
-	return nil
-}
-
-// This will check the users token to make sure they have the correct scope to access the handler.
-// It will evaluate if you have the admin scope or the required scope for the handler.
-func (j JwtToken) hasScope(scope string) error {
-	// they have the scope to access everything, so let them pass.
-	userScopes := strings.Join(j.Scopes, "")
-	if strings.Contains(domain.ScopeAll, userScopes) {
-		return nil
-	}
-
-	if strings.Contains(userScopes, scope) {
-		return nil
-	}
-
-	return errors.New(ErrJwtScopeMissing)
-}
-
-func (h *Handler) generateJwt(username, issuer, sessionToken string, userScopes []string, userId int64) (string, error) {
-	return h.generateJwtWithExp(username, issuer, sessionToken, userScopes, userId, time.Now().Add(10*time.Minute))
-}
-
-func (h *Handler) generateJwtWithExp(username, issuer, sessionToken string, userScopes []string, userId int64, expiresAt time.Time) (string, error) {
-	secret := []byte(h.config.JwtSecret)
-
-	// Anyone who wants to decrypt the key needs to use the same method
-	token := jwt.New(jwt.SigningMethodHS256)
-	claims := token.Claims.(jwt.MapClaims)
-	claims["exp"] = expiresAt
-	claims["authorized"] = true
-	claims["username"] = username
-	claims["iss"] = issuer
-	claims["userId"] = userId
-	claims["sessionToken"] = sessionToken
-
-	var scopes []string
-	scopes = append(scopes, userScopes...)
-	claims["scopes"] = scopes
-
-	tokenString, err := token.SignedString(secret)
-	if err != nil {
-		return "", err
-	}
-
-	return tokenString, nil
-}
-
-func (h *Handler) getJwtTokenFromContext(c echo.Context) (JwtToken, error) {
-	// Make sure that the request came with a jwtToken
-	token, ok := c.Get("user").(*jwt.Token)
-	if !ok {
-		return JwtToken{}, errors.New(ErrJwtMissing)
-	}
-
-	// Generate the claims from the token
-	claims, ok := token.Claims.(*JwtToken)
-	if !ok {
-		return JwtToken{}, errors.New(ErrJwtClaimsMissing)
-	}
-
-	return *claims, nil
-}

internal/handler/v1/sources.go

@@ -1,653 +0,0 @@
-package v1
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/labstack/echo/v4"
-)
-
-// ListSources
-//  @Summary   Lists the top 50 records
-//  @Param     page  query  string  false  "page number"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse  "Unable to reach SQL or Data problems"
-//  @Security  Bearer
-func (s *Handler) listSources(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		page = 0
-	}
-
-	// Default way of showing all sources
-	items, err := s.repo.Sources.List(c.Request().Context(), page, 25)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.SourcesToDto(items)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// ListSourcesBySource
-//  @Summary   Lists the top 50 records based on the name given. Example: reddit
-//  @Param     source  query  string  true   "Source Name"
-//  @Param     page    query  string  false  "page number"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources/by/source [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) listSourcesBySource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	source := c.QueryParam("source")
-	if source == "" {
-		p.BaseResponse.Message = fmt.Sprintf("%s source", ErrParameterMissing)
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	page, err := strconv.Atoi(c.QueryParam("page"))
-	if err != nil {
-		page = 0
-	}
-
-	// Shows the list by Sources.source
-	items, err := s.repo.Sources.ListBySource(c.Request().Context(), page, 25, source)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Payload = dtoconv.SourcesToDto(items)
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetSource
-//  @Summary   Returns a single entity by ID
-//  @Param     id  path  int  true  "id"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources/{id} [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) getSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("id"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// GetSourceByNameAndSource
-//  @Summary   Returns a single entity by ID
-//  @Param     name    query  string  true  "dadjokes"
-//  @Param     source  query  string  true  "reddit"
-//  @Produce   application/json
-//  @Tags      Source
-//  @Router    /v1/sources/by/sourceAndName [get]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (s *Handler) GetSourceBySourceAndName(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.GetSourceBySourceAndNameParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), param.Source, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewRedditSource
-//  @Summary   Creates a new reddit source to monitor.
-//  @Param     name  query  string  true  "name"
-//  @Param     url   query  string  true  "url"
-//  @Tags      Source
-//  @Router    /v1/sources/new/reddit [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newRedditSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.NewSourceParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if param.Url == "" {
-		p.BaseResponse.Message = "url is missing"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if !strings.Contains(param.Url, "reddit.com") {
-		p.BaseResponse.Message = "invalid url"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to see if we already have this record, if we do, return it.
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
-	if err == nil {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	tags := fmt.Sprintf("twitch, %v, %s", param.Name, param.Tags)
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorReddit, param.Name, param.Url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewYoutubeSource
-//  @Summary   Creates a new youtube source to monitor.
-//  @Param     name  query  string  true  "name"
-//  @Param     url   query  string  true  "url"
-//  @Tags      Source
-//  @Router    /v1/sources/new/youtube [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newYoutubeSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-		},
-	}
-
-	// Validate the jwt
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.NewSourceParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if param.Url == "" {
-		p.BaseResponse.Message = "url is missing a value"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	if !strings.Contains(param.Url, "youtube.com") {
-		p.BaseResponse.Message = "invalid url"
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
-	if err == nil {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	tags := fmt.Sprintf("twitch, %v", param.Name)
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorYoutube, param.Name, param.Url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewTwitchSource
-//  @Summary   Creates a new twitch source to monitor.
-//  @Param     name  query  string  true  "name"
-//  @Tags      Source
-//  @Router    /v1/sources/new/twitch [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newTwitchSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	var param domain.NewSourceParamRequest
-	err = c.Bind(&param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	tags := fmt.Sprintf("twitch, %v", param.Name)
-	url := fmt.Sprintf("https://twitch.tv/%v", param.Name)
-
-	// Check if the record already exists
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
-	if err == nil {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorTwitch, param.Name, url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, _ = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// NewRssSource
-//  @Summary   Creates a new rss source to monitor.
-//  @Param     request  body  domain.NewSourceParamRequest  true  "body"
-//  @Tags      Source
-//  @Router    /v1/sources/new/rss [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) newRssSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-
-	_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	param := domain.NewSourceParamRequest{}
-	err = c.Bind(&param)
-	//err = (&echo.DefaultBinder{}).BindBody(c, &param)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if param.Url == "" {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check if the record already exists
-	blankRecord := entity.SourceEntity{}
-	item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
-	if err != nil {
-		p.IsError = true
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// No record was found, but no error returned
-	if item != blankRecord {
-		var dto []domain.SourceDto
-		dto = append(dto, dtoconv.SourceToDto(item))
-		p.Payload = dto
-		p.BaseResponse.IsError = false
-		return c.JSON(http.StatusOK, p)
-	}
-
-	//if err == nil {
-	//	var dto []domain.SourceDto
-	//	dto = append(dto, dtoconv.SourceToDto(item))
-	//	p.Payload = dto
-	//	p.BaseResponse.IsError = false
-	//	return c.JSON(http.StatusOK, p)
-	//}
-
-	tags := fmt.Sprintf("rss, %v, %s", param.Name, param.Tags)
-	rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorRss, param.Name, param.Url, tags, true)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToCreateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DeleteSource
-//  @Summary   Marks a source as deleted based on its ID value.
-//  @Param     id  path  int  true  "id"
-//  @Tags      Source
-//  @Router    /v1/sources/{id} [POST]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) deleteSources(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// Delete the record
-	rows, err := s.repo.Sources.SoftDelete(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-	if rows != 1 {
-		p.BaseResponse.Message = ErrFailedToUpdateRecord
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// pull the record with its updated value
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var items []domain.SourceDto
-	items = append(items, dtoconv.SourceToDto(item))
-	p.Payload = items
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// DisableSource
-//  @Summary   Disables a source from processing.
-//  @Param     id  path  int  true  "id"
-//  @Tags      Source
-//  @Router    /v1/sources/{id}/disable [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) disableSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	_, err = s.repo.Sources.Disable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// EnableSource
-//  @Summary   Enables a source to continue processing.
-//  @Param     id  path  string  true  "id"
-//  @Tags      Source
-//  @Router    /v1/sources/{id}/enable [post]
-//  @Success   200  {object}  domain.SourcesResponse  "ok"
-//  @Failure   400  {object}  domain.SourcesResponse
-//  @Failure   500  {object}  domain.SourcesResponse
-//  @Security  Bearer
-func (s *Handler) enableSource(c echo.Context) error {
-	p := domain.SourcesResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-	}
-	_, err := s.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-
-	id, err := strconv.Atoi(c.Param("ID"))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	// Check to make sure we can find the record
-	_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	_, err = s.repo.Sources.Enable(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	var dto []domain.SourceDto
-	dto = append(dto, dtoconv.SourceToDto(item))
-	p.Payload = dto
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}

internal/handler/v1/users.go

@@ -1,351 +0,0 @@
-package v1
-
-import (
-	"net/http"
-	"strings"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-
-	"github.com/google/uuid"
-	"github.com/labstack/echo/v4"
-)
-
-const (
-	ErrUserNotFound          = "requested user does not exist"
-	ErrUsernameAlreadyExists = "the requested username already exists"
-)
-
-//  @Summary  Creates a new user
-//  @Router   /v1/users/register [post]
-//  @Param    request  formData  domain.LoginFormRequest  true  "form"
-//  @Accepts  x-www-form-urlencoded
-//  @Produce  json
-//  @Tags     Users
-//  @Success  201  {object}  domain.BaseResponse
-//  @Failure  400  {object}  domain.BaseResponse
-//  @Failure  500  {object}  domain.BaseResponse
-func (h *Handler) AuthRegister(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	username := c.FormValue("username")
-	password := c.FormValue("password")
-
-	//username := c.QueryParam("username")
-	exists, err := h.repo.Users.GetUser(c.Request().Context(), username)
-	if err != nil {
-		// if we have an err, validate that if its not user not found.
-		// if the user is not found, we can use that name
-		if err.Error() != repository.ErrUserNotFound {
-			p.Message = err.Error()
-			return c.JSON(http.StatusBadRequest, p)
-		}
-	}
-	if exists.Username == username {
-		p.Message = ErrUsernameAlreadyExists
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	//password := c.QueryParam("password")
-	err = h.repo.Users.CheckPasswordForRequirements(password)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	_, err = h.repo.Users.Create(c.Request().Context(), username, password, domain.ScopeArticleRead)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	return c.JSON(http.StatusCreated, p)
-}
-
-//  @Summary  Logs into the API and returns a bearer token if successful
-//  @Router   /v1/users/login [post]
-//  @Param    request  formData  domain.LoginFormRequest  true  "form"
-//  @Accepts  x-www-form-urlencoded
-//  @Produce  json
-//  @Tags     Users
-//  @Success  200  {object}  domain.LoginResponse
-//  @Failure  400  {object}  domain.LoginResponse
-//  @Failure  500  {object}  domain.LoginResponse
-func (h *Handler) AuthLogin(c echo.Context) error {
-	p := domain.LoginResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-		//Token:        jwt,
-		Type: "Bearer",
-		//RefreshToken: refresh,
-	}
-
-	username := c.FormValue("username")
-	password := c.FormValue("password")
-
-	// Check to see if they are trying to login with the admin token
-	if username == "" {
-		return h.createAdminToken(c, password)
-	}
-
-	// check if the user exists
-	user, err := h.repo.Users.GetUser(c.Request().Context(), username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// make sure the hash matches
-	err = h.repo.Users.DoesPasswordMatchHash(c.Request().Context(), username, password)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	// TODO think about moving this down some?
-	expiresAt := time.Now().Add(time.Hour * 48)
-	userScopes := strings.Split(user.Scopes, ",")
-
-	jwt, err := h.generateJwtWithExp(username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, expiresAt)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	refresh, err := h.repo.RefreshTokens.Create(c.Request().Context(), username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Token = jwt
-	p.RefreshToken = refresh
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-func (h *Handler) createAdminToken(c echo.Context, password string) error {
-	p := domain.LoginResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: ResponseMessageSuccess,
-			IsError: true,
-		},
-		//Token: token,
-		Type: "Bearer",
-	}
-
-	// if the admin token is blank, then the admin wanted this disabled.
-	// this will fail right away and not progress.
-	if h.config.AdminSecret == "" {
-		p.BaseResponse.Message = ErrUserNotFound
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	if h.config.AdminSecret != password {
-		p.BaseResponse.Message = ErrUserNotFound
-		return c.JSON(http.StatusBadRequest, p)
-	}
-	var userScopes []string
-	userScopes = append(userScopes, domain.ScopeAll)
-	sessionToken, err := uuid.NewV7()
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	token, err := h.generateJwt("admin", h.config.ServerAddress, sessionToken.String(), userScopes, -1)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Token = token
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-// This will take collect some information about the requested refresh, validate and then return a new jwt token if approved.
-// Register
-//  @Summary   Generates a new token
-//  @Router    /v1/users/refresh/token [post]
-//  @Param     request  body  domain.RefreshTokenRequest  true  "body"
-//  @Tags      Users
-//  @Success   200  {object}  domain.LoginResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) RefreshJwtToken(c echo.Context) error {
-	p := domain.LoginResponse{
-		BaseResponse: domain.BaseResponse{
-			Message: "OK",
-		},
-		//Token:        jwt,
-		Type: "Bearer",
-		//RefreshToken: newRefreshToken,
-	}
-
-	_, err := h.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	// Check the context for the refresh token
-	var request domain.RefreshTokenRequest
-	err = (&echo.DefaultBinder{}).BindBody(c, &request)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	err = h.repo.RefreshTokens.IsRequestValid(c.Request().Context(), request.Username, request.RefreshToken)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	user, err := h.repo.Users.GetUser(c.Request().Context(), request.Username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-	userScopes := strings.Split(user.Scopes, ",")
-
-	jwt, err := h.generateJwtWithExp(request.Username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, time.Now().Add(time.Hour*48))
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	newRefreshToken, err := h.repo.RefreshTokens.Create(c.Request().Context(), request.Username)
-	if err != nil {
-		p.BaseResponse.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.Token = jwt
-	p.RefreshToken = newRefreshToken
-	p.BaseResponse.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-//  @Summary   Adds a new scope to a user account
-//  @Router    /v1/users/scopes/add [post]
-//  @Param     request  body  domain.UpdateScopesRequest  true  "body"
-//  @Tags      Users
-//  @Accept    json
-//  @Produce   json
-//  @Success   200  {object}  domain.BaseResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) AddScopes(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	_, err := h.ValidateJwtToken(c, domain.ScopeAll)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	request := domain.UpdateScopesRequest{}
-	err = (&echo.DefaultBinder{}).BindBody(c, &request)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	err = h.repo.Users.AddScopes(c.Request().Context(), request.Username, request.Scopes)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-//  @Summary   Adds a new scope to a user account
-//  @Router    /v1/users/scopes/remove [post]
-//  @Param     request  body  domain.UpdateScopesRequest  true  "body"
-//  @Tags      Users
-//  @Accept    json
-//  @Produce   json
-//  @Success   200  {object}  domain.BaseResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) RemoveScopes(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	token, err := h.getJwtTokenFromContext(c)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	err = token.IsValid(domain.ScopeAll)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	request := domain.UpdateScopesRequest{}
-	err = (&echo.DefaultBinder{}).BindBody(c, &request)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusBadRequest, p)
-	}
-
-	err = h.repo.Users.RemoveScopes(c.Request().Context(), request.Username, request.Scopes)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.IsError = false
-	return c.JSON(http.StatusOK, p)
-}
-
-//  @Summary   Revokes the current session token and replaces it with a new one.
-//  @Router    /v1/users/refresh/sessionToken [post]
-//  @Tags      Users
-//  @Accept    json
-//  @Produce   json
-//  @Success   200  {object}  domain.BaseResponse
-//  @Failure   400  {object}  domain.BaseResponse
-//  @Failure   500  {object}  domain.BaseResponse
-//  @Security  Bearer
-func (h *Handler) NewSessionToken(c echo.Context) error {
-	p := domain.BaseResponse{
-		Message: ResponseMessageSuccess,
-		IsError: true,
-	}
-
-	token, err := h.getJwtTokenFromContext(c)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusUnauthorized, p)
-	}
-
-	_, err = h.repo.Users.NewSessionToken(c.Request().Context(), token.UserName)
-	if err != nil {
-		p.Message = err.Error()
-		return c.JSON(http.StatusInternalServerError, p)
-	}
-
-	p.IsError = false
-	return c.JSON(http.StatusInternalServerError, p)
-}

internal/repository/alertDiscord.go

@@ -1,122 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type AlertDiscordRepo interface {
-	Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error)
-}
-
-type alertDiscordRepository struct {
-	conn          *sql.DB
-	defaultLimit  int
-	defaultOffset int
-}
-
-func NewAlertDiscordRepository(conn *sql.DB) alertDiscordRepository {
-	return alertDiscordRepository{
-		conn:          conn,
-		defaultLimit:  50,
-		defaultOffset: 50,
-	}
-}
-
-func (r alertDiscordRepository) Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("AlertDiscord")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID", "DiscordWebHookID")
-	queryBuilder.Values(dt, dt, timeZero, userId, sourceId, webhookId)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r alertDiscordRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "AlertDiscord", id)
-}
-
-func (r alertDiscordRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "AlertDiscord", id)
-}
-
-func (r alertDiscordRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "AlertDiscord", id)
-}
-
-func (r alertDiscordRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("AlertDiscord")
-	builder.Where(
-		builder.Equal("UserID", userId),
-	)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.AlertDiscordEntity{}, err
-	}
-
-	data := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.AlertDiscordEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ur alertDiscordRepository) processRows(rows *sql.Rows) []entity.AlertDiscordEntity {
-	items := []entity.AlertDiscordEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var userId int64
-		var sourceId int64
-		var webhookId int64
-
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt, &deletedAt,
-			&userId, &sourceId, &webhookId,
-		)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.AlertDiscordEntity{
-			ID:               id,
-			CreatedAt:        createdAt,
-			UpdatedAt:        updatedAt,
-			DeletedAt:        deletedAt,
-			UserID:           userId,
-			SourceID:         sourceId,
-			DiscordWebHookId: webhookId,
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/alertDiscord_test.go

@@ -1,63 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestAlertDiscordCreate(t *testing.T) {
-	t.Log(time.Time{})
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	r := repository.NewAlertDiscordRepository(db)
-	created, err := r.Create(context.Background(), 1, 1, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-}
-
-func TestAlertDiscordCreateAndValidate(t *testing.T) {
-	t.Log(time.Time{})
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	source := repository.NewSourceRepository(db)
-	source.Create(context.Background(), domain.SourceCollectorRss, "Unit Testing", "www.fake.com", "testing,units", true)
-	sourceRecord, _ := source.GetBySourceAndName(context.Background(), domain.SourceCollectorRss, "Unit Testing")
-
-	webhookRepo := repository.NewDiscordWebHookRepository(db)
-	webhookRepo.Create(context.Background(), 999, "discord.com", "Unit Testing", "memes", true)
-	webhook, _ := webhookRepo.GetByUrl(context.Background(), "discord.com")
-	
-	r := repository.NewAlertDiscordRepository(db)
-	r.Create(context.Background(), 999, sourceRecord.ID, webhook.ID)
-	alert, err := r.ListByUser(context.Background(), 0, 10, 999)
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	if len(alert) != 1 {
-		t.Error("got the incorrect number of rows back")
-		t.FailNow()
-	}
-}

internal/repository/article.go

@@ -1,261 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-const (
-	ArticleOrderByPublishDateDesc = "pubDate desc"
-	ArticleOrderByPublishDateAsc  = "pubDate asc"
-)
-
-type ArticlesRepo interface {
-	GetById(ctx context.Context, id int64) (entity.ArticleEntity, error)
-	GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error)
-	ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error)
-	ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error)
-	ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error)
-	ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error)
-	Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error)
-	CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error)
-}
-
-type ArticleRepository struct {
-	conn          *sql.DB
-	defaultLimit  int
-	defaultOffset int
-}
-
-func NewArticleRepository(conn *sql.DB) ArticleRepository {
-	return ArticleRepository{
-		conn:          conn,
-		defaultLimit:  50,
-		defaultOffset: 50,
-	}
-}
-
-func (ar ArticleRepository) GetById(ctx context.Context, id int64) (entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles").Where(
-		builder.E("id", id),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data[0], nil
-}
-
-func (ar ArticleRepository) GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles").Where(
-		builder.E("url", url),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data[0], nil
-}
-
-func (ar ArticleRepository) ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ar ArticleRepository) ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	builder.OrderBy(ArticleOrderByPublishDateDesc)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ar ArticleRepository) ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	if orderBy != "" {
-		builder.OrderBy(orderBy)
-	}
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-	return data, nil
-}
-
-func (ar ArticleRepository) ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("articles")
-	builder.JoinWithOption("InnerJoin", "sources", "articles.sourceId=sources.Id")
-	if orderBy != "" {
-		builder.OrderBy(orderBy)
-	}
-	builder.Where(
-		builder.Equal("SourceId", sourceId),
-	)
-	builder.Offset(50)
-	builder.Limit(page * limit)
-
-	query, args := builder.Build()
-	rows, err := ar.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.ArticleEntity{}, err
-	}
-
-	data := ar.processRows(rows)
-	if len(data) == 0 {
-		return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
-	}
-	return data, nil
-}
-
-func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("articles")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
-	queryBuilder.Values(dt, dt, timeZero, sourceId, tags, title, url, pubDate, isVideo, thumbnailUrl, description, authorName, authorImageUrl)
-	query, args := queryBuilder.Build()
-
-	_, err := ar.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("articles")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
-	queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl)
-	query, args := queryBuilder.Build()
-
-	_, err := ar.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (ur ArticleRepository) processRows(rows *sql.Rows) []entity.ArticleEntity {
-	items := []entity.ArticleEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var sourceId int64
-		var tags string
-		var title string
-		var url string
-		var pubDate time.Time
-		var isVideo bool
-		var thumbnail string
-		var description string
-		var authorName string
-		var authorImageUrl string
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt,
-			&deletedAt, &sourceId, &tags,
-			&title, &url, &pubDate,
-			&isVideo, &thumbnail, &description,
-			&authorName, &authorImageUrl)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.ArticleEntity{
-			ID:             id,
-			CreatedAt:      createdAt,
-			UpdatedAt:      updatedAt,
-			DeletedAt:      deletedAt,
-			SourceID:       sourceId,
-			Tags:           tags,
-			Title:          title,
-			Url:            url,
-			PubDate:        pubDate,
-			IsVideo:        isVideo,
-			Thumbnail:      thumbnail,
-			Description:    description,
-			AuthorName:     authorName,
-			AuthorImageUrl: authorImageUrl,
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/article_test.go

@@ -1,156 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-const (
-	articleFakeDotCom = "www.fake.com"
-)
-
-func TestCreateArticle(t *testing.T) {
-	t.Log(time.Time{})
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	r := repository.NewArticleRepository(db)
-	created, err := r.Create(context.Background(), 1, "", "unit test", articleFakeDotCom, "", "testing", "", "", time.Now(), false)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-}
-
-func TestArticleByUrl(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-
-	err = insertFakeArticles(r, "u1", 0)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	article, err := r.GetByUrl(context.Background(), articleFakeDotCom)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if article.Url != "www.fake.com" {
-		t.Log("failed to find the requested record")
-		t.FailNow()
-	}
-}
-
-func TestPullingMultipleArticlesWithLimit(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-	insertFakeArticles(r, "u1", 0)
-	insertFakeArticles(r, "u2", 0)
-	insertFakeArticles(r, "u3", 0)
-	insertFakeArticles(r, "u4", 0)
-
-	items, err := r.ListTop(context.Background(), 3)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if len(items) != 3 {
-		t.Log("expected 3 rows back")
-		t.FailNow()
-	}
-}
-
-func TestPullingMultipleArticlesWithPaging(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-	insertFakeArticles(r, "u1", 0)
-	insertFakeArticles(r, "u2", 0)
-	insertFakeArticles(r, "u3", 0)
-	insertFakeArticles(r, "u4", 0)
-
-	items, err := r.ListByPage(context.Background(), 2, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if items[0].Title != "u2" {
-		t.Log("pulled the wrong record with paging")
-		t.FailNow()
-	}
-}
-
-func TestPullingByPublishDate(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	r := repository.NewArticleRepository(db)
-
-	today := time.Now()
-
-	insertFakeArticles(r, "u1", 0)
-	insertFakeArticles(r, "u1", -1)
-	insertFakeArticles(r, "u1", -2)
-
-	items, err := r.ListByPublishDate(context.Background(), 0, 2, repository.ArticleOrderByPublishDateDesc)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if len(items) != 2 {
-		t.Log("expected two items back")
-		t.FailNow()
-	}
-	pubDate := items[1].PubDate.Day()
-	expectedDay := today.Day() - 1
-	if pubDate != expectedDay {
-		t.Log("expected a record that was 2 days old")
-		t.FailNow()
-	}
-}
-
-//func TestArticleBySource
-
-func insertFakeArticles(r repository.ArticleRepository, title string, daysOld int) error {
-	pubDate := time.Now().AddDate(0,0, daysOld)
-	_, err := r.Create(context.Background(), 1, "", title, articleFakeDotCom, "", "testing", "", "", pubDate, false)
-	if err != nil {
-		return err
-	}
-	return nil
-}

internal/repository/common.go

@@ -1,71 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"github.com/huandu/go-sqlbuilder"
-)
-
-var (
-	timeZero = time.Time{}
-)
-
-func deleteFromTable(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
-	b := sqlbuilder.NewDeleteBuilder()
-	b.DeleteFrom(tableName)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func restoreRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
-	timeZero := time.Time{}
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update(tableName)
-	b.Set(
-		b.Assign("UpdatedAt", time.Now()),
-		b.Assign("DeletedAt", timeZero),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func softDeleteRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
-	now := time.Now()
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update(tableName)
-	b.Set(
-		b.Assign("UpdatedAt", now),
-		b.Assign("DeletedAt", now),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}

internal/repository/discordWebHooks.go

@@ -1,228 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type DiscordWebHookRepo interface {
-	Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error)
-	Enable(ctx context.Context, id int64) (int64, error)
-	Disable(ctx context.Context, id int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error)
-	GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error)
-	ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error)
-	ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error)
-}
-
-type discordWebHookRepository struct {
-	conn *sql.DB
-}
-
-func NewDiscordWebHookRepository(conn *sql.DB) discordWebHookRepository {
-	return discordWebHookRepository{
-		conn: conn,
-	}
-}
-
-func (r discordWebHookRepository) Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("DiscordWebHooks")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "Url", "Server", "Channel", "Enabled")
-	queryBuilder.Values(dt, dt, timeZero, userId, url, server, channel, enabled)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r discordWebHookRepository) Enable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("DiscordWebHooks")
-	b.Set(
-		b.Assign("Enabled", true),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r discordWebHookRepository) Disable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("DiscordWebHooks")
-	b.Set(
-		b.Assign("Enabled", false),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r discordWebHookRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "DiscordWebHooks", id)
-}
-
-func (r discordWebHookRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "DiscordWebHooks", id)
-}
-
-func (r discordWebHookRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "DiscordWebHooks", id)
-}
-
-func (r discordWebHookRepository) GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.E("id", id),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r discordWebHookRepository) GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.E("Url", url),
-	)
-	builder.Limit(1)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.DiscordWebHookEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r discordWebHookRepository) ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.E("Server", name),
-	)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r discordWebHookRepository) ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("DiscordWebHooks").Where(
-		builder.Equal("Server", server),
-		builder.Equal("Channel", channel),
-	)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.DiscordWebHookEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r discordWebHookRepository) processRows(rows *sql.Rows) ([]entity.DiscordWebHookEntity, error) {
-	items := []entity.DiscordWebHookEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var userId int64
-		var url string
-		var server string
-		var channel string
-		var enabled bool
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt,
-			&deletedAt, &userId, &url, &server,
-			&channel, &enabled,
-		)
-		if err != nil {
-			return items, err
-		}
-
-		item := entity.DiscordWebHookEntity{
-			ID:        id,
-			CreatedAt: createdAt,
-			UpdatedAt: updatedAt,
-			DeletedAt: deletedAt,
-			UserID:    userId,
-			Url:       url,
-			Server:    server,
-			Channel:   channel,
-			Enabled:   enabled,
-		}
-
-		items = append(items, item)
-	}
-
-	return items, nil
-}

internal/repository/discordWebHooks_test.go

@@ -1,287 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestCreateDiscordWebHookRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	r := repository.NewDiscordWebHookRepository(db)
-	created, err := r.Create(context.Background(), 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookGetById(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewDiscordWebHookRepository(db)
-	created, err := r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if created != 1 {
-		t.Log("failed to create the record")
-		t.FailNow()
-	}
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.ID != 1 {
-		t.Log("got the wrong record back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookGetByUrl(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
-	item, err := r.GetByUrl(ctx, "www.discord.com/bad/webhook")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Url != "www.discord.com/bad/webhook" {
-		t.Log("got the wrong record back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookListByServerName(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, "memes", true)
-
-	item, err := r.ListByServerName(ctx, serverName)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item[0].Server != serverName {
-		t.Log("got the wrong record back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookListByServerAndChannel(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-
-	item, err := r.ListByServerAndChannel(ctx, serverName, channel)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item[0].Server != serverName {
-		t.Log("got the wrong wrong server back")
-		t.FailNow()
-	}
-
-	if item[0].Channel != channel {
-		t.Log("got the wrong channel back")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookEnableRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, false)
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != false {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Enable(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookDisableRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != true {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Disable(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}
-
-func TestDiscordWebHookSoftDelete(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-	_, err = r.SoftDelete(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, _ := r.GetById(ctx, 1)
-	t.Log(updated.DeletedAt)
-}
-
-func TestDiscordWebHookRestore(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	serverName := "Unit Testing"
-	channel := "memes"
-	timeZero := time.Time{}
-
-	r := repository.NewDiscordWebHookRepository(db)
-	_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
-	item, _ := r.GetById(ctx, 1)
-	if item.DeletedAt != timeZero {
-		t.Log("DeletedAt was not zero")
-		t.FailNow()
-	}
-	_, _ = r.SoftDelete(ctx, 1)
-	softDeleted, _ := r.GetById(ctx, 1)
-	if softDeleted.ID != 1 {
-		t.Log("record went boom")
-		t.FailNow()
-	}
-
-	_, err = r.Restore(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, _ := r.GetById(ctx, 1)
-	t.Log(updated.DeletedAt)
-}

internal/repository/refreshTokens.go

@@ -1,120 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-const (
-	refreshTokenTableName = "RefreshTokens"
-)
-
-type RefreshToken interface {
-	Create(ctx context.Context, username string, token string) (int64, error)
-	GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
-	DeleteById(ctx context.Context, id int64) (int64, error)
-}
-
-type RefreshTokenRepository struct {
-	connection *sql.DB
-}
-
-func NewRefreshTokenRepository(conn *sql.DB) RefreshTokenRepository {
-	return RefreshTokenRepository{
-		connection: conn,
-	}
-}
-
-func (rt RefreshTokenRepository) Create(ctx context.Context, username string, token string) (int64, error) {
-	dt := time.Now()
-	builder := sqlbuilder.NewInsertBuilder()
-	builder.InsertInto(refreshTokenTableName)
-	builder.Cols("Username", "Token", "CreatedAt", "UpdatedAt", "DeletedAt")
-	builder.Values(username, token, dt, dt, time.Time{})
-	query, args := builder.Build()
-
-	_, err := rt.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (rt RefreshTokenRepository) GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*").From(refreshTokenTableName).Where(
-		builder.E("Username", name),
-	)
-
-	query, args := builder.Build()
-	rows, err := rt.connection.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.RefreshTokenEntity{}, err
-	}
-
-	data := rt.processRows(rows)
-	if len(data) == 0 {
-		return entity.RefreshTokenEntity{}, errors.New("no token found for user")
-	}
-
-	return data[0], nil
-}
-
-func (rt RefreshTokenRepository) DeleteById(ctx context.Context, id int64) (int64, error) {
-	builder := sqlbuilder.NewDeleteBuilder()
-	builder.DeleteFrom(refreshTokenTableName)
-	builder.Where(
-		builder.EQ("Id", id),
-	)
-
-	query, args := builder.Build()
-	rows, err := rt.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return -1, err
-	}
-
-	return rows.RowsAffected()
-}
-
-func (rd RefreshTokenRepository) processRows(rows *sql.Rows) []entity.RefreshTokenEntity {
-	items := []entity.RefreshTokenEntity{}
-
-	for rows.Next() {
-		var id int64
-		var username string
-		var token string
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt sql.NullTime
-
-		err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &token)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.RefreshTokenEntity{
-			ID:        id,
-			Username:  username,
-			Token:     token,
-			CreatedAt: createdAt,
-			UpdatedAt: updatedAt,
-		}
-
-		if deletedAt.Valid {
-			item.DeletedAt = deletedAt.Time
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}
-
-//func (rt RefreshTokenRepository) Delete()

internal/repository/refreshTokens_test.go

@@ -1,93 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestRefreshTokenCreate(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	client := repository.NewRefreshTokenRepository(conn)
-	rows, err := client.Create(context.Background(), "tester", "BadTokenDontUse")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if rows == 0 {
-		t.Log("expected one row to come back but got 0")
-	}
-}
-
-func TestRefreshTokenGetByUsername(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	client := repository.NewRefreshTokenRepository(conn)
-	rows, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if rows != 1 {
-		t.Log("expected a row to be added but not the wrong value back")
-		t.FailNow()
-	}
-
-	model, err := client.GetByUsername(context.Background(), "tester")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if model.Username != "tester" {
-		t.Log("got the wrong user back")
-		t.FailNow()
-	}
-}
-
-func TestRefreshTokenDeleteById(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	client := repository.NewRefreshTokenRepository(conn)
-	created, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if created != 1 {
-		t.Log("Unexpected number back for rows created")
-	}
-
-	model, err := client.GetByUsername(context.Background(), "tester")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := client.DeleteById(context.Background(), model.ID)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if updated != 1 {
-		t.Log("deleted the wrong number of records")
-		t.FailNow()
-	}
-}

internal/repository/source.go

@@ -1,277 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type Sources interface {
-	Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error)
-	GetById(ctx context.Context, id int64) (entity.SourceEntity, error)
-	GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error)
-	GetBySource(ctx context.Context, source string) (entity.SourceEntity, error)
-	GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error)
-	List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error)
-	ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error)
-	Enable(ctx context.Context, id int64) (int64, error)
-	Disable(ctx context.Context, id int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-}
-
-type sourceRepository struct {
-	conn *sql.DB
-}
-
-func NewSourceRepository(conn *sql.DB) sourceRepository {
-	return sourceRepository{
-		conn: conn,
-	}
-}
-
-func (r sourceRepository) Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("Sources")
-	queryBuilder.Cols("CreatedAt", "UpdatedAt", "DeletedAt", "DisplayName", "Source", "Url", "Tags", "Enabled")
-	queryBuilder.Values(dt, dt, timeZero, displayName, source, url, tags, enabled)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r sourceRepository) GetById(ctx context.Context, id int64) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("Id", id),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("DisplayName", displayName),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) GetBySource(ctx context.Context, source string) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("Source", source),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error) {
-	b := sqlbuilder.NewSelectBuilder()
-	b.Select("*")
-	b.From("Sources").Where(
-		b.Equal("Source", source),
-		b.Equal("DisplayName", name),
-	)
-	b.Limit(1)
-	query, args := b.Build()
-
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return entity.SourceEntity{}, err
-	}
-
-	return data[0], nil
-}
-
-func (r sourceRepository) List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("Sources")
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.SourceEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r sourceRepository) ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("Sources")
-	builder.Where(
-		builder.Equal("Source", source),
-	)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.SourceEntity{}, err
-	}
-
-	data, err := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.SourceEntity{}, err
-	}
-
-	return data, nil
-}
-
-func (r sourceRepository) Enable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("Sources")
-	b.Set(
-		b.Assign("Enabled", true),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r sourceRepository) Disable(ctx context.Context, id int64) (int64, error) {
-	b := sqlbuilder.NewUpdateBuilder()
-	b.Update("Sources")
-	b.Set(
-		b.Assign("Enabled", false),
-		b.Assign("UpdatedAt", time.Now()),
-	)
-	b.Where(
-		b.Equal("Id", id),
-	)
-	query, args := b.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r sourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "Sources", id)
-}
-
-func (r sourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "Sources", id)
-}
-
-func (r sourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "Sources", id)
-}
-
-func (r sourceRepository) processRows(rows *sql.Rows) ([]entity.SourceEntity, error) {
-	items := []entity.SourceEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var displayName string
-		var source string
-		var enabled bool
-		var url string
-		var tags string
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt,
-			&deletedAt, &displayName, &source,
-			&enabled, &url, &tags,
-		)
-		if err != nil {
-			return items, err
-		}
-
-		item := entity.SourceEntity{
-			ID:          id,
-			CreatedAt:   createdAt,
-			UpdatedAt:   updatedAt,
-			DeletedAt:   deletedAt,
-			DisplayName: displayName,
-			Source:      source,
-			Enabled:     enabled,
-			Url:         url,
-			Tags:        tags,
-		}
-
-		items = append(items, item)
-	}
-
-	return items, nil
-}

internal/repository/source_test.go

@@ -1,246 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-)
-
-func TestSourceCreate(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	rows, err := r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if rows != 1 {
-		t.Log("failed to create a record, why")
-		t.FailNow()
-	}
-}
-
-func TestSourceGetById(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	item, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if item.ID != 1 {
-		t.Log("got no record or the wrong one")
-		t.FailNow()
-	}
-}
-
-func TestSourceGetByDisplayName(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	item, err := r.GetByDisplayName(ctx, "Test")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if item.DisplayName != "Test" {
-		t.Log("got no record or the wrong one")
-		t.FailNow()
-	}
-}
-
-func TestSourceGetBySource(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	item, err := r.GetBySource(ctx, domain.SourceCollectorRss)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if item.Source != domain.SourceCollectorRss {
-		t.Log("got no record or the wrong one")
-		t.FailNow()
-	}
-}
-
-func TestSourceList(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-
-	items, err := r.List(ctx, 0, 4)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if len(items ) != 4 {
-		t.Log("something bad happened here")
-		t.FailNow()
-	}
-}
-
-func TestSourceListBySource(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-
-	items, err := r.ListBySource(ctx, 0, 4, domain.SourceCollectorRss)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	if len(items ) != 4 {
-		t.Log("something bad happened here")
-		t.FailNow()
-	}
-}
-
-func TestSourcesEnableRecord(t *testing.T) {
-	// This depends on the seed migration
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-
-	r := repository.NewSourceRepository(db)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", false)
-	item, err := r.GetByDisplayName(ctx, "Test")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != false {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Enable(ctx, item.ID)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, item.ID)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}
-
-func TestSourcesDisableRecord(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	ctx := context.Background()
-	r := repository.NewSourceRepository(db)
-	_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
-	item, err := r.GetByDisplayName(ctx, "Test")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled != true {
-		t.Log("the initial record was created wrong")
-		t.FailNow()
-	}
-
-	_, err = r.Disable(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	updated, err := r.GetById(ctx, 1)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if item.Enabled == updated.Enabled {
-		t.Log("failed to update the enabled value")
-		t.FailNow()
-	}
-}

internal/repository/userSourceSubscription.go

@@ -1,120 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-type UserSourceRepo interface {
-	Create(ctx context.Context, userId, sourceId int64) (int64, error)
-	SoftDelete(ctx context.Context, id int64) (int64, error)
-	Restore(ctx context.Context, id int64) (int64, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error)
-}
-
-type userSourceRepository struct {
-	conn          *sql.DB
-	defaultLimit  int
-	defaultOffset int
-}
-
-func NewUserSourceRepository(conn *sql.DB) userSourceRepository {
-	return userSourceRepository{
-		conn:          conn,
-		defaultLimit:  50,
-		defaultOffset: 50,
-	}
-}
-
-func (r userSourceRepository) Create(ctx context.Context, userId, sourceId int64) (int64, error) {
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("UserSourceSubscriptions")
-	queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID")
-	queryBuilder.Values(dt, dt, timeZero, userId, sourceId)
-	query, args := queryBuilder.Build()
-
-	_, err := r.conn.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (r userSourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
-	return softDeleteRow(ctx, r.conn, "UserSourceSubscriptions", id)
-}
-
-func (r userSourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
-	return restoreRow(ctx, r.conn, "UserSourceSubscriptions", id)
-}
-
-func (r userSourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
-	return deleteFromTable(ctx, r.conn, "UserSourceSubscriptions", id)
-}
-
-func (r userSourceRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*")
-	builder.From("UserSourceSubscriptions")
-	builder.Where(
-		builder.Equal("UserID", userId),
-	)
-	builder.Offset(page * limit)
-	builder.Limit(limit)
-
-	query, args := builder.Build()
-	rows, err := r.conn.QueryContext(ctx, query, args...)
-	if err != nil {
-		return []entity.UserSourceSubscriptionEntity{}, err
-	}
-
-	data := r.processRows(rows)
-	if len(data) == 0 {
-		return []entity.UserSourceSubscriptionEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data, nil
-}
-
-func (ur userSourceRepository) processRows(rows *sql.Rows) []entity.UserSourceSubscriptionEntity {
-	items := []entity.UserSourceSubscriptionEntity{}
-
-	for rows.Next() {
-		var id int64
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt time.Time
-		var userId int64
-		var sourceId int64
-
-		err := rows.Scan(
-			&id, &createdAt, &updatedAt, &deletedAt,
-			&userId, &sourceId,
-		)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.UserSourceSubscriptionEntity{
-			ID:        id,
-			CreatedAt: createdAt,
-			UpdatedAt: updatedAt,
-			DeletedAt: deletedAt,
-			UserID:    userId,
-			SourceID:  sourceId,
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/users.go

@@ -1,192 +0,0 @@
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-
-	"github.com/huandu/go-sqlbuilder"
-	"golang.org/x/crypto/bcrypt"
-)
-
-const (
-	usersTableName  string = "users"
-	ErrUserNotFound string = "requested user was not found"
-)
-
-type Users interface {
-	GetByName(ctx context.Context, name string) (entity.UserEntity, error)
-	Create(ctx context.Context, name, password, sessionTOken, scope string) (int64, error)
-	Update(ctx context.Context, id int, entity entity.UserEntity) error
-	UpdatePassword(ctx context.Context, name, password string) error
-	CheckUserHash(ctx context.Context, name, password string) error
-	UpdateScopes(ctx context.Context, name, scope string) error
-	UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error)
-}
-
-// Creates a new instance of UserRepository with the bound sql
-func NewUserRepository(conn *sql.DB) userRepository {
-	return userRepository{
-		connection: conn,
-	}
-}
-
-type userRepository struct {
-	connection *sql.DB
-}
-
-func (ur userRepository) GetByName(ctx context.Context, name string) (entity.UserEntity, error) {
-	builder := sqlbuilder.NewSelectBuilder()
-	builder.Select("*").From("users").Where(
-		builder.E("Name", name),
-	)
-	query, args := builder.Build()
-
-	rows, err := ur.connection.QueryContext(ctx, query, args...)
-	if err != nil {
-		return entity.UserEntity{}, err
-	}
-
-	data := ur.processRows(rows)
-	if len(data) == 0 {
-		return entity.UserEntity{}, errors.New(ErrUserNotFound)
-	}
-
-	return data[0], nil
-}
-
-func (ur userRepository) Create(ctx context.Context, name, password, sessionToken, scope string) (int64, error) {
-	passwordBytes := []byte(password)
-	hash, err := bcrypt.GenerateFromPassword(passwordBytes, bcrypt.DefaultCost)
-	if err != nil {
-		return 0, err
-	}
-
-	dt := time.Now()
-	queryBuilder := sqlbuilder.NewInsertBuilder()
-	queryBuilder.InsertInto("users")
-	queryBuilder.Cols("Name", "Hash", "UpdatedAt", "CreatedAt", "DeletedAt", "Scopes", "SessionToken")
-	queryBuilder.Values(name, string(hash), dt, dt, time.Time{}, scope, sessionToken)
-	query, args := queryBuilder.Build()
-
-	_, err = ur.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return 1, nil
-}
-
-func (ur userRepository) Update(ctx context.Context, id int, entity entity.UserEntity) error {
-	return errors.New("not implemented")
-}
-
-func (ur userRepository) UpdatePassword(ctx context.Context, name, password string) error {
-	_, err := ur.GetByName(ctx, name)
-	if err != nil {
-		return nil
-	}
-
-	queryBuilder := sqlbuilder.NewUpdateBuilder()
-	queryBuilder.Update(usersTableName)
-	//queryBuilder.Set
-	return nil
-}
-
-func (ur userRepository) UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error) {
-	_, err := ur.GetByName(ctx, name)
-	if err != nil {
-		return 0, err
-	}
-
-	q := sqlbuilder.NewUpdateBuilder()
-	q.Update(usersTableName)
-	q.Set(
-		q.Equal("SessionToken", sessionToken),
-	)
-	q.Where(
-		q.Equal("Name", name),
-	)
-
-	query, args := q.Build()
-	rowsUpdates, err := ur.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return 0, err
-	}
-
-	return rowsUpdates.RowsAffected()
-}
-
-// If the hash matches what we have in the database, an error will not be returned.
-// If the user does not exist or the hash does not match, an error will be returned
-func (ur userRepository) CheckUserHash(ctx context.Context, name, password string) error {
-	record, err := ur.GetByName(ctx, name)
-	if err != nil {
-		return err
-	}
-
-	err = bcrypt.CompareHashAndPassword([]byte(record.Hash), []byte(password))
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (ur userRepository) UpdateScopes(ctx context.Context, name, scope string) error {
-	builder := sqlbuilder.NewUpdateBuilder()
-	builder.Update("users")
-	builder.Set(
-		builder.Assign("Scopes", scope),
-	)
-	builder.Where(
-		builder.Equal("Name", name),
-	)
-	query, args := builder.Build()
-
-	_, err := ur.connection.ExecContext(ctx, query, args...)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (ur userRepository) processRows(rows *sql.Rows) []entity.UserEntity {
-	items := []entity.UserEntity{}
-
-	for rows.Next() {
-		var id int64
-		var username string
-		var hash string
-		var createdAt time.Time
-		var updatedAt time.Time
-		var deletedAt sql.NullTime
-		var scopes string
-		var sessionToken string
-		err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &hash, &scopes, &sessionToken)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		item := entity.UserEntity{
-			ID:           id,
-			UpdatedAt:    updatedAt,
-			Username:     username,
-			Hash:         hash,
-			Scopes:       scopes,
-			CreatedAt:    createdAt,
-			SessionToken: sessionToken,
-		}
-		if deletedAt.Valid {
-			item.DeletedAt = deletedAt.Time
-		}
-
-		items = append(items, item)
-	}
-
-	return items
-}

internal/repository/users_test.go

@@ -1,88 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"database/sql"
-	"log"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-
-	_ "github.com/glebarez/go-sqlite"
-	"github.com/pressly/goose/v3"
-)
-
-func TestCanCreateNewUser(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	defer db.Close()
-
-	repo := repository.NewUserRepository(db)
-	updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
-	if err != nil {
-		log.Println(err)
-		t.FailNow()
-	}
-	log.Println(updated)
-}
-
-func TestCanFindUserInTable(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		log.Println("unable to open connection")
-		t.FailNow()
-	}
-	defer db.Close()
-
-	repo := repository.NewUserRepository(db)
-	updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	if updated != 1 {
-		t.Log("expected a row to come back")
-		t.FailNow()
-	}
-
-	user, err := repo.GetByName(context.Background(), "testing")
-	if err != nil {
-		log.Println(err)
-		t.FailNow()
-	}
-	log.Println(user)
-}
-
-func TestCheckUserHash(t *testing.T) {
-	db, err := setupInMemoryDb()
-	if err != nil {
-		log.Println("unable to open connection")
-		t.FailNow()
-	}
-	defer db.Close()
-
-	repo := repository.NewUserRepository(db)
-	repo.CheckUserHash(context.Background(), "testing", "NotSecure")
-}
-
-func setupInMemoryDb() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", ":memory:")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.SetDialect("sqlite3")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.Up(db, "../database/migrations")
-	if err != nil {
-		return nil, err
-	}
-	return db, nil
-}

internal/repositoryServices/refreshTokens.go

@@ -1,87 +0,0 @@
-package repositoryservices
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-	"github.com/google/uuid"
-)
-
-const (
-	ErrUnexpectedAmountOfRowsUpdated = "got a unexpected of rows updated"
-)
-
-type RefreshToken interface {
-	Create(ctx context.Context, username string) (string, error)
-	GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
-	Delete(ctx context.Context, id int64) (int64, error)
-	IsRequestValid(ctx context.Context, username, refreshToken string) error
-}
-
-// A new jwt token can be made if the user has the correct refresh token for the user.
-// It will also require the old JWT token so the expire time is pulled and part of the validation
-type RefreshTokenService struct {
-	table repository.RefreshTokenRepository
-}
-
-func NewRefreshTokenService(conn *sql.DB) RefreshTokenService {
-	return RefreshTokenService{
-		table: repository.NewRefreshTokenRepository(conn),
-	}
-}
-
-func (rt RefreshTokenService) Create(ctx context.Context, username string) (string, error) {
-	//if a refresh token already exists for a user, reuse
-	existingToken, err := rt.GetByName(ctx, username)
-	if err == nil {
-		rowsRemoved, err := rt.Delete(ctx, existingToken.ID)
-		if err != nil {
-			return "", err
-		}
-
-		if rowsRemoved != 1 {
-			return "", errors.New(ErrUnexpectedAmountOfRowsUpdated)
-		}
-	}
-
-	token, err := uuid.NewV7()
-	if err != nil {
-		return "", err
-	}
-
-	rows, err := rt.table.Create(ctx, username, token.String())
-	if err != nil {
-		return "", err
-	}
-
-	if rows != 1 {
-		return "", errors.New("expected one row but got none")
-	}
-	return token.String(), nil
-}
-
-// Find the saved refresh token for a user and return it if it exists
-func (rt RefreshTokenService) GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
-	return rt.table.GetByUsername(ctx, name)
-}
-
-// This will request that a object is removed from the database
-func (rt RefreshTokenService) Delete(ctx context.Context, id int64) (int64, error) {
-	return rt.table.DeleteById(ctx, id)
-}
-
-func (rt RefreshTokenService) IsRequestValid(ctx context.Context, username, refreshToken string) error {
-	token, err := rt.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	if token.Token != refreshToken {
-		return errors.New("the refresh token given does not match")
-	}
-
-	return nil
-}

internal/repositoryServices/userService.go

@@ -1,198 +0,0 @@
-package repositoryservices
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-	"github.com/google/uuid"
-
-	"golang.org/x/crypto/bcrypt"
-)
-
-const (
-	ErrPasswordNotLongEnough           = "password needs to be 8 character or longer"
-	ErrPasswordMissingSpecialCharacter = "password needs to contain one of the following: !, @, #"
-	ErrInvalidPassword                 = "invalid password"
-)
-
-type UserServices interface {
-	DoesUserExist(ctx context.Context, username string) error
-	DoesPasswordMatchHash(ctx context.Context, username, password string) error
-	GetUser(ctx context.Context, username string) (entity.UserEntity, error)
-	AddScopes(ctx context.Context, username string, scopes []string) error
-	RemoveScopes(ctx context.Context, username string, scopes []string) error
-	Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error)
-	NewSessionToken(ctx context.Context, name string) (string, error)
-	CheckPasswordForRequirements(password string) error
-}
-
-// This will handle operations that are user related, but one layer higher then the repository
-type UserService struct {
-	repo repository.Users
-}
-
-// This is a layer on top of the Users Repository.
-// Use this over directly talking to the table when ever possible.
-func NewUserService(conn *sql.DB) UserService {
-	return UserService{
-		repo: repository.NewUserRepository(conn),
-	}
-}
-
-func (us UserService) DoesUserExist(ctx context.Context, username string) error {
-	_, err := us.repo.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (us UserService) DoesPasswordMatchHash(ctx context.Context, username, password string) error {
-	model, err := us.GetUser(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	err = bcrypt.CompareHashAndPassword([]byte(model.Hash), []byte(password))
-	if err != nil {
-		return errors.New(ErrInvalidPassword)
-	}
-
-	return nil
-}
-
-func (us UserService) GetUser(ctx context.Context, username string) (entity.UserEntity, error) {
-	return us.repo.GetByName(ctx, username)
-}
-
-func (us UserService) AddScopes(ctx context.Context, username string, scopes []string) error {
-	usr, err := us.repo.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	if usr.Username != username {
-		return errors.New(repository.ErrUserNotFound)
-	}
-
-	currentScopes := strings.Split(usr.Scopes, ",")
-
-	// check the current scopes
-	for _, item := range scopes {
-		if !strings.Contains(usr.Scopes, item) {
-			currentScopes = append(currentScopes, item)
-		}
-	}
-	return us.repo.UpdateScopes(ctx, username, strings.Join(currentScopes, ","))
-}
-
-func (us UserService) RemoveScopes(ctx context.Context, username string, scopes []string) error {
-	usr, err := us.repo.GetByName(ctx, username)
-	if err != nil {
-		return err
-	}
-
-	if usr.Username != username {
-		return errors.New(repository.ErrUserNotFound)
-	}
-
-	var newScopes []string
-
-	// check all the scopes that are currently assigned
-	for _, item := range strings.Split(usr.Scopes, ",") {
-
-		// check the scopes given, if one matches skip it
-		if us.doesScopeExist(scopes, item) {
-			continue
-		}
-
-		// did not match, add it
-		newScopes = append(newScopes, item)
-	}
-
-	return us.repo.UpdateScopes(ctx, username, strings.Join(newScopes, ","))
-}
-
-func (us UserService) doesScopeExist(scopes []string, target string) bool {
-	for _, item := range scopes {
-		if item == target {
-			return true
-		}
-	}
-	return false
-}
-
-func (us UserService) Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error) {
-	err := us.CheckPasswordForRequirements(password)
-	if err != nil {
-		return entity.UserEntity{}, err
-	}
-
-	token, err := uuid.NewV7()
-	if err != nil {
-		return entity.UserEntity{}, err
-	}
-
-	us.repo.Create(ctx, name, password, token.String(), domain.ScopeArticleRead)
-	return entity.UserEntity{}, nil
-}
-
-func (us UserService) NewSessionToken(ctx context.Context, name string) (string, error) {
-	token, err := uuid.NewV7()
-	if err != nil {
-		return "", err
-	}
-
-	rows, err := us.repo.UpdateSessionToken(ctx, name, token.String())
-	if err != nil {
-		return "", err
-	}
-
-	if rows != 1 {
-		return "", fmt.Errorf("UserService.NewSessionToken %w", err)
-	}
-
-	return token.String(), nil
-}
-
-func (us UserService) CheckPasswordForRequirements(password string) error {
-	err := us.checkPasswordLength(password)
-	if err != nil {
-		return err
-	}
-
-	err = us.checkPasswordForSpecialCharacters(password)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (us UserService) checkPasswordLength(password string) error {
-	if len(password) < 8 {
-		return errors.New(ErrPasswordNotLongEnough)
-	}
-	return nil
-}
-
-func (us UserService) checkPasswordForSpecialCharacters(password string) error {
-	var chars []string
-	chars = append(chars, "!")
-	chars = append(chars, "@")
-	chars = append(chars, "#")
-
-	for _, char := range chars {
-		if strings.Contains(password, char) {
-			return nil
-		}
-	}
-
-	return errors.New(ErrPasswordMissingSpecialCharacter)
-}

internal/services/config.go

@@ -1,154 +0,0 @@
-package services
-
-import (
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"strconv"
-
-	"github.com/joho/godotenv"
-)
-
-const (
-	ServerAddress = "ServerAddress"
-
-	//Sql_Connection_String = "SQL_CONNECTION_STRING"
-
-	FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
-	REDDIT_PULL_TOP               = "REDDIT_PULL_TOP"
-	REDDIT_PULL_HOT               = "REDDIT_PULL_HOT"
-	REDDIT_PULL_NSFW              = "REDDIT_PULL_NSFW"
-
-	FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
-	YOUTUBE_DEBUG                  = "YOUTUBE_DEBUG"
-
-	FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
-	TWITCH_CLIENT_ID              = "TWITCH_CLIENT_ID"
-	TWITCH_CLIENT_SECRET          = "TWITCH_CLIENT_SECRET"
-	TWITCH_MONITOR_CLIPS          = "TWITCH_MONITOR_CLIPS"
-	TWITCH_MONITOR_VOD            = "TWITCH_MONITOR_VOD"
-
-	FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
-)
-
-type Configs struct {
-	ServerAddress string
-	JwtSecret     string
-	AdminSecret   string
-
-	RedditEnabled  bool
-	RedditPullTop  bool
-	RedditPullHot  bool
-	RedditPullNsfw bool
-
-	YoutubeEnabled bool
-	YoutubeDebug   bool
-
-	TwitchEnabled      bool
-	TwitchClientId     string
-	TwitchClientSecret string
-	TwitchMonitorClips bool
-	TwitchMonitorVOD   bool
-
-	FfxivEnabled bool
-}
-
-type ConfigClient struct{}
-
-func NewConfig() ConfigClient {
-	c := ConfigClient{}
-	c.RefreshEnv()
-
-	return c
-}
-
-func GetEnvConfig() Configs {
-	return Configs{
-		ServerAddress: os.Getenv(ServerAddress),
-		JwtSecret:     os.Getenv("JwtSecret"),
-		AdminSecret:   os.Getenv("AdminSecret"),
-
-		RedditEnabled:  processBoolConfig(os.Getenv(FEATURE_ENABLE_REDDIT_BACKEND)),
-		RedditPullTop:  processBoolConfig(os.Getenv(REDDIT_PULL_TOP)),
-		RedditPullHot:  processBoolConfig(os.Getenv(REDDIT_PULL_HOT)),
-		RedditPullNsfw: processBoolConfig(os.Getenv(REDDIT_PULL_NSFW)),
-
-		YoutubeEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_YOUTUBE_BACKEND)),
-		YoutubeDebug:   processBoolConfig(os.Getenv(YOUTUBE_DEBUG)),
-
-		TwitchEnabled:      processBoolConfig(os.Getenv(FEATURE_ENABLE_TWITCH_BACKEND)),
-		TwitchClientId:     os.Getenv(TWITCH_CLIENT_ID),
-		TwitchClientSecret: os.Getenv(TWITCH_CLIENT_SECRET),
-		TwitchMonitorClips: processBoolConfig(TWITCH_MONITOR_CLIPS),
-		TwitchMonitorVOD:   processBoolConfig(os.Getenv(TWITCH_MONITOR_VOD)),
-
-		FfxivEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_FFXIV_BACKEND)),
-	}
-}
-
-// This will parse a string and convert it to a bool.
-// If it runs into any errors, it will default to false
-func processBoolConfig(value string) bool {
-	b, err := strconv.ParseBool(value)
-	if err != nil {
-		return false
-	}
-	return b
-}
-
-func (cc *ConfigClient) GetConfig(key string) string {
-	res, filled := os.LookupEnv(key)
-	if !filled {
-		log.Printf("Missing the a value for '%v'.  Could generate errors.", key)
-	}
-	return res
-}
-
-// Looks for a value in the env and will panic if it does not exist.
-func (c ConfigClient) MustGetString(key string) string {
-	res, filled := os.LookupEnv(key)
-	if !filled {
-		msg := fmt.Sprintf("No value was found for '%v'", key)
-		panic(msg)
-	}
-
-	return res
-}
-
-func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
-	cc.RefreshEnv()
-
-	res, filled := os.LookupEnv(flag)
-	if !filled {
-		errorMessage := fmt.Sprintf("'%v' was not found", flag)
-		return false, errors.New(errorMessage)
-	}
-
-	b, err := strconv.ParseBool(res)
-	if err != nil {
-		return false, err
-	}
-	return b, nil
-}
-
-// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
-func (cc *ConfigClient) RefreshEnv() {
-	// Check to see if we have the env file on the system
-	_, err := os.Stat(".env")
-
-	// We have the file, load it.
-	if err == nil {
-		_, err := os.Open(".env")
-		if err == nil {
-			loadEnvFile()
-		}
-	}
-}
-
-func loadEnvFile() {
-	err := godotenv.Load()
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

internal/services/config_test.go

@@ -1,22 +0,0 @@
-package services_test
-
-import (
-	"os"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-)
-
-func TestNewClient(t *testing.T) {
-	services.NewConfig()
-}
-
-func TestGetConfigExpectNull(t *testing.T) {
-	cc := services.NewConfig()
-	os.Setenv(services.REDDIT_PULL_HOT, "")
-	res := cc.GetConfig(services.REDDIT_PULL_HOT)
-	if res != "" {
-		panic("expected blank")
-	}
-
-}

internal/services/cron/collectors.go

@@ -1,220 +0,0 @@
-package cron
-
-import (
-	"log"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-func (c *Cron) CollectRssPosts() {
-	log.Println("Starting ")
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
-	if err != nil {
-		log.Println(err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		rssClient := input.NewRssClient(source)
-		articles, err := rssClient.GetArticles()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range articles {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Println(err)
-			}
-			if rowsCreated != 1 {
-				log.Println("Got back the wrong number of rows")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-}
-
-func (c *Cron) CollectRedditPosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
-	if err != nil {
-		log.Printf("[Reddit] No sources found to query - %v\r", err)
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
-		rc := input.NewRedditClient(source)
-		raw, err := rc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		redditArticles := rc.ConvertToArticles(raw)
-		for _, article := range redditArticles {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new reddit article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-	}
-	log.Print("[Reddit] Done!")
-}
-
-func (c *Cron) CollectYoutubePosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
-	if err != nil {
-		log.Printf("[Youtube] No sources found to query - %v\r", err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
-		yc := input.NewYoutubeClient(source)
-		raw, err := yc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range raw {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new youtube article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-	log.Print("[YouTube] Done!")
-}
-
-func (c *Cron) CollectFfxivPosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
-	if err != nil {
-		log.Printf("[FFXIV] No sources found to query - %v\r", err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		fc := input.NewFFXIVClient(source)
-		items, err := fc.CheckSource()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range items {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new FFXIV article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-	log.Printf("[FFXIV Done!]")
-}
-
-func (c *Cron) CollectTwitchPosts() {
-	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
-	if err != nil {
-		log.Printf("[Twitch] No sources found to query - %v\r", err)
-	}
-
-	tc, err := input.NewTwitchClient()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	err = tc.Login()
-	if err != nil {
-		log.Println(err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
-		tc.ReplaceSourceRecord(source)
-		items, err := tc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range items {
-			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new Twitch article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-
-	log.Print("[Twitch] Done!")
-}

internal/services/cron/collectors_test.go

@@ -1,43 +0,0 @@
-package cron_test
-
-import (
-	"context"
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
-)
-
-func TestRssPullsCorrectly(t *testing.T) {
-	conn, err := setupInMemoryDb()
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-	defer conn.Close()
-
-	ctx := context.Background()
-	db := services.NewRepositoryService(conn)
-	rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	if rowsCreated != 1 {
-		t.Error("failed to create the source record")
-		t.FailNow()
-	}
-
-	client := cron.NewScheduler(ctx, conn)
-	client.CollectRssPosts()
-
-	articles, err := db.Articles.ListByPage(ctx, 0, 100)
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	t.Log(len(articles))
-}

internal/services/cron/scheduler.go

@@ -1,131 +0,0 @@
-package cron
-
-import (
-	"context"
-	"database/sql"
-
-	_ "github.com/lib/pq"
-	"github.com/robfig/cron/v3"
-
-	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-)
-
-type Cron struct {
-	//Db    *database.Queries
-	ctx   context.Context
-	timer *cron.Cron
-	repo  services.RepositoryService
-}
-
-func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
-	c := &Cron{
-		ctx:  ctx,
-		repo: services.NewRepositoryService(conn),
-	}
-	timer := cron.New()
-
-	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
-	//features := services.GetEnvConfig()
-
-	timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
-	//timer.AddFunc("10 * * * *", c.CollectRedditPosts)
-	//timer.AddFunc("15 * * * *", c.CheckYoutube)
-	//timer.AddFunc("20 * * * *", c.CheckFfxiv)
-	//timer.AddFunc("25 * * * *", c.CheckTwitch)
-	//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
-
-	c.timer = timer
-	return c
-}
-
-func (c *Cron) Start() {
-	c.timer.Start()
-}
-
-func (c *Cron) Stop() {
-	c.timer.Stop()
-}
-
-/*
-func (c *Cron) CheckDiscordQueue() {
-	// Get items from the table
-	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
-	if err != nil {
-		return err
-	}
-
-	for _, queue := range queueItems {
-		// Get the articleByID
-		article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
-		if err != nil {
-			return err
-		}
-
-		var endpoints []string
-		// List Subscription by SourceID
-		subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
-		if err != nil {
-			return err
-		}
-
-		// if no one is subscribed to it, remove it from the index.
-		if len(subs) == 0 {
-			log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
-			err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		// Get the webhhooks to send to
-		for _, sub := range subs {
-			webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
-			if err != nil {
-				return err
-			}
-
-			// store them in an array
-			endpoints = append(endpoints, webhook.Url)
-		}
-
-		// Create Discord Message
-		dwh := output.NewDiscordWebHookMessage(article)
-		msg, err := dwh.GeneratePayload()
-		if err != nil {
-			return err
-		}
-
-		// Send Message(s)
-		for _, i := range endpoints {
-			err = dwh.SendPayload(msg, i)
-
-			if err != nil {
-				return err
-			}
-		}
-
-		// Remove the item from the queue, given we sent our notification.
-		err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
-		if err != nil {
-			return err
-		}
-
-		time.Sleep(10 * time.Second)
-	}
-
-	return nil
-}
-*/
-
-//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
-//	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
-//		ID:        uuid.New(),
-//		Articleid: Id,
-//	})
-//	if err != nil {
-//		return err
-//	}
-//	return nil
-//}

internal/services/cron/scheduler_test.go

@@ -1,53 +0,0 @@
-package cron_test
-
-import (
-	"database/sql"
-
-	"github.com/pressly/goose/v3"
-)
-
-/*
-func TestInvokeTwitch(t *testing.T) {
-
-}
-
-// TODO add database mocks but not sure how to do that yet.
-func TestCheckReddit(t *testing.T) {
-	ctx := context.Background()
-	c := cron.NewScheduler(ctx)
-	c.Col()
-}
-
-func TestCheckYouTube(t *testing.T) {
-	ctx := context.Background()
-	c := cron.NewScheduler(ctx)
-	c.CheckYoutube()
-}
-
-func TestCheckTwitch(t *testing.T) {
-	ctx := context.Background()
-	c := cron.NewScheduler(ctx)
-	err := c.CheckTwitch()
-	if err != nil {
-		t.Error(err)
-	}
-}
-*/
-
-func setupInMemoryDb() (*sql.DB, error) {
-	db, err := sql.Open("sqlite", ":memory:")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.SetDialect("sqlite3")
-	if err != nil {
-		return nil, err
-	}
-
-	err = goose.Up(db, "../../database/migrations")
-	if err != nil {
-		return nil, err
-	}
-	return db, nil
-}

internal/services/database.go

@@ -1,30 +0,0 @@
-package services
-
-import (
-	"database/sql"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
-	repositoryservices "git.jamestombleson.com/jtom38/newsbot-api/internal/repositoryServices"
-)
-
-type RepositoryService struct {
-	AlertDiscord            repository.AlertDiscordRepo
-	Articles                repository.ArticlesRepo
-	DiscordWebHooks         repository.DiscordWebHookRepo
-	RefreshTokens           repositoryservices.RefreshToken
-	Sources                 repository.Sources
-	Users                   repositoryservices.UserServices
-	UserSourceSubscriptions repository.UserSourceRepo
-}
-
-func NewRepositoryService(conn *sql.DB) RepositoryService {
-	return RepositoryService{
-		AlertDiscord:            repository.NewAlertDiscordRepository(conn),
-		Articles:                repository.NewArticleRepository(conn),
-		DiscordWebHooks:         repository.NewDiscordWebHookRepository(conn),
-		RefreshTokens:           repositoryservices.NewRefreshTokenService(conn),
-		Sources:                 repository.NewSourceRepository(conn),
-		Users:                   repositoryservices.NewUserService(conn),
-		UserSourceSubscriptions: repository.NewUserSourceRepository(conn),
-	}
-}

internal/services/input/httpClient.go

@@ -1,44 +0,0 @@
-package input
-
-import (
-	"crypto/tls"
-	"io"
-	"log"
-	"net/http"
-)
-
-// This will use the net/http client reach out to a site and collect the content.
-func getHttpContent(uri string) ([]byte, error) {
-	// Code to disable the http2 client for reddit.
-	// https://github.com/golang/go/issues/39302
-	tr := http.DefaultTransport.(*http.Transport).Clone()
-	tr.ForceAttemptHTTP2 = false
-	tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
-	tr.TLSClientConfig = &tls.Config{}
-
-	client := &http.Client{
-		Transport: tr,
-	}
-
-	req, err := http.NewRequest("GET", uri, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// set the user agent header to avoid kick backs.. as much
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
-
-	log.Printf("Requesting content from %v\n", uri)
-	resp, err := client.Do(req)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	return body, nil
-}

internal/services/input/reddit.go

@@ -1,187 +0,0 @@
-package input
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
-	"github.com/go-rod/rod"
-	"github.com/go-rod/rod/lib/launcher"
-)
-
-type RedditClient struct {
-	config RedditConfig
-	record entity.SourceEntity
-}
-
-type RedditConfig struct {
-	PullTop  string
-	PullHot  string
-	PullNSFW string
-}
-
-func NewRedditClient(Record entity.SourceEntity) *RedditClient {
-	rc := RedditClient{
-		record: Record,
-	}
-	cc := services.NewConfig()
-	rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
-	rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
-	rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
-
-	//rc.disableHttp2Client()
-
-	return &rc
-}
-
-// This is needed for to get modern go to talk to the endpoint.
-// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
-//func (rc *RedditClient) disableHttp2Client() {
-//	os.Setenv("GODEBUG", "http2client=0")
-//}
-
-func (rc *RedditClient) GetBrowser() *rod.Browser {
-	var browser *rod.Browser
-	if path, exists := launcher.LookPath(); exists {
-		u := launcher.New().Bin(path).MustLaunch()
-		browser = rod.New().ControlURL(u).MustConnect()
-	}
-	return browser
-}
-
-func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
-	page := parser.MustPage(url)
-	return page
-}
-
-//func (rc RedditClient)
-
-// GetContent() reaches out to Reddit and pulls the Json data.
-// It will then convert the data to a struct and return the struct.
-func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
-	var items domain.RedditJsonContent = domain.RedditJsonContent{}
-
-	// TODO Wire this to support the config options
-	Url := fmt.Sprintf("%v.json", rc.record.Url)
-
-	log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
-
-	content, err := getHttpContent(Url)
-	if err != nil {
-		return items, err
-	}
-	if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
-		return items, errors.New("did not get json data from the server")
-	}
-
-	json.Unmarshal(content, &items)
-	if len(items.Data.Children) == 0 {
-		return items, errors.New("failed to unmarshal the data")
-	}
-	return items, nil
-}
-
-func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []entity.ArticleEntity {
-	var redditArticles []entity.ArticleEntity
-	for _, item := range items.Data.Children {
-		var article entity.ArticleEntity
-		article, err := rc.convertToArticle(item.Data)
-		if err != nil {
-			log.Printf("[Reddit] %v", err)
-			continue
-		}
-		redditArticles = append(redditArticles, article)
-	}
-	return redditArticles
-}
-
-// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
-// This data can be passed to the database.
-func (rc *RedditClient) convertToArticle(source domain.RedditPost) (entity.ArticleEntity, error) {
-	var item entity.ArticleEntity
-
-	if source.Content == "" && source.Url != "" {
-		item = rc.convertPicturePost(source)
-	}
-
-	if source.Media.RedditVideo.FallBackUrl != "" {
-		item = rc.convertVideoPost(source)
-	}
-
-	if source.Content != "" {
-		item = rc.convertTextPost(source)
-	}
-
-	if source.UrlOverriddenByDest != "" {
-		item = rc.convertRedirectPost(source)
-	}
-
-	if item.Description == "" && item.Title == "" {
-		var err = errors.New("post failed to parse correctly")
-		return item, err
-	}
-
-	return item, nil
-}
-
-func (rc *RedditClient) convertPicturePost(source domain.RedditPost) entity.ArticleEntity {
-	var item = entity.ArticleEntity{
-		SourceID:       rc.record.ID,
-		Title:          source.Title,
-		Tags:           fmt.Sprintf("%v", rc.record.Tags),
-		Url:            fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		PubDate:        time.Now(),
-		IsVideo:        false,
-		Thumbnail:      source.Thumbnail,
-		Description:    source.Content,
-		AuthorName:     source.Author,
-		AuthorImageUrl: "null",
-	}
-	return item
-}
-
-func (rc *RedditClient) convertTextPost(source domain.RedditPost) entity.ArticleEntity {
-	var item = entity.ArticleEntity{
-		SourceID:    rc.record.ID,
-		Tags:        "a",
-		Title:       source.Title,
-		PubDate:     time.Now(),
-		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
-		Description: source.Content,
-	}
-	return item
-}
-
-func (rc *RedditClient) convertVideoPost(source domain.RedditPost) entity.ArticleEntity {
-	var item = entity.ArticleEntity{
-		SourceID:    rc.record.ID,
-		Tags:        "a",
-		Title:       source.Title,
-		PubDate:     time.Now(),
-		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
-		Description: source.Media.RedditVideo.FallBackUrl,
-	}
-	return item
-}
-
-// This post is nothing more then a redirect to another location.
-func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) entity.ArticleEntity {
-	var item = entity.ArticleEntity{
-		SourceID:    rc.record.ID,
-		Tags:        "a",
-		Title:       source.Title,
-		PubDate:     time.Now(),
-		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
-		Description: source.UrlOverriddenByDest,
-	}
-	return item
-}

internal/services/input/reddit_test.go

@@ -1,32 +0,0 @@
-package input_test
-
-import (
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-var RedditRecord entity.SourceEntity = entity.SourceEntity{
-	ID:          9999,
-	DisplayName: "dadjokes",
-	Source:      domain.SourceCollectorRss,
-	Url:         "https://reddit.com/r/dadjokes",
-	Tags:        "reddit, dadjokes",
-}
-
-func TestGetContent(t *testing.T) {
-	//This test is flaky right now due to the http changes in 1.17
-	rc := input.NewRedditClient(RedditRecord)
-	raw, err := rc.GetContent()
-	if err != nil {
-		t.Error(err)
-	}
-	redditArticles := rc.ConvertToArticles(raw)
-	for _, posts := range redditArticles {
-		if posts.Title == "" {
-			t.Error("Title is missing")
-		}
-	}
-}

internal/services/input/rss.go

@@ -1,77 +0,0 @@
-package input
-
-import (
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"github.com/mmcdole/gofeed"
-)
-
-type FeedInput interface {
-	GetArticles() (entity.ArticleEntity, error)
-}
-
-type rssClient struct {
-	SourceRecord entity.SourceEntity
-}
-
-func NewRssClient(sourceRecord entity.SourceEntity) rssClient {
-	client := rssClient{
-		SourceRecord: sourceRecord,
-	}
-
-	return client
-}
-
-func (rc rssClient) GetArticles() ([]entity.ArticleEntity, error) {
-	parser := gofeed.NewParser()
-	feed, err := parser.ParseURL(rc.SourceRecord.Url)
-	if err != nil {
-		return nil, err
-	}
-
-	sourceTags := strings.Split(rc.SourceRecord.Tags, ",")
-	var articles []entity.ArticleEntity
-	for _, post := range feed.Items {
-		article := entity.ArticleEntity{
-			SourceID:    rc.SourceRecord.ID,
-			Title:       post.Title,
-			Description: post.Content,
-			Url:         post.Link,
-			PubDate:     *post.PublishedParsed,
-			//AuthorName:  post.Authors[0].Email,
-		}
-
-		if len(post.Authors) != 0 {
-			article.AuthorName = post.Authors[0].Email
-		}
-
-		var postTags []string
-		postTags = append(postTags, sourceTags...)
-		postTags = append(postTags, post.Categories...)
-		article.Tags = strings.Join(postTags, ",")
-
-		/*
-			pageContent, err := getHttpContent(article.Url)
-			if err != nil {
-				continue
-			}
-
-			htmlNode, err := html.Parse(bytes.NewReader(pageContent))
-			if err != nil {
-				continue
-			}
-			htmlNode.
-
-			fmt.Println(htmlNode)
-		*/
-
-		if post.Image == nil {
-			article.Thumbnail = ""
-		}
-
-		articles = append(articles, article)
-	}
-
-	return articles, nil
-}

internal/services/input/rss_test.go

@@ -1,43 +0,0 @@
-package input_test
-
-import (
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-var rssRecord = entity.SourceEntity{
-	ID:          1,
-	DisplayName: "ArsTechnica",
-	Url:         "https://feeds.arstechnica.com/arstechnica/index",
-	Source:      domain.SourceCollectorRss,
-}
-
-func TestRssClientConstructor(t *testing.T) {
-	input.NewRssClient(rssRecord)
-}
-
-func TestRssGetFeed(t *testing.T) {
-	client := input.NewRssClient(rssRecord)
-	_, err := client.GetArticles()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestRssAgainstGita(t *testing.T) {
-	client := input.NewRssClient(entity.SourceEntity{
-		ID:          2,
-		DisplayName: "Gitea - Newsbot-api",
-		Source:      domain.SourceCollectorRss,
-		Url:         "https://git.jamestombleson.com/jtom38/newsbot-api.rss",
-		Tags:        "rss,gitea,newsbot-api",
-	})
-	_, err := client.GetArticles()
-	if err != nil {
-		t.Error(err)
-	}
-
-}

internal/services/input/youtube_test.go

@@ -1,149 +0,0 @@
-package input_test
-
-import (
-	"testing"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-var YouTubeRecord = entity.SourceEntity{
-	ID:          9999,
-	DisplayName: "dadjokes",
-	Source:      domain.SourceCollectorReddit,
-	Url:         "https://youtube.com/gamegrumps",
-}
-
-func TestGetPageParser(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	_, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestGetChannelId(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	parser, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil {
-		t.Error(err)
-	}
-
-	_, err = yc.GetChannelId(parser)
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestPullFeed(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	parser, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil {
-		t.Error(err)
-	}
-
-	_, err = yc.GetChannelId(parser)
-	if err != nil {
-		t.Error(err)
-	}
-
-	_, err = yc.PullFeed()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestGetAvatarUri(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	res, err := yc.GetAvatarUri()
-	if err != nil {
-		t.Error(err)
-	}
-	if res == "" {
-		t.Error(input.ErrMissingAuthorImage)
-	}
-}
-
-func TestGetVideoTags(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-
-	var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
-
-	parser, err := yc.GetParser(videoUri)
-	if err != nil {
-		t.Error(err)
-	}
-
-	tags, err := yc.GetTags(parser)
-	if err == nil && tags == "" {
-		t.Error("err was empty but value was missing.")
-	}
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestGetChannelTags(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-
-	parser, err := yc.GetParser(YouTubeRecord.Url)
-	if err != nil {
-		t.Error(err)
-	}
-
-	tags, err := yc.GetTags(parser)
-	if err == nil && tags == "" {
-		t.Error("no err but expected value was missing.")
-	}
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestGetVideoThumbnail(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
-	if err != nil {
-		t.Error(err)
-	}
-
-	thumb, err := yc.GetVideoThumbnail(parser)
-	if err == nil && thumb == "" {
-		t.Error("no err but expected result was missing")
-	}
-	if err != nil {
-		t.Error(err)
-	}
-
-}
-
-func TestCheckSource(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	_, err := yc.GetContent()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestCheckUriCache(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	item := "demo"
-
-	input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
-	res := yc.CheckUriCache(&item)
-	if res == false {
-		t.Error("expected a value to come back")
-	}
-}
-
-func TestCheckUriCacheFails(t *testing.T) {
-	yc := input.NewYoutubeClient(YouTubeRecord)
-	item := "demo1"
-
-	res := yc.CheckUriCache(&item)
-	if res == true {
-		t.Error("expected no value to come back")
-	}
-
-}

internal/services/output/discordwebhook.go

@@ -1,191 +0,0 @@
-package output
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
-)
-
-type discordField struct {
-	Name   *string `json:"name,omitempty"`
-	Value  *string `json:"value,omitempty"`
-	Inline *bool   `json:"inline,omitempty"`
-}
-
-type discordFooter struct {
-	Value   *string `json:"text,omitempty"`
-	IconUrl *string `json:"icon_url,omitempty"`
-}
-
-type discordAuthor struct {
-	Name    *string `json:"name,omitempty"`
-	Url     *string `json:"url,omitempty"`
-	IconUrl *string `json:"icon_url,omitempty"`
-}
-
-type discordImage struct {
-	Url *string `json:"url,omitempty"`
-}
-
-type DiscordEmbed struct {
-	Title       *string `json:"title,omitempty"`
-	Description *string `json:"description,omitempty"`
-	Url         *string `json:"url,omitempty"`
-	Color       *int32  `json:"color,omitempty"`
-	//Timestamp   time.Time      `json:"timestamp,omitempty"`
-	Fields    []*discordField `json:"fields,omitempty"`
-	Author    discordAuthor   `json:"author,omitempty"`
-	Image     discordImage    `json:"image,omitempty"`
-	Thumbnail discordImage    `json:"thumbnail,omitempty"`
-	Footer    *discordFooter  `json:"footer,omitempty"`
-}
-
-// Root object for Discord Webhook messages
-type DiscordMessage struct {
-	Username *string         `json:"username,omitempty"`
-	Content  *string         `json:"content,omitempty"`
-	Embeds   *[]DiscordEmbed `json:"embeds,omitempty"`
-}
-
-const (
-	DefaultColor = 0
-	YoutubeColor = 16711680
-	TwitchColor  = 0
-	RedditColor  = 0
-	TwitterColor = 0
-	FfxivColor   = 0
-)
-
-type Discord struct {
-	Subscriptions []string
-	article       entity.ArticleEntity
-	Message       *DiscordMessage
-}
-
-func NewDiscordWebHookMessage(Article entity.ArticleEntity) Discord {
-	return Discord{
-		article: Article,
-	}
-}
-
-// Generates the link field to expose in the message
-func (dwh Discord) getFields() []*discordField {
-	var fields []*discordField
-
-	key := "Link"
-	linkField := discordField{
-		Name:  &key,
-		Value: &dwh.article.Url,
-	}
-
-	fields = append(fields, &linkField)
-
-	return fields
-}
-
-// This will create the message that will be sent out.
-func (dwh Discord) GeneratePayload() (*DiscordMessage, error) {
-
-	// Create the embed
-	footerMessage := "Brought to you by Newsbot"
-	footerUrl := ""
-	description := dwh.convertFromHtml(dwh.article.Description)
-	color := dwh.getColor(dwh.article.Url)
-
-	embed := DiscordEmbed{
-		Title:       &dwh.article.Title,
-		Description: &description,
-		Image: discordImage{
-			Url: &dwh.article.Thumbnail,
-		},
-		Fields: dwh.getFields(),
-		Footer: &discordFooter{
-			Value:   &footerMessage,
-			IconUrl: &footerUrl,
-		},
-		Color: &color,
-	}
-
-	// attach the embed to an array
-	var embedArray []DiscordEmbed
-	embedArray = append(embedArray, embed)
-
-	// create the base message
-	msg := DiscordMessage{
-		Embeds: &embedArray,
-	}
-
-	return &msg, nil
-}
-
-func (dwh Discord) SendPayload(Message *DiscordMessage, Url string) error {
-	// Convert the message to a io.reader object
-	buffer := new(bytes.Buffer)
-	json.NewEncoder(buffer).Encode(Message)
-
-	// Send the message
-	resp, err := http.Post(Url, "application/json", buffer)
-	if err != nil {
-		return err
-	}
-
-	// Check for 204
-	if resp.StatusCode != 204 {
-		defer resp.Body.Close()
-
-		errMsg, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-
-		return fmt.Errorf(string(errMsg))
-	}
-
-	return nil
-}
-
-func (dwh Discord) convertFromHtml(body string) string {
-	clean := body
-	clean = strings.ReplaceAll(clean, "<h2>", "**")
-	clean = strings.ReplaceAll(clean, "</h2>", "**")
-	clean = strings.ReplaceAll(clean, "<h3>", "**")
-	clean = strings.ReplaceAll(clean, "</h3>", "**\r\n")
-	clean = strings.ReplaceAll(clean, "<strong>", "**")
-	clean = strings.ReplaceAll(clean, "</strong>", "**\r\n")
-	clean = strings.ReplaceAll(clean, "<ul>", "\r\n")
-	clean = strings.ReplaceAll(clean, "</ul>", "")
-	clean = strings.ReplaceAll(clean, "</li>", "\r\n")
-	clean = strings.ReplaceAll(clean, "<li>", "> ")
-	clean = strings.ReplaceAll(clean, "&#8220;", "\"")
-	clean = strings.ReplaceAll(clean, "&#8221;", "\"")
-	clean = strings.ReplaceAll(clean, "&#8230;", "...")
-	clean = strings.ReplaceAll(clean, "<b>", "**")
-	clean = strings.ReplaceAll(clean, "</b>", "**")
-	clean = strings.ReplaceAll(clean, "<br>", "\r\n")
-	clean = strings.ReplaceAll(clean, "<br/>", "\r\n")
-	clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*")
-	clean = strings.ReplaceAll(clean, "\xa0", "\r\n")
-	clean = strings.ReplaceAll(clean, "<p>", "")
-	clean = strings.ReplaceAll(clean, "</p>", "\r\n")
-	return clean
-}
-
-func (dwh *Discord) getColor(Url string) int32 {
-	if strings.Contains(Url, "youtube.com") {
-		return YoutubeColor
-	}
-
-	return DefaultColor
-
-}
-
-func (dwh *Discord) convertLinks(body string) string {
-	//items := regexp.MustCompile("<a(.*?)a>")
-	return ""
-}

internal/services/output/discordwebhook_test.go

@@ -1,134 +0,0 @@
-package output_test
-
-import (
-	"os"
-	"strings"
-	"testing"
-
-	//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
-	"github.com/joho/godotenv"
-)
-
-var (
-	article entity.ArticleEntity = entity.ArticleEntity{
-		ID:       999,
-		SourceID: 1,
-		Tags:     "unit, testing",
-		Title:    "Demo",
-		Url:      "https://github.com/jtom38/newsbot.collector.api",
-		Description: "Hello World",
-	}
-	blank string = ""
-)
-
-func TestDiscordMessageContainsTitle(t *testing.T) {
-	d := output.NewDiscordWebHookMessage(article)
-	msg, err := d.GeneratePayload()
-	if err != nil {
-		t.Error(err)
-	}
-
-	for _, i := range *msg.Embeds {
-		if i.Title == &blank {
-			t.Error("title missing")
-		}
-	}
-}
-
-func TestDiscordMessageContainsDescription(t *testing.T) {
-	d := output.NewDiscordWebHookMessage(article)
-	msg, err := d.GeneratePayload()
-	if err != nil {
-		t.Error(err)
-	}
-
-	for _, i := range *msg.Embeds {
-		if i.Description == &blank {
-			t.Error("description missing")
-		}
-	}
-}
-
-func TestDiscordMessageFooter(t *testing.T) {
-	d := output.NewDiscordWebHookMessage(article)
-	msg, err := d.GeneratePayload()
-	if err != nil {
-		t.Error(err)
-	}
-	for _, i := range *msg.Embeds {
-		blank := ""
-		if i.Footer.Value == &blank {
-			t.Error("missing footer vlue")
-		}
-		if i.Footer.IconUrl == &blank {
-			t.Error("missing footer url")
-		}
-	}
-}
-
-func TestDiscordMessageFields(t *testing.T) {
-	header := "Link"
-	d := output.NewDiscordWebHookMessage(article)
-	msg, err := d.GeneratePayload()
-	if err != nil {
-		t.Error(err)
-	}
-	for _, embed := range *msg.Embeds {
-		for _, field := range embed.Fields {
-			var fName string
-			if field.Name != nil {
-				fName = *field.Name
-			} else {
-				t.Error("missing link field value")
-			}
-
-			if fName != header {
-				t.Error("missing link field key")
-			}
-
-			var fValue string
-			if field.Value != nil {
-				fValue = *field.Value
-			}
-
-			if fValue == blank {
-				t.Error("missing link field value")
-			}
-		}
-	}
-}
-
-// This test requires a env value to be present to work
-func TestDiscordMessagePost(t *testing.T) {
-	_, err := os.Open(".env")
-	if err != nil {
-		t.Error(err)
-	}
-
-	err = godotenv.Load()
-	if err != nil {
-		t.Error(err)
-	}
-
-	res := os.Getenv("TESTS_DISCORD_WEBHOOK")
-	if res == "" {
-		t.Error("TESTS_DISCORD_WEBHOOK is missing")
-	}
-	endpoints := strings.Split(res, " ")
-	if err != nil {
-		t.Error(err)
-	}
-
-	d := output.NewDiscordWebHookMessage(article)
-	msg, err := d.GeneratePayload()
-	if err != nil {
-		t.Error(err)
-	}
-
-	err = d.SendPayload(msg, endpoints[0])
-	if err != nil {
-		t.Error(err)
-	}
-}

internal/services/output/interface.go

@@ -1,6 +0,0 @@
-package output
-
-type Output interface {
-	GeneratePayload() error
-	SendPayload() error
-}

main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"context"
+	"log"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
+
+	"github.com/jtom38/newsbot/collector/routes"
+	"github.com/jtom38/newsbot/collector/services/cron"
+)
+
+func main() {
+	ctx := context.Background()
+
+	cron.EnableScheduler(ctx)
+
+	app := chi.NewRouter()
+	app.Use(middleware.Logger)
+	app.Use(middleware.Recoverer)
+
+	//app.Mount("/swagger", httpSwagger.WrapHandler)
+	app.Mount("/api", routes.RootRoutes())
+	
+	log.Println("API is online and waiting for requests.")
+	log.Println("API: http://localhost:8081/api")
+	//log.Println("Swagger: http://localhost:8080/swagger/index.html")
+	err := http.ListenAndServe(":8081", app)
+	if err != nil { log.Fatalln(err) }
+}

makefile

@@ -3,29 +3,14 @@ help: ## Shows this help command
 	@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
 build: ## builds the application with the current go runtime
-	~/go/bin/swag f
-	~/go/bin/swag init -g cmd/server.go
-	go build cmd/server.go
-	ls -lh server
+	go build .
 	
 docker-build: ## Generates the docker image
 	docker build -t "newsbot.collector.api" .
 	docker image ls | grep newsbot.collector.api
 
 migrate-dev: ## Apply sql migrations to dev db
-	goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up 
+	goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up 
 
 migrate-dev-down: ## revert sql migrations to dev db
-	goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down 
-
-swag: ## Generates the swagger documentation with the swag tool
-	~/go/bin/swag f -g cmd/server.go
-	~/go/bin/swag init -g cmd/server.go
-	go run tools/swaggertoopenapi/main.go
-	oapi-codegen -config api/client.yaml docs/openapi.json
-	mv api.gen.go api/
-
-install-tools: ## Installs the required tools for this project
-	go install github.com/swaggo/swag/cmd/swag@v1.8.1
-	go install github.com/pressly/goose/v3/cmd/goose@latest
-	go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.3.0
+	goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down 

routes/root.go

@@ -0,0 +1,25 @@
+package routes
+
+import (
+	"net/http"
+	"fmt"
+	"github.com/go-chi/chi/v5"
+)
+
+func RootRoutes() chi.Router {
+	app := chi.NewRouter()
+	app.Get("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("Hello World!"))
+	})
+
+	app.Get("/ping", func(w http.ResponseWriter, r *http.Request) {
+		msg := "pong"
+		w.Write([]byte(msg))
+	})
+
+	app.Get("/hello/{world}", func(w http.ResponseWriter, r *http.Request) {
+		msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "world"))
+		w.Write([]byte(msg))
+	})
+	return app
+}

services/cache/cache.go

@@ -3,7 +3,7 @@ package cache
 import (
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"github.com/jtom38/newsbot/collector/domain/model"
 )
 
 type CacheClient struct{
@@ -19,7 +19,7 @@ func NewCacheClient(group string) CacheClient {
 }
 
 func (cc *CacheClient) Insert(key string, value string) {
-	item := domain.CacheItem{
+	item := model.CacheItem{
 		Key: key,
 		Value: value,
 		Group: cc.group,
@@ -29,14 +29,10 @@ func (cc *CacheClient) Insert(key string, value string) {
 	cacheStorage = append(cacheStorage, &item)
 }
 
-func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
+func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
 	for _, item := range cacheStorage {
-		if item.Group != cc.group {
-			continue
-		}
-		if item.Key != key {
-			continue
-		}
+		if item.Group != cc.group { continue }
+		if item.Key != key { continue }
 
 		// if it was tainted, renew the timer and remove the taint as this record was still needed
 		if item.IsTainted {
@@ -46,17 +42,13 @@ func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
 		return item, nil
 	}
 
-	return &domain.CacheItem{}, ErrCacheRecordMissing
+	return &model.CacheItem{}, ErrCacheRecordMissing
 }
 
-func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
+func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
 	for _, item := range cacheStorage {
-		if item.Group != cc.group {
-			continue
-		}
-		if item.Value != value {
-			continue
-		}
+		if item.Group != cc.group { continue }
+		if item.Value != value { continue }
 
 		// if it was tainted, renew the timer and remove the taint as this record was still needed
 		if item.IsTainted {
@@ -65,5 +57,6 @@ func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
 		}
 		return item, nil
 	}
-	return &domain.CacheItem{}, ErrCacheRecordMissing
+	return &model.CacheItem{}, ErrCacheRecordMissing
 }
+

services/cache/cache_test.go

@@ -3,7 +3,7 @@ package cache_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
+	"github.com/jtom38/newsbot/collector/services/cache"
 )
 
 func TestNewCacheClient(t *testing.T) {
@@ -18,18 +18,14 @@ func TestInsert(t *testing.T) {
 func TestFindGroupMissing(t *testing.T) {
 	cache := cache.NewCacheClient("faker")
 	_, err := cache.FindByKey("UnitTesting")
-	if err == nil {
-		panic("Nothing was appended with the requested group.")
-	}
+	if err == nil { panic("Nothing was appended with the requested group.") }
 }
 
 func TestFindGroupExists(t *testing.T) {
 	cache := cache.NewCacheClient("Testing")
 	cache.Insert("UnitTesting", "Something")
 	_, err := cache.FindByKey("UnitTesting")
-	if err != nil {
-		panic("")
-	}
+	if err != nil { panic("") }
 }
 
 func TestCacheStorage(t *testing.T) {
@@ -39,7 +35,6 @@ func TestCacheStorage(t *testing.T) {
 
 	cache := cache.NewCacheClient("Testing")
 	_, err := cache.FindByKey("UnitTesting02")
-	if err != nil {
-		panic("expected to find the value")
-	}
+	if err != nil { panic("expected to find the value")}
 }
+

services/cache/common.go

@@ -3,11 +3,11 @@ package cache
 import (
 	"errors"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"github.com/jtom38/newsbot/collector/domain/model"
 )
 
 var (
-	cacheStorage []*domain.CacheItem
+	cacheStorage []*model.CacheItem
 
 	ErrCacheRecordMissing = errors.New("unable to find the requested record")
 )

services/cache/monitor.go

@@ -3,7 +3,7 @@ package cache
 import (
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"github.com/jtom38/newsbot/collector/domain/model"
 )
 
 // When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
@@ -36,12 +36,10 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
 }
 
 // This creates a new slice and skips over the item that needs to be dropped
-func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
-	var temp []*domain.CacheItem
+func (cam CacheAgeMonitor) removeEntry(index int) []*model.CacheItem {
+	var temp []*model.CacheItem
 	for i, item := range cacheStorage {
-		if i != index {
-			temp = append(temp, item)
-		}
+		if i != index { temp = append(temp, item )}
 	}
 	return temp
 }

services/cache/monitor_test.go

@@ -3,7 +3,7 @@ package cache_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
+	"github.com/jtom38/newsbot/collector/services/cache"
 )
 
 func TestCacheTaintItem(t *testing.T) {

services/common.go

@@ -1,4 +1,4 @@
-package input
+package services
 
 import "errors"
 

services/config/config.go

@@ -0,0 +1,56 @@
+package config
+
+import (
+	"os"
+	"log"
+
+	"github.com/joho/godotenv"
+)
+
+const (
+	DB_URI string	= "DB_URI"
+	
+	Sql_Connection_String string = "SQL_CONNECTION_STRING"
+
+	REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
+	REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
+	REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
+
+	YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
+
+	TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
+	TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
+	TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
+	TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
+)
+
+type ConfigClient struct {}
+
+func New() ConfigClient {
+	_, err := os.Open(".env")
+	if err == nil {
+		loadEnvFile()
+	}
+
+	return ConfigClient{}
+}
+
+func (cc *ConfigClient) GetConfig(key string) string {
+	res, filled := os.LookupEnv(key)
+	if !filled {
+		log.Printf("Missing the a value for '%v'.  Could generate errors.", key)
+	}
+	return res
+}
+
+// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
+func (cc *ConfigClient) RefreshEnv() {
+	loadEnvFile()
+}
+
+func loadEnvFile() {
+	err := godotenv.Load()
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

services/config/config_test.go

@@ -0,0 +1,20 @@
+package config_test
+
+import (
+	"testing"
+	"os"
+
+	"github.com/jtom38/newsbot/collector/services/config"
+)
+
+func TestNewClient(t *testing.T) {
+	config.New()
+}
+
+func TestGetConfigExpectNull(t *testing.T) {
+	cc := config.New()
+	os.Setenv(config.REDDIT_PULL_HOT, "")
+	res := cc.GetConfig(config.REDDIT_PULL_HOT)
+	if res != "" { panic("expected blank")}
+
+}

services/cron/scheduler.go

@@ -0,0 +1,167 @@
+package cron
+
+import (
+	"context"
+	"database/sql"
+	"log"
+	"time"
+
+	"github.com/google/uuid"
+	_ "github.com/lib/pq"
+	"github.com/robfig/cron/v3"
+
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services"
+	"github.com/jtom38/newsbot/collector/services/config"
+)
+
+var _env config.ConfigClient
+var _connString string
+var _queries *database.Queries
+
+func EnableScheduler(ctx context.Context) {
+	c := cron.New()
+	OpenDatabase(ctx)
+
+	//c.AddFunc("*/5 * * * *", func()  { go CheckCache() })
+	c.AddFunc("* */1 * * *", func() { go CheckReddit(ctx) })
+	//c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
+	//c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
+	//c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
+
+	c.Start()
+}
+
+// Open the connection to the database and share it with the package so all of them are able to share.
+func OpenDatabase(ctx context.Context) error {
+	_env = config.New()
+	_connString = _env.GetConfig(config.Sql_Connection_String)
+	db, err := sql.Open("postgres", _connString)
+	if err != nil {
+		panic(err)
+	}
+
+	queries := database.New(db)
+	_queries = queries
+	return err
+}
+
+// This is the main entry point to query all the reddit services
+func CheckReddit(ctx context.Context) {
+	sources, err := _queries.ListSourcesBySource(ctx, "reddit")
+	if err != nil {
+		log.Printf("No defines sources for reddit to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		rc := services.NewRedditClient(source)
+		raw, err := rc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		redditArticles := rc.ConvertToArticles(raw)
+		checkPosts(ctx, redditArticles)
+	}
+}
+
+func CheckYoutube(ctx context.Context) {
+	// Add call to the db to request youtube sources.
+	sources, err := _queries.ListSourcesBySource(ctx, "youtube")
+	if err != nil {
+		log.Printf("Youtube - No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		yc := services.NewYoutubeClient(source)
+		raw, err := yc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		checkPosts(ctx, raw)
+	}
+}
+
+func CheckFfxiv(ctx context.Context) {
+	sources, err := _queries.ListSourcesBySource(ctx, "ffxiv")
+	if err != nil {
+		log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		fc := services.NewFFXIVClient(source)
+		items, err := fc.CheckSource()
+		if err != nil {
+			log.Println(err)
+		}
+		checkPosts(ctx, items)
+	}
+}
+
+func CheckTwitch(ctx context.Context) error {
+	sources, err := _queries.ListSourcesBySource(ctx, "twitch")
+	if err != nil {
+		log.Printf("Twitch - No sources found to query - %v\r", err)
+	}
+	
+	tc, err := services.NewTwitchClient()
+	if err != nil {
+		return err
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		tc.ReplaceSourceRecord(source)
+		items, err := tc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		checkPosts(ctx, items)
+	}
+
+	return nil
+}
+
+func checkPosts(ctx context.Context, posts []database.Article) {
+	for _, item := range posts {
+		_, err := _queries.GetArticleByUrl(ctx, item.Url)
+		if err != nil {
+			err = postArticle(ctx, item)
+			if err != nil {
+				log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
+			} else {
+				log.Printf("Reddit - Posted article - %v\r", item.Url)
+			}
+		}
+	}
+	time.Sleep(30 * time.Second)
+}
+
+func postArticle(ctx context.Context, item database.Article) error {
+	err := _queries.CreateArticle(ctx, database.CreateArticleParams{
+		ID:          uuid.New(),
+		Sourceid:    item.Sourceid,
+		Tags:        item.Tags,
+		Title:       item.Title,
+		Url:         item.Url,
+		Pubdate:     item.Pubdate,
+		Video:       item.Video,
+		Videoheight: item.Videoheight,
+		Videowidth:  item.Videowidth,
+		Thumbnail:   item.Thumbnail,
+		Description: item.Description,
+		Authorname:  item.Authorname,
+		Authorimage: item.Authorimage,
+	})
+	return err
+}

services/cron/scheduler_test.go

@@ -0,0 +1,37 @@
+package cron_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/jtom38/newsbot/collector/services/cron"
+)
+
+func TestInvokeTwitch(t *testing.T) {
+
+}
+
+// TODO add database mocks but not sure how to do that yet.
+func TestCheckReddit(t *testing.T) {
+	ctx := context.Background()
+	cron.OpenDatabase(ctx)
+	cron.CheckReddit(ctx)
+}
+
+func TestCheckYouTube(t *testing.T) {
+	ctx := context.Background()
+	cron.OpenDatabase(ctx)
+	cron.CheckYoutube(ctx)
+}
+
+func TestCheckTwitch(t *testing.T) {
+	ctx := context.Background()
+	err := cron.OpenDatabase(ctx)
+	if err != nil {
+		t.Error(err)
+	}
+	err = cron.CheckTwitch(ctx)
+	if err != nil {
+		t.Error(err)
+	}
+}

services/ffxiv.go

@@ -1,6 +1,7 @@
-package input
+package services
 
 import (
+	"database/sql"
 	"errors"
 	"log"
 	"net/http"
@@ -9,11 +10,10 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/go-rod/rod"
-	"github.com/go-rod/rod/lib/launcher"
 	"github.com/google/uuid"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services/cache"
 )
 
 const (
@@ -24,7 +24,7 @@ const (
 )
 
 type FFXIVClient struct {
-	record entity.SourceEntity
+	record database.Source
 	//SourceID uint
 	//Url string
 	//Region string
@@ -32,80 +32,65 @@ type FFXIVClient struct {
 	cacheGroup string
 }
 
-func NewFFXIVClient(Record entity.SourceEntity) FFXIVClient {
+func NewFFXIVClient(Record database.Source) FFXIVClient {
 	return FFXIVClient{
 		record: Record,
 		cacheGroup: "ffxiv",
 	}
 }
 
-func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
-	var articles []entity.ArticleEntity
+func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
+	var articles []database.Article
 
 	parser := fc.GetBrowser()
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		return articles, err
-	}
+	if err != nil { return articles, err }
 
 	cache := cache.NewCacheClient(fc.cacheGroup)
 
 	for _, link := range links {
 		// Check cache/db if this link has been seen already, skip
 		_, err := cache.FindByValue(link)
-		if err == nil {
-			continue
-		}
+		if err == nil { continue }
+		
 
 		page := fc.GetPage(parser, link)
 
 		title, err := fc.ExtractTitle(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err }
 
 		thumb, err := fc.ExtractThumbnail(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err }
 
 		pubDate, err := fc.ExtractPubDate(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err }
 
 		description, err := fc.ExtractDescription(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err }
 
 		authorName, err := fc.ExtractAuthor(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err }
 
 		authorImage, err := fc.ExtractAuthorImage(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err }
 
 		tags, err := fc.ExtractTags(page)
-		if err != nil {
-			return articles, err
-		}
+		if err != nil { return articles, err } 
 
-		article := entity.ArticleEntity{
-			SourceID:       fc.record.ID,
+		article := database.Article{
+			Sourceid: fc.record.ID,
 			Tags: tags,
 			Title: title,
 			Url: link,
-			PubDate:        pubDate,
+			Pubdate: pubDate,
+			Videoheight: 0,
+			Videowidth: 0,
 			Thumbnail: thumb,
 			Description: description,
-			AuthorName:     authorName,
-			AuthorImageUrl: authorImage,
+			Authorname: sql.NullString{String: authorName},
+			Authorimage: sql.NullString{String: authorImage},
 		}
 		log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
 
@@ -119,24 +104,16 @@ func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
 
 func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
 	html, err := http.Get(fc.record.Url)
-	if err != nil {
-		return nil, err
-	}
+	if err != nil { return nil, err }
 	defer html.Body.Close()
 	
 	doc, err := goquery.NewDocumentFromReader(html.Body)
-	if err != nil {
-		return nil, err
-	}
+	if err != nil { return nil, err }
 	return doc, nil
 }
 
-func (fc *FFXIVClient) GetBrowser() *rod.Browser {
-	var browser *rod.Browser
-	if path, exists := launcher.LookPath(); exists {
-		u := launcher.New().Bin(path).MustLaunch()
-		browser = rod.New().ControlURL(u).MustConnect()
-	}
+func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
+	browser := rod.New().MustConnect()
 	return browser
 }
 
@@ -184,9 +161,7 @@ func (rc *FFXIVClient) GetPage(parser *rod.Browser, url string) *rod.Page {
 
 func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
 	thumbnail := page.MustElementX("/html/body/div[3]/div[2]/div[1]/article/div[1]/img").MustProperty("src").String()
-	if thumbnail == "" {
-		return "", errors.New("unable to find thumbnail")
-	}
+	if thumbnail == "" { return "", errors.New("unable to find thumbnail")}
 	
 	title := page.MustElement(".news__header > h1:nth-child(2)").MustText()
 	log.Println(title)
@@ -196,23 +171,17 @@ func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
 
 func (fc *FFXIVClient) ExtractPubDate(page *rod.Page) (time.Time, error) {
 	stringDate := page.MustElement(".news__ic--topics").MustText()
-	if stringDate == "" {
-		return time.Now(), errors.New("unable to locate the publish date on the post")
-	}
+	if stringDate == "" { return time.Now(), errors.New("unable to locate the publish date on the post")}
 
 	PubDate, err := time.Parse(FFXIV_TIME_FORMAT, stringDate)
-	if err != nil {
-		return time.Now(), err
-	}
+	if err != nil { return time.Now(), err }
 
 	return PubDate, nil
 }
 
 func (fc *FFXIVClient) ExtractDescription(page *rod.Page) (string, error) {
 	res := page.MustElement(".news__detail__wrapper").MustText()
-	if res == "" {
-		return "", errors.New("unable to locate the description on the post")
-	}
+	if res == "" { return "", errors.New("unable to locate the description on the post")}
 
 	return res, nil
 }
@@ -221,17 +190,11 @@ func (fc *FFXIVClient) ExtractAuthor(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > meta")
 	for _, item := range meta {
 		name, err := item.Property("name")
-		if err != nil {
-			return "", err
-		}
+		if err != nil { return "", err }
 
-		if name.String() != "author" {
-			continue
-		}
+		if name.String() != "author" { continue }
 		content, err := item.Property("content")
-		if err != nil {
-			return "", err
-		}
+		if err != nil { return "", err }
 		
 		return content.String(), nil
 	}
@@ -243,17 +206,11 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > meta")
 	for _, item := range meta {
 		name, err := item.Property("name")
-		if err != nil {
-			return "", err
-		}
+		if err != nil { return "", err }
 
-		if name.String() != "keywords" {
-			continue
-		}
+		if name.String() != "keywords" { continue }
 		content, err := item.Property("content")
-		if err != nil {
-			return "", err
-		}
+		if err != nil { return "", err }
 		
 		return content.String(), nil
 	}
@@ -263,18 +220,12 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
 
 func (fc *FFXIVClient) ExtractTitle(page *rod.Page) (string, error) {
 	title, err := page.MustElement("head > title").Text()
-	if err != nil {
-		return "", err
-	}
+	if err != nil { return "", err }
 
-	if !strings.Contains(title, "|") {
-		return "", errors.New("unable to split the title, missing | in the string")
-	}
+	if !strings.Contains(title, "|") { return "", errors.New("unable to split the title, missing | in the string")}
 
 	res := strings.Split(title, "|")
-	if title != "" {
-		return res[0], nil
-	}
+	if title != "" { return res[0], nil }
 	
 	//log.Println(meta)
 	return "", errors.New("unable to find the author on the page")
@@ -284,20 +235,15 @@ func (fc *FFXIVClient) ExtractAuthorImage(page *rod.Page) (string, error) {
 	meta := page.MustElements("head > link")
 	for _, item := range meta {
 		name, err := item.Property("rel")
-		if err != nil {
-			return "", err
-		}
+		if err != nil { return "", err }
 
-		if name.String() != "apple-touch-icon-precomposed" {
-			continue
-		}
+		if name.String() != "apple-touch-icon-precomposed" { continue }
 		content, err := item.Property("href")
-		if err != nil {
-			return "", err
-		}
+		if err != nil { return "", err }
 		
 		return content.String(), nil
 	}
 	//log.Println(meta)
 	return "", errors.New("unable to find the author image on the page")
 }
+

services/ffxiv_test.go

@@ -1,17 +1,18 @@
-package input_test
+package services_test
 
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"github.com/google/uuid"
+	"github.com/jtom38/newsbot/collector/database"
+	ffxiv "github.com/jtom38/newsbot/collector/services"
 )
 
-var FFXIVRecord entity.SourceEntity = entity.SourceEntity{
-	ID:          9999,
-	DisplayName: "Final Fantasy XIV - NA",
-	Source:      domain.SourceCollectorFfxiv,
+var FFXIVRecord database.Source = database.Source{
+	ID: uuid.New(),
+	Site: "ffxiv",
+	Name: "Final Fantasy XIV - NA",
+	Source: "ffxiv",
 	Url: "https://na.finalfantasyxiv.com/lodestone/",
 	Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
 }
@@ -19,9 +20,7 @@ var FFXIVRecord entity.SourceEntity = entity.SourceEntity{
 func TestFfxivGetParser(t *testing.T) {
 	fc := ffxiv.NewFFXIVClient(FFXIVRecord)
 	_, err := fc.GetParser()
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 }
 
 func TestFfxivPullFeed(t *testing.T) {
@@ -31,12 +30,8 @@ func TestFfxivPullFeed(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
-	if len(links) == 0 {
-		t.Error("expected links to come back but got 0")
-	}
+	if err != nil { panic(err) }
+	if len(links) == 0 { panic("expected links to come back but got 0") }
 
 }
 
@@ -47,20 +42,14 @@ func TestFfxivExtractThumbnail(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	thumb, err := fc.ExtractThumbnail(page)
-	if err != nil {
-		t.Error(err)
-	}
-	if thumb == "" {
-		t.Error("expected a link but got nothing.")
-	}
+	if err != nil { panic(err) }
+	if thumb == "" { panic("expected a link but got nothing.")}
 }
 
 func TestFfxivExtractPubDate(t *testing.T) {
@@ -70,17 +59,13 @@ func TestFfxivExtractPubDate(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	_, err = fc.ExtractPubDate(page)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 }
 
 func TestFfxivExtractDescription(t *testing.T) {
@@ -90,17 +75,13 @@ func TestFfxivExtractDescription(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	_, err = fc.ExtractDescription(page)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 }
 
 func TestFfxivExtractAuthor(t *testing.T) {
@@ -110,20 +91,14 @@ func TestFfxivExtractAuthor(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	author, err := fc.ExtractAuthor(page)
-	if err != nil {
-		t.Error(err)
-	}
-	if author == "" {
-		t.Error("failed to locate the author name")
-	}
+	if err != nil { panic(err) }
+	if author == "" { panic("failed to locate the author name") }
 }
 
 func TestFfxivExtractTags(t *testing.T) {
@@ -133,20 +108,14 @@ func TestFfxivExtractTags(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractTags(page)
-	if err != nil {
-		t.Error(err)
-	}
-	if res == "" {
-		t.Error("failed to locate the tags")
-	}
+	if err != nil { panic(err) }
+	if res == "" {panic("failed to locate the tags")}
 }
 
 func TestFfxivExtractTitle(t *testing.T) {
@@ -156,20 +125,14 @@ func TestFfxivExtractTitle(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractTitle(page)
-	if err != nil {
-		t.Error(err)
-	}
-	if res == "" {
-		t.Error("failed to locate the tags")
-	}
+	if err != nil { panic(err) }
+	if res == "" { panic("failed to locate the tags") }
 }
 
 func TestFFxivExtractAuthorIamge(t *testing.T) {
@@ -179,20 +142,14 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
 	defer parser.Close()
 
 	links, err := fc.PullFeed(parser)
-	if err != nil {
-		t.Error(err)
-	}
+	if err != nil { panic(err) }
 
 	page := fc.GetPage(parser, links[0])
 	defer page.Close()
 
 	res, err := fc.ExtractAuthorImage(page)
-	if err != nil {
-		t.Error(err)
-	}
-	if res == "" {
-		t.Error("failed to locate the tags")
-	}
+	if err != nil { panic(err) }
+	if res == "" { panic("failed to locate the tags") }
 }
 
 func TestFfxivCheckSource(t *testing.T) {

services/httpClient.go

@@ -0,0 +1,29 @@
+package services
+
+import (
+	"net/http"
+	"log"
+	"io/ioutil"
+)
+
+// This will use the net/http client reach out to a site and collect the content.
+func getHttpContent(uri string) ([]byte, error) {
+
+	client := &http.Client{}
+	
+	req, err := http.NewRequest("GET", uri, nil)
+	if err != nil { return nil, err }
+
+	// set the user agent header to avoid kick backs.. as much
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
+
+	log.Printf("Requesting content from %v\n", uri)
+	resp, err := client.Do(req)
+	if err != nil { log.Fatalln(err) }
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil { return nil, err }
+
+	return body, nil
+}

services/reddit.go

@@ -0,0 +1,192 @@
+package services
+
+import (
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/go-rod/rod"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/domain/model"
+	"github.com/jtom38/newsbot/collector/services/config"
+)
+
+type RedditClient struct {
+	config RedditConfig
+	record database.Source
+}
+
+type RedditConfig struct {
+	PullTop  string
+	PullHot  string
+	PullNSFW string
+}
+
+func NewRedditClient(Record database.Source) RedditClient {
+	rc := RedditClient{
+		record: Record,
+	}
+	cc := config.New()
+	rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
+	rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
+	rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
+
+	rc.disableHttp2Client()
+
+	return rc
+}
+
+// This is needed for to get modern go to talk to the endpoint.
+// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
+func (rc RedditClient) disableHttp2Client() {
+	os.Setenv("GODEBUG", "http2client=0")
+}
+
+func (rc RedditClient) GetBrowser() *rod.Browser {
+	browser := rod.New().MustConnect()
+	return browser
+}
+
+func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
+	page := parser.MustPage(url)
+	return page
+}
+
+//func (rc RedditClient) 
+
+// GetContent() reaches out to Reddit and pulls the Json data.
+// It will then convert the data to a struct and return the struct.
+func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
+	var items model.RedditJsonContent = model.RedditJsonContent{}
+
+	// TODO Wire this to support the config options
+	Url := fmt.Sprintf("%v.json", rc.record.Url)
+
+	log.Printf("Collecting results on '%v'", rc.record.Name)
+
+	content, err := getHttpContent(Url)
+	if err != nil {
+		return items, err
+	}
+	if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
+		return items, errors.New("did not get json data from the server")
+	}
+
+	json.Unmarshal(content, &items)
+	if len(items.Data.Children) == 0 {
+		return items, errors.New("failed to unmarshal the data")
+	}
+	return items, nil
+}
+
+func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
+	var redditArticles []database.Article
+	for _, item := range items.Data.Children {
+		var article database.Article
+		article, err := rc.convertToArticle(item.Data)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		redditArticles = append(redditArticles, article)
+	}
+	return redditArticles
+}
+
+// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
+// This data can be passed to the database.
+func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
+	var item database.Article
+
+	if source.Content == "" && source.Url != "" {
+		item = rc.convertPicturePost(source)
+	}
+
+	if source.Media.RedditVideo.FallBackUrl != "" {
+		item = rc.convertVideoPost(source)
+	}
+
+	if source.Content != "" {
+		item = rc.convertTextPost(source)
+	}
+
+	if source.UrlOverriddenByDest != "" {
+		item = rc.convertRedirectPost(source)
+	}
+
+	if item.Description == "" {
+		var err = errors.New("reddit post failed to parse correctly")
+		return item, err
+	}
+
+	return item, nil
+}
+
+func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
+	var item = database.Article{
+		Sourceid:    rc.record.ID,
+		Title:       source.Title,
+		Tags:        fmt.Sprintf("%v", rc.record.Tags),
+		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		Pubdate:     time.Now(),
+		Video:       sql.NullString{String: "null"},
+		Videoheight: 0,
+		Videowidth:  0,
+		Thumbnail:   source.Thumbnail,
+		Description: source.Content,
+		Authorname:  sql.NullString{String: source.Author},
+		Authorimage: sql.NullString{String: "null"},
+	}
+	return item
+}
+
+func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
+	var item = database.Article{
+		Sourceid:    rc.record.ID,
+		Tags:        "a",
+		Title:       source.Title,
+		Pubdate:     time.Now(),
+		Videoheight: 0,
+		Videowidth:  0,
+		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		Authorname:  sql.NullString{String: source.Author},
+		Description: source.Content,
+	}
+	return item
+}
+
+func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
+	var item = database.Article{
+		Sourceid:    rc.record.ID,
+		Tags:        "a",
+		Title:       source.Title,
+		Pubdate:     time.Now(),
+		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		Videoheight: 0,
+		Videowidth:  0,
+		Authorname:  sql.NullString{String: source.Author},
+		Description: source.Media.RedditVideo.FallBackUrl,
+	}
+	return item
+}
+
+// This post is nothing more then a redirect to another location.
+func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
+	var item = database.Article{
+		Sourceid:    rc.record.ID,
+		Tags:        "a",
+		Title:       source.Title,
+		Pubdate:     time.Now(),
+		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		Videoheight: 0,
+		Videowidth:  0,
+		Authorname:  sql.NullString{String: source.Author},
+		Description: source.UrlOverriddenByDest,
+	}
+	return item
+}

services/reddit_test.go

@@ -0,0 +1,33 @@
+package services_test
+
+import (
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services"
+)
+
+var RedditRecord database.Source = database.Source{
+	ID: uuid.New(),
+	Name: "dadjokes",
+	Source: "reddit",
+	Site: "reddit",
+	Url: "https://reddit.com/r/dadjokes",
+	Tags: "reddit, dadjokes",
+}
+
+func TestGetContent(t *testing.T) {
+	//This test is flaky right now due to the http changes in 1.17
+	rc := services.NewRedditClient(RedditRecord)
+	raw, err := rc.GetContent()
+	if err != nil {
+		t.Error(err)
+	}
+	redditArticles := rc.ConvertToArticles(raw)
+	for _, posts := range redditArticles {
+		if posts.Title == "" {
+			t.Error("Title is missing")
+		}
+	}
+}

services/twitch.go

@@ -1,18 +1,19 @@
-package input
+package services
 
 import (
+	"database/sql"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services/config"
 	"github.com/nicklaw5/helix/v2"
 )
 
 type TwitchClient struct {
-	SourceRecord entity.SourceEntity
+	SourceRecord database.Source
 
 	// config
 	monitorClips string
@@ -30,14 +31,14 @@ var (
 )
 
 func NewTwitchClient() (TwitchClient, error) {
-	c := services.NewConfig()
+	c := config.New()
 
-	id := c.GetConfig(services.TWITCH_CLIENT_ID)
+	id := c.GetConfig(config.TWITCH_CLIENT_ID)
 	if id == "" {
 		return TwitchClient{}, ErrTwitchClientIdMissing
 	}
 
-	secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
+	secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
 	if secret == "" {
 		return TwitchClient{}, ErrTwitchClientSecretMissing
 	}
@@ -49,8 +50,8 @@ func NewTwitchClient() (TwitchClient, error) {
 
 	client := TwitchClient{
 		//SourceRecord: &source,
-		monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
-		monitorVod:   c.GetConfig(services.TWITCH_MONITOR_VOD),
+		monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
+		monitorVod:   c.GetConfig(config.TWITCH_MONITOR_VOD),
 		api:          &api,
 	}
 
@@ -71,12 +72,12 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
 }
 
 // This will let you replace the bound source record to keep the same session alive.
-func (tc *TwitchClient) ReplaceSourceRecord(source entity.SourceEntity) {
+func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
 	tc.SourceRecord = source
 }
 
 // Invokes Logon request to the API
-func (tc *TwitchClient) Login() error {
+func (tc TwitchClient) Login() error {
 	token, err := tc.api.RequestAppAccessToken([]string{twitchScopes})
 	if err != nil {
 		return err
@@ -86,8 +87,8 @@ func (tc *TwitchClient) Login() error {
 	return nil
 }
 
-func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
-	var items []entity.ArticleEntity
+func (tc TwitchClient) GetContent() ([]database.Article, error) {
+	var items []database.Article
 
 	user, err := tc.GetUserDetails()
 	if err != nil {
@@ -100,50 +101,34 @@ func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
 	}
 
 	for _, video := range posts {
-		var article entity.ArticleEntity
+		var article database.Article
 
 		AuthorName, err := tc.ExtractAuthor(video)
-		if err != nil {
-			return items, err
-		}
-		article.AuthorName = AuthorName
+		if err != nil { return items, err }
+		article.Authorname = sql.NullString{String: AuthorName}
 		
 		Authorimage, err := tc.ExtractAuthorImage(user)
-		if err != nil {
-			return items, err
-		}
-		article.AuthorImageUrl = Authorimage
+		if err != nil { return items, err }
+		article.Authorimage = sql.NullString{String: Authorimage}
 
 		article.Description, err = tc.ExtractDescription(video)
-		if err != nil {
-			return items, err
-		}
+		if err != nil {return items, err }
 
-		article.PubDate, err = tc.ExtractPubDate(video)
-		if err != nil {
-			return items, err
-		}
+		article.Pubdate, err = tc.ExtractPubDate(video)
+		if err != nil { return items, err }
 
-		article.SourceID = tc.SourceRecord.ID
+		article.Sourceid = tc.SourceRecord.ID
 		article.Tags, err = tc.ExtractTags(video, user)
-		if err != nil {
-			return items, err
-		}
+		if err != nil { return items, err }
 
 		article.Thumbnail, err = tc.ExtractThumbnail(video)
-		if err != nil {
-			return items, err
-		}
+		if err != nil { return items, err }
 
 		article.Title, err = tc.ExtractTitle(video)
-		if err != nil {
-			return items, err
-		}
+		if err != nil { return items, err }
 		
 		article.Url, err = tc.ExtractUrl(video)
-		if err != nil {
-			return items, err
-		}
+		if err != nil { return items, err }
 
 		items = append(items, article)
 	}
@@ -151,25 +136,20 @@ func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
 	return items, nil
 }
 
-func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
+func (tc TwitchClient) GetUserDetails() (helix.User, error) {
 	var blank helix.User
 
 	users, err := tc.api.GetUsers(&helix.UsersParams{
-		Logins: []string{tc.SourceRecord.DisplayName},
+		Logins: []string{tc.SourceRecord.Name},
 	})
 	if err != nil {
 		return blank, err
 	}
-
-	if len(users.Data.Users) == 0 {
-		return blank, errors.New("no results have been returned")
-	}
-
 	return users.Data.Users[0], nil
 }
 
 // This will reach out and collect the posts made by the user.
-func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
+func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
 	var blank []helix.Video
 
 	videos, err := tc.api.GetVideos(&helix.VideosParams{
@@ -183,14 +163,14 @@ func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
 	return videos.Data.Videos, nil
 }
 
-func (tc *TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
+func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
 	if post.UserName == "" {
 		return "", ErrMissingAuthorName
 	}
 	return post.UserName, nil
 }
 
-func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
+func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
 	if post.ThumbnailURL == "" {
 		return "", ErrMissingThumbnail
 	}
@@ -200,7 +180,7 @@ func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
 	return thumb, nil
 }
 
-func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
+func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
 	if post.PublishedAt == "" {
 		return time.Now(), ErrMissingPublishDate
 	}
@@ -211,7 +191,7 @@ func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
 	return pubDate, nil
 }
 
-func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
+func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
 	// Check if the description is null but we have a title.
 	// The poster didnt add a description but this isnt an error.
 	if post.Description == "" && post.Title == "" {
@@ -224,34 +204,28 @@ func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
 }
 
 // Extracts the avatar of the author with some validation.
-func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
-	if user.ProfileImageURL == "" {
-		return "", ErrMissingAuthorImage
-	}
-	if !strings.Contains(user.ProfileImageURL, "-profile_image-") {
-		return "", ErrInvalidAuthorImage
-	}
+func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
+	if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
+	if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
 	return user.ProfileImageURL, nil
 }
 
 // Generate tags based on the video metadata.
 // TODO Figure out how to query what game is played
-func (tc *TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
+func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
 	res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName)
 	return res, nil
 }
 
 // Extracts the title from a post with some validation.
-func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
+func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) {
 	if post.Title == "" {
 		return "", errors.New("unable to find the title on the requested post")
 	}
 	return post.Title, nil
 }
 
-func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
-	if post.URL == "" {
-		return "", ErrMissingUrl
-	}
+func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) {
+	if post.URL == "" { return "", ErrMissingUrl }
 	return post.URL, nil
 }