Compare commits

..

1 Commits

Author SHA1 Message Date
c00b1446d1 version bump 2022-06-14 10:15:27 -07:00
106 changed files with 2473 additions and 15391 deletions

View File

@ -1,60 +0,0 @@
---
kind: pipeline
type: docker
name: buildLatestImage
steps:
- name: buildLatestImage
image: plugins/docker
settings:
repo: jtom38/newsbot-collector
username: jtom38
password:
from_secret: DockerPushPat
trigger:
branch:
include:
- main
event:
exclude:
- pull_request
---
kind: pipeline
type: docker
name: buildReleaseImage
steps:
- name: buildReleaseImage
image: plugins/docker
settings:
repo: jtom38/newsbot-collector
username: jtom38
password:
from_secret: DockerPushPat
trigger:
branch:
include:
- releases/*
ref:
include:
- refs/tags/**
event:
exclude:
- pull_request
---
kind: pipeline
type: docker
name: PullRequestCompileTest
steps:
- name: Compile project
image: golang:1.22
commands:
- go test ./internal/repository
- go build ./cmd/server.go
-
trigger:
event:
- pull_request

64
.github/workflows/docker.build.yaml vendored Normal file
View File

@ -0,0 +1,64 @@
name: Docker
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
on:
#schedule:
# - cron: '21 19 * * *'
push:
branches: [ master ]
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
#pull_request:
# branches: [ master ]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
#images: ${{ env.REGISTRY }}/newsbot.worker
# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

25
.github/workflows/go-build.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.18
- name: Build
run: go build -v ./...
#- name: Test
# run: go test -v ./...

7
.gitignore vendored
View File

@ -1,9 +1,5 @@
.env
dev.session.sql
__debug_bin
server
.vscode
openapi.json
# Binaries for programs and plugins
*.exe
@ -12,13 +8,10 @@ openapi.json
*.so
*.dylib
collector
newsbot.db
tmp/
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out

15
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,15 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "."
}
]
}

View File

@ -1,23 +1,16 @@
FROM golang:1.22 as build
FROM golang:1.18.3 as build
COPY . /app
WORKDIR /app
RUN go build .
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
# Always make sure that swagger docs are updated
RUN go install github.com/swaggo/swag/cmd/swag@latest
RUN /go/bin/swag init -g cmd/server.go
FROM alpine
#RUN go build .
#RUN go install github.com/pressly/goose/v3/cmd/goose@latest
RUN mkdir /app && \
mkdir /app/migrations
COPY --from=build /app/collector /app
COPY --from=build /go/bin/goose /app
COPY ./database/migrations/ /app/migrations
FROM alpine:latest as app
RUN apk --no-cache add bash
RUN apk --no-cache add libc6-compat
RUN apk --no-cache add chromium
RUN mkdir /app && mkdir /app/migrations
COPY --from=build /app/server /app
COPY ./internal/database/migrations/ /app/migrations
CMD [ "/app/collector" ]
ENTRYPOINT [ "/app/collector" ]

View File

@ -1,25 +1,2 @@
# newsbot.collector.api
This is the collection service of newsbot to pull articles from the web.
## Deployment
1. Create a copy of the docker compose file and make it local
2. Update the `docker-compose.yaml` with your secrets
3. Run migrations
2. `docker compose run api /app/goose -dir "/app/migrations" up`
4. Run app
1. `docker compose up -d`
5. Once the app is running go to the swagger page and validate that you see the seeded sources.
1. `http://localhost:8081/swagger/index.html#/Source/get_config_sources`
2. `curl -X 'GET' 'http://localhost:8081/api/config/sources' -H 'accept: application/json'`
6. Add any new sources
7. Add a Discord Web Hook
1. `curl -X 'POST' 'http://localhost:8081/api/discord/webhooks/new?url=WEBHOOKURL&server=SERVERNAME&channel=CHANNELNAME' -H 'accept: application/json' -d ''`
8. Create your subscription links
1. This is a link between a source and a discord web hook. Without this, the app will not send a notification about new posts.
### Errors
- pq: permission denied to create extension "uuid-ossp"
- Might need to grant your account `ALTER USER root WITH SUPERUSER;` to create the 'uuid-ossp' for uuid creations

View File

@ -1,6 +0,0 @@
### Select Sources fro mthe top
GET http://localhost:8081/api/v1/sources/
### Select Sources by type
GET http://localhost:8081/api/v1/sources/by/source?source=rss

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +0,0 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/deepmap/oapi-codegen/HEAD/configuration-schema.json
package: api
output: api.gen.go
generate:
models: true
client: true

View File

@ -1,90 +0,0 @@
package main
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
_ "github.com/glebarez/go-sqlite"
"github.com/pressly/goose/v3"
"git.jamestombleson.com/jtom38/newsbot-api/docs"
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
)
// @title NewsBot collector
// @version 0.1
// @BasePath /api
// @securityDefinitions.apikey Bearer
// @in header
// @name Authorization
// @description Type "Bearer" followed by a space and JWT token.
func main() {
ctx := context.Background()
cfg := services.NewConfig()
configs := services.GetEnvConfig()
address := cfg.GetConfig(services.ServerAddress)
docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
db, err := sql.Open("sqlite", "newsbot.db")
if err != nil {
panic(err)
}
err = migrateDatabase(db)
if err != nil {
fmt.Print(err)
}
c := cron.NewScheduler(ctx, db)
c.Start()
server := v1.NewServer(ctx, configs, db)
fmt.Println("API is online and waiting for requests.")
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress)
server.Router.Start(":8081")
}
func migrateDatabase(db *sql.DB) error {
err := goose.SetDialect("sqlite3")
if err != nil {
panic(err)
}
err = goose.Up(db, "../internal/database/migrations")
if err != nil {
panic(err)
}
_, err = os.Stat("./migrations")
if err == nil {
err = goose.Up(db, "../internal/database/migrations")
if err != nil {
panic(err)
}
return nil
}
_, err = os.Stat("../internal/database/migrations")
if err == nil {
err = goose.Up(db, "../internal/database/migrations")
if err != nil {
panic(err)
}
return nil
}
return errors.New("failed to find the migration files")
}

31
database/db.go Normal file
View File

@ -0,0 +1,31 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package database
import (
"context"
"database/sql"
)
type DBTX interface {
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
PrepareContext(context.Context, string) (*sql.Stmt, error)
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
return &Queries{
db: tx,
}
}

View File

@ -0,0 +1,72 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
CREATE TABLE Articles (
ID uuid PRIMARY KEY,
SourceId uuid NOT null,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate timestamp NOT NULL,
Video TEXT,
VideoHeight int NOT NULL,
VideoWidth int NOT NULL,
Thumbnail TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT,
AuthorImage TEXT
);
CREATE Table DiscordQueue (
ID uuid PRIMARY KEY,
ArticleId uuid NOT NULL
);
CREATE Table DiscordWebHooks (
ID uuid PRIMARY KEY,
Name TEXT NOT NULL, -- Defines webhook purpose
Key TEXT,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID uuid PRIMARY Key,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID uuid PRIMARY Key,
Key TEXT NOT NULL, -- How you search for a entry
Value TEXT NOT NULL, -- The value for one
Options TEXT -- any notes about the entry
);
Create Table Sources (
ID uuid PRIMARY Key,
Site TEXT NOT NULL, -- Vanity name
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
Value TEXT,
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
Drop Table Articles;
Drop Table DiscordQueue;
Drop Table DiscordWebHooks;
Drop Table Icons;
Drop Table Settings;
Drop Table Sources;
-- +goose StatementEnd

View File

@ -0,0 +1,50 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
-- Enable UUID's
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Final Fantasy XIV Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
--SELECT 'down SQL query';
DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
DELETE FROM sources where source = 'ffxiv';
DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
-- +goose StatementEnd

68
database/models.go Normal file
View File

@ -0,0 +1,68 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package database
import (
"database/sql"
"time"
"github.com/google/uuid"
)
type Article struct {
ID uuid.UUID
Sourceid uuid.UUID
Tags string
Title string
Url string
Pubdate time.Time
Video sql.NullString
Videoheight int32
Videowidth int32
Thumbnail string
Description string
Authorname sql.NullString
Authorimage sql.NullString
}
type Discordqueue struct {
ID uuid.UUID
Articleid uuid.UUID
}
type Discordwebhook struct {
ID uuid.UUID
Name string
Key sql.NullString
Url string
Server string
Channel string
Enabled bool
}
type Icon struct {
ID uuid.UUID
Filename string
Site string
}
type Setting struct {
ID uuid.UUID
Key string
Value string
Options sql.NullString
}
type Source struct {
ID uuid.UUID
Site string
Name string
Source string
Type string
Value sql.NullString
Enabled bool
Url string
Tags string
}

521
database/query.sql.go Normal file
View File

@ -0,0 +1,521 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
// source: query.sql
package database
import (
"context"
"database/sql"
"time"
"github.com/google/uuid"
)
const createArticle = `-- name: CreateArticle :exec
INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
`
type CreateArticleParams struct {
ID uuid.UUID
Sourceid uuid.UUID
Tags string
Title string
Url string
Pubdate time.Time
Video sql.NullString
Videoheight int32
Videowidth int32
Thumbnail string
Description string
Authorname sql.NullString
Authorimage sql.NullString
}
func (q *Queries) CreateArticle(ctx context.Context, arg CreateArticleParams) error {
_, err := q.db.ExecContext(ctx, createArticle,
arg.ID,
arg.Sourceid,
arg.Tags,
arg.Title,
arg.Url,
arg.Pubdate,
arg.Video,
arg.Videoheight,
arg.Videowidth,
arg.Thumbnail,
arg.Description,
arg.Authorname,
arg.Authorimage,
)
return err
}
const createDiscordQueue = `-- name: CreateDiscordQueue :exec
Insert into DiscordQueue
(ID, ArticleId)
Values
($1, $2)
`
type CreateDiscordQueueParams struct {
ID uuid.UUID
Articleid uuid.UUID
}
// DiscordQueue
func (q *Queries) CreateDiscordQueue(ctx context.Context, arg CreateDiscordQueueParams) error {
_, err := q.db.ExecContext(ctx, createDiscordQueue, arg.ID, arg.Articleid)
return err
}
const createDiscordWebHook = `-- name: CreateDiscordWebHook :exec
Insert Into DiscordWebHooks
(ID, Name, Key, Url, Server, Channel, Enabled)
Values
($1, $2, $3, $4, $5, $6, $7)
`
type CreateDiscordWebHookParams struct {
ID uuid.UUID
Name string
Key sql.NullString
Url string
Server string
Channel string
Enabled bool
}
// DiscordWebHooks
func (q *Queries) CreateDiscordWebHook(ctx context.Context, arg CreateDiscordWebHookParams) error {
_, err := q.db.ExecContext(ctx, createDiscordWebHook,
arg.ID,
arg.Name,
arg.Key,
arg.Url,
arg.Server,
arg.Channel,
arg.Enabled,
)
return err
}
const createIcon = `-- name: CreateIcon :exec
INSERT INTO Icons
(ID, FileName, Site)
VALUES
($1,$2,$3)
`
type CreateIconParams struct {
ID uuid.UUID
Filename string
Site string
}
// Icons
func (q *Queries) CreateIcon(ctx context.Context, arg CreateIconParams) error {
_, err := q.db.ExecContext(ctx, createIcon, arg.ID, arg.Filename, arg.Site)
return err
}
const createSettings = `-- name: CreateSettings :one
Insert Into settings
(ID, Key, Value, OPTIONS)
Values
($1,$2,$3,$4)
RETURNING id, key, value, options
`
type CreateSettingsParams struct {
ID uuid.UUID
Key string
Value string
Options sql.NullString
}
// Settings
func (q *Queries) CreateSettings(ctx context.Context, arg CreateSettingsParams) (Setting, error) {
row := q.db.QueryRowContext(ctx, createSettings,
arg.ID,
arg.Key,
arg.Value,
arg.Options,
)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const createSource = `-- name: CreateSource :exec
Insert Into Sources
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
Values
($1,$2,$3,$4,$5,$6,$7,$8,$9)
`
type CreateSourceParams struct {
ID uuid.UUID
Site string
Name string
Source string
Type string
Value sql.NullString
Enabled bool
Url string
Tags string
}
// Sources
func (q *Queries) CreateSource(ctx context.Context, arg CreateSourceParams) error {
_, err := q.db.ExecContext(ctx, createSource,
arg.ID,
arg.Site,
arg.Name,
arg.Source,
arg.Type,
arg.Value,
arg.Enabled,
arg.Url,
arg.Tags,
)
return err
}
const deleteDiscordQueueItem = `-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1
`
func (q *Queries) DeleteDiscordQueueItem(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteDiscordQueueItem, id)
return err
}
const deleteDiscordWebHooks = `-- name: DeleteDiscordWebHooks :exec
Delete From discordwebhooks Where ID = $1
`
func (q *Queries) DeleteDiscordWebHooks(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteDiscordWebHooks, id)
return err
}
const deleteIcon = `-- name: DeleteIcon :exec
Delete From Icons where ID = $1
`
func (q *Queries) DeleteIcon(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteIcon, id)
return err
}
const deleteSetting = `-- name: DeleteSetting :exec
Delete From settings Where ID = $1
`
func (q *Queries) DeleteSetting(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSetting, id)
return err
}
const deleteSource = `-- name: DeleteSource :exec
DELETE From sources where id = $1
`
func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSource, id)
return err
}
const getArticleByID = `-- name: GetArticleByID :one
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
WHERE ID = $1 LIMIT 1
`
// Articles
func (q *Queries) GetArticleByID(ctx context.Context, id uuid.UUID) (Article, error) {
row := q.db.QueryRowContext(ctx, getArticleByID, id)
var i Article
err := row.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
)
return i, err
}
const getArticleByUrl = `-- name: GetArticleByUrl :one
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
Where Url = $1 LIMIT 1
`
func (q *Queries) GetArticleByUrl(ctx context.Context, url string) (Article, error) {
row := q.db.QueryRowContext(ctx, getArticleByUrl, url)
var i Article
err := row.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
)
return i, err
}
const getDiscordQueueByID = `-- name: GetDiscordQueueByID :one
Select id, articleid from DiscordQueue
Where ID = $1 LIMIT 1
`
func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discordqueue, error) {
row := q.db.QueryRowContext(ctx, getDiscordQueueByID, id)
var i Discordqueue
err := row.Scan(&i.ID, &i.Articleid)
return i, err
}
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, name, key, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1
`
func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Discordwebhook, error) {
row := q.db.QueryRowContext(ctx, getDiscordWebHooksByID, id)
var i Discordwebhook
err := row.Scan(
&i.ID,
&i.Name,
&i.Key,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
)
return i, err
}
const getIconByID = `-- name: GetIconByID :one
Select id, filename, site FROM Icons
Where ID = $1 Limit 1
`
func (q *Queries) GetIconByID(ctx context.Context, id uuid.UUID) (Icon, error) {
row := q.db.QueryRowContext(ctx, getIconByID, id)
var i Icon
err := row.Scan(&i.ID, &i.Filename, &i.Site)
return i, err
}
const getIconBySite = `-- name: GetIconBySite :one
Select id, filename, site FROM Icons
Where Site = $1 Limit 1
`
func (q *Queries) GetIconBySite(ctx context.Context, site string) (Icon, error) {
row := q.db.QueryRowContext(ctx, getIconBySite, site)
var i Icon
err := row.Scan(&i.ID, &i.Filename, &i.Site)
return i, err
}
const getSettingByID = `-- name: GetSettingByID :one
Select id, key, value, options From settings
Where ID = $1 Limit 1
`
func (q *Queries) GetSettingByID(ctx context.Context, id uuid.UUID) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByID, id)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSettingByKey = `-- name: GetSettingByKey :one
Select id, key, value, options From settings Where
Key = $1 Limit 1
`
func (q *Queries) GetSettingByKey(ctx context.Context, key string) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByKey, key)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSettingByValue = `-- name: GetSettingByValue :one
Select id, key, value, options From settings Where
Value = $1 Limit 1
`
func (q *Queries) GetSettingByValue(ctx context.Context, value string) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByValue, value)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSourceByID = `-- name: GetSourceByID :one
Select id, site, name, source, type, value, enabled, url, tags From Sources where ID = $1 Limit 1
`
func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, error) {
row := q.db.QueryRowContext(ctx, getSourceByID, id)
var i Source
err := row.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
)
return i, err
}
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
Select id, name, key, url, server, channel, enabled From DiscordWebHooks
Where Server = $1
`
func (q *Queries) ListDiscordWebHooksByServer(ctx context.Context, server string) ([]Discordwebhook, error) {
rows, err := q.db.QueryContext(ctx, listDiscordWebHooksByServer, server)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordwebhook
for rows.Next() {
var i Discordwebhook
if err := rows.Scan(
&i.ID,
&i.Name,
&i.Key,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listSourcesBySource = `-- name: ListSourcesBySource :many
Select id, site, name, source, type, value, enabled, url, tags From Sources where Source = $1
`
func (q *Queries) ListSourcesBySource(ctx context.Context, source string) ([]Source, error) {
rows, err := q.db.QueryContext(ctx, listSourcesBySource, source)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Source
for rows.Next() {
var i Source
if err := rows.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}

112
database/schema/query.sql Normal file
View File

@ -0,0 +1,112 @@
/* Articles */
-- name: GetArticleByID :one
Select * from Articles
WHERE ID = $1 LIMIT 1;
-- name: GetArticleByUrl :one
Select * from Articles
Where Url = $1 LIMIT 1;
-- name: CreateArticle :exec
INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
/* DiscordQueue */
-- name: CreateDiscordQueue :exec
Insert into DiscordQueue
(ID, ArticleId)
Values
($1, $2);
-- name: GetDiscordQueueByID :one
Select * from DiscordQueue
Where ID = $1 LIMIT 1;
-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1;
-- name: GetDiscordQueueItems :many
Select * from DiscordQueue LIMIT $1;
/* DiscordWebHooks */
-- name: CreateDiscordWebHook :exec
Insert Into DiscordWebHooks
(ID, Name, Key, Url, Server, Channel, Enabled)
Values
($1, $2, $3, $4, $5, $6, $7);
-- name: GetDiscordWebHooksByID :one
Select * from DiscordWebHooks
Where ID = $1 LIMIT 1;
-- name: ListDiscordWebHooksByServer :many
Select * From DiscordWebHooks
Where Server = $1;
-- name: DeleteDiscordWebHooks :exec
Delete From discordwebhooks Where ID = $1;
/* Icons */
-- name: CreateIcon :exec
INSERT INTO Icons
(ID, FileName, Site)
VALUES
($1,$2,$3);
-- name: GetIconByID :one
Select * FROM Icons
Where ID = $1 Limit 1;
-- name: GetIconBySite :one
Select * FROM Icons
Where Site = $1 Limit 1;
-- name: DeleteIcon :exec
Delete From Icons where ID = $1;
/* Settings */
-- name: CreateSettings :one
Insert Into settings
(ID, Key, Value, OPTIONS)
Values
($1,$2,$3,$4)
RETURNING *;
-- name: GetSettingByID :one
Select * From settings
Where ID = $1 Limit 1;
-- name: GetSettingByKey :one
Select * From settings Where
Key = $1 Limit 1;
-- name: GetSettingByValue :one
Select * From settings Where
Value = $1 Limit 1;
-- name: DeleteSetting :exec
Delete From settings Where ID = $1;
/* Sources */
-- name: CreateSource :exec
Insert Into Sources
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
Values
($1,$2,$3,$4,$5,$6,$7,$8,$9);
-- name: GetSourceByID :one
Select * From Sources where ID = $1 Limit 1;
-- name: ListSourcesBySource :many
Select * From Sources where Source = $1;
-- name: DeleteSource :exec
DELETE From sources where id = $1;

View File

@ -0,0 +1,56 @@
CREATE TABLE Articles (
ID uuid PRIMARY KEY,
SourceId uuid NOT null,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate timestamp NOT NULL,
Video TEXT,
VideoHeight int NOT NULL,
VideoWidth int NOT NULL,
Thumbnail TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT,
AuthorImage TEXT
);
CREATE Table DiscordQueue (
ID uuid PRIMARY KEY,
ArticleId uuid NOT NULL
);
CREATE Table DiscordWebHooks (
ID uuid PRIMARY KEY,
Name TEXT NOT NULL, -- Defines webhook purpose
Key TEXT,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID uuid PRIMARY Key,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID uuid PRIMARY Key,
Key TEXT NOT NULL,
Value TEXT NOT NULL,
Options TEXT
);
Create Table Sources (
ID uuid PRIMARY Key,
Site TEXT NOT NULL, -- Vanity name
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
Value TEXT,
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);

100
databaseRest/articles.go Normal file
View File

@ -0,0 +1,100 @@
package databaseRest
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/jtom38/newsbot/collector/domain/model"
)
// Generate this struct fr
type ArticlesClient struct {
rootUri string
}
func (ac *ArticlesClient) List() ([]model.Articles, error) {
var items []model.Articles
url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return []model.Articles{}, err
}
return items, nil
}
func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
var items model.Articles
url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return items, err
}
return items, nil
}
func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
var item model.Articles
get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
resp, err := getContent(get)
if err != nil {
return item, err
}
err = json.Unmarshal(resp, &item)
if err != nil {
return item, err
}
return item, nil
}
func (ac *ArticlesClient) Delete(id int32) error {
return errors.New("not implemented")
}
func (ac *ArticlesClient) Add(item model.Articles) error {
//return errors.New("not implemented")
url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
bItem, err := json.Marshal(item)
if err != nil {
return err
}
client := &http.Client{}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.New("failed to post to the DB")
}
return nil
//body, err := ioutil.ReadAll(resp.Body)
//if err != nil { return err }
}

74
databaseRest/common.go Normal file
View File

@ -0,0 +1,74 @@
package databaseRest
import (
"errors"
"io/ioutil"
"log"
"net/http"
"github.com/jtom38/newsbot/collector/services/config"
)
type DatabaseClient struct {
Diagnosis DiagnosisClient
Articles ArticlesClient
Sources SourcesClient
}
// This will generate a new client to interface with the API Database.
func NewDatabaseClient() DatabaseClient {
cc := config.New()
dbUri := cc.GetConfig(config.DB_URI)
var client = DatabaseClient{}
client.Diagnosis.rootUri = dbUri
client.Sources.rootUri = dbUri
client.Articles.rootUri = dbUri
return client
}
func getContent(url string) ([]byte, error) {
client := &http.Client{}
var blank []byte
req, err := http.NewRequest("GET", url, nil)
if err != nil { return blank, err }
// set the user agent header to avoid kick backs.. as much
req.Header.Set("User-Agent", getUserAgent())
log.Printf("Requesting content from %v\n", url)
resp, err := client.Do(req)
if err != nil { return blank, err }
if resp.StatusCode == 404 {
err = errors.New("404 not found")
return blank, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil { return blank, err }
//log.Println(string(body))
return body, nil
}
func httpDelete(url string) error {
client := &http.Client{}
req, err := http.NewRequest("DELETE", url, nil)
if err != nil { return err }
//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
req.Header.Set("User-Agent", getUserAgent())
_, err = client.Do(req)
if err != nil { return err }
return nil
}
func getUserAgent() string {
return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
}

24
databaseRest/diagnosis.go Normal file
View File

@ -0,0 +1,24 @@
package databaseRest
import (
"fmt"
"io/ioutil"
"net/http"
//"github.com/jtom38/newsbot/collector/services"
)
type DiagnosisClient struct {
rootUri string
}
func (dc *DiagnosisClient) Ping() error {
dbPing := fmt.Sprintf("%v/ping", dc.rootUri)
resp, err := http.Get(dbPing)
if err != nil { return err }
_, err = ioutil.ReadAll(resp.Body)
if err != nil { return err }
return nil
}

38
databaseRest/sources.go Normal file
View File

@ -0,0 +1,38 @@
package databaseRest
import (
"encoding/json"
"fmt"
"log"
"github.com/jtom38/newsbot/collector/domain/model"
)
type SourcesClient struct {
rootUri string
}
func (sb *SourcesClient) List() ([]model.Sources, error) {
var items []model.Sources
url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
resp, err := getContent(url)
if err != nil { return items, err }
err = json.Unmarshal(resp, &items)
if err != nil { return []model.Sources{}, err }
return items, nil
}
func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) {
items, err := sb.List()
if err != nil { log.Panicln(err) }
var res []model.Sources
for _, item := range(items) {
if item.Source == SourceType {
res = append(res, item)
}
}
return res, nil
}

View File

@ -1,36 +0,0 @@
version: "3"
networks:
newsbot:
services:
api:
image: ghcr.io/jtom38/newsbot.collector.api:master
environment:
SERVER_ADDRESS: "localhost"
SQL_CONNECTION_STRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
# Used for database migrations
GOOSE_DRIVER: "postgres"
# Connection String to Postgresql
GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
# Enable/Disable Reddit monitoring
FEATURE_ENABLE_REDDIT_BACKEND: true
# Enable/Disable YouTube monitoring
FEATURE_ENABLE_YOUTUBE_BACKEND: false
FEATURE_ENABLE_TWITCH_BACKEND: false
# Set your Twitch Developer ID and Secrets here and they will be used to collect updates.
TWITCH_CLIENT_ID: ""
TWITCH_CLIENT_SECRET: ""
# If you want to collect news on Final Fantasy XIV, set this to true
FEATURE_ENABLE_FFXIV_BACKEND: false
ports:
- 8081:8081
networks:
- newsbot

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,945 +0,0 @@
definitions:
domain.ArticleAndSourceModel:
properties:
article:
$ref: '#/definitions/domain.ArticleDto'
source:
$ref: '#/definitions/domain.SourceDto'
type: object
domain.ArticleDetailedResponse:
properties:
isError:
type: boolean
message:
type: string
payload:
$ref: '#/definitions/domain.ArticleAndSourceModel'
type: object
domain.ArticleDto:
properties:
authorImage:
type: string
authorName:
type: string
description:
type: string
id:
type: integer
isVideo:
type: boolean
pubDate:
type: string
sourceId:
type: integer
tags:
type: string
thumbnail:
type: string
title:
type: string
url:
type: string
type: object
domain.ArticleResponse:
properties:
isError:
type: boolean
message:
type: string
payload:
items:
$ref: '#/definitions/domain.ArticleDto'
type: array
type: object
domain.BaseResponse:
properties:
isError:
type: boolean
message:
type: string
type: object
domain.DiscordWebHookDto:
properties:
channel:
type: string
enabled:
type: boolean
id:
type: integer
server:
type: string
url:
description: |-
Name string `json:"name"`
Key string `json:"key"`
type: string
type: object
domain.DiscordWebhookResponse:
properties:
isError:
type: boolean
message:
type: string
payload:
items:
$ref: '#/definitions/domain.DiscordWebHookDto'
type: array
type: object
domain.LoginResponse:
properties:
isError:
type: boolean
message:
type: string
refreshToken:
type: string
token:
type: string
type:
type: string
type: object
domain.NewSourceParamRequest:
properties:
name:
type: string
tags:
type: string
url:
type: string
type: object
domain.RefreshTokenRequest:
properties:
refreshToken:
type: string
username:
type: string
type: object
domain.SourceDto:
properties:
enabled:
type: boolean
id:
type: integer
name:
type: string
source:
type: string
tags:
type: string
url:
type: string
type: object
domain.SourcesResponse:
properties:
isError:
type: boolean
message:
type: string
payload:
items:
$ref: '#/definitions/domain.SourceDto'
type: array
type: object
domain.UpdateScopesRequest:
properties:
scopes:
items:
type: string
type: array
username:
type: string
required:
- scopes
type: object
info:
contact: {}
paths:
/v1/articles:
get:
parameters:
- description: page number
in: query
name: page
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.ArticleResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Lists the top 25 records ordering from newest to oldest.
tags:
- Articles
/v1/articles/{id}:
get:
parameters:
- description: int
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.ArticleResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Returns an article based on defined ID.
tags:
- Articles
/v1/articles/{id}/details:
get:
parameters:
- description: int
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.ArticleDetailedResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Returns an article and source based on defined ID.
tags:
- Articles
/v1/articles/by/source/{id}:
get:
parameters:
- description: source id
in: path
name: id
required: true
type: string
- description: Page to query
in: query
name: page
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.ArticleResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Finds the articles based on the SourceID provided. Returns the top
25.
tags:
- Articles
/v1/discord/webhooks:
get:
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Returns the top 100
tags:
- DiscordWebhook
/v1/discord/webhooks/{id}:
delete:
parameters:
- description: id
in: path
name: id
required: true
type: string
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Deletes a record by ID.
tags:
- DiscordWebhook
get:
parameters:
- description: id
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Returns the top 100 entries from the queue to be processed.
tags:
- DiscordWebhook
/v1/discord/webhooks/{id}/disable:
post:
parameters:
- description: id
in: path
name: id
required: true
type: integer
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Disables a Webhook from being used.
tags:
- DiscordWebhook
/v1/discord/webhooks/{id}/enable:
post:
parameters:
- description: id
in: path
name: id
required: true
type: integer
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Enables a source to continue processing.
tags:
- DiscordWebhook
/v1/discord/webhooks/by/serverAndChannel:
get:
parameters:
- description: Fancy Server
in: query
name: server
required: true
type: string
- description: memes
in: query
name: channel
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Returns all the known web hooks based on the Server and Channel given.
tags:
- DiscordWebhook
/v1/discord/webhooks/new:
post:
parameters:
- description: url
in: query
name: url
required: true
type: string
- description: Server name
in: query
name: server
required: true
type: string
- description: Channel name
in: query
name: channel
required: true
type: string
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.DiscordWebhookResponse'
security:
- Bearer: []
summary: Creates a new record for a discord web hook to post data to.
tags:
- DiscordWebhook
/v1/sources:
get:
parameters:
- description: page number
in: query
name: page
type: string
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Unable to reach SQL or Data problems
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Lists the top 50 records
tags:
- Source
/v1/sources/{id}:
get:
parameters:
- description: id
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Returns a single entity by ID
tags:
- Source
post:
parameters:
- description: id
in: path
name: id
required: true
type: integer
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Marks a source as deleted based on its ID value.
tags:
- Source
/v1/sources/{id}/disable:
post:
parameters:
- description: id
in: path
name: id
required: true
type: integer
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Disables a source from processing.
tags:
- Source
/v1/sources/{id}/enable:
post:
parameters:
- description: id
in: path
name: id
required: true
type: string
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Enables a source to continue processing.
tags:
- Source
/v1/sources/by/source:
get:
parameters:
- description: Source Name
in: query
name: source
required: true
type: string
- description: page number
in: query
name: page
type: string
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: 'Lists the top 50 records based on the name given. Example: reddit'
tags:
- Source
/v1/sources/by/sourceAndName:
get:
parameters:
- description: dadjokes
in: query
name: name
required: true
type: string
- description: reddit
in: query
name: source
required: true
type: string
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Returns a single entity by ID
tags:
- Source
/v1/sources/new/reddit:
post:
parameters:
- description: name
in: query
name: name
required: true
type: string
- description: url
in: query
name: url
required: true
type: string
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Creates a new reddit source to monitor.
tags:
- Source
/v1/sources/new/rss:
post:
parameters:
- description: body
in: body
name: request
required: true
schema:
$ref: '#/definitions/domain.NewSourceParamRequest'
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Creates a new rss source to monitor.
tags:
- Source
/v1/sources/new/twitch:
post:
parameters:
- description: name
in: query
name: name
required: true
type: string
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Creates a new twitch source to monitor.
tags:
- Source
/v1/sources/new/youtube:
post:
parameters:
- description: name
in: query
name: name
required: true
type: string
- description: url
in: query
name: url
required: true
type: string
responses:
"200":
description: ok
schema:
$ref: '#/definitions/domain.SourcesResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.SourcesResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.SourcesResponse'
security:
- Bearer: []
summary: Creates a new youtube source to monitor.
tags:
- Source
/v1/users/login:
post:
parameters:
- in: formData
name: password
type: string
- in: formData
name: username
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.LoginResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.LoginResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.LoginResponse'
summary: Logs into the API and returns a bearer token if successful
tags:
- Users
/v1/users/refresh/sessionToken:
post:
consumes:
- application/json
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.BaseResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Revokes the current session token and replaces it with a new one.
tags:
- Users
/v1/users/refresh/token:
post:
parameters:
- description: body
in: body
name: request
required: true
schema:
$ref: '#/definitions/domain.RefreshTokenRequest'
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.LoginResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Generates a new token
tags:
- Users
/v1/users/register:
post:
parameters:
- in: formData
name: password
type: string
- in: formData
name: username
type: string
produces:
- application/json
responses:
"201":
description: Created
schema:
$ref: '#/definitions/domain.BaseResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
summary: Creates a new user
tags:
- Users
/v1/users/scopes/add:
post:
consumes:
- application/json
parameters:
- description: body
in: body
name: request
required: true
schema:
$ref: '#/definitions/domain.UpdateScopesRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.BaseResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Adds a new scope to a user account
tags:
- Users
/v1/users/scopes/remove:
post:
consumes:
- application/json
parameters:
- description: body
in: body
name: request
required: true
schema:
$ref: '#/definitions/domain.UpdateScopesRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/domain.BaseResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/domain.BaseResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/domain.BaseResponse'
security:
- Bearer: []
summary: Adds a new scope to a user account
tags:
- Users
swagger: "2.0"

View File

@ -1,9 +0,0 @@
package domain
const (
SourceCollectorRss = "rss"
SourceCollectorFfxiv = "ffxiv"
SourceCollectorTwitch = "twitch"
SourceCollectorYoutube = "youtube"
SourceCollectorReddit = "reddit"
)

View File

@ -1,64 +0,0 @@
package domain
import "time"
type ArticleDto struct {
ID int64 `json:"id"`
SourceID int64 `json:"sourceId"`
Tags string `json:"tags"`
Title string `json:"title"`
Url string `json:"url"`
PubDate time.Time `json:"pubDate"`
IsVideo bool `json:"isVideo"`
Thumbnail string `json:"thumbnail"`
Description string `json:"description"`
AuthorName string `json:"authorName"`
AuthorImageUrl string `json:"authorImage"`
}
type DiscordQueueDto struct {
ID int64 `json:"id"`
ArticleId int64 `json:"articleId"`
SourceId int64 `json:"sourceId"`
}
type DiscordWebHookDto struct {
ID int64 `json:"id"`
//Name string `json:"name"`
//Key string `json:"key"`
Url string `json:"url"`
Server string `json:"server"`
Channel string `json:"channel"`
Enabled bool `json:"enabled"`
}
type IconDto struct {
ID int64 `json:"id"`
FileName string `json:"fileName"`
Site string `json:"site"`
}
type SettingDto struct {
ID int64 `json:"id"`
Key string `json:"key"`
Value string `json:"value"`
Options string `json:"options"`
}
type SubscriptionDto struct {
ID int64 `json:"id"`
SourceID int64 `json:"sourceId"`
SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"`
DiscordID int64 `json:"discordId"`
DiscordName string `json:"discordName"`
}
type SourceDto struct {
ID int64 `json:"id"`
Source string `json:"source"`
DisplayName string `json:"name"`
Url string `json:"url"`
Tags string `json:"tags"`
Enabled bool `json:"enabled"`
}

View File

@ -0,0 +1,23 @@
package interfaces
import (
"github.com/go-rod/rod"
"github.com/mmcdole/gofeed"
)
type Sources interface {
CheckSource() error
PullFeed() (*gofeed.Feed, error)
GetBrowser() *rod.Browser
GetPage(parser *rod.Browser, url string) *rod.Page
ExtractThumbnail(page *rod.Page) (string, error)
ExtractPubDate(page *rod.Page) (string, error)
ExtractDescription(page *rod.Page) (string, error)
ExtractAuthor(page *rod.Page) (string, error)
ExtractAuthorImage(page *rod.Page) (string, error)
ExtractTags(page *rod.Page) (string, error)
ExtractTitle(page *rod.Page) (string, error)
}

View File

@ -1,6 +1,8 @@
package domain
package model
import "time"
import (
"time"
)
type CacheItem struct {
Key string
@ -8,7 +10,7 @@ type CacheItem struct {
// Group defines what it should be a reference to.
// youtube, reddit, ect
Group string
Expires time.Time
Group string
Expires time.Time
IsTainted bool
}
}

90
domain/model/database.go Normal file
View File

@ -0,0 +1,90 @@
package model
import (
"time"
)
// Articles represents the model for an Article
type Articles struct {
ID uint `json:"ID"`
SourceID uint `json:"sourceId"`
Tags string `json:"tags"`
Title string `json:"title"`
Url string `json:"url"`
PubDate time.Time `json:"pubdate"`
Video string `json:"video"`
VideoHeight uint16 `json:"videoHeight"`
VideoWidth uint16 `json:"videoWidth"`
Thumbnail string `json:"thumbnail"`
Description string `json:"description"`
AuthorName string `json:"authorName"`
AuthorImage string `json:"authorImage"`
}
type DiscordQueue struct {
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
ArticleId string `json:"articleId"`
}
type DiscordWebHooks struct {
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
Name string `json:"name"`
Key string `json:"key"`
Url string `json:"url"`
Server string `json:"server"`
Channel string `json:"channel"`
Enabled bool `json:"enabled"`
}
type Icons struct {
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
FileName string `json:"fileName"`
Site string `json:"site"`
}
type Settings struct {
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
Key string `json:"key"`
Value string `json:"value"`
Options string `json:"options"`
}
type Sources struct {
ID uint `json:"ID"`
Site string `json:"site"`
Name string `json:"name"`
Source string `json:"source"`
Type string `json:"type"`
Value string `json:"value"`
Enabled bool `json:"enabled"`
Url string `json:"url"`
Tags string `json:"tags"`
}
type SourceLinks struct {
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
SourceID uint `json:"sourceId"`
SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"`
DiscordID uint `json:"discordId"`
DiscordName string `json:"discordName"`
}

54
domain/model/reddit.go Normal file
View File

@ -0,0 +1,54 @@
package model
// This is the root Json object. It does not contain data that we care about though.
type RedditJsonContent struct {
Kind string `json:"kind"`
Data RedditJsonContentData `json:"data"`
}
type RedditJsonContentData struct {
After string `json:"after"`
Dist int `json:"dist"`
Modhash string `json:"modhash"`
Children []RedditJsonContentChildren `json:"children"`
}
type RedditJsonContentChildren struct {
Kind string `json:"kind"`
Data RedditPost `json:"data"`
}
// RedditPost contains the information that was posted by a user.
type RedditPost struct {
Subreddit string `json:"subreddit"`
Title string `json:"title"`
Content string `json:"selftext"`
ContentHtml string `json:"selftext_html"`
Author string `json:"author"`
Permalink string `json:"permalink"`
IsVideo bool `json:"is_video"`
Media RedditPostMedia `json:"media"`
Url string `json:"url"`
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
Thumbnail string `json:"thumbnail"`
}
// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
type RedditPostMedia struct {
RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
}
// RedditVideo contains information about the video in the post.
type RedditPostMediaRedditVideo struct {
BitrateKbps int `json:"bitrate_kpbs"`
FallBackUrl string `json:"fallback_url"`
Height int `json:"height"`
Width int `json:"width"`
ScrubberMediaUrl string `json:"scrubber_media_url"`
DashUrl string `json:"dash_url"`
Duration int `json:"duration"`
HlsUrl string `json:"hls_url"`
IsGif bool `json:"is_gif"`
TranscodingStatus string `json:"transcoding_status"`
}

View File

@ -1,27 +0,0 @@
package domain
type LoginFormRequest struct {
Username string `form:"username"`
Password string `form:"password"`
}
type GetSourceBySourceAndNameParamRequest struct {
Name string `query:"name"`
Source string `query:"source"`
}
type NewSourceParamRequest struct {
Name string `json:"name"`
Url string `json:"url"`
Tags string `json:"tags"`
}
type RefreshTokenRequest struct {
Username string `json:"username"`
RefreshToken string `json:"refreshToken"`
}
type UpdateScopesRequest struct {
Username string `json:"username"`
Scopes []string `json:"scopes" validate:"required"`
}

View File

@ -1,38 +0,0 @@
package domain
type BaseResponse struct {
Message string `json:"message"`
IsError bool `json:"isError"`
}
type LoginResponse struct {
BaseResponse
Token string `json:"token"`
Type string `json:"type"`
RefreshToken string `json:"refreshToken"`
}
type ArticleResponse struct {
BaseResponse
Payload []ArticleDto `json:"payload"`
}
type ArticleAndSourceModel struct {
Article ArticleDto `json:"article"`
Source SourceDto `json:"source"`
}
type ArticleDetailedResponse struct {
BaseResponse
Payload ArticleAndSourceModel `json:"payload"`
}
type DiscordWebhookResponse struct {
BaseResponse
Payload []DiscordWebHookDto `json:"payload"`
}
type SourcesResponse struct {
BaseResponse
Payload []SourceDto `json:"payload"`
}

View File

@ -1,14 +0,0 @@
package domain
const (
ScopeAll = "newsbot:all"
ScopeArticleRead = "newsbot:article:read"
ScopeArticleDisable = "newsbot:article:disable"
ScopeSourceRead = "newsbot:source:read"
ScopeSourceCreate = "newsbot:source:create"
ScopeDiscordWebHookCreate = "newsbot:discordwebhook:create"
ScopeDiscordWebhookRead = "newsbot:discordwebhook:read"
)

55
go.mod
View File

@ -1,48 +1,19 @@
module git.jamestombleson.com/jtom38/newsbot-api
module github.com/jtom38/newsbot/collector
go 1.22
go 1.18
require (
github.com/PuerkitoBio/goquery v1.8.0
github.com/glebarez/go-sqlite v1.22.0
github.com/go-chi/chi/v5 v5.0.7
github.com/go-rod/rod v0.107.1
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/huandu/go-sqlbuilder v1.27.1
github.com/google/uuid v1.3.0
github.com/joho/godotenv v1.4.0
github.com/labstack/echo-jwt/v4 v4.2.0
github.com/labstack/echo/v4 v4.12.0
github.com/lib/pq v1.10.6
github.com/mmcdole/gofeed v1.1.3
github.com/nicklaw5/helix/v2 v2.4.0
github.com/pressly/goose/v3 v3.20.0
github.com/robfig/cron/v3 v3.0.1
github.com/swaggo/echo-swagger v1.4.1
github.com/swaggo/swag v1.8.12
golang.org/x/crypto v0.22.0
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sethvargo/go-retry v0.2.4 // indirect
github.com/swaggo/files/v2 v2.0.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/time v0.5.0 // indirect
modernc.org/libc v1.41.0 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.7.2 // indirect
modernc.org/sqlite v1.29.6 // indirect
github.com/swaggo/http-swagger v1.3.0
github.com/swaggo/swag v1.8.2
)
require (
@ -52,20 +23,20 @@ require (
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/spec v0.20.6 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/lib/pq v1.10.6
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/gson v0.7.2 // indirect
github.com/ysmood/leakless v0.7.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.17.0 // indirect
golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.11 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

136
go.sum
View File

@ -12,12 +12,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@ -29,31 +25,20 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU=
github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-rod/rod v0.105.1 h1:r0bNmO9siOe13lG6Vbkaak11u48rYmWGl/Hk4MJdOiE=
github.com/go-rod/rod v0.105.1/go.mod h1:Wrnn6HokFHskwaIVke3ML1y/NBVp7XPIeB8eDzR9vuw=
github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY=
github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
github.com/huandu/go-sqlbuilder v1.27.1 h1:7UU/3EMIQYYX8wn+L7BNcGVz1aEs5TPNOVFd7ryrPos=
github.com/huandu/go-sqlbuilder v1.27.1/go.mod h1:nUVmMitjOmn/zacMLXT0d3Yd3RHoO2K+vy906JzqxMI=
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@ -62,12 +47,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/labstack/echo-jwt/v4 v4.2.0 h1:odSISV9JgcSCuhgQSV/6Io3i7nUmfM/QkBeR5GVJj5c=
github.com/labstack/echo-jwt/v4 v4.2.0/go.mod h1:MA2RqdXdEn4/uEglx0HcUOgQSyBaTh5JcaHIan3biwU=
github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@ -75,103 +54,87 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 h1:Z6i7ND25ixRtXFBylIUggqpvLMV1I15yprcqMVB7WZA=
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/nicklaw5/helix/v2 v2.4.0 h1:ZvqCKVqza1eJYyqgTRrZ/xjDq0w/EQVFNkN067Utls0=
github.com/nicklaw5/helix/v2 v2.4.0/go.mod h1:0ONzvVi1cH+k3a7EDIFNNqxfW0podhf+CqlmFvuexq8=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0=
github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk=
github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc=
github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w=
github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM=
github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc=
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
github.com/swaggo/http-swagger v1.2.8 h1:TVjxLU7qoqofJ9qynJazmpTGs/p4Kx9FTp7YYwOkJb0=
github.com/swaggo/http-swagger v1.2.8/go.mod h1:FrQwV7rx+A5t11PIX8d+tFJa2GKx11RdAXQptllPQHg=
github.com/swaggo/http-swagger v1.3.0 h1:1+6M4qRorIbdyTWTsGrwnb0r9jGK5dcWN82O6oY/yHQ=
github.com/swaggo/http-swagger v1.3.0/go.mod h1:9glekdg40lwclrrKNRGgj/IMDxpNPZ3kzab4oPcF8EM=
github.com/swaggo/swag v1.8.2 h1:D4aBiVS2a65zhyk3WFqOUz7Rz0sOaUcgeErcid5uGL4=
github.com/swaggo/swag v1.8.2/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
github.com/ysmood/got v0.23.2 h1:U2U0vyQ/gDaawkKJZK/hyza8UUXbWCurbmazK7AcAfY=
github.com/ysmood/got v0.23.2/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
github.com/ysmood/got v0.29.5 h1:+wMnm8UjoyYFMfeAsr57a1bahWTkloysc0Hxsu2gmnM=
github.com/ysmood/got v0.29.5/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/gson v0.7.2 h1:1iWUvpi5DPvd2j59W7ifRPR9DiAZ3Ga+fmMl1mJrRbM=
github.com/ysmood/gson v0.7.2/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 h1:z8Hj/bl9cOV2grsOpEaQFUaly0JWN3i97mo3jXKJNp0=
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 h1:PgOr27OhUx2IRqGJ2RxAWI4dJQ7bi9cSrB82uzFzfUA=
golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
@ -180,20 +143,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4=
modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

View File

@ -1,115 +0,0 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
CREATE TABLE Articles (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
SourceId NUMBER NOT NULL,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate DATETIME NOT NULL,
IsVideo TEXT NOT NULL,
ThumbnailUrl TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT NOT NULL,
AuthorImageUrl TEXT NOT NULL
);
CREATE Table DiscordWebHooks (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
UserID INTEGER NOT NULL,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for reference
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for reference
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME,
Key TEXT NOT NULL, -- How you search for a entry
Value TEXT NOT NULL, -- The value for one
Options TEXT -- any notes about the entry
);
CREATE Table Sources (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
DisplayName TEXT NOT NULL, -- Vanity name
Source TEXT NOT NULL, -- Defines the service that will use this record. IE reddit or youtube
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);
CREATE TABLE UserSourceSubscriptions (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
UserID NUMBER NOT NULL,
SourceID NUMBER NOT NULL
);
CREATE TABLE AlertDiscord (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
UserID NUMBER NOT NULL,
SourceID NUMBER NOT NULL,
DiscordWebHookID NUMBER NOT NULL
);
CREATE TABLE Users (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
Name TEXT NOT NULL,
Hash TEXT NOT NULL,
Scopes TEXT NOT NULL
);
CREATE TABLE RefreshTokens (
ID INTEGER PRIMARY KEY AUTOINCREMENT,
CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL,
Username TEXT NOT NULL,
Token TEXT NOT NULL
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE AlertDiscord;
Drop Table Articles;
Drop Table DiscordWebHooks;
Drop Table Icons;
DROP TABLE RefreshTokens;
Drop Table Sources;
DROP TABLE Users;
DROP TABLE UserSourceSubscriptions;
-- +goose StatementEnd

View File

@ -1,50 +0,0 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
-- Enable UUID's
--CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Final Fantasy XIV Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
--SELECT 'down SQL query';
DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes';
DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck';
DELETE FROM sources where Source = 'ffxiv';
DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo';
DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps';
DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck';
-- +goose StatementEnd

View File

@ -1,9 +0,0 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE Users ADD SessionToken TEXT;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE Users DROP SessionToken;
-- +goose StatementEnd

View File

@ -1,54 +0,0 @@
package domain
// This is the root Json object. It does not contain data that we care about though.
type RedditJsonContent struct {
Kind string `json:"kind"`
Data RedditJsonContentData `json:"data"`
}
type RedditJsonContentData struct {
After string `json:"after"`
Dist int `json:"dist"`
Modhash string `json:"modhash"`
Children []RedditJsonContentChildren `json:"children"`
}
type RedditJsonContentChildren struct {
Kind string `json:"kind"`
Data RedditPost `json:"data"`
}
// RedditPost contains the information that was posted by a user.
type RedditPost struct {
Subreddit string `json:"subreddit"`
Title string `json:"title"`
Content string `json:"selftext"`
ContentHtml string `json:"selftext_html"`
Author string `json:"author"`
Permalink string `json:"permalink"`
IsVideo bool `json:"is_video"`
Media RedditPostMedia `json:"media"`
Url string `json:"url"`
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
Thumbnail string `json:"thumbnail"`
}
// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
type RedditPostMedia struct {
RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
}
// RedditVideo contains information about the video in the post.
type RedditPostMediaRedditVideo struct {
BitrateKbps int `json:"bitrate_kpbs"`
FallBackUrl string `json:"fallback_url"`
Height int `json:"height"`
Width int `json:"width"`
ScrubberMediaUrl string `json:"scrubber_media_url"`
DashUrl string `json:"dash_url"`
Duration int `json:"duration"`
HlsUrl string `json:"hls_url"`
IsGif bool `json:"is_gif"`
TranscodingStatus string `json:"transcoding_status"`
}

View File

@ -1,67 +0,0 @@
package dtoconv
import (
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
)
func ArticlesToDto(items []entity.ArticleEntity) []domain.ArticleDto {
var dtos []domain.ArticleDto
for _, item := range items {
dtos = append(dtos, ArticleToDto(item))
}
return dtos
}
func ArticleToDto(item entity.ArticleEntity) domain.ArticleDto {
return domain.ArticleDto{
ID: item.ID,
SourceID: item.SourceID,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
PubDate: item.PubDate,
IsVideo: item.IsVideo,
Thumbnail: item.Thumbnail,
Description: item.Description,
AuthorName: item.AuthorName,
AuthorImageUrl: item.AuthorImageUrl,
}
}
func DiscordWebhooksToDto(items []entity.DiscordWebHookEntity) []domain.DiscordWebHookDto{
var dtos []domain.DiscordWebHookDto
for _, item := range items {
dtos = append(dtos, DiscordWebhookToDto(item))
}
return dtos
}
func DiscordWebhookToDto(item entity.DiscordWebHookEntity) domain.DiscordWebHookDto {
return domain.DiscordWebHookDto{
ID: item.ID,
Server: item.Server,
Channel: item.Channel,
Url: item.Url,
Enabled: item.Enabled,
}
}
func SourcesToDto(items []entity.SourceEntity) []domain.SourceDto {
var dtos []domain.SourceDto
for _, item := range items {
dtos = append(dtos, SourceToDto(item))
}
return dtos
}
func SourceToDto(item entity.SourceEntity) domain.SourceDto {
return domain.SourceDto{
ID: item.ID,
Source: item.Source,
DisplayName: item.DisplayName,
Url: item.Url,
Tags: item.Tags,
Enabled: item.Enabled,
}
}

View File

@ -1,141 +0,0 @@
package entity
import (
"time"
)
// This links a source to a discord webhook.
// It is owned by a user so they can remove the link
type AlertDiscordEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
UserID int64
SourceID int64
DiscordWebHookId int64
}
type ArticleEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
SourceID int64
Tags string
Title string
Url string
PubDate time.Time
IsVideo bool
Thumbnail string
Description string
AuthorName string
AuthorImageUrl string
}
type DiscordQueueEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
ArticleId int64
SourceId int64
}
type DiscordWebHookEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
UserID int64
Url string
Server string
Channel string
Enabled bool
}
type IconEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
FileName string
Site string
}
type SettingEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
Key string
Value string
Options string
}
type SourceEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
// Who will collect from it. Used
// domain.SourceCollector...
Source string
// Human Readable value to state what is getting collected
DisplayName string
// Tells the parser where to look for data
Url string
// Static tags for this defined record
Tags string
// If the record is disabled, then it will be skipped on processing
Enabled bool
}
//type SubscriptionEntity struct {
// ID int64
// CreatedAt time.Time
// UpdatedAt time.Time
// DeletedAt time.Time
// UserID int64
// SourceID int64
// //SourceType string
// //SourceName string
// DiscordID int64
// //DiscordName string
//}
// This defines what sources a user wants to follow.
// These will show up for the user as a front page
type UserSourceSubscriptionEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
UserID int64
SourceID int64
}
type UserEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
Username string
Hash string
Scopes string
SessionToken string
}
type RefreshTokenEntity struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt time.Time
Username string
Token string
}

View File

@ -1,181 +0,0 @@
package v1
import (
"net/http"
"strconv"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
"github.com/labstack/echo/v4"
)
// ListArticles
// @Summary Lists the top 25 records ordering from newest to oldest.
// @Produce application/json
// @Param page query int true "page number" test
// @Tags Articles
// @Router /v1/articles [get]
// @Success 200 {object} domain.ArticleResponse
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (s *Handler) listArticles(c echo.Context) error {
resp := domain.ArticleResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
if err != nil {
return c.JSON(http.StatusUnauthorized, resp)
}
page, err := strconv.Atoi(c.QueryParam("page"))
if err != nil {
page = 0
}
res, err := s.repo.Articles.ListByPage(c.Request().Context(), page, 25)
if err != nil {
return c.JSON(http.StatusInternalServerError, resp)
}
resp.Payload = dtoconv.ArticlesToDto(res)
resp.BaseResponse.IsError = false
return c.JSON(http.StatusOK, resp)
}
// GetArticle
// @Summary Returns an article based on defined ID.
// @Param id path string true "int"
// @Produce application/json
// @Tags Articles
// @Router /v1/articles/{id} [get]
// @Success 200 {object} domain.ArticleResponse "OK"
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (s *Handler) getArticle(c echo.Context) error {
p := domain.ArticleResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
if err != nil {
return c.JSON(http.StatusUnauthorized, p)
}
id := c.Param("ID")
idNumber, err := strconv.Atoi(id)
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
item, err := s.repo.Articles.GetById(c.Request().Context(), int64(idNumber))
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
var dtos []domain.ArticleDto
dtos = append(dtos, dtoconv.ArticleToDto(item))
p.Payload = dtos
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// GetArticleDetails
// @Summary Returns an article and source based on defined ID.
// @Param id path string true "int"
// @Produce application/json
// @Tags Articles
// @Router /v1/articles/{id}/details [get]
// @Success 200 {object} domain.ArticleDetailedResponse "OK"
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (s *Handler) getArticleDetails(c echo.Context) error {
p := domain.ArticleDetailedResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
Payload: domain.ArticleAndSourceModel{},
}
_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
if err != nil {
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
article, err := s.repo.Articles.GetById(c.Request().Context(), int64(id))
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
source, err := s.repo.Sources.GetById(c.Request().Context(), article.SourceID)
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
p.Payload.Article = dtoconv.ArticleToDto(article)
p.Payload.Source = dtoconv.SourceToDto(source)
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// ListArticlesBySourceID
// @Summary Finds the articles based on the SourceID provided. Returns the top 25.
// @Param id path string true "source id"
// @Param page query int false "Page to query"
// @Produce application/json
// @Tags Articles
// @Router /v1/articles/by/source/{id} [get]
// @Success 200 {object} domain.ArticleResponse "OK"
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (s *Handler) ListArticlesBySourceId(c echo.Context) error {
p := domain.ArticleResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeArticleRead)
if err != nil {
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.QueryParam("id"))
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
// if the page number is missing, default to 0
_page, err := strconv.Atoi(c.QueryParam("page"))
if err != nil {
_page = 0
}
items, err := s.repo.Articles.ListBySource(c.Request().Context(), _page, 25, id, "")
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
p.Payload = dtoconv.ArticlesToDto(items)
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}

View File

@ -1,443 +0,0 @@
package v1
import (
"net/http"
"strconv"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
"github.com/labstack/echo/v4"
)
// ListDiscordWebhooks
// @Summary Returns the top 100
// @Produce application/json
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks [get]
// @Success 200 {object} domain.DiscordWebhookResponse
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) ListDiscordWebHooks(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
if err != nil {
return c.JSON(http.StatusUnauthorized, p)
}
res, err := s.repo.DiscordWebHooks.ListByServerName(c.Request().Context(), "")
if err != nil {
return c.JSON(http.StatusInternalServerError, p)
}
p.Payload = dtoconv.DiscordWebhooksToDto(res)
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// GetDiscordWebHook
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Param id path int true "id"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/{id} [get]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) GetDiscordWebHooksById(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
if err != nil {
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
return c.JSON(http.StatusBadRequest, p)
}
res, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
return c.JSON(http.StatusInternalServerError, p)
}
var dtos []domain.DiscordWebHookDto
dtos = append(dtos, dtoconv.DiscordWebhookToDto(res))
p.Payload = dtos
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// GetDiscordWebHookByServerAndChannel
// @Summary Returns all the known web hooks based on the Server and Channel given.
// @Produce application/json
// @Param server query string true "Fancy Server"
// @Param channel query string true "memes"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/by/serverAndChannel [get]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebhookRead)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
_server := c.QueryParam("server")
if _server == "" {
p.BaseResponse.Message = "server was not defined"
return c.JSON(http.StatusBadRequest, p)
}
_channel := c.QueryParam("channel")
if _channel == "" {
p.BaseResponse.Message = "channel was not defined"
return c.JSON(http.StatusBadRequest, p)
}
res, err := s.repo.DiscordWebHooks.ListByServerAndChannel(c.Request().Context(), _server, _channel)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.Payload = dtoconv.DiscordWebhooksToDto(res)
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// NewDiscordWebHook
// @Summary Creates a new record for a discord web hook to post data to.
// @Param url query string true "url"
// @Param server query string true "Server name"
// @Param channel query string true "Channel name"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/new [post]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) NewDiscordWebHook(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
token, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
_url := c.QueryParam("url")
_server := c.QueryParam("server")
_channel := c.QueryParam("channel")
if _url == "" {
p.Message = "url is missing a value"
return c.JSON(http.StatusBadRequest, p)
}
if !strings.Contains(_url, "discord.com/api/webhooks") {
p.Message = "invalid url"
return c.JSON(http.StatusBadRequest, p)
}
if _server == "" {
p.Message = "server is missing"
return c.JSON(http.StatusBadRequest, p)
}
if _channel == "" {
p.Message = "channel is missing"
return c.JSON(http.StatusBadRequest, p)
}
user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
rows, err := s.repo.DiscordWebHooks.Create(c.Request().Context(), user.ID, _url, _server, _channel, true)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if rows != 1 {
p.Message = "data was not written to database"
return c.JSON(http.StatusInternalServerError, p)
}
item, err := s.repo.DiscordWebHooks.GetByUrl(c.Request().Context(), _url)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dtos []domain.DiscordWebHookDto
dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
p.Payload = dtos
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// DisableDiscordWebHooks
// @Summary Disables a Webhook from being used.
// @Param id path int true "id"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/{id}/disable [post]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) disableDiscordWebHook(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check to make sure we can find the record
record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if record.UserID != s.GetUserIdFromJwtToken(c) {
p.BaseResponse.Message = ErrYouDontOwnTheRecord
return c.JSON(http.StatusBadRequest, p)
}
// flip the it
updated, err := s.repo.DiscordWebHooks.Disable(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
// make sure we got a row updated
if updated != 1 {
p.BaseResponse.Message = "unexpected number of updates found"
return c.JSON(http.StatusInternalServerError, p)
}
item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dtos []domain.DiscordWebHookDto
dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
p.Payload = dtos
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// EnableDiscordWebHook
// @Summary Enables a source to continue processing.
// @Param id path int true "id"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/{id}/enable [post]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) enableDiscordWebHook(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check to make sure we can find the record
record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
if record.UserID != s.GetUserIdFromJwtToken(c) {
p.BaseResponse.Message = ErrYouDontOwnTheRecord
return c.JSON(http.StatusBadRequest, p)
}
updated, err := s.repo.DiscordWebHooks.Enable(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if updated != 1 {
p.BaseResponse.Message = ErrFailedToUpdateRecord
return c.JSON(http.StatusInternalServerError, p)
}
item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dtos []domain.DiscordWebHookDto
dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
p.Payload = dtos
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// DeleteDiscordWebHook
// @Summary Deletes a record by ID.
// @Param id path string true "id"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/{id} [delete]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.DiscordWebhookResponse
// @Failure 500 {object} domain.DiscordWebhookResponse
// @Security Bearer
func (s *Handler) deleteDiscordWebHook(c echo.Context) error {
p := domain.DiscordWebhookResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check to make sure we can find the record
record, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
if record.UserID != s.GetUserIdFromJwtToken(c) {
p.BaseResponse.Message = ErrYouDontOwnTheRecord
return c.JSON(http.StatusBadRequest, p)
}
// Soft delete the record
updated, err := s.repo.DiscordWebHooks.SoftDelete(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if updated != 1 {
p.BaseResponse.Message = ErrFailedToUpdateRecord
return c.JSON(http.StatusInternalServerError, p)
}
item, err := s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dtos []domain.DiscordWebHookDto
dtos = append(dtos, dtoconv.DiscordWebhookToDto(item))
p.Payload = dtos
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// UpdateDiscordWebHook
// @Summary Updates a valid discord webhook ID based on the body given.
// @Param id path string true "id"
// @Tags DiscordWebhook
// @Router /v1/discord/webhooks/{id} [patch]
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
//func (s *Handler) UpdateDiscordWebHook(c echo.Context) error {
// id, err := strconv.Atoi(c.Param("ID"))
// if err != nil {
// return c.JSON(http.StatusInternalServerError, err.Error())
// }
//
// // Check to make sure we can find the record
// _, err = s.repo.DiscordWebHooks.GetById(c.Request().Context(), int64(id))
// if err != nil {
// return c.JSON(http.StatusInternalServerError, err.Error())
// }
//
// // Soft delete the record
// updated, err := s.repo.DiscordWebHooks(c.Request().Context(), int64(id))
// if err != nil {
// return c.JSON(http.StatusInternalServerError, err.Error())
// }
//
// _, err = s.Db.GetDiscordQueueByID(c.Request().Context(), uuid)
// if err != nil {
// return c.JSON(http.StatusInternalServerError, err.Error())
// }
//
// return nil
//}

View File

@ -1,197 +0,0 @@
package v1
import (
"context"
"database/sql"
"errors"
"github.com/golang-jwt/jwt/v5"
echojwt "github.com/labstack/echo-jwt/v4"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
swagger "github.com/swaggo/echo-swagger"
_ "git.jamestombleson.com/jtom38/newsbot-api/docs"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
)
type Handler struct {
Router *echo.Echo
//Db *database.Queries
config services.Configs
repo services.RepositoryService
}
const (
ErrParameterIdMissing = "The requested parameter ID was not found."
ErrParameterMissing = "The requested parameter was not found found:"
ErrUnableToParseId = "Unable to parse the requested ID"
ErrRecordMissing = "The requested record was not found"
ErrFailedToCreateRecord = "The record was not created due to a database problem"
ErrFailedToUpdateRecord = "The requested record was not updated due to a database problem"
ErrUserUnknown = "User is unknown"
ErrYouDontOwnTheRecord = "The record requested does not belong to you"
ResponseMessageSuccess = "Success"
)
var (
ErrIdValueMissing string = "id value is missing"
ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct."
ErrNoRecordFound string = "no record was found."
ErrUnableToConvertToJson string = "Unable to convert to json"
)
func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
s := &Handler{
config: configs,
repo: services.NewRepositoryService(conn),
}
jwtConfig := echojwt.Config{
NewClaimsFunc: func(c echo.Context) jwt.Claims {
return new(JwtToken)
},
SigningKey: []byte(configs.JwtSecret),
}
router := echo.New()
router.Pre(middleware.RemoveTrailingSlash())
router.Pre(middleware.Logger())
router.Pre(middleware.Recover())
router.GET("/swagger/*", swagger.WrapHandler)
v1 := router.Group("/api/v1")
articles := v1.Group("/articles")
articles.Use(echojwt.WithConfig(jwtConfig))
articles.GET("", s.listArticles)
articles.GET(":id", s.getArticle)
articles.GET(":id/details", s.getArticleDetails)
articles.GET("by/source/:id", s.ListArticlesBySourceId)
//dwh := v1.Group("/discord/webhooks")
//dwh.GET("/", s.ListDiscordWebHooks)
//dwh.POST("/new", s.NewDiscordWebHook)
//dwh.GET("/by/serverAndChannel", s.GetDiscordWebHooksByServerAndChannel)
//dwh.GET("/:ID", s.GetDiscordWebHooksById)
//dwh.DELETE("/:ID", s.deleteDiscordWebHook)
//dwh.POST("/:ID/disable", s.disableDiscordWebHook)
//dwh.POST("/:ID/enable", s.enableDiscordWebHook)
//queue := v1.Group("/queue")
//queue.GET("/discord/webhooks", s.ListDiscordWebhookQueue) // TODO this needs to be reworked
//settings := v1.Group("/settings")
//settings.GET("/", s.getSettings)
sources := v1.Group("/sources")
sources.Use(echojwt.WithConfig(jwtConfig))
sources.GET("", s.listSources)
sources.GET("/by/source", s.listSourcesBySource)
sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName)
//sources.POST("/new/reddit", s.newRedditSource)
//sources.POST("/new/youtube", s.newYoutubeSource)
//sources.POST("/new/twitch", s.newTwitchSource)
sources.POST("/new/rss", s.newRssSource)
sources.GET("/:id", s.getSource)
sources.DELETE("/:id", s.deleteSources)
sources.POST("/:id/disable", s.disableSource)
sources.POST("/:id/enable", s.enableSource)
users := v1.Group("/users")
users.POST("/login", s.AuthLogin)
users.POST("/register", s.AuthRegister)
users.Use(echojwt.WithConfig(jwtConfig))
users.POST("/scopes/add", s.AddScopes)
users.POST("/scopes/remove", s.RemoveScopes)
users.POST("/refresh/token", s.RefreshJwtToken)
users.POST("/refresh/sessionToken", s.NewSessionToken)
s.Router = router
return s
}
//type ApiStatusModel struct {
// StatusCode int `json:"status"`
// Message string `json:"message"`
//}
//type ApiError struct {
// *ApiStatusModel
//}
//func (s *Handler) WriteError(c echo.Context, errMessage error, HttpStatusCode int) error {
// return c.JSON(HttpStatusCode, domain.BaseResponse{
// Message: errMessage.Error(),
// IsError: true,
// })
//}
//func (s *Handler) WriteMessage(c echo.Context, msg string, HttpStatusCode int) error {
// return c.JSON(HttpStatusCode, domain.BaseResponse{
// Message: msg,
// })
//}
//func (s *Handler) InternalServerErrorResponse(c echo.Context, msg string) error {
// return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
// Message: msg,
// })
//}
//func (s *Handler) UnauthorizedResponse(c echo.Context, msg string) error {
// return c.JSON(http.StatusUnauthorized, domain.BaseResponse{
// Message: msg,
// })
//}
// If the token is not valid then an json error will be returned.
// If the token has the wrong scope, a json error will be returned.
// If the token passes all the checks, it is valid and is returned back to the caller.
func (s *Handler) ValidateJwtToken(c echo.Context, requiredScope string) (JwtToken, error) {
token, err := s.getJwtTokenFromContext(c)
if err != nil {
return JwtToken{}, errors.New(ErrJwtMissing)
}
err = token.hasExpired()
if err != nil {
return JwtToken{}, errors.New(ErrJwtExpired)
}
err = token.hasScope(requiredScope)
if err != nil {
return JwtToken{}, errors.New(ErrJwtScopeMissing)
}
if token.Iss != s.config.ServerAddress {
return JwtToken{}, errors.New(ErrJwtInvalidIssuer)
}
// If you are the built in admin account, skip the username and session token check
if token.UserName == "admin" {
return token, nil
}
user, err := s.repo.Users.GetUser(c.Request().Context(), token.UserName)
if err != nil {
return JwtToken{}, errors.New("user record not found")
}
if user.SessionToken != token.SessionToken {
return JwtToken{}, errors.New("invalid session token")
}
return token, nil
}
func (s *Handler) GetUserIdFromJwtToken(c echo.Context) int64 {
token, err := s.getJwtTokenFromContext(c)
if err != nil {
return -1
}
return token.GetUserId()
}

View File

@ -1,130 +0,0 @@
package v1
import (
"errors"
"strings"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"github.com/golang-jwt/jwt/v5"
"github.com/labstack/echo/v4"
)
const (
ErrJwtMissing = "auth token is missing"
ErrJwtClaimsMissing = "claims missing on token"
ErrJwtExpired = "auth token has expired"
ErrJwtScopeMissing = "required scope is missing"
ErrJwtInvalidIssuer = "incorrect server issued the token"
)
type JwtToken struct {
Exp time.Time `json:"exp"`
Iss string `json:"iss"`
Authorized bool `json:"authorized"`
UserName string `json:"username"`
UserId int64 `json:"userId"`
Scopes []string `json:"scopes"`
SessionToken string `json:"sessionToken"`
jwt.RegisteredClaims
}
func (j JwtToken) IsValid(scope string) error {
err := j.hasExpired()
if err != nil {
return err
}
// Check to see if they have the scope to do anything
// if they do, let them pass
err = j.hasScope(domain.ScopeAll)
if err == nil {
return nil
}
err = j.hasScope(scope)
if err != nil {
return err
}
return nil
}
func (j JwtToken) GetUsername() string {
return j.UserName
}
func (j JwtToken) GetUserId() int64 {
return j.UserId
}
func (j JwtToken) hasExpired() error {
// Check to see if the token has expired
//hasExpired := j.Exp.Compare(time.Now())
hasExpired := time.Now().Compare(j.Exp)
if hasExpired == 1 {
return errors.New(ErrJwtExpired)
}
return nil
}
// This will check the users token to make sure they have the correct scope to access the handler.
// It will evaluate if you have the admin scope or the required scope for the handler.
func (j JwtToken) hasScope(scope string) error {
// they have the scope to access everything, so let them pass.
userScopes := strings.Join(j.Scopes, "")
if strings.Contains(domain.ScopeAll, userScopes) {
return nil
}
if strings.Contains(userScopes, scope) {
return nil
}
return errors.New(ErrJwtScopeMissing)
}
func (h *Handler) generateJwt(username, issuer, sessionToken string, userScopes []string, userId int64) (string, error) {
return h.generateJwtWithExp(username, issuer, sessionToken, userScopes, userId, time.Now().Add(10*time.Minute))
}
func (h *Handler) generateJwtWithExp(username, issuer, sessionToken string, userScopes []string, userId int64, expiresAt time.Time) (string, error) {
secret := []byte(h.config.JwtSecret)
// Anyone who wants to decrypt the key needs to use the same method
token := jwt.New(jwt.SigningMethodHS256)
claims := token.Claims.(jwt.MapClaims)
claims["exp"] = expiresAt
claims["authorized"] = true
claims["username"] = username
claims["iss"] = issuer
claims["userId"] = userId
claims["sessionToken"] = sessionToken
var scopes []string
scopes = append(scopes, userScopes...)
claims["scopes"] = scopes
tokenString, err := token.SignedString(secret)
if err != nil {
return "", err
}
return tokenString, nil
}
func (h *Handler) getJwtTokenFromContext(c echo.Context) (JwtToken, error) {
// Make sure that the request came with a jwtToken
token, ok := c.Get("user").(*jwt.Token)
if !ok {
return JwtToken{}, errors.New(ErrJwtMissing)
}
// Generate the claims from the token
claims, ok := token.Claims.(*JwtToken)
if !ok {
return JwtToken{}, errors.New(ErrJwtClaimsMissing)
}
return *claims, nil
}

View File

@ -1,653 +0,0 @@
package v1
import (
"fmt"
"net/http"
"strconv"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/dtoconv"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/labstack/echo/v4"
)
// ListSources
// @Summary Lists the top 50 records
// @Param page query string false "page number"
// @Produce application/json
// @Tags Source
// @Router /v1/sources [get]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse "Unable to reach SQL or Data problems"
// @Security Bearer
func (s *Handler) listSources(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
page, err := strconv.Atoi(c.QueryParam("page"))
if err != nil {
page = 0
}
// Default way of showing all sources
items, err := s.repo.Sources.List(c.Request().Context(), page, 25)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.Payload = dtoconv.SourcesToDto(items)
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// ListSourcesBySource
// @Summary Lists the top 50 records based on the name given. Example: reddit
// @Param source query string true "Source Name"
// @Param page query string false "page number"
// @Produce application/json
// @Tags Source
// @Router /v1/sources/by/source [get]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) listSourcesBySource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
source := c.QueryParam("source")
if source == "" {
p.BaseResponse.Message = fmt.Sprintf("%s source", ErrParameterMissing)
return c.JSON(http.StatusBadRequest, p)
}
page, err := strconv.Atoi(c.QueryParam("page"))
if err != nil {
page = 0
}
// Shows the list by Sources.source
items, err := s.repo.Sources.ListBySource(c.Request().Context(), page, 25, source)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.Payload = dtoconv.SourcesToDto(items)
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// GetSource
// @Summary Returns a single entity by ID
// @Param id path int true "id"
// @Produce application/json
// @Tags Source
// @Router /v1/sources/{id} [get]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) getSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("id"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// GetSourceByNameAndSource
// @Summary Returns a single entity by ID
// @Param name query string true "dadjokes"
// @Param source query string true "reddit"
// @Produce application/json
// @Tags Source
// @Router /v1/sources/by/sourceAndName [get]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (s *Handler) GetSourceBySourceAndName(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceRead)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
var param domain.GetSourceBySourceAndNameParamRequest
err = c.Bind(&param)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), param.Source, param.Name)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// NewRedditSource
// @Summary Creates a new reddit source to monitor.
// @Param name query string true "name"
// @Param url query string true "url"
// @Tags Source
// @Router /v1/sources/new/reddit [post]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) newRedditSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
var param domain.NewSourceParamRequest
err = c.Bind(&param)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
if param.Url == "" {
p.BaseResponse.Message = "url is missing"
return c.JSON(http.StatusBadRequest, p)
}
if !strings.Contains(param.Url, "reddit.com") {
p.BaseResponse.Message = "invalid url"
return c.JSON(http.StatusBadRequest, p)
}
// Check to see if we already have this record, if we do, return it.
item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
if err == nil {
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
tags := fmt.Sprintf("twitch, %v, %s", param.Name, param.Tags)
rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorReddit, param.Name, param.Url, tags, true)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if rows != 1 {
p.BaseResponse.Message = ErrFailedToCreateRecord
return c.JSON(http.StatusInternalServerError, p)
}
item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorReddit, param.Name)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// NewYoutubeSource
// @Summary Creates a new youtube source to monitor.
// @Param name query string true "name"
// @Param url query string true "url"
// @Tags Source
// @Router /v1/sources/new/youtube [post]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) newYoutubeSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
},
}
// Validate the jwt
_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
var param domain.NewSourceParamRequest
err = c.Bind(&param)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
if param.Url == "" {
p.BaseResponse.Message = "url is missing a value"
return c.JSON(http.StatusBadRequest, p)
}
if !strings.Contains(param.Url, "youtube.com") {
p.BaseResponse.Message = "invalid url"
return c.JSON(http.StatusBadRequest, p)
}
item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
if err == nil {
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
tags := fmt.Sprintf("twitch, %v", param.Name)
rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorYoutube, param.Name, param.Url, tags, true)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if rows != 1 {
p.BaseResponse.Message = ErrFailedToCreateRecord
return c.JSON(http.StatusInternalServerError, p)
}
item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorYoutube, param.Name)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// NewTwitchSource
// @Summary Creates a new twitch source to monitor.
// @Param name query string true "name"
// @Tags Source
// @Router /v1/sources/new/twitch [post]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) newTwitchSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
var param domain.NewSourceParamRequest
err = c.Bind(&param)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
tags := fmt.Sprintf("twitch, %v", param.Name)
url := fmt.Sprintf("https://twitch.tv/%v", param.Name)
// Check if the record already exists
item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
if err == nil {
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorTwitch, param.Name, url, tags, true)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if rows != 1 {
p.BaseResponse.Message = ErrFailedToCreateRecord
return c.JSON(http.StatusInternalServerError, p)
}
item, _ = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorTwitch, param.Name)
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// NewRssSource
// @Summary Creates a new rss source to monitor.
// @Param request body domain.NewSourceParamRequest true "body"
// @Tags Source
// @Router /v1/sources/new/rss [post]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) newRssSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeSourceCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
param := domain.NewSourceParamRequest{}
err = c.Bind(&param)
//err = (&echo.DefaultBinder{}).BindBody(c, &param)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
if param.Url == "" {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check if the record already exists
blankRecord := entity.SourceEntity{}
item, err := s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
if err != nil {
p.IsError = true
return c.JSON(http.StatusInternalServerError, p)
}
// No record was found, but no error returned
if item != blankRecord {
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
//if err == nil {
// var dto []domain.SourceDto
// dto = append(dto, dtoconv.SourceToDto(item))
// p.Payload = dto
// p.BaseResponse.IsError = false
// return c.JSON(http.StatusOK, p)
//}
tags := fmt.Sprintf("rss, %v, %s", param.Name, param.Tags)
rows, err := s.repo.Sources.Create(c.Request().Context(), domain.SourceCollectorRss, param.Name, param.Url, tags, true)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if rows != 1 {
p.BaseResponse.Message = ErrFailedToCreateRecord
return c.JSON(http.StatusInternalServerError, p)
}
item, err = s.repo.Sources.GetBySourceAndName(c.Request().Context(), domain.SourceCollectorRss, param.Name)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// DeleteSource
// @Summary Marks a source as deleted based on its ID value.
// @Param id path int true "id"
// @Tags Source
// @Router /v1/sources/{id} [POST]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) deleteSources(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeAll)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check to make sure we can find the record
_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
// Delete the record
rows, err := s.repo.Sources.SoftDelete(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
if rows != 1 {
p.BaseResponse.Message = ErrFailedToUpdateRecord
return c.JSON(http.StatusInternalServerError, p)
}
// pull the record with its updated value
item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var items []domain.SourceDto
items = append(items, dtoconv.SourceToDto(item))
p.Payload = items
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// DisableSource
// @Summary Disables a source from processing.
// @Param id path int true "id"
// @Tags Source
// @Router /v1/sources/{id}/disable [post]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) disableSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeAll)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check to make sure we can find the record
_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
_, err = s.repo.Sources.Disable(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// EnableSource
// @Summary Enables a source to continue processing.
// @Param id path string true "id"
// @Tags Source
// @Router /v1/sources/{id}/enable [post]
// @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.SourcesResponse
// @Failure 500 {object} domain.SourcesResponse
// @Security Bearer
func (s *Handler) enableSource(c echo.Context) error {
p := domain.SourcesResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
}
_, err := s.ValidateJwtToken(c, domain.ScopeAll)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
id, err := strconv.Atoi(c.Param("ID"))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
// Check to make sure we can find the record
_, err = s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
_, err = s.repo.Sources.Enable(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
item, err := s.repo.Sources.GetById(c.Request().Context(), int64(id))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
var dto []domain.SourceDto
dto = append(dto, dtoconv.SourceToDto(item))
p.Payload = dto
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}

View File

@ -1,351 +0,0 @@
package v1
import (
"net/http"
"strings"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
)
const (
ErrUserNotFound = "requested user does not exist"
ErrUsernameAlreadyExists = "the requested username already exists"
)
// @Summary Creates a new user
// @Router /v1/users/register [post]
// @Param request formData domain.LoginFormRequest true "form"
// @Accepts x-www-form-urlencoded
// @Produce json
// @Tags Users
// @Success 201 {object} domain.BaseResponse
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
func (h *Handler) AuthRegister(c echo.Context) error {
p := domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
}
username := c.FormValue("username")
password := c.FormValue("password")
//username := c.QueryParam("username")
exists, err := h.repo.Users.GetUser(c.Request().Context(), username)
if err != nil {
// if we have an err, validate that if its not user not found.
// if the user is not found, we can use that name
if err.Error() != repository.ErrUserNotFound {
p.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
}
if exists.Username == username {
p.Message = ErrUsernameAlreadyExists
return c.JSON(http.StatusInternalServerError, p)
}
//password := c.QueryParam("password")
err = h.repo.Users.CheckPasswordForRequirements(password)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
_, err = h.repo.Users.Create(c.Request().Context(), username, password, domain.ScopeArticleRead)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
return c.JSON(http.StatusCreated, p)
}
// @Summary Logs into the API and returns a bearer token if successful
// @Router /v1/users/login [post]
// @Param request formData domain.LoginFormRequest true "form"
// @Accepts x-www-form-urlencoded
// @Produce json
// @Tags Users
// @Success 200 {object} domain.LoginResponse
// @Failure 400 {object} domain.LoginResponse
// @Failure 500 {object} domain.LoginResponse
func (h *Handler) AuthLogin(c echo.Context) error {
p := domain.LoginResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
//Token: jwt,
Type: "Bearer",
//RefreshToken: refresh,
}
username := c.FormValue("username")
password := c.FormValue("password")
// Check to see if they are trying to login with the admin token
if username == "" {
return h.createAdminToken(c, password)
}
// check if the user exists
user, err := h.repo.Users.GetUser(c.Request().Context(), username)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
// make sure the hash matches
err = h.repo.Users.DoesPasswordMatchHash(c.Request().Context(), username, password)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
// TODO think about moving this down some?
expiresAt := time.Now().Add(time.Hour * 48)
userScopes := strings.Split(user.Scopes, ",")
jwt, err := h.generateJwtWithExp(username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, expiresAt)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
refresh, err := h.repo.RefreshTokens.Create(c.Request().Context(), username)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.Token = jwt
p.RefreshToken = refresh
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
func (h *Handler) createAdminToken(c echo.Context, password string) error {
p := domain.LoginResponse{
BaseResponse: domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
},
//Token: token,
Type: "Bearer",
}
// if the admin token is blank, then the admin wanted this disabled.
// this will fail right away and not progress.
if h.config.AdminSecret == "" {
p.BaseResponse.Message = ErrUserNotFound
return c.JSON(http.StatusBadRequest, p)
}
if h.config.AdminSecret != password {
p.BaseResponse.Message = ErrUserNotFound
return c.JSON(http.StatusBadRequest, p)
}
var userScopes []string
userScopes = append(userScopes, domain.ScopeAll)
sessionToken, err := uuid.NewV7()
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
token, err := h.generateJwt("admin", h.config.ServerAddress, sessionToken.String(), userScopes, -1)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.Token = token
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// This will take collect some information about the requested refresh, validate and then return a new jwt token if approved.
// Register
// @Summary Generates a new token
// @Router /v1/users/refresh/token [post]
// @Param request body domain.RefreshTokenRequest true "body"
// @Tags Users
// @Success 200 {object} domain.LoginResponse
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (h *Handler) RefreshJwtToken(c echo.Context) error {
p := domain.LoginResponse{
BaseResponse: domain.BaseResponse{
Message: "OK",
},
//Token: jwt,
Type: "Bearer",
//RefreshToken: newRefreshToken,
}
_, err := h.ValidateJwtToken(c, domain.ScopeDiscordWebHookCreate)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
// Check the context for the refresh token
var request domain.RefreshTokenRequest
err = (&echo.DefaultBinder{}).BindBody(c, &request)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
err = h.repo.RefreshTokens.IsRequestValid(c.Request().Context(), request.Username, request.RefreshToken)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
user, err := h.repo.Users.GetUser(c.Request().Context(), request.Username)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
userScopes := strings.Split(user.Scopes, ",")
jwt, err := h.generateJwtWithExp(request.Username, h.config.ServerAddress, user.SessionToken, userScopes, user.ID, time.Now().Add(time.Hour*48))
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
newRefreshToken, err := h.repo.RefreshTokens.Create(c.Request().Context(), request.Username)
if err != nil {
p.BaseResponse.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.Token = jwt
p.RefreshToken = newRefreshToken
p.BaseResponse.IsError = false
return c.JSON(http.StatusOK, p)
}
// @Summary Adds a new scope to a user account
// @Router /v1/users/scopes/add [post]
// @Param request body domain.UpdateScopesRequest true "body"
// @Tags Users
// @Accept json
// @Produce json
// @Success 200 {object} domain.BaseResponse
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (h *Handler) AddScopes(c echo.Context) error {
p := domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
}
_, err := h.ValidateJwtToken(c, domain.ScopeAll)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
request := domain.UpdateScopesRequest{}
err = (&echo.DefaultBinder{}).BindBody(c, &request)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
err = h.repo.Users.AddScopes(c.Request().Context(), request.Username, request.Scopes)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// @Summary Adds a new scope to a user account
// @Router /v1/users/scopes/remove [post]
// @Param request body domain.UpdateScopesRequest true "body"
// @Tags Users
// @Accept json
// @Produce json
// @Success 200 {object} domain.BaseResponse
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (h *Handler) RemoveScopes(c echo.Context) error {
p := domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
}
token, err := h.getJwtTokenFromContext(c)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
err = token.IsValid(domain.ScopeAll)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
request := domain.UpdateScopesRequest{}
err = (&echo.DefaultBinder{}).BindBody(c, &request)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusBadRequest, p)
}
err = h.repo.Users.RemoveScopes(c.Request().Context(), request.Username, request.Scopes)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.IsError = false
return c.JSON(http.StatusOK, p)
}
// @Summary Revokes the current session token and replaces it with a new one.
// @Router /v1/users/refresh/sessionToken [post]
// @Tags Users
// @Accept json
// @Produce json
// @Success 200 {object} domain.BaseResponse
// @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse
// @Security Bearer
func (h *Handler) NewSessionToken(c echo.Context) error {
p := domain.BaseResponse{
Message: ResponseMessageSuccess,
IsError: true,
}
token, err := h.getJwtTokenFromContext(c)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusUnauthorized, p)
}
_, err = h.repo.Users.NewSessionToken(c.Request().Context(), token.UserName)
if err != nil {
p.Message = err.Error()
return c.JSON(http.StatusInternalServerError, p)
}
p.IsError = false
return c.JSON(http.StatusInternalServerError, p)
}

View File

@ -1,122 +0,0 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
)
type AlertDiscordRepo interface {
Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error)
SoftDelete(ctx context.Context, id int64) (int64, error)
Restore(ctx context.Context, id int64) (int64, error)
Delete(ctx context.Context, id int64) (int64, error)
ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error)
}
type alertDiscordRepository struct {
conn *sql.DB
defaultLimit int
defaultOffset int
}
func NewAlertDiscordRepository(conn *sql.DB) alertDiscordRepository {
return alertDiscordRepository{
conn: conn,
defaultLimit: 50,
defaultOffset: 50,
}
}
func (r alertDiscordRepository) Create(ctx context.Context, userId, sourceId, webhookId int64) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("AlertDiscord")
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID", "DiscordWebHookID")
queryBuilder.Values(dt, dt, timeZero, userId, sourceId, webhookId)
query, args := queryBuilder.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r alertDiscordRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
return softDeleteRow(ctx, r.conn, "AlertDiscord", id)
}
func (r alertDiscordRepository) Restore(ctx context.Context, id int64) (int64, error) {
return restoreRow(ctx, r.conn, "AlertDiscord", id)
}
func (r alertDiscordRepository) Delete(ctx context.Context, id int64) (int64, error) {
return deleteFromTable(ctx, r.conn, "AlertDiscord", id)
}
func (r alertDiscordRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.AlertDiscordEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("AlertDiscord")
builder.Where(
builder.Equal("UserID", userId),
)
builder.Offset(page * limit)
builder.Limit(limit)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.AlertDiscordEntity{}, err
}
data := r.processRows(rows)
if len(data) == 0 {
return []entity.AlertDiscordEntity{}, errors.New(ErrUserNotFound)
}
return data, nil
}
func (ur alertDiscordRepository) processRows(rows *sql.Rows) []entity.AlertDiscordEntity {
items := []entity.AlertDiscordEntity{}
for rows.Next() {
var id int64
var createdAt time.Time
var updatedAt time.Time
var deletedAt time.Time
var userId int64
var sourceId int64
var webhookId int64
err := rows.Scan(
&id, &createdAt, &updatedAt, &deletedAt,
&userId, &sourceId, &webhookId,
)
if err != nil {
fmt.Println(err)
}
item := entity.AlertDiscordEntity{
ID: id,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
DeletedAt: deletedAt,
UserID: userId,
SourceID: sourceId,
DiscordWebHookId: webhookId,
}
items = append(items, item)
}
return items
}

View File

@ -1,63 +0,0 @@
package repository_test
import (
"context"
"testing"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
)
func TestAlertDiscordCreate(t *testing.T) {
t.Log(time.Time{})
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewAlertDiscordRepository(db)
created, err := r.Create(context.Background(), 1, 1, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if created != 1 {
t.Log("failed to create the record")
t.FailNow()
}
}
func TestAlertDiscordCreateAndValidate(t *testing.T) {
t.Log(time.Time{})
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
source := repository.NewSourceRepository(db)
source.Create(context.Background(), domain.SourceCollectorRss, "Unit Testing", "www.fake.com", "testing,units", true)
sourceRecord, _ := source.GetBySourceAndName(context.Background(), domain.SourceCollectorRss, "Unit Testing")
webhookRepo := repository.NewDiscordWebHookRepository(db)
webhookRepo.Create(context.Background(), 999, "discord.com", "Unit Testing", "memes", true)
webhook, _ := webhookRepo.GetByUrl(context.Background(), "discord.com")
r := repository.NewAlertDiscordRepository(db)
r.Create(context.Background(), 999, sourceRecord.ID, webhook.ID)
alert, err := r.ListByUser(context.Background(), 0, 10, 999)
if err != nil {
t.Error(err)
t.FailNow()
}
if len(alert) != 1 {
t.Error("got the incorrect number of rows back")
t.FailNow()
}
}

View File

@ -1,261 +0,0 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
)
const (
ArticleOrderByPublishDateDesc = "pubDate desc"
ArticleOrderByPublishDateAsc = "pubDate asc"
)
type ArticlesRepo interface {
GetById(ctx context.Context, id int64) (entity.ArticleEntity, error)
GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error)
ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error)
ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error)
ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error)
ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error)
Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error)
CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error)
}
type ArticleRepository struct {
conn *sql.DB
defaultLimit int
defaultOffset int
}
func NewArticleRepository(conn *sql.DB) ArticleRepository {
return ArticleRepository{
conn: conn,
defaultLimit: 50,
defaultOffset: 50,
}
}
func (ar ArticleRepository) GetById(ctx context.Context, id int64) (entity.ArticleEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("articles").Where(
builder.E("id", id),
)
builder.Limit(1)
query, args := builder.Build()
rows, err := ar.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.ArticleEntity{}, err
}
data := ar.processRows(rows)
if len(data) == 0 {
return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
}
return data[0], nil
}
func (ar ArticleRepository) GetByUrl(ctx context.Context, url string) (entity.ArticleEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("articles").Where(
builder.E("url", url),
)
builder.Limit(1)
query, args := builder.Build()
rows, err := ar.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.ArticleEntity{}, err
}
data := ar.processRows(rows)
if len(data) == 0 {
return entity.ArticleEntity{}, errors.New(ErrUserNotFound)
}
return data[0], nil
}
func (ar ArticleRepository) ListTop(ctx context.Context, limit int) ([]entity.ArticleEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("articles")
builder.Limit(limit)
query, args := builder.Build()
rows, err := ar.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.ArticleEntity{}, err
}
data := ar.processRows(rows)
if len(data) == 0 {
return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
}
return data, nil
}
func (ar ArticleRepository) ListByPage(ctx context.Context, page, limit int) ([]entity.ArticleEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("articles")
builder.OrderBy(ArticleOrderByPublishDateDesc)
builder.Offset(page * limit)
builder.Limit(limit)
query, args := builder.Build()
rows, err := ar.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.ArticleEntity{}, err
}
data := ar.processRows(rows)
if len(data) == 0 {
return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
}
return data, nil
}
func (ar ArticleRepository) ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]entity.ArticleEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("articles")
if orderBy != "" {
builder.OrderBy(orderBy)
}
builder.Offset(page * limit)
builder.Limit(limit)
query, args := builder.Build()
rows, err := ar.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.ArticleEntity{}, err
}
data := ar.processRows(rows)
if len(data) == 0 {
return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
}
return data, nil
}
func (ar ArticleRepository) ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]entity.ArticleEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("articles")
builder.JoinWithOption("InnerJoin", "sources", "articles.sourceId=sources.Id")
if orderBy != "" {
builder.OrderBy(orderBy)
}
builder.Where(
builder.Equal("SourceId", sourceId),
)
builder.Offset(50)
builder.Limit(page * limit)
query, args := builder.Build()
rows, err := ar.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.ArticleEntity{}, err
}
data := ar.processRows(rows)
if len(data) == 0 {
return []entity.ArticleEntity{}, errors.New(ErrUserNotFound)
}
return data, nil
}
func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("articles")
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
queryBuilder.Values(dt, dt, timeZero, sourceId, tags, title, url, pubDate, isVideo, thumbnailUrl, description, authorName, authorImageUrl)
query, args := queryBuilder.Build()
_, err := ar.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity entity.ArticleEntity) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("articles")
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl)
query, args := queryBuilder.Build()
_, err := ar.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (ur ArticleRepository) processRows(rows *sql.Rows) []entity.ArticleEntity {
items := []entity.ArticleEntity{}
for rows.Next() {
var id int64
var createdAt time.Time
var updatedAt time.Time
var deletedAt time.Time
var sourceId int64
var tags string
var title string
var url string
var pubDate time.Time
var isVideo bool
var thumbnail string
var description string
var authorName string
var authorImageUrl string
err := rows.Scan(
&id, &createdAt, &updatedAt,
&deletedAt, &sourceId, &tags,
&title, &url, &pubDate,
&isVideo, &thumbnail, &description,
&authorName, &authorImageUrl)
if err != nil {
fmt.Println(err)
}
item := entity.ArticleEntity{
ID: id,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
DeletedAt: deletedAt,
SourceID: sourceId,
Tags: tags,
Title: title,
Url: url,
PubDate: pubDate,
IsVideo: isVideo,
Thumbnail: thumbnail,
Description: description,
AuthorName: authorName,
AuthorImageUrl: authorImageUrl,
}
items = append(items, item)
}
return items
}

View File

@ -1,156 +0,0 @@
package repository_test
import (
"context"
"testing"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
)
const (
articleFakeDotCom = "www.fake.com"
)
func TestCreateArticle(t *testing.T) {
t.Log(time.Time{})
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewArticleRepository(db)
created, err := r.Create(context.Background(), 1, "", "unit test", articleFakeDotCom, "", "testing", "", "", time.Now(), false)
if err != nil {
t.Log(err)
t.FailNow()
}
if created != 1 {
t.Log("failed to create the record")
t.FailNow()
}
}
func TestArticleByUrl(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewArticleRepository(db)
err = insertFakeArticles(r, "u1", 0)
if err != nil {
t.Log(err)
t.FailNow()
}
article, err := r.GetByUrl(context.Background(), articleFakeDotCom)
if err != nil {
t.Log(err)
t.FailNow()
}
if article.Url != "www.fake.com" {
t.Log("failed to find the requested record")
t.FailNow()
}
}
func TestPullingMultipleArticlesWithLimit(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewArticleRepository(db)
insertFakeArticles(r, "u1", 0)
insertFakeArticles(r, "u2", 0)
insertFakeArticles(r, "u3", 0)
insertFakeArticles(r, "u4", 0)
items, err := r.ListTop(context.Background(), 3)
if err != nil {
t.Log(err)
t.FailNow()
}
if len(items) != 3 {
t.Log("expected 3 rows back")
t.FailNow()
}
}
func TestPullingMultipleArticlesWithPaging(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewArticleRepository(db)
insertFakeArticles(r, "u1", 0)
insertFakeArticles(r, "u2", 0)
insertFakeArticles(r, "u3", 0)
insertFakeArticles(r, "u4", 0)
items, err := r.ListByPage(context.Background(), 2, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if items[0].Title != "u2" {
t.Log("pulled the wrong record with paging")
t.FailNow()
}
}
func TestPullingByPublishDate(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewArticleRepository(db)
today := time.Now()
insertFakeArticles(r, "u1", 0)
insertFakeArticles(r, "u1", -1)
insertFakeArticles(r, "u1", -2)
items, err := r.ListByPublishDate(context.Background(), 0, 2, repository.ArticleOrderByPublishDateDesc)
if err != nil {
t.Log(err)
t.FailNow()
}
if len(items) != 2 {
t.Log("expected two items back")
t.FailNow()
}
pubDate := items[1].PubDate.Day()
expectedDay := today.Day() - 1
if pubDate != expectedDay {
t.Log("expected a record that was 2 days old")
t.FailNow()
}
}
//func TestArticleBySource
func insertFakeArticles(r repository.ArticleRepository, title string, daysOld int) error {
pubDate := time.Now().AddDate(0,0, daysOld)
_, err := r.Create(context.Background(), 1, "", title, articleFakeDotCom, "", "testing", "", "", pubDate, false)
if err != nil {
return err
}
return nil
}

View File

@ -1,71 +0,0 @@
package repository
import (
"context"
"database/sql"
"time"
"github.com/huandu/go-sqlbuilder"
)
var (
timeZero = time.Time{}
)
func deleteFromTable(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
b := sqlbuilder.NewDeleteBuilder()
b.DeleteFrom(tableName)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func restoreRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
timeZero := time.Time{}
b := sqlbuilder.NewUpdateBuilder()
b.Update(tableName)
b.Set(
b.Assign("UpdatedAt", time.Now()),
b.Assign("DeletedAt", timeZero),
)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func softDeleteRow(ctx context.Context, conn *sql.DB, tableName string, id int64) (int64, error) {
now := time.Now()
b := sqlbuilder.NewUpdateBuilder()
b.Update(tableName)
b.Set(
b.Assign("UpdatedAt", now),
b.Assign("DeletedAt", now),
)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}

View File

@ -1,228 +0,0 @@
package repository
import (
"context"
"database/sql"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
)
type DiscordWebHookRepo interface {
Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error)
Enable(ctx context.Context, id int64) (int64, error)
Disable(ctx context.Context, id int64) (int64, error)
SoftDelete(ctx context.Context, id int64) (int64, error)
Restore(ctx context.Context, id int64) (int64, error)
Delete(ctx context.Context, id int64) (int64, error)
GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error)
GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error)
ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error)
ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error)
}
type discordWebHookRepository struct {
conn *sql.DB
}
func NewDiscordWebHookRepository(conn *sql.DB) discordWebHookRepository {
return discordWebHookRepository{
conn: conn,
}
}
func (r discordWebHookRepository) Create(ctx context.Context, userId int64, url, server, channel string, enabled bool) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("DiscordWebHooks")
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "Url", "Server", "Channel", "Enabled")
queryBuilder.Values(dt, dt, timeZero, userId, url, server, channel, enabled)
query, args := queryBuilder.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r discordWebHookRepository) Enable(ctx context.Context, id int64) (int64, error) {
b := sqlbuilder.NewUpdateBuilder()
b.Update("DiscordWebHooks")
b.Set(
b.Assign("Enabled", true),
b.Assign("UpdatedAt", time.Now()),
)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r discordWebHookRepository) Disable(ctx context.Context, id int64) (int64, error) {
b := sqlbuilder.NewUpdateBuilder()
b.Update("DiscordWebHooks")
b.Set(
b.Assign("Enabled", false),
b.Assign("UpdatedAt", time.Now()),
)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r discordWebHookRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
return softDeleteRow(ctx, r.conn, "DiscordWebHooks", id)
}
func (r discordWebHookRepository) Restore(ctx context.Context, id int64) (int64, error) {
return restoreRow(ctx, r.conn, "DiscordWebHooks", id)
}
func (r discordWebHookRepository) Delete(ctx context.Context, id int64) (int64, error) {
return deleteFromTable(ctx, r.conn, "DiscordWebHooks", id)
}
func (r discordWebHookRepository) GetById(ctx context.Context, id int64) (entity.DiscordWebHookEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("DiscordWebHooks").Where(
builder.E("id", id),
)
builder.Limit(1)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.DiscordWebHookEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return entity.DiscordWebHookEntity{}, err
}
return data[0], nil
}
func (r discordWebHookRepository) GetByUrl(ctx context.Context, url string) (entity.DiscordWebHookEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("DiscordWebHooks").Where(
builder.E("Url", url),
)
builder.Limit(1)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.DiscordWebHookEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return entity.DiscordWebHookEntity{}, err
}
return data[0], nil
}
func (r discordWebHookRepository) ListByServerName(ctx context.Context, name string) ([]entity.DiscordWebHookEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("DiscordWebHooks").Where(
builder.E("Server", name),
)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.DiscordWebHookEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return []entity.DiscordWebHookEntity{}, err
}
return data, nil
}
func (r discordWebHookRepository) ListByServerAndChannel(ctx context.Context, server, channel string) ([]entity.DiscordWebHookEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("DiscordWebHooks").Where(
builder.Equal("Server", server),
builder.Equal("Channel", channel),
)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.DiscordWebHookEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return []entity.DiscordWebHookEntity{}, err
}
return data, nil
}
func (r discordWebHookRepository) processRows(rows *sql.Rows) ([]entity.DiscordWebHookEntity, error) {
items := []entity.DiscordWebHookEntity{}
for rows.Next() {
var id int64
var createdAt time.Time
var updatedAt time.Time
var deletedAt time.Time
var userId int64
var url string
var server string
var channel string
var enabled bool
err := rows.Scan(
&id, &createdAt, &updatedAt,
&deletedAt, &userId, &url, &server,
&channel, &enabled,
)
if err != nil {
return items, err
}
item := entity.DiscordWebHookEntity{
ID: id,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
DeletedAt: deletedAt,
UserID: userId,
Url: url,
Server: server,
Channel: channel,
Enabled: enabled,
}
items = append(items, item)
}
return items, nil
}

View File

@ -1,287 +0,0 @@
package repository_test
import (
"context"
"testing"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
)
func TestCreateDiscordWebHookRecord(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
r := repository.NewDiscordWebHookRepository(db)
created, err := r.Create(context.Background(), 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
if err != nil {
t.Log(err)
t.FailNow()
}
if created != 1 {
t.Log("failed to create the record")
t.FailNow()
}
}
func TestDiscordWebHookGetById(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewDiscordWebHookRepository(db)
created, err := r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
if err != nil {
t.Log(err)
t.FailNow()
}
if created != 1 {
t.Log("failed to create the record")
t.FailNow()
}
item, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.ID != 1 {
t.Log("got the wrong record back")
t.FailNow()
}
}
func TestDiscordWebHookGetByUrl(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", "Unit Testing", "memes", true)
item, err := r.GetByUrl(ctx, "www.discord.com/bad/webhook")
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Url != "www.discord.com/bad/webhook" {
t.Log("got the wrong record back")
t.FailNow()
}
}
func TestDiscordWebHookListByServerName(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
serverName := "Unit Testing"
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, "memes", true)
item, err := r.ListByServerName(ctx, serverName)
if err != nil {
t.Log(err)
t.FailNow()
}
if item[0].Server != serverName {
t.Log("got the wrong record back")
t.FailNow()
}
}
func TestDiscordWebHookListByServerAndChannel(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
serverName := "Unit Testing"
channel := "memes"
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
item, err := r.ListByServerAndChannel(ctx, serverName, channel)
if err != nil {
t.Log(err)
t.FailNow()
}
if item[0].Server != serverName {
t.Log("got the wrong wrong server back")
t.FailNow()
}
if item[0].Channel != channel {
t.Log("got the wrong channel back")
t.FailNow()
}
}
func TestDiscordWebHookEnableRecord(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
serverName := "Unit Testing"
channel := "memes"
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, false)
item, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled != false {
t.Log("the initial record was created wrong")
t.FailNow()
}
_, err = r.Enable(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
updated, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled == updated.Enabled {
t.Log("failed to update the enabled value")
t.FailNow()
}
}
func TestDiscordWebHookDisableRecord(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
serverName := "Unit Testing"
channel := "memes"
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
item, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled != true {
t.Log("the initial record was created wrong")
t.FailNow()
}
_, err = r.Disable(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
updated, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled == updated.Enabled {
t.Log("failed to update the enabled value")
t.FailNow()
}
}
func TestDiscordWebHookSoftDelete(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
serverName := "Unit Testing"
channel := "memes"
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
_, err = r.SoftDelete(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
updated, _ := r.GetById(ctx, 1)
t.Log(updated.DeletedAt)
}
func TestDiscordWebHookRestore(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
serverName := "Unit Testing"
channel := "memes"
timeZero := time.Time{}
r := repository.NewDiscordWebHookRepository(db)
_, _ = r.Create(ctx, 999, "www.discord.com/bad/webhook", serverName, channel, true)
item, _ := r.GetById(ctx, 1)
if item.DeletedAt != timeZero {
t.Log("DeletedAt was not zero")
t.FailNow()
}
_, _ = r.SoftDelete(ctx, 1)
softDeleted, _ := r.GetById(ctx, 1)
if softDeleted.ID != 1 {
t.Log("record went boom")
t.FailNow()
}
_, err = r.Restore(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
updated, _ := r.GetById(ctx, 1)
t.Log(updated.DeletedAt)
}

View File

@ -1,120 +0,0 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
)
const (
refreshTokenTableName = "RefreshTokens"
)
type RefreshToken interface {
Create(ctx context.Context, username string, token string) (int64, error)
GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
DeleteById(ctx context.Context, id int64) (int64, error)
}
type RefreshTokenRepository struct {
connection *sql.DB
}
func NewRefreshTokenRepository(conn *sql.DB) RefreshTokenRepository {
return RefreshTokenRepository{
connection: conn,
}
}
func (rt RefreshTokenRepository) Create(ctx context.Context, username string, token string) (int64, error) {
dt := time.Now()
builder := sqlbuilder.NewInsertBuilder()
builder.InsertInto(refreshTokenTableName)
builder.Cols("Username", "Token", "CreatedAt", "UpdatedAt", "DeletedAt")
builder.Values(username, token, dt, dt, time.Time{})
query, args := builder.Build()
_, err := rt.connection.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (rt RefreshTokenRepository) GetByUsername(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*").From(refreshTokenTableName).Where(
builder.E("Username", name),
)
query, args := builder.Build()
rows, err := rt.connection.QueryContext(ctx, query, args...)
if err != nil {
return entity.RefreshTokenEntity{}, err
}
data := rt.processRows(rows)
if len(data) == 0 {
return entity.RefreshTokenEntity{}, errors.New("no token found for user")
}
return data[0], nil
}
func (rt RefreshTokenRepository) DeleteById(ctx context.Context, id int64) (int64, error) {
builder := sqlbuilder.NewDeleteBuilder()
builder.DeleteFrom(refreshTokenTableName)
builder.Where(
builder.EQ("Id", id),
)
query, args := builder.Build()
rows, err := rt.connection.ExecContext(ctx, query, args...)
if err != nil {
return -1, err
}
return rows.RowsAffected()
}
func (rd RefreshTokenRepository) processRows(rows *sql.Rows) []entity.RefreshTokenEntity {
items := []entity.RefreshTokenEntity{}
for rows.Next() {
var id int64
var username string
var token string
var createdAt time.Time
var updatedAt time.Time
var deletedAt sql.NullTime
err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &token)
if err != nil {
fmt.Println(err)
}
item := entity.RefreshTokenEntity{
ID: id,
Username: username,
Token: token,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}
if deletedAt.Valid {
item.DeletedAt = deletedAt.Time
}
items = append(items, item)
}
return items
}
//func (rt RefreshTokenRepository) Delete()

View File

@ -1,93 +0,0 @@
package repository_test
import (
"context"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
)
func TestRefreshTokenCreate(t *testing.T) {
conn, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
client := repository.NewRefreshTokenRepository(conn)
rows, err := client.Create(context.Background(), "tester", "BadTokenDontUse")
if err != nil {
t.Log(err)
t.FailNow()
}
if rows == 0 {
t.Log("expected one row to come back but got 0")
}
}
func TestRefreshTokenGetByUsername(t *testing.T) {
conn, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
client := repository.NewRefreshTokenRepository(conn)
rows, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
if err != nil {
t.Log(err)
t.FailNow()
}
if rows != 1 {
t.Log("expected a row to be added but not the wrong value back")
t.FailNow()
}
model, err := client.GetByUsername(context.Background(), "tester")
if err != nil {
t.Log(err)
t.FailNow()
}
if model.Username != "tester" {
t.Log("got the wrong user back")
t.FailNow()
}
}
func TestRefreshTokenDeleteById(t *testing.T) {
conn, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
client := repository.NewRefreshTokenRepository(conn)
created, err := client.Create(context.Background(), "tester", "BadTokenDoNotUse")
if err != nil {
t.Log(err)
t.FailNow()
}
if created != 1 {
t.Log("Unexpected number back for rows created")
}
model, err := client.GetByUsername(context.Background(), "tester")
if err != nil {
t.Log(err)
t.FailNow()
}
updated, err := client.DeleteById(context.Background(), model.ID)
if err != nil {
t.Log(err)
t.FailNow()
}
if updated != 1 {
t.Log("deleted the wrong number of records")
t.FailNow()
}
}

View File

@ -1,277 +0,0 @@
package repository
import (
"context"
"database/sql"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
)
type Sources interface {
Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error)
GetById(ctx context.Context, id int64) (entity.SourceEntity, error)
GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error)
GetBySource(ctx context.Context, source string) (entity.SourceEntity, error)
GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error)
List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error)
ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error)
Enable(ctx context.Context, id int64) (int64, error)
Disable(ctx context.Context, id int64) (int64, error)
SoftDelete(ctx context.Context, id int64) (int64, error)
Restore(ctx context.Context, id int64) (int64, error)
Delete(ctx context.Context, id int64) (int64, error)
}
type sourceRepository struct {
conn *sql.DB
}
func NewSourceRepository(conn *sql.DB) sourceRepository {
return sourceRepository{
conn: conn,
}
}
func (r sourceRepository) Create(ctx context.Context, source, displayName, url, tags string, enabled bool) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("Sources")
queryBuilder.Cols("CreatedAt", "UpdatedAt", "DeletedAt", "DisplayName", "Source", "Url", "Tags", "Enabled")
queryBuilder.Values(dt, dt, timeZero, displayName, source, url, tags, enabled)
query, args := queryBuilder.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r sourceRepository) GetById(ctx context.Context, id int64) (entity.SourceEntity, error) {
b := sqlbuilder.NewSelectBuilder()
b.Select("*")
b.From("Sources").Where(
b.Equal("Id", id),
)
b.Limit(1)
query, args := b.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.SourceEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return entity.SourceEntity{}, err
}
return data[0], nil
}
func (r sourceRepository) GetByDisplayName(ctx context.Context, displayName string) (entity.SourceEntity, error) {
b := sqlbuilder.NewSelectBuilder()
b.Select("*")
b.From("Sources").Where(
b.Equal("DisplayName", displayName),
)
b.Limit(1)
query, args := b.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.SourceEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return entity.SourceEntity{}, err
}
return data[0], nil
}
func (r sourceRepository) GetBySource(ctx context.Context, source string) (entity.SourceEntity, error) {
b := sqlbuilder.NewSelectBuilder()
b.Select("*")
b.From("Sources").Where(
b.Equal("Source", source),
)
b.Limit(1)
query, args := b.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.SourceEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return entity.SourceEntity{}, err
}
return data[0], nil
}
func (r sourceRepository) GetBySourceAndName(ctx context.Context, source, name string) (entity.SourceEntity, error) {
b := sqlbuilder.NewSelectBuilder()
b.Select("*")
b.From("Sources").Where(
b.Equal("Source", source),
b.Equal("DisplayName", name),
)
b.Limit(1)
query, args := b.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return entity.SourceEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return entity.SourceEntity{}, err
}
return data[0], nil
}
func (r sourceRepository) List(ctx context.Context, page, limit int) ([]entity.SourceEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("Sources")
builder.Offset(page * limit)
builder.Limit(limit)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.SourceEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return []entity.SourceEntity{}, err
}
return data, nil
}
func (r sourceRepository) ListBySource(ctx context.Context, page, limit int, source string) ([]entity.SourceEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("Sources")
builder.Where(
builder.Equal("Source", source),
)
builder.Offset(page * limit)
builder.Limit(limit)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.SourceEntity{}, err
}
data, err := r.processRows(rows)
if len(data) == 0 {
return []entity.SourceEntity{}, err
}
return data, nil
}
func (r sourceRepository) Enable(ctx context.Context, id int64) (int64, error) {
b := sqlbuilder.NewUpdateBuilder()
b.Update("Sources")
b.Set(
b.Assign("Enabled", true),
b.Assign("UpdatedAt", time.Now()),
)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r sourceRepository) Disable(ctx context.Context, id int64) (int64, error) {
b := sqlbuilder.NewUpdateBuilder()
b.Update("Sources")
b.Set(
b.Assign("Enabled", false),
b.Assign("UpdatedAt", time.Now()),
)
b.Where(
b.Equal("Id", id),
)
query, args := b.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r sourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
return softDeleteRow(ctx, r.conn, "Sources", id)
}
func (r sourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
return restoreRow(ctx, r.conn, "Sources", id)
}
func (r sourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
return deleteFromTable(ctx, r.conn, "Sources", id)
}
func (r sourceRepository) processRows(rows *sql.Rows) ([]entity.SourceEntity, error) {
items := []entity.SourceEntity{}
for rows.Next() {
var id int64
var createdAt time.Time
var updatedAt time.Time
var deletedAt time.Time
var displayName string
var source string
var enabled bool
var url string
var tags string
err := rows.Scan(
&id, &createdAt, &updatedAt,
&deletedAt, &displayName, &source,
&enabled, &url, &tags,
)
if err != nil {
return items, err
}
item := entity.SourceEntity{
ID: id,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
DeletedAt: deletedAt,
DisplayName: displayName,
Source: source,
Enabled: enabled,
Url: url,
Tags: tags,
}
items = append(items, item)
}
return items, nil
}

View File

@ -1,246 +0,0 @@
package repository_test
import (
"context"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
)
func TestSourceCreate(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
rows, err := r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
if err != nil {
t.Log(err)
t.FailNow()
}
if rows != 1 {
t.Log("failed to create a record, why")
t.FailNow()
}
}
func TestSourceGetById(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
if err != nil {
t.Log(err)
t.FailNow()
}
item, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.ID != 1 {
t.Log("got no record or the wrong one")
t.FailNow()
}
}
func TestSourceGetByDisplayName(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
if err != nil {
t.Log(err)
t.FailNow()
}
item, err := r.GetByDisplayName(ctx, "Test")
if err != nil {
t.Log(err)
t.FailNow()
}
if item.DisplayName != "Test" {
t.Log("got no record or the wrong one")
t.FailNow()
}
}
func TestSourceGetBySource(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, err = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
if err != nil {
t.Log(err)
t.FailNow()
}
item, err := r.GetBySource(ctx, domain.SourceCollectorRss)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Source != domain.SourceCollectorRss {
t.Log("got no record or the wrong one")
t.FailNow()
}
}
func TestSourceList(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
items, err := r.List(ctx, 0, 4)
if err != nil {
t.Log(err)
t.FailNow()
}
if len(items ) != 4 {
t.Log("something bad happened here")
t.FailNow()
}
}
func TestSourceListBySource(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
items, err := r.ListBySource(ctx, 0, 4, domain.SourceCollectorRss)
if err != nil {
t.Log(err)
t.FailNow()
}
if len(items ) != 4 {
t.Log("something bad happened here")
t.FailNow()
}
}
func TestSourcesEnableRecord(t *testing.T) {
// This depends on the seed migration
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", false)
item, err := r.GetByDisplayName(ctx, "Test")
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled != false {
t.Log("the initial record was created wrong")
t.FailNow()
}
_, err = r.Enable(ctx, item.ID)
if err != nil {
t.Log(err)
t.FailNow()
}
updated, err := r.GetById(ctx, item.ID)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled == updated.Enabled {
t.Log("failed to update the enabled value")
t.FailNow()
}
}
func TestSourcesDisableRecord(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background()
r := repository.NewSourceRepository(db)
_, _ = r.Create(ctx, domain.SourceCollectorRss, "Test", "www.badurl.com", "rss, badurl", true)
item, err := r.GetByDisplayName(ctx, "Test")
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled != true {
t.Log("the initial record was created wrong")
t.FailNow()
}
_, err = r.Disable(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
updated, err := r.GetById(ctx, 1)
if err != nil {
t.Log(err)
t.FailNow()
}
if item.Enabled == updated.Enabled {
t.Log("failed to update the enabled value")
t.FailNow()
}
}

View File

@ -1,120 +0,0 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
)
type UserSourceRepo interface {
Create(ctx context.Context, userId, sourceId int64) (int64, error)
SoftDelete(ctx context.Context, id int64) (int64, error)
Restore(ctx context.Context, id int64) (int64, error)
Delete(ctx context.Context, id int64) (int64, error)
ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error)
}
type userSourceRepository struct {
conn *sql.DB
defaultLimit int
defaultOffset int
}
func NewUserSourceRepository(conn *sql.DB) userSourceRepository {
return userSourceRepository{
conn: conn,
defaultLimit: 50,
defaultOffset: 50,
}
}
func (r userSourceRepository) Create(ctx context.Context, userId, sourceId int64) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("UserSourceSubscriptions")
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "UserID", "SourceID")
queryBuilder.Values(dt, dt, timeZero, userId, sourceId)
query, args := queryBuilder.Build()
_, err := r.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (r userSourceRepository) SoftDelete(ctx context.Context, id int64) (int64, error) {
return softDeleteRow(ctx, r.conn, "UserSourceSubscriptions", id)
}
func (r userSourceRepository) Restore(ctx context.Context, id int64) (int64, error) {
return restoreRow(ctx, r.conn, "UserSourceSubscriptions", id)
}
func (r userSourceRepository) Delete(ctx context.Context, id int64) (int64, error) {
return deleteFromTable(ctx, r.conn, "UserSourceSubscriptions", id)
}
func (r userSourceRepository) ListByUser(ctx context.Context, page, limit int, userId int64) ([]entity.UserSourceSubscriptionEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*")
builder.From("UserSourceSubscriptions")
builder.Where(
builder.Equal("UserID", userId),
)
builder.Offset(page * limit)
builder.Limit(limit)
query, args := builder.Build()
rows, err := r.conn.QueryContext(ctx, query, args...)
if err != nil {
return []entity.UserSourceSubscriptionEntity{}, err
}
data := r.processRows(rows)
if len(data) == 0 {
return []entity.UserSourceSubscriptionEntity{}, errors.New(ErrUserNotFound)
}
return data, nil
}
func (ur userSourceRepository) processRows(rows *sql.Rows) []entity.UserSourceSubscriptionEntity {
items := []entity.UserSourceSubscriptionEntity{}
for rows.Next() {
var id int64
var createdAt time.Time
var updatedAt time.Time
var deletedAt time.Time
var userId int64
var sourceId int64
err := rows.Scan(
&id, &createdAt, &updatedAt, &deletedAt,
&userId, &sourceId,
)
if err != nil {
fmt.Println(err)
}
item := entity.UserSourceSubscriptionEntity{
ID: id,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
DeletedAt: deletedAt,
UserID: userId,
SourceID: sourceId,
}
items = append(items, item)
}
return items
}

View File

@ -1,192 +0,0 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/huandu/go-sqlbuilder"
"golang.org/x/crypto/bcrypt"
)
const (
usersTableName string = "users"
ErrUserNotFound string = "requested user was not found"
)
type Users interface {
GetByName(ctx context.Context, name string) (entity.UserEntity, error)
Create(ctx context.Context, name, password, sessionTOken, scope string) (int64, error)
Update(ctx context.Context, id int, entity entity.UserEntity) error
UpdatePassword(ctx context.Context, name, password string) error
CheckUserHash(ctx context.Context, name, password string) error
UpdateScopes(ctx context.Context, name, scope string) error
UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error)
}
// Creates a new instance of UserRepository with the bound sql
func NewUserRepository(conn *sql.DB) userRepository {
return userRepository{
connection: conn,
}
}
type userRepository struct {
connection *sql.DB
}
func (ur userRepository) GetByName(ctx context.Context, name string) (entity.UserEntity, error) {
builder := sqlbuilder.NewSelectBuilder()
builder.Select("*").From("users").Where(
builder.E("Name", name),
)
query, args := builder.Build()
rows, err := ur.connection.QueryContext(ctx, query, args...)
if err != nil {
return entity.UserEntity{}, err
}
data := ur.processRows(rows)
if len(data) == 0 {
return entity.UserEntity{}, errors.New(ErrUserNotFound)
}
return data[0], nil
}
func (ur userRepository) Create(ctx context.Context, name, password, sessionToken, scope string) (int64, error) {
passwordBytes := []byte(password)
hash, err := bcrypt.GenerateFromPassword(passwordBytes, bcrypt.DefaultCost)
if err != nil {
return 0, err
}
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("users")
queryBuilder.Cols("Name", "Hash", "UpdatedAt", "CreatedAt", "DeletedAt", "Scopes", "SessionToken")
queryBuilder.Values(name, string(hash), dt, dt, time.Time{}, scope, sessionToken)
query, args := queryBuilder.Build()
_, err = ur.connection.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (ur userRepository) Update(ctx context.Context, id int, entity entity.UserEntity) error {
return errors.New("not implemented")
}
func (ur userRepository) UpdatePassword(ctx context.Context, name, password string) error {
_, err := ur.GetByName(ctx, name)
if err != nil {
return nil
}
queryBuilder := sqlbuilder.NewUpdateBuilder()
queryBuilder.Update(usersTableName)
//queryBuilder.Set
return nil
}
func (ur userRepository) UpdateSessionToken(ctx context.Context, name, sessionToken string) (int64, error) {
_, err := ur.GetByName(ctx, name)
if err != nil {
return 0, err
}
q := sqlbuilder.NewUpdateBuilder()
q.Update(usersTableName)
q.Set(
q.Equal("SessionToken", sessionToken),
)
q.Where(
q.Equal("Name", name),
)
query, args := q.Build()
rowsUpdates, err := ur.connection.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return rowsUpdates.RowsAffected()
}
// If the hash matches what we have in the database, an error will not be returned.
// If the user does not exist or the hash does not match, an error will be returned
func (ur userRepository) CheckUserHash(ctx context.Context, name, password string) error {
record, err := ur.GetByName(ctx, name)
if err != nil {
return err
}
err = bcrypt.CompareHashAndPassword([]byte(record.Hash), []byte(password))
if err != nil {
return err
}
return nil
}
func (ur userRepository) UpdateScopes(ctx context.Context, name, scope string) error {
builder := sqlbuilder.NewUpdateBuilder()
builder.Update("users")
builder.Set(
builder.Assign("Scopes", scope),
)
builder.Where(
builder.Equal("Name", name),
)
query, args := builder.Build()
_, err := ur.connection.ExecContext(ctx, query, args...)
if err != nil {
return err
}
return nil
}
func (ur userRepository) processRows(rows *sql.Rows) []entity.UserEntity {
items := []entity.UserEntity{}
for rows.Next() {
var id int64
var username string
var hash string
var createdAt time.Time
var updatedAt time.Time
var deletedAt sql.NullTime
var scopes string
var sessionToken string
err := rows.Scan(&id, &createdAt, &updatedAt, &deletedAt, &username, &hash, &scopes, &sessionToken)
if err != nil {
fmt.Println(err)
}
item := entity.UserEntity{
ID: id,
UpdatedAt: updatedAt,
Username: username,
Hash: hash,
Scopes: scopes,
CreatedAt: createdAt,
SessionToken: sessionToken,
}
if deletedAt.Valid {
item.DeletedAt = deletedAt.Time
}
items = append(items, item)
}
return items
}

View File

@ -1,88 +0,0 @@
package repository_test
import (
"context"
"database/sql"
"log"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
_ "github.com/glebarez/go-sqlite"
"github.com/pressly/goose/v3"
)
func TestCanCreateNewUser(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
repo := repository.NewUserRepository(db)
updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
if err != nil {
log.Println(err)
t.FailNow()
}
log.Println(updated)
}
func TestCanFindUserInTable(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
log.Println("unable to open connection")
t.FailNow()
}
defer db.Close()
repo := repository.NewUserRepository(db)
updated, err := repo.Create(context.Background(), "testing", "NotSecure", "sessionToken", "placeholder")
if err != nil {
t.Log(err)
t.FailNow()
}
if updated != 1 {
t.Log("expected a row to come back")
t.FailNow()
}
user, err := repo.GetByName(context.Background(), "testing")
if err != nil {
log.Println(err)
t.FailNow()
}
log.Println(user)
}
func TestCheckUserHash(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
log.Println("unable to open connection")
t.FailNow()
}
defer db.Close()
repo := repository.NewUserRepository(db)
repo.CheckUserHash(context.Background(), "testing", "NotSecure")
}
func setupInMemoryDb() (*sql.DB, error) {
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
return nil, err
}
err = goose.SetDialect("sqlite3")
if err != nil {
return nil, err
}
err = goose.Up(db, "../database/migrations")
if err != nil {
return nil, err
}
return db, nil
}

View File

@ -1,87 +0,0 @@
package repositoryservices
import (
"context"
"database/sql"
"errors"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
"github.com/google/uuid"
)
const (
ErrUnexpectedAmountOfRowsUpdated = "got a unexpected of rows updated"
)
type RefreshToken interface {
Create(ctx context.Context, username string) (string, error)
GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error)
Delete(ctx context.Context, id int64) (int64, error)
IsRequestValid(ctx context.Context, username, refreshToken string) error
}
// A new jwt token can be made if the user has the correct refresh token for the user.
// It will also require the old JWT token so the expire time is pulled and part of the validation
type RefreshTokenService struct {
table repository.RefreshTokenRepository
}
func NewRefreshTokenService(conn *sql.DB) RefreshTokenService {
return RefreshTokenService{
table: repository.NewRefreshTokenRepository(conn),
}
}
func (rt RefreshTokenService) Create(ctx context.Context, username string) (string, error) {
//if a refresh token already exists for a user, reuse
existingToken, err := rt.GetByName(ctx, username)
if err == nil {
rowsRemoved, err := rt.Delete(ctx, existingToken.ID)
if err != nil {
return "", err
}
if rowsRemoved != 1 {
return "", errors.New(ErrUnexpectedAmountOfRowsUpdated)
}
}
token, err := uuid.NewV7()
if err != nil {
return "", err
}
rows, err := rt.table.Create(ctx, username, token.String())
if err != nil {
return "", err
}
if rows != 1 {
return "", errors.New("expected one row but got none")
}
return token.String(), nil
}
// Find the saved refresh token for a user and return it if it exists
func (rt RefreshTokenService) GetByName(ctx context.Context, name string) (entity.RefreshTokenEntity, error) {
return rt.table.GetByUsername(ctx, name)
}
// This will request that a object is removed from the database
func (rt RefreshTokenService) Delete(ctx context.Context, id int64) (int64, error) {
return rt.table.DeleteById(ctx, id)
}
func (rt RefreshTokenService) IsRequestValid(ctx context.Context, username, refreshToken string) error {
token, err := rt.GetByName(ctx, username)
if err != nil {
return err
}
if token.Token != refreshToken {
return errors.New("the refresh token given does not match")
}
return nil
}

View File

@ -1,198 +0,0 @@
package repositoryservices
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
"github.com/google/uuid"
"golang.org/x/crypto/bcrypt"
)
const (
ErrPasswordNotLongEnough = "password needs to be 8 character or longer"
ErrPasswordMissingSpecialCharacter = "password needs to contain one of the following: !, @, #"
ErrInvalidPassword = "invalid password"
)
type UserServices interface {
DoesUserExist(ctx context.Context, username string) error
DoesPasswordMatchHash(ctx context.Context, username, password string) error
GetUser(ctx context.Context, username string) (entity.UserEntity, error)
AddScopes(ctx context.Context, username string, scopes []string) error
RemoveScopes(ctx context.Context, username string, scopes []string) error
Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error)
NewSessionToken(ctx context.Context, name string) (string, error)
CheckPasswordForRequirements(password string) error
}
// This will handle operations that are user related, but one layer higher then the repository
type UserService struct {
repo repository.Users
}
// This is a layer on top of the Users Repository.
// Use this over directly talking to the table when ever possible.
func NewUserService(conn *sql.DB) UserService {
return UserService{
repo: repository.NewUserRepository(conn),
}
}
func (us UserService) DoesUserExist(ctx context.Context, username string) error {
_, err := us.repo.GetByName(ctx, username)
if err != nil {
return err
}
return nil
}
func (us UserService) DoesPasswordMatchHash(ctx context.Context, username, password string) error {
model, err := us.GetUser(ctx, username)
if err != nil {
return err
}
err = bcrypt.CompareHashAndPassword([]byte(model.Hash), []byte(password))
if err != nil {
return errors.New(ErrInvalidPassword)
}
return nil
}
func (us UserService) GetUser(ctx context.Context, username string) (entity.UserEntity, error) {
return us.repo.GetByName(ctx, username)
}
func (us UserService) AddScopes(ctx context.Context, username string, scopes []string) error {
usr, err := us.repo.GetByName(ctx, username)
if err != nil {
return err
}
if usr.Username != username {
return errors.New(repository.ErrUserNotFound)
}
currentScopes := strings.Split(usr.Scopes, ",")
// check the current scopes
for _, item := range scopes {
if !strings.Contains(usr.Scopes, item) {
currentScopes = append(currentScopes, item)
}
}
return us.repo.UpdateScopes(ctx, username, strings.Join(currentScopes, ","))
}
func (us UserService) RemoveScopes(ctx context.Context, username string, scopes []string) error {
usr, err := us.repo.GetByName(ctx, username)
if err != nil {
return err
}
if usr.Username != username {
return errors.New(repository.ErrUserNotFound)
}
var newScopes []string
// check all the scopes that are currently assigned
for _, item := range strings.Split(usr.Scopes, ",") {
// check the scopes given, if one matches skip it
if us.doesScopeExist(scopes, item) {
continue
}
// did not match, add it
newScopes = append(newScopes, item)
}
return us.repo.UpdateScopes(ctx, username, strings.Join(newScopes, ","))
}
func (us UserService) doesScopeExist(scopes []string, target string) bool {
for _, item := range scopes {
if item == target {
return true
}
}
return false
}
func (us UserService) Create(ctx context.Context, name, password, scope string) (entity.UserEntity, error) {
err := us.CheckPasswordForRequirements(password)
if err != nil {
return entity.UserEntity{}, err
}
token, err := uuid.NewV7()
if err != nil {
return entity.UserEntity{}, err
}
us.repo.Create(ctx, name, password, token.String(), domain.ScopeArticleRead)
return entity.UserEntity{}, nil
}
func (us UserService) NewSessionToken(ctx context.Context, name string) (string, error) {
token, err := uuid.NewV7()
if err != nil {
return "", err
}
rows, err := us.repo.UpdateSessionToken(ctx, name, token.String())
if err != nil {
return "", err
}
if rows != 1 {
return "", fmt.Errorf("UserService.NewSessionToken %w", err)
}
return token.String(), nil
}
func (us UserService) CheckPasswordForRequirements(password string) error {
err := us.checkPasswordLength(password)
if err != nil {
return err
}
err = us.checkPasswordForSpecialCharacters(password)
if err != nil {
return err
}
return nil
}
func (us UserService) checkPasswordLength(password string) error {
if len(password) < 8 {
return errors.New(ErrPasswordNotLongEnough)
}
return nil
}
func (us UserService) checkPasswordForSpecialCharacters(password string) error {
var chars []string
chars = append(chars, "!")
chars = append(chars, "@")
chars = append(chars, "#")
for _, char := range chars {
if strings.Contains(password, char) {
return nil
}
}
return errors.New(ErrPasswordMissingSpecialCharacter)
}

View File

@ -1,154 +0,0 @@
package services
import (
"errors"
"fmt"
"log"
"os"
"strconv"
"github.com/joho/godotenv"
)
const (
ServerAddress = "ServerAddress"
//Sql_Connection_String = "SQL_CONNECTION_STRING"
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
)
type Configs struct {
ServerAddress string
JwtSecret string
AdminSecret string
RedditEnabled bool
RedditPullTop bool
RedditPullHot bool
RedditPullNsfw bool
YoutubeEnabled bool
YoutubeDebug bool
TwitchEnabled bool
TwitchClientId string
TwitchClientSecret string
TwitchMonitorClips bool
TwitchMonitorVOD bool
FfxivEnabled bool
}
type ConfigClient struct{}
func NewConfig() ConfigClient {
c := ConfigClient{}
c.RefreshEnv()
return c
}
func GetEnvConfig() Configs {
return Configs{
ServerAddress: os.Getenv(ServerAddress),
JwtSecret: os.Getenv("JwtSecret"),
AdminSecret: os.Getenv("AdminSecret"),
RedditEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_REDDIT_BACKEND)),
RedditPullTop: processBoolConfig(os.Getenv(REDDIT_PULL_TOP)),
RedditPullHot: processBoolConfig(os.Getenv(REDDIT_PULL_HOT)),
RedditPullNsfw: processBoolConfig(os.Getenv(REDDIT_PULL_NSFW)),
YoutubeEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_YOUTUBE_BACKEND)),
YoutubeDebug: processBoolConfig(os.Getenv(YOUTUBE_DEBUG)),
TwitchEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_TWITCH_BACKEND)),
TwitchClientId: os.Getenv(TWITCH_CLIENT_ID),
TwitchClientSecret: os.Getenv(TWITCH_CLIENT_SECRET),
TwitchMonitorClips: processBoolConfig(TWITCH_MONITOR_CLIPS),
TwitchMonitorVOD: processBoolConfig(os.Getenv(TWITCH_MONITOR_VOD)),
FfxivEnabled: processBoolConfig(os.Getenv(FEATURE_ENABLE_FFXIV_BACKEND)),
}
}
// This will parse a string and convert it to a bool.
// If it runs into any errors, it will default to false
func processBoolConfig(value string) bool {
b, err := strconv.ParseBool(value)
if err != nil {
return false
}
return b
}
func (cc *ConfigClient) GetConfig(key string) string {
res, filled := os.LookupEnv(key)
if !filled {
log.Printf("Missing the a value for '%v'. Could generate errors.", key)
}
return res
}
// Looks for a value in the env and will panic if it does not exist.
func (c ConfigClient) MustGetString(key string) string {
res, filled := os.LookupEnv(key)
if !filled {
msg := fmt.Sprintf("No value was found for '%v'", key)
panic(msg)
}
return res
}
func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
cc.RefreshEnv()
res, filled := os.LookupEnv(flag)
if !filled {
errorMessage := fmt.Sprintf("'%v' was not found", flag)
return false, errors.New(errorMessage)
}
b, err := strconv.ParseBool(res)
if err != nil {
return false, err
}
return b, nil
}
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
func (cc *ConfigClient) RefreshEnv() {
// Check to see if we have the env file on the system
_, err := os.Stat(".env")
// We have the file, load it.
if err == nil {
_, err := os.Open(".env")
if err == nil {
loadEnvFile()
}
}
}
func loadEnvFile() {
err := godotenv.Load()
if err != nil {
log.Fatalln(err)
}
}

View File

@ -1,22 +0,0 @@
package services_test
import (
"os"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
)
func TestNewClient(t *testing.T) {
services.NewConfig()
}
func TestGetConfigExpectNull(t *testing.T) {
cc := services.NewConfig()
os.Setenv(services.REDDIT_PULL_HOT, "")
res := cc.GetConfig(services.REDDIT_PULL_HOT)
if res != "" {
panic("expected blank")
}
}

View File

@ -1,220 +0,0 @@
package cron
import (
"log"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CollectRssPosts() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
rssClient := input.NewRssClient(source)
articles, err := rssClient.GetArticles()
if err != nil {
log.Println(err)
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Println(err)
}
if rowsCreated != 1 {
log.Println("Got back the wrong number of rows")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
}
func (c *Cron) CollectRedditPosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, article := range redditArticles {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new reddit article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CollectYoutubePosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range raw {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new youtube article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CollectFfxivPosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new FFXIV article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CollectTwitchPosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
log.Println(err)
return
}
err = tc.Login()
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new Twitch article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[Twitch] Done!")
}

View File

@ -1,43 +0,0 @@
package cron_test
import (
"context"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
)
func TestRssPullsCorrectly(t *testing.T) {
conn, err := setupInMemoryDb()
if err != nil {
t.Error(err)
t.FailNow()
}
defer conn.Close()
ctx := context.Background()
db := services.NewRepositoryService(conn)
rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
if err != nil {
t.Error(err)
t.FailNow()
}
if rowsCreated != 1 {
t.Error("failed to create the source record")
t.FailNow()
}
client := cron.NewScheduler(ctx, conn)
client.CollectRssPosts()
articles, err := db.Articles.ListByPage(ctx, 0, 100)
if err != nil {
t.Error(err)
t.FailNow()
}
t.Log(len(articles))
}

View File

@ -1,131 +0,0 @@
package cron
import (
"context"
"database/sql"
_ "github.com/lib/pq"
"github.com/robfig/cron/v3"
//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
)
type Cron struct {
//Db *database.Queries
ctx context.Context
timer *cron.Cron
repo services.RepositoryService
}
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
c := &Cron{
ctx: ctx,
repo: services.NewRepositoryService(conn),
}
timer := cron.New()
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//features := services.GetEnvConfig()
timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
//timer.AddFunc("10 * * * *", c.CollectRedditPosts)
//timer.AddFunc("15 * * * *", c.CheckYoutube)
//timer.AddFunc("20 * * * *", c.CheckFfxiv)
//timer.AddFunc("25 * * * *", c.CheckTwitch)
//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
c.timer = timer
return c
}
func (c *Cron) Start() {
c.timer.Start()
}
func (c *Cron) Stop() {
c.timer.Stop()
}
/*
func (c *Cron) CheckDiscordQueue() {
// Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
if err != nil {
return err
}
for _, queue := range queueItems {
// Get the articleByID
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
if err != nil {
return err
}
var endpoints []string
// List Subscription by SourceID
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
if err != nil {
return err
}
// if no one is subscribed to it, remove it from the index.
if len(subs) == 0 {
log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
if err != nil {
return err
}
continue
}
// Get the webhhooks to send to
for _, sub := range subs {
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
if err != nil {
return err
}
// store them in an array
endpoints = append(endpoints, webhook.Url)
}
// Create Discord Message
dwh := output.NewDiscordWebHookMessage(article)
msg, err := dwh.GeneratePayload()
if err != nil {
return err
}
// Send Message(s)
for _, i := range endpoints {
err = dwh.SendPayload(msg, i)
if err != nil {
return err
}
}
// Remove the item from the queue, given we sent our notification.
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
if err != nil {
return err
}
time.Sleep(10 * time.Second)
}
return nil
}
*/
//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
// err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
// ID: uuid.New(),
// Articleid: Id,
// })
// if err != nil {
// return err
// }
// return nil
//}

View File

@ -1,53 +0,0 @@
package cron_test
import (
"database/sql"
"github.com/pressly/goose/v3"
)
/*
func TestInvokeTwitch(t *testing.T) {
}
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) {
ctx := context.Background()
c := cron.NewScheduler(ctx)
c.Col()
}
func TestCheckYouTube(t *testing.T) {
ctx := context.Background()
c := cron.NewScheduler(ctx)
c.CheckYoutube()
}
func TestCheckTwitch(t *testing.T) {
ctx := context.Background()
c := cron.NewScheduler(ctx)
err := c.CheckTwitch()
if err != nil {
t.Error(err)
}
}
*/
func setupInMemoryDb() (*sql.DB, error) {
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
return nil, err
}
err = goose.SetDialect("sqlite3")
if err != nil {
return nil, err
}
err = goose.Up(db, "../../database/migrations")
if err != nil {
return nil, err
}
return db, nil
}

View File

@ -1,30 +0,0 @@
package services
import (
"database/sql"
"git.jamestombleson.com/jtom38/newsbot-api/internal/repository"
repositoryservices "git.jamestombleson.com/jtom38/newsbot-api/internal/repositoryServices"
)
type RepositoryService struct {
AlertDiscord repository.AlertDiscordRepo
Articles repository.ArticlesRepo
DiscordWebHooks repository.DiscordWebHookRepo
RefreshTokens repositoryservices.RefreshToken
Sources repository.Sources
Users repositoryservices.UserServices
UserSourceSubscriptions repository.UserSourceRepo
}
func NewRepositoryService(conn *sql.DB) RepositoryService {
return RepositoryService{
AlertDiscord: repository.NewAlertDiscordRepository(conn),
Articles: repository.NewArticleRepository(conn),
DiscordWebHooks: repository.NewDiscordWebHookRepository(conn),
RefreshTokens: repositoryservices.NewRefreshTokenService(conn),
Sources: repository.NewSourceRepository(conn),
Users: repositoryservices.NewUserService(conn),
UserSourceSubscriptions: repository.NewUserSourceRepository(conn),
}
}

View File

@ -1,44 +0,0 @@
package input
import (
"crypto/tls"
"io"
"log"
"net/http"
)
// This will use the net/http client reach out to a site and collect the content.
func getHttpContent(uri string) ([]byte, error) {
// Code to disable the http2 client for reddit.
// https://github.com/golang/go/issues/39302
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.ForceAttemptHTTP2 = false
tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
tr.TLSClientConfig = &tls.Config{}
client := &http.Client{
Transport: tr,
}
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, err
}
// set the user agent header to avoid kick backs.. as much
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
log.Printf("Requesting content from %v\n", uri)
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}

View File

@ -1,187 +0,0 @@
package input
import (
"encoding/json"
"errors"
"fmt"
"log"
"strings"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
)
type RedditClient struct {
config RedditConfig
record entity.SourceEntity
}
type RedditConfig struct {
PullTop string
PullHot string
PullNSFW string
}
func NewRedditClient(Record entity.SourceEntity) *RedditClient {
rc := RedditClient{
record: Record,
}
cc := services.NewConfig()
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
//rc.disableHttp2Client()
return &rc
}
// This is needed for to get modern go to talk to the endpoint.
// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
//func (rc *RedditClient) disableHttp2Client() {
// os.Setenv("GODEBUG", "http2client=0")
//}
func (rc *RedditClient) GetBrowser() *rod.Browser {
var browser *rod.Browser
if path, exists := launcher.LookPath(); exists {
u := launcher.New().Bin(path).MustLaunch()
browser = rod.New().ControlURL(u).MustConnect()
}
return browser
}
func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
page := parser.MustPage(url)
return page
}
//func (rc RedditClient)
// GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct.
func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
var items domain.RedditJsonContent = domain.RedditJsonContent{}
// TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
content, err := getHttpContent(Url)
if err != nil {
return items, err
}
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
return items, errors.New("did not get json data from the server")
}
json.Unmarshal(content, &items)
if len(items.Data.Children) == 0 {
return items, errors.New("failed to unmarshal the data")
}
return items, nil
}
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []entity.ArticleEntity {
var redditArticles []entity.ArticleEntity
for _, item := range items.Data.Children {
var article entity.ArticleEntity
article, err := rc.convertToArticle(item.Data)
if err != nil {
log.Printf("[Reddit] %v", err)
continue
}
redditArticles = append(redditArticles, article)
}
return redditArticles
}
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (entity.ArticleEntity, error) {
var item entity.ArticleEntity
if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source)
}
if source.Media.RedditVideo.FallBackUrl != "" {
item = rc.convertVideoPost(source)
}
if source.Content != "" {
item = rc.convertTextPost(source)
}
if source.UrlOverriddenByDest != "" {
item = rc.convertRedirectPost(source)
}
if item.Description == "" && item.Title == "" {
var err = errors.New("post failed to parse correctly")
return item, err
}
return item, nil
}
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) entity.ArticleEntity {
var item = entity.ArticleEntity{
SourceID: rc.record.ID,
Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
PubDate: time.Now(),
IsVideo: false,
Thumbnail: source.Thumbnail,
Description: source.Content,
AuthorName: source.Author,
AuthorImageUrl: "null",
}
return item
}
func (rc *RedditClient) convertTextPost(source domain.RedditPost) entity.ArticleEntity {
var item = entity.ArticleEntity{
SourceID: rc.record.ID,
Tags: "a",
Title: source.Title,
PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.Content,
}
return item
}
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) entity.ArticleEntity {
var item = entity.ArticleEntity{
SourceID: rc.record.ID,
Tags: "a",
Title: source.Title,
PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.Media.RedditVideo.FallBackUrl,
}
return item
}
// This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) entity.ArticleEntity {
var item = entity.ArticleEntity{
SourceID: rc.record.ID,
Tags: "a",
Title: source.Title,
PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.UrlOverriddenByDest,
}
return item
}

View File

@ -1,32 +0,0 @@
package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
var RedditRecord entity.SourceEntity = entity.SourceEntity{
ID: 9999,
DisplayName: "dadjokes",
Source: domain.SourceCollectorRss,
Url: "https://reddit.com/r/dadjokes",
Tags: "reddit, dadjokes",
}
func TestGetContent(t *testing.T) {
//This test is flaky right now due to the http changes in 1.17
rc := input.NewRedditClient(RedditRecord)
raw, err := rc.GetContent()
if err != nil {
t.Error(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, posts := range redditArticles {
if posts.Title == "" {
t.Error("Title is missing")
}
}
}

View File

@ -1,77 +0,0 @@
package input
import (
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"github.com/mmcdole/gofeed"
)
type FeedInput interface {
GetArticles() (entity.ArticleEntity, error)
}
type rssClient struct {
SourceRecord entity.SourceEntity
}
func NewRssClient(sourceRecord entity.SourceEntity) rssClient {
client := rssClient{
SourceRecord: sourceRecord,
}
return client
}
func (rc rssClient) GetArticles() ([]entity.ArticleEntity, error) {
parser := gofeed.NewParser()
feed, err := parser.ParseURL(rc.SourceRecord.Url)
if err != nil {
return nil, err
}
sourceTags := strings.Split(rc.SourceRecord.Tags, ",")
var articles []entity.ArticleEntity
for _, post := range feed.Items {
article := entity.ArticleEntity{
SourceID: rc.SourceRecord.ID,
Title: post.Title,
Description: post.Content,
Url: post.Link,
PubDate: *post.PublishedParsed,
//AuthorName: post.Authors[0].Email,
}
if len(post.Authors) != 0 {
article.AuthorName = post.Authors[0].Email
}
var postTags []string
postTags = append(postTags, sourceTags...)
postTags = append(postTags, post.Categories...)
article.Tags = strings.Join(postTags, ",")
/*
pageContent, err := getHttpContent(article.Url)
if err != nil {
continue
}
htmlNode, err := html.Parse(bytes.NewReader(pageContent))
if err != nil {
continue
}
htmlNode.
fmt.Println(htmlNode)
*/
if post.Image == nil {
article.Thumbnail = ""
}
articles = append(articles, article)
}
return articles, nil
}

View File

@ -1,43 +0,0 @@
package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
var rssRecord = entity.SourceEntity{
ID: 1,
DisplayName: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index",
Source: domain.SourceCollectorRss,
}
func TestRssClientConstructor(t *testing.T) {
input.NewRssClient(rssRecord)
}
func TestRssGetFeed(t *testing.T) {
client := input.NewRssClient(rssRecord)
_, err := client.GetArticles()
if err != nil {
t.Error(err)
}
}
func TestRssAgainstGita(t *testing.T) {
client := input.NewRssClient(entity.SourceEntity{
ID: 2,
DisplayName: "Gitea - Newsbot-api",
Source: domain.SourceCollectorRss,
Url: "https://git.jamestombleson.com/jtom38/newsbot-api.rss",
Tags: "rss,gitea,newsbot-api",
})
_, err := client.GetArticles()
if err != nil {
t.Error(err)
}
}

View File

@ -1,149 +0,0 @@
package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
var YouTubeRecord = entity.SourceEntity{
ID: 9999,
DisplayName: "dadjokes",
Source: domain.SourceCollectorReddit,
Url: "https://youtube.com/gamegrumps",
}
func TestGetPageParser(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetParser(YouTubeRecord.Url)
if err != nil {
t.Error(err)
}
}
func TestGetChannelId(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil {
t.Error(err)
}
_, err = yc.GetChannelId(parser)
if err != nil {
t.Error(err)
}
}
func TestPullFeed(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil {
t.Error(err)
}
_, err = yc.GetChannelId(parser)
if err != nil {
t.Error(err)
}
_, err = yc.PullFeed()
if err != nil {
t.Error(err)
}
}
func TestGetAvatarUri(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
res, err := yc.GetAvatarUri()
if err != nil {
t.Error(err)
}
if res == "" {
t.Error(input.ErrMissingAuthorImage)
}
}
func TestGetVideoTags(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
parser, err := yc.GetParser(videoUri)
if err != nil {
t.Error(err)
}
tags, err := yc.GetTags(parser)
if err == nil && tags == "" {
t.Error("err was empty but value was missing.")
}
if err != nil {
t.Error(err)
}
}
func TestGetChannelTags(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil {
t.Error(err)
}
tags, err := yc.GetTags(parser)
if err == nil && tags == "" {
t.Error("no err but expected value was missing.")
}
if err != nil {
t.Error(err)
}
}
func TestGetVideoThumbnail(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
if err != nil {
t.Error(err)
}
thumb, err := yc.GetVideoThumbnail(parser)
if err == nil && thumb == "" {
t.Error("no err but expected result was missing")
}
if err != nil {
t.Error(err)
}
}
func TestCheckSource(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetContent()
if err != nil {
t.Error(err)
}
}
func TestCheckUriCache(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
item := "demo"
input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
res := yc.CheckUriCache(&item)
if res == false {
t.Error("expected a value to come back")
}
}
func TestCheckUriCacheFails(t *testing.T) {
yc := input.NewYoutubeClient(YouTubeRecord)
item := "demo1"
res := yc.CheckUriCache(&item)
if res == true {
t.Error("expected no value to come back")
}
}

View File

@ -1,191 +0,0 @@
package output
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
)
type discordField struct {
Name *string `json:"name,omitempty"`
Value *string `json:"value,omitempty"`
Inline *bool `json:"inline,omitempty"`
}
type discordFooter struct {
Value *string `json:"text,omitempty"`
IconUrl *string `json:"icon_url,omitempty"`
}
type discordAuthor struct {
Name *string `json:"name,omitempty"`
Url *string `json:"url,omitempty"`
IconUrl *string `json:"icon_url,omitempty"`
}
type discordImage struct {
Url *string `json:"url,omitempty"`
}
type DiscordEmbed struct {
Title *string `json:"title,omitempty"`
Description *string `json:"description,omitempty"`
Url *string `json:"url,omitempty"`
Color *int32 `json:"color,omitempty"`
//Timestamp time.Time `json:"timestamp,omitempty"`
Fields []*discordField `json:"fields,omitempty"`
Author discordAuthor `json:"author,omitempty"`
Image discordImage `json:"image,omitempty"`
Thumbnail discordImage `json:"thumbnail,omitempty"`
Footer *discordFooter `json:"footer,omitempty"`
}
// Root object for Discord Webhook messages
type DiscordMessage struct {
Username *string `json:"username,omitempty"`
Content *string `json:"content,omitempty"`
Embeds *[]DiscordEmbed `json:"embeds,omitempty"`
}
const (
DefaultColor = 0
YoutubeColor = 16711680
TwitchColor = 0
RedditColor = 0
TwitterColor = 0
FfxivColor = 0
)
type Discord struct {
Subscriptions []string
article entity.ArticleEntity
Message *DiscordMessage
}
func NewDiscordWebHookMessage(Article entity.ArticleEntity) Discord {
return Discord{
article: Article,
}
}
// Generates the link field to expose in the message
func (dwh Discord) getFields() []*discordField {
var fields []*discordField
key := "Link"
linkField := discordField{
Name: &key,
Value: &dwh.article.Url,
}
fields = append(fields, &linkField)
return fields
}
// This will create the message that will be sent out.
func (dwh Discord) GeneratePayload() (*DiscordMessage, error) {
// Create the embed
footerMessage := "Brought to you by Newsbot"
footerUrl := ""
description := dwh.convertFromHtml(dwh.article.Description)
color := dwh.getColor(dwh.article.Url)
embed := DiscordEmbed{
Title: &dwh.article.Title,
Description: &description,
Image: discordImage{
Url: &dwh.article.Thumbnail,
},
Fields: dwh.getFields(),
Footer: &discordFooter{
Value: &footerMessage,
IconUrl: &footerUrl,
},
Color: &color,
}
// attach the embed to an array
var embedArray []DiscordEmbed
embedArray = append(embedArray, embed)
// create the base message
msg := DiscordMessage{
Embeds: &embedArray,
}
return &msg, nil
}
func (dwh Discord) SendPayload(Message *DiscordMessage, Url string) error {
// Convert the message to a io.reader object
buffer := new(bytes.Buffer)
json.NewEncoder(buffer).Encode(Message)
// Send the message
resp, err := http.Post(Url, "application/json", buffer)
if err != nil {
return err
}
// Check for 204
if resp.StatusCode != 204 {
defer resp.Body.Close()
errMsg, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf(string(errMsg))
}
return nil
}
func (dwh Discord) convertFromHtml(body string) string {
clean := body
clean = strings.ReplaceAll(clean, "<h2>", "**")
clean = strings.ReplaceAll(clean, "</h2>", "**")
clean = strings.ReplaceAll(clean, "<h3>", "**")
clean = strings.ReplaceAll(clean, "</h3>", "**\r\n")
clean = strings.ReplaceAll(clean, "<strong>", "**")
clean = strings.ReplaceAll(clean, "</strong>", "**\r\n")
clean = strings.ReplaceAll(clean, "<ul>", "\r\n")
clean = strings.ReplaceAll(clean, "</ul>", "")
clean = strings.ReplaceAll(clean, "</li>", "\r\n")
clean = strings.ReplaceAll(clean, "<li>", "> ")
clean = strings.ReplaceAll(clean, "&#8220;", "\"")
clean = strings.ReplaceAll(clean, "&#8221;", "\"")
clean = strings.ReplaceAll(clean, "&#8230;", "...")
clean = strings.ReplaceAll(clean, "<b>", "**")
clean = strings.ReplaceAll(clean, "</b>", "**")
clean = strings.ReplaceAll(clean, "<br>", "\r\n")
clean = strings.ReplaceAll(clean, "<br/>", "\r\n")
clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*")
clean = strings.ReplaceAll(clean, "\xa0", "\r\n")
clean = strings.ReplaceAll(clean, "<p>", "")
clean = strings.ReplaceAll(clean, "</p>", "\r\n")
return clean
}
func (dwh *Discord) getColor(Url string) int32 {
if strings.Contains(Url, "youtube.com") {
return YoutubeColor
}
return DefaultColor
}
func (dwh *Discord) convertLinks(body string) string {
//items := regexp.MustCompile("<a(.*?)a>")
return ""
}

View File

@ -1,134 +0,0 @@
package output_test
import (
"os"
"strings"
"testing"
//"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
"github.com/joho/godotenv"
)
var (
article entity.ArticleEntity = entity.ArticleEntity{
ID: 999,
SourceID: 1,
Tags: "unit, testing",
Title: "Demo",
Url: "https://github.com/jtom38/newsbot.collector.api",
Description: "Hello World",
}
blank string = ""
)
func TestDiscordMessageContainsTitle(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
if i.Title == &blank {
t.Error("title missing")
}
}
}
func TestDiscordMessageContainsDescription(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
if i.Description == &blank {
t.Error("description missing")
}
}
}
func TestDiscordMessageFooter(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
blank := ""
if i.Footer.Value == &blank {
t.Error("missing footer vlue")
}
if i.Footer.IconUrl == &blank {
t.Error("missing footer url")
}
}
}
func TestDiscordMessageFields(t *testing.T) {
header := "Link"
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, embed := range *msg.Embeds {
for _, field := range embed.Fields {
var fName string
if field.Name != nil {
fName = *field.Name
} else {
t.Error("missing link field value")
}
if fName != header {
t.Error("missing link field key")
}
var fValue string
if field.Value != nil {
fValue = *field.Value
}
if fValue == blank {
t.Error("missing link field value")
}
}
}
}
// This test requires a env value to be present to work
func TestDiscordMessagePost(t *testing.T) {
_, err := os.Open(".env")
if err != nil {
t.Error(err)
}
err = godotenv.Load()
if err != nil {
t.Error(err)
}
res := os.Getenv("TESTS_DISCORD_WEBHOOK")
if res == "" {
t.Error("TESTS_DISCORD_WEBHOOK is missing")
}
endpoints := strings.Split(res, " ")
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
err = d.SendPayload(msg, endpoints[0])
if err != nil {
t.Error(err)
}
}

View File

@ -1,6 +0,0 @@
package output
type Output interface {
GeneratePayload() error
SendPayload() error
}

32
main.go Normal file
View File

@ -0,0 +1,32 @@
package main
import (
"context"
"log"
"net/http"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/jtom38/newsbot/collector/routes"
"github.com/jtom38/newsbot/collector/services/cron"
)
func main() {
ctx := context.Background()
cron.EnableScheduler(ctx)
app := chi.NewRouter()
app.Use(middleware.Logger)
app.Use(middleware.Recoverer)
//app.Mount("/swagger", httpSwagger.WrapHandler)
app.Mount("/api", routes.RootRoutes())
log.Println("API is online and waiting for requests.")
log.Println("API: http://localhost:8081/api")
//log.Println("Swagger: http://localhost:8080/swagger/index.html")
err := http.ListenAndServe(":8081", app)
if err != nil { log.Fatalln(err) }
}

View File

@ -3,29 +3,14 @@ help: ## Shows this help command
@egrep -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
build: ## builds the application with the current go runtime
~/go/bin/swag f
~/go/bin/swag init -g cmd/server.go
go build cmd/server.go
ls -lh server
go build .
docker-build: ## Generates the docker image
docker build -t "newsbot.collector.api" .
docker image ls | grep newsbot.collector.api
migrate-dev: ## Apply sql migrations to dev db
goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down
swag: ## Generates the swagger documentation with the swag tool
~/go/bin/swag f -g cmd/server.go
~/go/bin/swag init -g cmd/server.go
go run tools/swaggertoopenapi/main.go
oapi-codegen -config api/client.yaml docs/openapi.json
mv api.gen.go api/
install-tools: ## Installs the required tools for this project
go install github.com/swaggo/swag/cmd/swag@v1.8.1
go install github.com/pressly/goose/v3/cmd/goose@latest
go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.3.0
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down

25
routes/root.go Normal file
View File

@ -0,0 +1,25 @@
package routes
import (
"net/http"
"fmt"
"github.com/go-chi/chi/v5"
)
func RootRoutes() chi.Router {
app := chi.NewRouter()
app.Get("/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello World!"))
})
app.Get("/ping", func(w http.ResponseWriter, r *http.Request) {
msg := "pong"
w.Write([]byte(msg))
})
app.Get("/hello/{world}", func(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "world"))
w.Write([]byte(msg))
})
return app
}

View File

@ -3,40 +3,36 @@ package cache
import (
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/domain/model"
)
type CacheClient struct {
group string
type CacheClient struct{
group string
DefaultTimer time.Duration
}
func NewCacheClient(group string) CacheClient {
return CacheClient{
group: group,
group: group,
DefaultTimer: time.Hour,
}
}
func (cc *CacheClient) Insert(key string, value string) {
item := domain.CacheItem{
Key: key,
Value: value,
Group: cc.group,
Expires: time.Now().Add(1 * time.Hour),
item := model.CacheItem{
Key: key,
Value: value,
Group: cc.group,
Expires: time.Now().Add(1 * time.Hour),
IsTainted: false,
}
cacheStorage = append(cacheStorage, &item)
}
func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
for _, item := range cacheStorage {
if item.Group != cc.group {
continue
}
if item.Key != key {
continue
}
if item.Group != cc.group { continue }
if item.Key != key { continue }
// if it was tainted, renew the timer and remove the taint as this record was still needed
if item.IsTainted {
@ -46,17 +42,13 @@ func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
return item, nil
}
return &domain.CacheItem{}, ErrCacheRecordMissing
return &model.CacheItem{}, ErrCacheRecordMissing
}
func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
for _, item := range cacheStorage {
if item.Group != cc.group {
continue
}
if item.Value != value {
continue
}
if item.Group != cc.group { continue }
if item.Value != value { continue }
// if it was tainted, renew the timer and remove the taint as this record was still needed
if item.IsTainted {
@ -65,5 +57,6 @@ func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
}
return item, nil
}
return &domain.CacheItem{}, ErrCacheRecordMissing
return &model.CacheItem{}, ErrCacheRecordMissing
}

View File

@ -3,7 +3,7 @@ package cache_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/services/cache"
)
func TestNewCacheClient(t *testing.T) {
@ -18,18 +18,14 @@ func TestInsert(t *testing.T) {
func TestFindGroupMissing(t *testing.T) {
cache := cache.NewCacheClient("faker")
_, err := cache.FindByKey("UnitTesting")
if err == nil {
panic("Nothing was appended with the requested group.")
}
if err == nil { panic("Nothing was appended with the requested group.") }
}
func TestFindGroupExists(t *testing.T) {
cache := cache.NewCacheClient("Testing")
cache.Insert("UnitTesting", "Something")
_, err := cache.FindByKey("UnitTesting")
if err != nil {
panic("")
}
if err != nil { panic("") }
}
func TestCacheStorage(t *testing.T) {
@ -39,7 +35,6 @@ func TestCacheStorage(t *testing.T) {
cache := cache.NewCacheClient("Testing")
_, err := cache.FindByKey("UnitTesting02")
if err != nil {
panic("expected to find the value")
}
if err != nil { panic("expected to find the value")}
}

View File

@ -3,11 +3,11 @@ package cache
import (
"errors"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/domain/model"
)
var (
cacheStorage []*domain.CacheItem
cacheStorage []*model.CacheItem
ErrCacheRecordMissing = errors.New("unable to find the requested record")
)
)

View File

@ -3,13 +3,13 @@ package cache
import (
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/domain/model"
)
// When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
// If a record is tainted and used again, the taint will be removed and a new Expires value will be set.
// If its not renewed, it will be dropped.
type CacheAgeMonitor struct{}
type CacheAgeMonitor struct {}
func NewCacheAgeMonitor() CacheAgeMonitor {
return CacheAgeMonitor{}
@ -20,10 +20,10 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
now := time.Now()
for index, item := range cacheStorage {
if now.After(item.Expires) {
// the timer expired, and its not tainted, taint it
if !item.IsTainted {
item.IsTainted = true
item.IsTainted = true
item.Expires = now.Add(1 * time.Hour)
}
@ -36,12 +36,10 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
}
// This creates a new slice and skips over the item that needs to be dropped
func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
var temp []*domain.CacheItem
func (cam CacheAgeMonitor) removeEntry(index int) []*model.CacheItem {
var temp []*model.CacheItem
for i, item := range cacheStorage {
if i != index {
temp = append(temp, item)
}
if i != index { temp = append(temp, item )}
}
return temp
}

View File

@ -3,11 +3,11 @@ package cache_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/services/cache"
)
func TestCacheTaintItem(t *testing.T) {
cc := cache.NewCacheClient("Testing")
cc.Insert("UnitTesting01", "test")
}
}

View File

@ -1,4 +1,4 @@
package input
package services
import "errors"
@ -14,4 +14,4 @@ var (
ErrInvalidAuthorImage = errors.New("expected value looks to be wrong, something is missing")
)
const DATETIME_FORMAT string = "1/2/2006 3:4 PM"
const DATETIME_FORMAT string = "1/2/2006 3:4 PM"

56
services/config/config.go Normal file
View File

@ -0,0 +1,56 @@
package config
import (
"os"
"log"
"github.com/joho/godotenv"
)
const (
DB_URI string = "DB_URI"
Sql_Connection_String string = "SQL_CONNECTION_STRING"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
)
type ConfigClient struct {}
func New() ConfigClient {
_, err := os.Open(".env")
if err == nil {
loadEnvFile()
}
return ConfigClient{}
}
func (cc *ConfigClient) GetConfig(key string) string {
res, filled := os.LookupEnv(key)
if !filled {
log.Printf("Missing the a value for '%v'. Could generate errors.", key)
}
return res
}
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
func (cc *ConfigClient) RefreshEnv() {
loadEnvFile()
}
func loadEnvFile() {
err := godotenv.Load()
if err != nil {
log.Fatalln(err)
}
}

View File

@ -0,0 +1,20 @@
package config_test
import (
"testing"
"os"
"github.com/jtom38/newsbot/collector/services/config"
)
func TestNewClient(t *testing.T) {
config.New()
}
func TestGetConfigExpectNull(t *testing.T) {
cc := config.New()
os.Setenv(config.REDDIT_PULL_HOT, "")
res := cc.GetConfig(config.REDDIT_PULL_HOT)
if res != "" { panic("expected blank")}
}

167
services/cron/scheduler.go Normal file
View File

@ -0,0 +1,167 @@
package cron
import (
"context"
"database/sql"
"log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq"
"github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
"github.com/jtom38/newsbot/collector/services/config"
)
var _env config.ConfigClient
var _connString string
var _queries *database.Queries
func EnableScheduler(ctx context.Context) {
c := cron.New()
OpenDatabase(ctx)
//c.AddFunc("*/5 * * * *", func() { go CheckCache() })
c.AddFunc("* */1 * * *", func() { go CheckReddit(ctx) })
//c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
//c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
//c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
c.Start()
}
// Open the connection to the database and share it with the package so all of them are able to share.
func OpenDatabase(ctx context.Context) error {
_env = config.New()
_connString = _env.GetConfig(config.Sql_Connection_String)
db, err := sql.Open("postgres", _connString)
if err != nil {
panic(err)
}
queries := database.New(db)
_queries = queries
return err
}
// This is the main entry point to query all the reddit services
func CheckReddit(ctx context.Context) {
sources, err := _queries.ListSourcesBySource(ctx, "reddit")
if err != nil {
log.Printf("No defines sources for reddit to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
rc := services.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
checkPosts(ctx, redditArticles)
}
}
func CheckYoutube(ctx context.Context) {
// Add call to the db to request youtube sources.
sources, err := _queries.ListSourcesBySource(ctx, "youtube")
if err != nil {
log.Printf("Youtube - No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
yc := services.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
checkPosts(ctx, raw)
}
}
func CheckFfxiv(ctx context.Context) {
sources, err := _queries.ListSourcesBySource(ctx, "ffxiv")
if err != nil {
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := services.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
checkPosts(ctx, items)
}
}
func CheckTwitch(ctx context.Context) error {
sources, err := _queries.ListSourcesBySource(ctx, "twitch")
if err != nil {
log.Printf("Twitch - No sources found to query - %v\r", err)
}
tc, err := services.NewTwitchClient()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
checkPosts(ctx, items)
}
return nil
}
func checkPosts(ctx context.Context, posts []database.Article) {
for _, item := range posts {
_, err := _queries.GetArticleByUrl(ctx, item.Url)
if err != nil {
err = postArticle(ctx, item)
if err != nil {
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
} else {
log.Printf("Reddit - Posted article - %v\r", item.Url)
}
}
}
time.Sleep(30 * time.Second)
}
func postArticle(ctx context.Context, item database.Article) error {
err := _queries.CreateArticle(ctx, database.CreateArticleParams{
ID: uuid.New(),
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}

View File

@ -0,0 +1,37 @@
package cron_test
import (
"context"
"testing"
"github.com/jtom38/newsbot/collector/services/cron"
)
func TestInvokeTwitch(t *testing.T) {
}
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) {
ctx := context.Background()
cron.OpenDatabase(ctx)
cron.CheckReddit(ctx)
}
func TestCheckYouTube(t *testing.T) {
ctx := context.Background()
cron.OpenDatabase(ctx)
cron.CheckYoutube(ctx)
}
func TestCheckTwitch(t *testing.T) {
ctx := context.Background()
err := cron.OpenDatabase(ctx)
if err != nil {
t.Error(err)
}
err = cron.CheckTwitch(ctx)
if err != nil {
t.Error(err)
}
}

View File

@ -1,6 +1,7 @@
package input
package services
import (
"database/sql"
"errors"
"log"
"net/http"
@ -9,11 +10,10 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/cache"
)
const (
@ -24,7 +24,7 @@ const (
)
type FFXIVClient struct {
record entity.SourceEntity
record database.Source
//SourceID uint
//Url string
//Region string
@ -32,80 +32,65 @@ type FFXIVClient struct {
cacheGroup string
}
func NewFFXIVClient(Record entity.SourceEntity) FFXIVClient {
func NewFFXIVClient(Record database.Source) FFXIVClient {
return FFXIVClient{
record: Record,
record: Record,
cacheGroup: "ffxiv",
}
}
func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
var articles []entity.ArticleEntity
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
var articles []database.Article
parser := fc.GetBrowser()
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
cache := cache.NewCacheClient(fc.cacheGroup)
for _, link := range links {
// Check cache/db if this link has been seen already, skip
_, err := cache.FindByValue(link)
if err == nil {
continue
}
if err == nil { continue }
page := fc.GetPage(parser, link)
title, err := fc.ExtractTitle(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
thumb, err := fc.ExtractThumbnail(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
pubDate, err := fc.ExtractPubDate(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
description, err := fc.ExtractDescription(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
authorName, err := fc.ExtractAuthor(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
authorImage, err := fc.ExtractAuthorImage(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
tags, err := fc.ExtractTags(page)
if err != nil {
return articles, err
}
if err != nil { return articles, err }
article := entity.ArticleEntity{
SourceID: fc.record.ID,
Tags: tags,
Title: title,
Url: link,
PubDate: pubDate,
Thumbnail: thumb,
Description: description,
AuthorName: authorName,
AuthorImageUrl: authorImage,
article := database.Article{
Sourceid: fc.record.ID,
Tags: tags,
Title: title,
Url: link,
Pubdate: pubDate,
Videoheight: 0,
Videowidth: 0,
Thumbnail: thumb,
Description: description,
Authorname: sql.NullString{String: authorName},
Authorimage: sql.NullString{String: authorImage},
}
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
@ -119,24 +104,16 @@ func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) {
func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
html, err := http.Get(fc.record.Url)
if err != nil {
return nil, err
}
if err != nil { return nil, err }
defer html.Body.Close()
doc, err := goquery.NewDocumentFromReader(html.Body)
if err != nil {
return nil, err
}
if err != nil { return nil, err }
return doc, nil
}
func (fc *FFXIVClient) GetBrowser() *rod.Browser {
var browser *rod.Browser
if path, exists := launcher.LookPath(); exists {
u := launcher.New().Bin(path).MustLaunch()
browser = rod.New().ControlURL(u).MustConnect()
}
func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
browser := rod.New().MustConnect()
return browser
}
@ -151,26 +128,26 @@ func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) {
// find all the li items
items := res.MustElements("li")
for _, item := range items {
// in each li, find the a items
a, err := item.Element("a")
if err != nil {
if err != nil {
log.Println("Unable to find the a item, skipping")
continue
continue
}
// find the href behind the a
// find the href behind the a
url, err := a.Property("href")
if err != nil {
if err != nil {
log.Println("Unable to find a href link, skipping")
continue
continue
}
urlString := url.String()
isTopic := strings.Contains(urlString, "topics")
if isTopic {
links = append(links, urlString)
links = append(links, urlString)
}
}
@ -184,10 +161,8 @@ func (rc *FFXIVClient) GetPage(parser *rod.Browser, url string) *rod.Page {
func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
thumbnail := page.MustElementX("/html/body/div[3]/div[2]/div[1]/article/div[1]/img").MustProperty("src").String()
if thumbnail == "" {
return "", errors.New("unable to find thumbnail")
}
if thumbnail == "" { return "", errors.New("unable to find thumbnail")}
title := page.MustElement(".news__header > h1:nth-child(2)").MustText()
log.Println(title)
@ -196,23 +171,17 @@ func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
func (fc *FFXIVClient) ExtractPubDate(page *rod.Page) (time.Time, error) {
stringDate := page.MustElement(".news__ic--topics").MustText()
if stringDate == "" {
return time.Now(), errors.New("unable to locate the publish date on the post")
}
if stringDate == "" { return time.Now(), errors.New("unable to locate the publish date on the post")}
PubDate, err := time.Parse(FFXIV_TIME_FORMAT, stringDate)
if err != nil {
return time.Now(), err
}
if err != nil { return time.Now(), err }
return PubDate, nil
}
func (fc *FFXIVClient) ExtractDescription(page *rod.Page) (string, error) {
res := page.MustElement(".news__detail__wrapper").MustText()
if res == "" {
return "", errors.New("unable to locate the description on the post")
}
if res == "" { return "", errors.New("unable to locate the description on the post")}
return res, nil
}
@ -221,18 +190,12 @@ func (fc *FFXIVClient) ExtractAuthor(page *rod.Page) (string, error) {
meta := page.MustElements("head > meta")
for _, item := range meta {
name, err := item.Property("name")
if err != nil {
return "", err
}
if err != nil { return "", err }
if name.String() != "author" {
continue
}
if name.String() != "author" { continue }
content, err := item.Property("content")
if err != nil {
return "", err
}
if err != nil { return "", err }
return content.String(), nil
}
//log.Println(meta)
@ -243,18 +206,12 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
meta := page.MustElements("head > meta")
for _, item := range meta {
name, err := item.Property("name")
if err != nil {
return "", err
}
if err != nil { return "", err }
if name.String() != "keywords" {
continue
}
if name.String() != "keywords" { continue }
content, err := item.Property("content")
if err != nil {
return "", err
}
if err != nil { return "", err }
return content.String(), nil
}
//log.Println(meta)
@ -263,19 +220,13 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
func (fc *FFXIVClient) ExtractTitle(page *rod.Page) (string, error) {
title, err := page.MustElement("head > title").Text()
if err != nil {
return "", err
}
if err != nil { return "", err }
if !strings.Contains(title, "|") {
return "", errors.New("unable to split the title, missing | in the string")
}
if !strings.Contains(title, "|") { return "", errors.New("unable to split the title, missing | in the string")}
res := strings.Split(title, "|")
if title != "" {
return res[0], nil
}
if title != "" { return res[0], nil }
//log.Println(meta)
return "", errors.New("unable to find the author on the page")
}
@ -284,20 +235,15 @@ func (fc *FFXIVClient) ExtractAuthorImage(page *rod.Page) (string, error) {
meta := page.MustElements("head > link")
for _, item := range meta {
name, err := item.Property("rel")
if err != nil {
return "", err
}
if err != nil { return "", err }
if name.String() != "apple-touch-icon-precomposed" {
continue
}
if name.String() != "apple-touch-icon-precomposed" { continue }
content, err := item.Property("href")
if err != nil {
return "", err
}
if err != nil { return "", err }
return content.String(), nil
}
//log.Println(meta)
return "", errors.New("unable to find the author image on the page")
}

View File

@ -1,27 +1,26 @@
package input_test
package services_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services"
)
var FFXIVRecord entity.SourceEntity = entity.SourceEntity{
ID: 9999,
DisplayName: "Final Fantasy XIV - NA",
Source: domain.SourceCollectorFfxiv,
Url: "https://na.finalfantasyxiv.com/lodestone/",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
var FFXIVRecord database.Source = database.Source{
ID: uuid.New(),
Site: "ffxiv",
Name: "Final Fantasy XIV - NA",
Source: "ffxiv",
Url: "https://na.finalfantasyxiv.com/lodestone/",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
}
func TestFfxivGetParser(t *testing.T) {
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
_, err := fc.GetParser()
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
}
func TestFfxivPullFeed(t *testing.T) {
@ -31,12 +30,8 @@ func TestFfxivPullFeed(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if len(links) == 0 {
t.Error("expected links to come back but got 0")
}
if err != nil { panic(err) }
if len(links) == 0 { panic("expected links to come back but got 0") }
}
@ -47,20 +42,14 @@ func TestFfxivExtractThumbnail(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
thumb, err := fc.ExtractThumbnail(page)
if err != nil {
t.Error(err)
}
if thumb == "" {
t.Error("expected a link but got nothing.")
}
if err != nil { panic(err) }
if thumb == "" { panic("expected a link but got nothing.")}
}
func TestFfxivExtractPubDate(t *testing.T) {
@ -70,17 +59,13 @@ func TestFfxivExtractPubDate(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
_, err = fc.ExtractPubDate(page)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
}
func TestFfxivExtractDescription(t *testing.T) {
@ -90,17 +75,13 @@ func TestFfxivExtractDescription(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
_, err = fc.ExtractDescription(page)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
}
func TestFfxivExtractAuthor(t *testing.T) {
@ -110,20 +91,14 @@ func TestFfxivExtractAuthor(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
author, err := fc.ExtractAuthor(page)
if err != nil {
t.Error(err)
}
if author == "" {
t.Error("failed to locate the author name")
}
if err != nil { panic(err) }
if author == "" { panic("failed to locate the author name") }
}
func TestFfxivExtractTags(t *testing.T) {
@ -133,20 +108,14 @@ func TestFfxivExtractTags(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
res, err := fc.ExtractTags(page)
if err != nil {
t.Error(err)
}
if res == "" {
t.Error("failed to locate the tags")
}
if err != nil { panic(err) }
if res == "" {panic("failed to locate the tags")}
}
func TestFfxivExtractTitle(t *testing.T) {
@ -156,20 +125,14 @@ func TestFfxivExtractTitle(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
res, err := fc.ExtractTitle(page)
if err != nil {
t.Error(err)
}
if res == "" {
t.Error("failed to locate the tags")
}
if err != nil { panic(err) }
if res == "" { panic("failed to locate the tags") }
}
func TestFFxivExtractAuthorIamge(t *testing.T) {
@ -179,24 +142,18 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil {
t.Error(err)
}
if err != nil { panic(err) }
page := fc.GetPage(parser, links[0])
defer page.Close()
res, err := fc.ExtractAuthorImage(page)
if err != nil {
t.Error(err)
}
if res == "" {
t.Error("failed to locate the tags")
}
if err != nil { panic(err) }
if res == "" { panic("failed to locate the tags") }
}
func TestFfxivCheckSource(t *testing.T) {
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
fc.CheckSource()
}
}

29
services/httpClient.go Normal file
View File

@ -0,0 +1,29 @@
package services
import (
"net/http"
"log"
"io/ioutil"
)
// This will use the net/http client reach out to a site and collect the content.
func getHttpContent(uri string) ([]byte, error) {
client := &http.Client{}
req, err := http.NewRequest("GET", uri, nil)
if err != nil { return nil, err }
// set the user agent header to avoid kick backs.. as much
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
log.Printf("Requesting content from %v\n", uri)
resp, err := client.Do(req)
if err != nil { log.Fatalln(err) }
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil { return nil, err }
return body, nil
}

192
services/reddit.go Normal file
View File

@ -0,0 +1,192 @@
package services
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/config"
)
type RedditClient struct {
config RedditConfig
record database.Source
}
type RedditConfig struct {
PullTop string
PullHot string
PullNSFW string
}
func NewRedditClient(Record database.Source) RedditClient {
rc := RedditClient{
record: Record,
}
cc := config.New()
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
rc.disableHttp2Client()
return rc
}
// This is needed for to get modern go to talk to the endpoint.
// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
func (rc RedditClient) disableHttp2Client() {
os.Setenv("GODEBUG", "http2client=0")
}
func (rc RedditClient) GetBrowser() *rod.Browser {
browser := rod.New().MustConnect()
return browser
}
func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
page := parser.MustPage(url)
return page
}
//func (rc RedditClient)
// GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct.
func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
var items model.RedditJsonContent = model.RedditJsonContent{}
// TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("Collecting results on '%v'", rc.record.Name)
content, err := getHttpContent(Url)
if err != nil {
return items, err
}
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
return items, errors.New("did not get json data from the server")
}
json.Unmarshal(content, &items)
if len(items.Data.Children) == 0 {
return items, errors.New("failed to unmarshal the data")
}
return items, nil
}
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
var redditArticles []database.Article
for _, item := range items.Data.Children {
var article database.Article
article, err := rc.convertToArticle(item.Data)
if err != nil {
log.Println(err)
continue
}
redditArticles = append(redditArticles, article)
}
return redditArticles
}
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database.
func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
var item database.Article
if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source)
}
if source.Media.RedditVideo.FallBackUrl != "" {
item = rc.convertVideoPost(source)
}
if source.Content != "" {
item = rc.convertTextPost(source)
}
if source.UrlOverriddenByDest != "" {
item = rc.convertRedirectPost(source)
}
if item.Description == "" {
var err = errors.New("reddit post failed to parse correctly")
return item, err
}
return item, nil
}
func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Pubdate: time.Now(),
Video: sql.NullString{String: "null"},
Videoheight: 0,
Videowidth: 0,
Thumbnail: source.Thumbnail,
Description: source.Content,
Authorname: sql.NullString{String: source.Author},
Authorimage: sql.NullString{String: "null"},
}
return item
}
func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title,
Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Authorname: sql.NullString{String: source.Author},
Description: source.Content,
}
return item
}
func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title,
Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl,
}
return item
}
// This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title,
Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest,
}
return item
}

33
services/reddit_test.go Normal file
View File

@ -0,0 +1,33 @@
package services_test
import (
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
)
var RedditRecord database.Source = database.Source{
ID: uuid.New(),
Name: "dadjokes",
Source: "reddit",
Site: "reddit",
Url: "https://reddit.com/r/dadjokes",
Tags: "reddit, dadjokes",
}
func TestGetContent(t *testing.T) {
//This test is flaky right now due to the http changes in 1.17
rc := services.NewRedditClient(RedditRecord)
raw, err := rc.GetContent()
if err != nil {
t.Error(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, posts := range redditArticles {
if posts.Title == "" {
t.Error("Title is missing")
}
}
}

View File

@ -1,18 +1,19 @@
package input
package services
import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/entity"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/nicklaw5/helix/v2"
)
type TwitchClient struct {
SourceRecord entity.SourceEntity
SourceRecord database.Source
// config
monitorClips string
@ -30,14 +31,14 @@ var (
)
func NewTwitchClient() (TwitchClient, error) {
c := services.NewConfig()
c := config.New()
id := c.GetConfig(services.TWITCH_CLIENT_ID)
id := c.GetConfig(config.TWITCH_CLIENT_ID)
if id == "" {
return TwitchClient{}, ErrTwitchClientIdMissing
}
secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
if secret == "" {
return TwitchClient{}, ErrTwitchClientSecretMissing
}
@ -49,8 +50,8 @@ func NewTwitchClient() (TwitchClient, error) {
client := TwitchClient{
//SourceRecord: &source,
monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(services.TWITCH_MONITOR_VOD),
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
api: &api,
}
@ -71,12 +72,12 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
}
// This will let you replace the bound source record to keep the same session alive.
func (tc *TwitchClient) ReplaceSourceRecord(source entity.SourceEntity) {
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
tc.SourceRecord = source
}
// Invokes Logon request to the API
func (tc *TwitchClient) Login() error {
func (tc TwitchClient) Login() error {
token, err := tc.api.RequestAppAccessToken([]string{twitchScopes})
if err != nil {
return err
@ -86,8 +87,8 @@ func (tc *TwitchClient) Login() error {
return nil
}
func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
var items []entity.ArticleEntity
func (tc TwitchClient) GetContent() ([]database.Article, error) {
var items []database.Article
user, err := tc.GetUserDetails()
if err != nil {
@ -100,50 +101,34 @@ func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
}
for _, video := range posts {
var article entity.ArticleEntity
var article database.Article
AuthorName, err := tc.ExtractAuthor(video)
if err != nil {
return items, err
}
article.AuthorName = AuthorName
if err != nil { return items, err }
article.Authorname = sql.NullString{String: AuthorName}
Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil {
return items, err
}
article.AuthorImageUrl = Authorimage
if err != nil { return items, err }
article.Authorimage = sql.NullString{String: Authorimage}
article.Description, err = tc.ExtractDescription(video)
if err != nil {
return items, err
}
if err != nil {return items, err }
article.PubDate, err = tc.ExtractPubDate(video)
if err != nil {
return items, err
}
article.Pubdate, err = tc.ExtractPubDate(video)
if err != nil { return items, err }
article.SourceID = tc.SourceRecord.ID
article.Sourceid = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user)
if err != nil {
return items, err
}
if err != nil { return items, err }
article.Thumbnail, err = tc.ExtractThumbnail(video)
if err != nil {
return items, err
}
if err != nil { return items, err }
article.Title, err = tc.ExtractTitle(video)
if err != nil {
return items, err
}
if err != nil { return items, err }
article.Url, err = tc.ExtractUrl(video)
if err != nil {
return items, err
}
if err != nil { return items, err }
items = append(items, article)
}
@ -151,25 +136,20 @@ func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) {
return items, nil
}
func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
func (tc TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{
Logins: []string{tc.SourceRecord.DisplayName},
Logins: []string{tc.SourceRecord.Name},
})
if err != nil {
return blank, err
}
if len(users.Data.Users) == 0 {
return blank, errors.New("no results have been returned")
}
return users.Data.Users[0], nil
}
// This will reach out and collect the posts made by the user.
func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
var blank []helix.Video
videos, err := tc.api.GetVideos(&helix.VideosParams{
@ -183,14 +163,14 @@ func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
return videos.Data.Videos, nil
}
func (tc *TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
if post.UserName == "" {
return "", ErrMissingAuthorName
}
return post.UserName, nil
}
func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
if post.ThumbnailURL == "" {
return "", ErrMissingThumbnail
}
@ -200,7 +180,7 @@ func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
return thumb, nil
}
func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
if post.PublishedAt == "" {
return time.Now(), ErrMissingPublishDate
}
@ -211,7 +191,7 @@ func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
return pubDate, nil
}
func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
// Check if the description is null but we have a title.
// The poster didnt add a description but this isnt an error.
if post.Description == "" && post.Title == "" {
@ -224,34 +204,28 @@ func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
}
// Extracts the avatar of the author with some validation.
func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
if user.ProfileImageURL == "" {
return "", ErrMissingAuthorImage
}
if !strings.Contains(user.ProfileImageURL, "-profile_image-") {
return "", ErrInvalidAuthorImage
}
func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
return user.ProfileImageURL, nil
}
// Generate tags based on the video metadata.
// TODO Figure out how to query what game is played
func (tc *TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName)
return res, nil
}
// Extracts the title from a post with some validation.
func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) {
if post.Title == "" {
return "", errors.New("unable to find the title on the requested post")
}
return post.Title, nil
}
func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
if post.URL == "" {
return "", ErrMissingUrl
}
func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) {
if post.URL == "" { return "", ErrMissingUrl }
return post.URL, nil
}
}

Some files were not shown because too many files have changed in this diff Show More