Feature/sql (#8)

* Using sqlc to generate queries and goose for migrations. The intial tests look good.

* moving the old calls away for now.  Might use this in a package later on.

* Added postgres driver

* Updated the dockerfile to support sql migrations

* added sqlc config file

* updated schema and starting a seed script

* updated models to use the database ones

* updated reddit cron to talk to the db

* added env for sql connection string

* got the reddit source working with the db and posting articles

* added sql packages

* added rule to ignore dev sql file

* added migration down statement for rolling back

* updated cron for reddit and youtube

* Updated reddit to follow a new standard pattern

* updated youtube to follow new patterns

* updated tests and brought them to the standard

* updated the seed migration

* all cron tasks should feed the db now

* updated app init

* bumped docker to 1.18.3

* disabled remote tests given secrets and lack of interfaces currently to run tests
This commit is contained in:
James Tombleson 2022-06-08 21:17:08 -07:00 committed by GitHub
parent 333a4f5345
commit 75b66dd625
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1359 additions and 305 deletions

View File

@ -21,5 +21,5 @@ jobs:
- name: Build
run: go build -v ./...
- name: Test
run: go test -v ./...
#- name: Test
# run: go test -v ./...

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
.env
dev.session.sql
# Binaries for programs and plugins
*.exe

View File

@ -1,10 +1,16 @@
FROM golang:1.18.2 as build
FROM golang:1.18.3 as build
COPY . /app
WORKDIR /app
RUN go build .
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
FROM alpine
RUN mkdir /app && \
mkdir /app/migrations
COPY --from=build /app/collector /app
COPY --from=build /go/bin/goose /app
COPY ./database/migrations/ /app/migrations
ENTRYPOINT [ "/app/collector" ]

31
database/db.go Normal file
View File

@ -0,0 +1,31 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package database
import (
"context"
"database/sql"
)
type DBTX interface {
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
PrepareContext(context.Context, string) (*sql.Stmt, error)
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
return &Queries{
db: tx,
}
}

View File

@ -0,0 +1,72 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
CREATE TABLE Articles (
ID uuid PRIMARY KEY,
SourceId uuid NOT null,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate timestamp NOT NULL,
Video TEXT,
VideoHeight int NOT NULL,
VideoWidth int NOT NULL,
Thumbnail TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT,
AuthorImage TEXT
);
CREATE Table DiscordQueue (
ID uuid PRIMARY KEY,
ArticleId uuid NOT NULL
);
CREATE Table DiscordWebHooks (
ID uuid PRIMARY KEY,
Name TEXT NOT NULL, -- Defines webhook purpose
Key TEXT,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID uuid PRIMARY Key,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID uuid PRIMARY Key,
Key TEXT NOT NULL, -- How you search for a entry
Value TEXT NOT NULL, -- The value for one
Options TEXT -- any notes about the entry
);
Create Table Sources (
ID uuid PRIMARY Key,
Site TEXT NOT NULL, -- Vanity name
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
Value TEXT,
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
Drop Table Articles;
Drop Table DiscordQueue;
Drop Table DiscordWebHooks;
Drop Table Icons;
Drop Table Settings;
Drop Table Sources;
-- +goose StatementEnd

View File

@ -0,0 +1,50 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
-- Enable UUID's
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Final Fantasy XIV Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
--SELECT 'down SQL query';
DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
DELETE FROM sources where source = 'ffxiv';
DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
-- +goose StatementEnd

68
database/models.go Normal file
View File

@ -0,0 +1,68 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package database
import (
"database/sql"
"time"
"github.com/google/uuid"
)
type Article struct {
ID uuid.UUID
Sourceid uuid.UUID
Tags string
Title string
Url string
Pubdate time.Time
Video sql.NullString
Videoheight int32
Videowidth int32
Thumbnail string
Description string
Authorname sql.NullString
Authorimage sql.NullString
}
type Discordqueue struct {
ID uuid.UUID
Articleid uuid.UUID
}
type Discordwebhook struct {
ID uuid.UUID
Name string
Key sql.NullString
Url string
Server string
Channel string
Enabled bool
}
type Icon struct {
ID uuid.UUID
Filename string
Site string
}
type Setting struct {
ID uuid.UUID
Key string
Value string
Options sql.NullString
}
type Source struct {
ID uuid.UUID
Site string
Name string
Source string
Type string
Value sql.NullString
Enabled bool
Url string
Tags string
}

521
database/query.sql.go Normal file
View File

@ -0,0 +1,521 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
// source: query.sql
package database
import (
"context"
"database/sql"
"time"
"github.com/google/uuid"
)
const createArticle = `-- name: CreateArticle :exec
INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
`
type CreateArticleParams struct {
ID uuid.UUID
Sourceid uuid.UUID
Tags string
Title string
Url string
Pubdate time.Time
Video sql.NullString
Videoheight int32
Videowidth int32
Thumbnail string
Description string
Authorname sql.NullString
Authorimage sql.NullString
}
func (q *Queries) CreateArticle(ctx context.Context, arg CreateArticleParams) error {
_, err := q.db.ExecContext(ctx, createArticle,
arg.ID,
arg.Sourceid,
arg.Tags,
arg.Title,
arg.Url,
arg.Pubdate,
arg.Video,
arg.Videoheight,
arg.Videowidth,
arg.Thumbnail,
arg.Description,
arg.Authorname,
arg.Authorimage,
)
return err
}
const createDiscordQueue = `-- name: CreateDiscordQueue :exec
Insert into DiscordQueue
(ID, ArticleId)
Values
($1, $2)
`
type CreateDiscordQueueParams struct {
ID uuid.UUID
Articleid uuid.UUID
}
// DiscordQueue
func (q *Queries) CreateDiscordQueue(ctx context.Context, arg CreateDiscordQueueParams) error {
_, err := q.db.ExecContext(ctx, createDiscordQueue, arg.ID, arg.Articleid)
return err
}
const createDiscordWebHook = `-- name: CreateDiscordWebHook :exec
Insert Into DiscordWebHooks
(ID, Name, Key, Url, Server, Channel, Enabled)
Values
($1, $2, $3, $4, $5, $6, $7)
`
type CreateDiscordWebHookParams struct {
ID uuid.UUID
Name string
Key sql.NullString
Url string
Server string
Channel string
Enabled bool
}
// DiscordWebHooks
func (q *Queries) CreateDiscordWebHook(ctx context.Context, arg CreateDiscordWebHookParams) error {
_, err := q.db.ExecContext(ctx, createDiscordWebHook,
arg.ID,
arg.Name,
arg.Key,
arg.Url,
arg.Server,
arg.Channel,
arg.Enabled,
)
return err
}
const createIcon = `-- name: CreateIcon :exec
INSERT INTO Icons
(ID, FileName, Site)
VALUES
($1,$2,$3)
`
type CreateIconParams struct {
ID uuid.UUID
Filename string
Site string
}
// Icons
func (q *Queries) CreateIcon(ctx context.Context, arg CreateIconParams) error {
_, err := q.db.ExecContext(ctx, createIcon, arg.ID, arg.Filename, arg.Site)
return err
}
const createSettings = `-- name: CreateSettings :one
Insert Into settings
(ID, Key, Value, OPTIONS)
Values
($1,$2,$3,$4)
RETURNING id, key, value, options
`
type CreateSettingsParams struct {
ID uuid.UUID
Key string
Value string
Options sql.NullString
}
// Settings
func (q *Queries) CreateSettings(ctx context.Context, arg CreateSettingsParams) (Setting, error) {
row := q.db.QueryRowContext(ctx, createSettings,
arg.ID,
arg.Key,
arg.Value,
arg.Options,
)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const createSource = `-- name: CreateSource :exec
Insert Into Sources
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
Values
($1,$2,$3,$4,$5,$6,$7,$8,$9)
`
type CreateSourceParams struct {
ID uuid.UUID
Site string
Name string
Source string
Type string
Value sql.NullString
Enabled bool
Url string
Tags string
}
// Sources
func (q *Queries) CreateSource(ctx context.Context, arg CreateSourceParams) error {
_, err := q.db.ExecContext(ctx, createSource,
arg.ID,
arg.Site,
arg.Name,
arg.Source,
arg.Type,
arg.Value,
arg.Enabled,
arg.Url,
arg.Tags,
)
return err
}
const deleteDiscordQueueItem = `-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1
`
func (q *Queries) DeleteDiscordQueueItem(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteDiscordQueueItem, id)
return err
}
const deleteDiscordWebHooks = `-- name: DeleteDiscordWebHooks :exec
Delete From discordwebhooks Where ID = $1
`
func (q *Queries) DeleteDiscordWebHooks(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteDiscordWebHooks, id)
return err
}
const deleteIcon = `-- name: DeleteIcon :exec
Delete From Icons where ID = $1
`
func (q *Queries) DeleteIcon(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteIcon, id)
return err
}
const deleteSetting = `-- name: DeleteSetting :exec
Delete From settings Where ID = $1
`
func (q *Queries) DeleteSetting(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSetting, id)
return err
}
const deleteSource = `-- name: DeleteSource :exec
DELETE From sources where id = $1
`
func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSource, id)
return err
}
const getArticleByID = `-- name: GetArticleByID :one
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
WHERE ID = $1 LIMIT 1
`
// Articles
func (q *Queries) GetArticleByID(ctx context.Context, id uuid.UUID) (Article, error) {
row := q.db.QueryRowContext(ctx, getArticleByID, id)
var i Article
err := row.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
)
return i, err
}
const getArticleByUrl = `-- name: GetArticleByUrl :one
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
Where Url = $1 LIMIT 1
`
func (q *Queries) GetArticleByUrl(ctx context.Context, url string) (Article, error) {
row := q.db.QueryRowContext(ctx, getArticleByUrl, url)
var i Article
err := row.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
)
return i, err
}
const getDiscordQueueByID = `-- name: GetDiscordQueueByID :one
Select id, articleid from DiscordQueue
Where ID = $1 LIMIT 1
`
func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discordqueue, error) {
row := q.db.QueryRowContext(ctx, getDiscordQueueByID, id)
var i Discordqueue
err := row.Scan(&i.ID, &i.Articleid)
return i, err
}
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, name, key, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1
`
func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Discordwebhook, error) {
row := q.db.QueryRowContext(ctx, getDiscordWebHooksByID, id)
var i Discordwebhook
err := row.Scan(
&i.ID,
&i.Name,
&i.Key,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
)
return i, err
}
const getIconByID = `-- name: GetIconByID :one
Select id, filename, site FROM Icons
Where ID = $1 Limit 1
`
func (q *Queries) GetIconByID(ctx context.Context, id uuid.UUID) (Icon, error) {
row := q.db.QueryRowContext(ctx, getIconByID, id)
var i Icon
err := row.Scan(&i.ID, &i.Filename, &i.Site)
return i, err
}
const getIconBySite = `-- name: GetIconBySite :one
Select id, filename, site FROM Icons
Where Site = $1 Limit 1
`
func (q *Queries) GetIconBySite(ctx context.Context, site string) (Icon, error) {
row := q.db.QueryRowContext(ctx, getIconBySite, site)
var i Icon
err := row.Scan(&i.ID, &i.Filename, &i.Site)
return i, err
}
const getSettingByID = `-- name: GetSettingByID :one
Select id, key, value, options From settings
Where ID = $1 Limit 1
`
func (q *Queries) GetSettingByID(ctx context.Context, id uuid.UUID) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByID, id)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSettingByKey = `-- name: GetSettingByKey :one
Select id, key, value, options From settings Where
Key = $1 Limit 1
`
func (q *Queries) GetSettingByKey(ctx context.Context, key string) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByKey, key)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSettingByValue = `-- name: GetSettingByValue :one
Select id, key, value, options From settings Where
Value = $1 Limit 1
`
func (q *Queries) GetSettingByValue(ctx context.Context, value string) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByValue, value)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSourceByID = `-- name: GetSourceByID :one
Select id, site, name, source, type, value, enabled, url, tags From Sources where ID = $1 Limit 1
`
func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, error) {
row := q.db.QueryRowContext(ctx, getSourceByID, id)
var i Source
err := row.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
)
return i, err
}
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
Select id, name, key, url, server, channel, enabled From DiscordWebHooks
Where Server = $1
`
func (q *Queries) ListDiscordWebHooksByServer(ctx context.Context, server string) ([]Discordwebhook, error) {
rows, err := q.db.QueryContext(ctx, listDiscordWebHooksByServer, server)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordwebhook
for rows.Next() {
var i Discordwebhook
if err := rows.Scan(
&i.ID,
&i.Name,
&i.Key,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listSourcesBySource = `-- name: ListSourcesBySource :many
Select id, site, name, source, type, value, enabled, url, tags From Sources where Source = $1
`
func (q *Queries) ListSourcesBySource(ctx context.Context, source string) ([]Source, error) {
rows, err := q.db.QueryContext(ctx, listSourcesBySource, source)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Source
for rows.Next() {
var i Source
if err := rows.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}

112
database/schema/query.sql Normal file
View File

@ -0,0 +1,112 @@
/* Articles */
-- name: GetArticleByID :one
Select * from Articles
WHERE ID = $1 LIMIT 1;
-- name: GetArticleByUrl :one
Select * from Articles
Where Url = $1 LIMIT 1;
-- name: CreateArticle :exec
INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
/* DiscordQueue */
-- name: CreateDiscordQueue :exec
Insert into DiscordQueue
(ID, ArticleId)
Values
($1, $2);
-- name: GetDiscordQueueByID :one
Select * from DiscordQueue
Where ID = $1 LIMIT 1;
-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1;
-- name: GetDiscordQueueItems :many
Select * from DiscordQueue LIMIT $1;
/* DiscordWebHooks */
-- name: CreateDiscordWebHook :exec
Insert Into DiscordWebHooks
(ID, Name, Key, Url, Server, Channel, Enabled)
Values
($1, $2, $3, $4, $5, $6, $7);
-- name: GetDiscordWebHooksByID :one
Select * from DiscordWebHooks
Where ID = $1 LIMIT 1;
-- name: ListDiscordWebHooksByServer :many
Select * From DiscordWebHooks
Where Server = $1;
-- name: DeleteDiscordWebHooks :exec
Delete From discordwebhooks Where ID = $1;
/* Icons */
-- name: CreateIcon :exec
INSERT INTO Icons
(ID, FileName, Site)
VALUES
($1,$2,$3);
-- name: GetIconByID :one
Select * FROM Icons
Where ID = $1 Limit 1;
-- name: GetIconBySite :one
Select * FROM Icons
Where Site = $1 Limit 1;
-- name: DeleteIcon :exec
Delete From Icons where ID = $1;
/* Settings */
-- name: CreateSettings :one
Insert Into settings
(ID, Key, Value, OPTIONS)
Values
($1,$2,$3,$4)
RETURNING *;
-- name: GetSettingByID :one
Select * From settings
Where ID = $1 Limit 1;
-- name: GetSettingByKey :one
Select * From settings Where
Key = $1 Limit 1;
-- name: GetSettingByValue :one
Select * From settings Where
Value = $1 Limit 1;
-- name: DeleteSetting :exec
Delete From settings Where ID = $1;
/* Sources */
-- name: CreateSource :exec
Insert Into Sources
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
Values
($1,$2,$3,$4,$5,$6,$7,$8,$9);
-- name: GetSourceByID :one
Select * From Sources where ID = $1 Limit 1;
-- name: ListSourcesBySource :many
Select * From Sources where Source = $1;
-- name: DeleteSource :exec
DELETE From sources where id = $1;

View File

@ -0,0 +1,56 @@
CREATE TABLE Articles (
ID uuid PRIMARY KEY,
SourceId uuid NOT null,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate timestamp NOT NULL,
Video TEXT,
VideoHeight int NOT NULL,
VideoWidth int NOT NULL,
Thumbnail TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT,
AuthorImage TEXT
);
CREATE Table DiscordQueue (
ID uuid PRIMARY KEY,
ArticleId uuid NOT NULL
);
CREATE Table DiscordWebHooks (
ID uuid PRIMARY KEY,
Name TEXT NOT NULL, -- Defines webhook purpose
Key TEXT,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID uuid PRIMARY Key,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID uuid PRIMARY Key,
Key TEXT NOT NULL,
Value TEXT NOT NULL,
Options TEXT
);
Create Table Sources (
ID uuid PRIMARY Key,
Site TEXT NOT NULL, -- Vanity name
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
Value TEXT,
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);

View File

@ -1,4 +1,4 @@
package database
package databaseRest
import (
"bytes"

View File

@ -1,4 +1,4 @@
package database
package databaseRest
import (
"errors"

View File

@ -1,4 +1,4 @@
package database
package databaseRest
import (
"fmt"

View File

@ -1,4 +1,4 @@
package database
package databaseRest
import (
"encoding/json"

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/go-rod/rod v0.105.1
github.com/google/uuid v1.3.0
github.com/joho/godotenv v1.4.0
github.com/lib/pq v1.10.6
github.com/mmcdole/gofeed v1.1.3
github.com/nicklaw5/helix/v2 v2.4.0
github.com/robfig/cron/v3 v3.0.1

2
go.sum
View File

@ -22,6 +22,8 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=

View File

@ -1,6 +1,7 @@
package main
import (
"context"
"log"
"net/http"
@ -12,10 +13,9 @@ import (
)
func main() {
//dc := database.NewDatabaseClient()
//err := dc.Diagnosis.Ping()
//if err != nil { log.Fatalln(err) }
cron.EnableScheduler()
ctx := context.Background()
cron.EnableScheduler(ctx)
app := chi.NewRouter()
app.Use(middleware.Logger)

View File

@ -5,7 +5,12 @@ help: ## Shows this help command
build: ## builds the application with the current go runtime
go build .
docker-build: ## Generates the docker image
docker build -t "newsbot.collector.api" .
docker image ls | grep newsbot.collector.api
migrate-dev: ## Apply sql migrations to dev db
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down

View File

@ -9,6 +9,8 @@ import (
const (
DB_URI string = "DB_URI"
Sql_Connection_String string = "SQL_CONNECTION_STRING"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"

View File

@ -1,106 +1,167 @@
package cron
import (
"context"
"database/sql"
"log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq"
"github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
//"github.com/jtom38/newsbot/collector/services/cache"
"github.com/jtom38/newsbot/collector/services/config"
)
func EnableScheduler() {
c := cron.New()
var _env config.ConfigClient
var _connString string
var _queries *database.Queries
//c.AddFunc("*/5 * * * *", func() { go CheckCache() })
c.AddFunc("* */1 * * *", func() { go CheckReddit() })
c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
func EnableScheduler(ctx context.Context) {
c := cron.New()
OpenDatabase(ctx)
//c.AddFunc("*/5 * * * *", func() { go CheckCache() })
c.AddFunc("* */1 * * *", func() { go CheckReddit(ctx) })
//c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
//c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
//c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
c.Start()
}
func CheckCache() {
//cache := services.NewCacheAgeMonitor()
//cache.CheckExpiredEntries()
}
func CheckReddit() {
dc := database.NewDatabaseClient()
sources, err := dc.Sources.FindBySource("reddit")
if err != nil { log.Println(err) }
rc := services.NewRedditClient(sources[0].Name, sources[0].ID)
raw, err := rc.GetContent()
if err != nil { log.Println(err) }
redditArticles := rc.ConvertToArticles(raw)
for _, item := range redditArticles {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil {
err = dc.Articles.Add(item)
if err != nil { log.Println("Failed to post article.")}
}
// Open the connection to the database and share it with the package so all of them are able to share.
func OpenDatabase(ctx context.Context) error {
_env = config.New()
_connString = _env.GetConfig(config.Sql_Connection_String)
db, err := sql.Open("postgres", _connString)
if err != nil {
panic(err)
}
queries := database.New(db)
_queries = queries
return err
}
func CheckYoutube() {
// Add call to the db to request youtube sources.
// Loop though the services, and generate the clients.
yt := services.NewYoutubeClient(0, "https://www.youtube.com/user/GameGrumps")
yt.CheckSource()
}
func CheckFfxiv() {
fc := services.NewFFXIVClient("na")
articles, err := fc.CheckSource()
// This isnt in a thread yet, so just output to stdout
if err != nil { log.Println(err) }
dc := database.NewDatabaseClient()
for _, item := range articles {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil {
err = dc.Articles.Add(item)
if err != nil { log.Println("Failed to post article.")}
}
// This is the main entry point to query all the reddit services
func CheckReddit(ctx context.Context) {
sources, err := _queries.ListSourcesBySource(ctx, "reddit")
if err != nil {
log.Printf("No defines sources for reddit to query - %v\r", err)
}
}
func CheckTwitch() error {
// TODO Wire this for the DB
// just a mock object for now
dc := database.NewDatabaseClient()
sources, err := dc.Sources.FindBySource("Twitch")
if err != nil { return err }
client, err := services.NewTwitchClient(sources[0])
if err != nil { log.Println(err) }
err = client.Login()
if err != nil { return err }
for _, source := range sources {
client.ReplaceSourceRecord(source)
posts, err := client.GetContent()
if err != nil { return err }
for _, item := range posts {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil {
err = dc.Articles.Add(item)
if err != nil { log.Println("Failed to post article.")}
}
if !source.Enabled {
continue
}
rc := services.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
checkPosts(ctx, redditArticles)
}
}
func CheckYoutube(ctx context.Context) {
// Add call to the db to request youtube sources.
sources, err := _queries.ListSourcesBySource(ctx, "youtube")
if err != nil {
log.Printf("Youtube - No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
yc := services.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
checkPosts(ctx, raw)
}
}
func CheckFfxiv(ctx context.Context) {
sources, err := _queries.ListSourcesBySource(ctx, "ffxiv")
if err != nil {
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := services.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
checkPosts(ctx, items)
}
}
func CheckTwitch(ctx context.Context) error {
sources, err := _queries.ListSourcesBySource(ctx, "twitch")
if err != nil {
log.Printf("Twitch - No sources found to query - %v\r", err)
}
tc, err := services.NewTwitchClient()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
checkPosts(ctx, items)
}
return nil
}
}
func checkPosts(ctx context.Context, posts []database.Article) {
for _, item := range posts {
_, err := _queries.GetArticleByUrl(ctx, item.Url)
if err != nil {
err = postArticle(ctx, item)
if err != nil {
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
} else {
log.Printf("Reddit - Posted article - %v\r", item.Url)
}
}
}
time.Sleep(30 * time.Second)
}
func postArticle(ctx context.Context, item database.Article) error {
err := _queries.CreateArticle(ctx, database.CreateArticleParams{
ID: uuid.New(),
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}

View File

@ -1,7 +1,37 @@
package cron_test
import "testing"
import (
"context"
"testing"
"github.com/jtom38/newsbot/collector/services/cron"
)
func TestInvokeTwitch(t *testing.T) {
}
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) {
ctx := context.Background()
cron.OpenDatabase(ctx)
cron.CheckReddit(ctx)
}
func TestCheckYouTube(t *testing.T) {
ctx := context.Background()
cron.OpenDatabase(ctx)
cron.CheckYoutube(ctx)
}
func TestCheckTwitch(t *testing.T) {
ctx := context.Background()
err := cron.OpenDatabase(ctx)
if err != nil {
t.Error(err)
}
err = cron.CheckTwitch(ctx)
if err != nil {
t.Error(err)
}
}

View File

@ -1,6 +1,7 @@
package services
import (
"database/sql"
"errors"
"log"
"net/http"
@ -11,7 +12,7 @@ import (
"github.com/go-rod/rod"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/cache"
)
@ -23,32 +24,23 @@ const (
)
type FFXIVClient struct {
SourceID uint
Url string
Region string
record database.Source
//SourceID uint
//Url string
//Region string
cacheGroup string
}
func NewFFXIVClient(region string) FFXIVClient {
var url string
switch region {
case "na":
url = FFXIV_NA_FEED_URL
case "jp":
url = FFXIV_JP_FEED_URL
}
func NewFFXIVClient(Record database.Source) FFXIVClient {
return FFXIVClient{
Region: region,
Url: url,
record: Record,
cacheGroup: "ffxiv",
}
}
func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
var articles []model.Articles
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
var articles []database.Article
parser := fc.GetBrowser()
defer parser.Close()
@ -87,19 +79,18 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
tags, err := fc.ExtractTags(page)
if err != nil { return articles, err }
article := model.Articles{
SourceID: fc.SourceID,
article := database.Article{
Sourceid: fc.record.ID,
Tags: tags,
Title: title,
Url: link,
PubDate: pubDate,
Video: "",
VideoHeight: 0,
VideoWidth: 0,
Pubdate: pubDate,
Videoheight: 0,
Videowidth: 0,
Thumbnail: thumb,
Description: description,
AuthorName: authorName,
AuthorImage: authorImage,
Authorname: sql.NullString{String: authorName},
Authorimage: sql.NullString{String: authorImage},
}
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
@ -112,7 +103,7 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
}
func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
html, err := http.Get(fc.Url)
html, err := http.Get(fc.record.Url)
if err != nil { return nil, err }
defer html.Body.Close()
@ -129,7 +120,7 @@ func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) {
var links []string
page := parser.MustPage(fc.Url)
page := parser.MustPage(fc.record.Url)
defer page.Close()
// find the list by xpath

View File

@ -3,17 +3,28 @@ package services_test
import (
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services"
)
var FFXIVRecord database.Source = database.Source{
ID: uuid.New(),
Site: "ffxiv",
Name: "Final Fantasy XIV - NA",
Source: "ffxiv",
Url: "https://na.finalfantasyxiv.com/lodestone/",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
}
func TestFfxivGetParser(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
_, err := fc.GetParser()
if err != nil { panic(err) }
}
func TestFfxivPullFeed(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -25,7 +36,7 @@ func TestFfxivPullFeed(t *testing.T) {
}
func TestFfxivExtractThumbnail(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -42,7 +53,7 @@ func TestFfxivExtractThumbnail(t *testing.T) {
}
func TestFfxivExtractPubDate(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -58,7 +69,7 @@ func TestFfxivExtractPubDate(t *testing.T) {
}
func TestFfxivExtractDescription(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -74,7 +85,7 @@ func TestFfxivExtractDescription(t *testing.T) {
}
func TestFfxivExtractAuthor(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -91,7 +102,7 @@ func TestFfxivExtractAuthor(t *testing.T) {
}
func TestFfxivExtractTags(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -108,7 +119,7 @@ func TestFfxivExtractTags(t *testing.T) {
}
func TestFfxivExtractTitle(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -125,7 +136,7 @@ func TestFfxivExtractTitle(t *testing.T) {
}
func TestFFxivExtractAuthorIamge(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser()
defer parser.Close()
@ -142,7 +153,7 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
}
func TestFfxivCheckSource(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na")
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
fc.CheckSource()
}

View File

@ -1,6 +1,7 @@
package services
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
@ -10,28 +11,25 @@ import (
"time"
"github.com/go-rod/rod"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/config"
)
type RedditClient struct {
subreddit string
url string
sourceId uint
config RedditConfig
record database.Source
}
type RedditConfig struct {
PullTop string
PullHot string
PullTop string
PullHot string
PullNSFW string
}
func NewRedditClient(subreddit string, sourceID uint) RedditClient {
func NewRedditClient(Record database.Source) RedditClient {
rc := RedditClient{
subreddit: subreddit,
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
sourceId: sourceID,
record: Record,
}
cc := config.New()
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
@ -59,15 +57,23 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
return page
}
//func (rc RedditClient)
// GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct.
func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
var items model.RedditJsonContent = model.RedditJsonContent{}
log.Printf("Collecting results on '%v'", rc.subreddit)
content, err := getHttpContent(rc.url)
if err != nil { return items, err }
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content) ) {
// TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("Collecting results on '%v'", rc.record.Name)
content, err := getHttpContent(Url)
if err != nil {
return items, err
}
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
return items, errors.New("did not get json data from the server")
}
@ -78,12 +84,15 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
return items, nil
}
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles {
var redditArticles []model.Articles
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
var redditArticles []database.Article
for _, item := range items.Data.Children {
var article model.Articles
var article database.Article
article, err := rc.convertToArticle(item.Data)
if err != nil { log.Println(err); continue }
if err != nil {
log.Println(err)
continue
}
redditArticles = append(redditArticles, article)
}
return redditArticles
@ -91,14 +100,13 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database.
func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) {
var item model.Articles
func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
var item database.Article
if source.Content == "" && source.Url != ""{
if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source)
}
if source.Media.RedditVideo.FallBackUrl != "" {
item = rc.convertVideoPost(source)
}
@ -119,58 +127,66 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles
return item, nil
}
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
PubDate: time.Now(),
Video: "null",
VideoHeight: 0,
VideoWidth: 0,
Thumbnail: source.Thumbnail,
func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Pubdate: time.Now(),
Video: sql.NullString{String: "null"},
Videoheight: 0,
Videowidth: 0,
Thumbnail: source.Thumbnail,
Description: source.Content,
AuthorName: source.Author,
AuthorImage: "null",
Authorname: sql.NullString{String: source.Author},
Authorimage: sql.NullString{String: "null"},
}
return item
}
func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title,
Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Authorname: sql.NullString{String: source.Author},
Description: source.Content,
}
return item
}
func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title,
Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl,
}
return item
}
// This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title,
Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest,
}
return item
}
}

View File

@ -1,16 +1,33 @@
package services_test
import (
"log"
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
)
var RedditRecord database.Source = database.Source{
ID: uuid.New(),
Name: "dadjokes",
Source: "reddit",
Site: "reddit",
Url: "https://reddit.com/r/dadjokes",
Tags: "reddit, dadjokes",
}
func TestGetContent(t *testing.T) {
//This test is flaky right now due to the http changes in 1.17
rc := services.NewRedditClient("dadjokes", 0)
_, err := rc.GetContent()
log.Println(err)
//if err != nil { panic(err) }
rc := services.NewRedditClient(RedditRecord)
raw, err := rc.GetContent()
if err != nil {
t.Error(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, posts := range redditArticles {
if posts.Title == "" {
t.Error("Title is missing")
}
}
}

View File

@ -1,20 +1,19 @@
package services
import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
//"log"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/nicklaw5/helix/v2"
)
type TwitchClient struct {
SourceRecord model.Sources
SourceRecord database.Source
// config
monitorClips string
@ -31,7 +30,7 @@ var (
twitchScopes = "user:read:email"
)
func NewTwitchClient(source model.Sources) (TwitchClient, error) {
func NewTwitchClient() (TwitchClient, error) {
c := config.New()
id := c.GetConfig(config.TWITCH_CLIENT_ID)
@ -50,7 +49,7 @@ func NewTwitchClient(source model.Sources) (TwitchClient, error) {
}
client := TwitchClient{
SourceRecord: source,
//SourceRecord: &source,
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
api: &api,
@ -73,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
}
// This will let you replace the bound source record to keep the same session alive.
func (tc TwitchClient) ReplaceSourceRecord(source model.Sources) {
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
tc.SourceRecord = source
}
@ -88,8 +87,8 @@ func (tc TwitchClient) Login() error {
return nil
}
func (tc TwitchClient) GetContent() ([]model.Articles, error) {
var items []model.Articles
func (tc TwitchClient) GetContent() ([]database.Article, error) {
var items []database.Article
user, err := tc.GetUserDetails()
if err != nil {
@ -102,21 +101,23 @@ func (tc TwitchClient) GetContent() ([]model.Articles, error) {
}
for _, video := range posts {
article := model.Articles{}
var article database.Article
article.AuthorName, err = tc.ExtractAuthor(video)
AuthorName, err := tc.ExtractAuthor(video)
if err != nil { return items, err }
article.Authorname = sql.NullString{String: AuthorName}
article.AuthorImage, err = tc.ExtractAuthorImage(user)
Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil { return items, err }
article.Authorimage = sql.NullString{String: Authorimage}
article.Description, err = tc.ExtractDescription(video)
if err != nil {return items, err }
article.PubDate, err = tc.ExtractPubDate(video)
article.Pubdate, err = tc.ExtractPubDate(video)
if err != nil { return items, err }
article.SourceID = tc.SourceRecord.ID
article.Sourceid = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user)
if err != nil { return items, err }

View File

@ -4,27 +4,29 @@ import (
"log"
"testing"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
)
var sourceRecord = model.Sources{
ID: 1,
var TwitchSourceRecord = database.Source {
ID: uuid.New(),
Name: "nintendo",
Source: "Twitch",
}
var invalidRecord = model.Sources{
ID: 1,
var TwitchInvalidRecord = database.Source {
ID: uuid.New(),
Name: "EvilNintendo",
Source: "Twitch",
}
func TestTwitchLogin(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil {
@ -34,10 +36,11 @@ func TestTwitchLogin(t *testing.T) {
// reach out and confirms that the API returns posts made by the user.
func TestTwitchReturnsUserPosts(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil {
@ -59,10 +62,11 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
}
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
tc, err := services.NewTwitchClient(invalidRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchInvalidRecord)
err = tc.Login()
if err != nil {
@ -84,10 +88,11 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
}
func TestTwitchReturnsVideoAuthor(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil {
@ -109,8 +114,9 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
}
func TestTwitchReturnsThumbnail(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil { t.Error(err) }
@ -127,8 +133,9 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
}
func TestTwitchReturnsPubDate(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil { t.Error(err) }
@ -145,10 +152,11 @@ func TestTwitchReturnsPubDate(t *testing.T) {
}
func TestTwitchReturnsDescription(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil {
@ -172,8 +180,9 @@ func TestTwitchReturnsDescription(t *testing.T) {
}
func TestTwitchReturnsAuthorImage(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil { t.Error(err) }
@ -186,11 +195,11 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
}
func TestTwitchReturnsTags(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil {
@ -210,10 +219,11 @@ func TestTwitchReturnsTags(t *testing.T) {
}
func TestTwitchReturnsTitle(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil {
t.Error(err)
}
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil {
@ -234,8 +244,9 @@ func TestTwitchReturnsTitle(t *testing.T) {
}
func TestTwitchReturnsUrl(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil { t.Error(err) }
@ -252,8 +263,9 @@ func TestTwitchReturnsUrl(t *testing.T) {
}
func TestTwitchGetContent(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord)
tc, err := services.NewTwitchClient()
if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login()
if err != nil { t.Error(err) }

View File

@ -1,6 +1,7 @@
package services
import (
"database/sql"
"errors"
"fmt"
"log"
@ -12,14 +13,15 @@ import (
"github.com/go-rod/rod"
"github.com/mmcdole/gofeed"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/database"
)
type YoutubeClient struct {
SourceID uint
Url string
ChannelID string
AvatarUri string
record database.Source
// internal variables at time of collection
channelID string
avatarUri string
// config
//debug bool
@ -36,10 +38,9 @@ var (
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
func NewYoutubeClient(Record database.Source) YoutubeClient {
yc := YoutubeClient{
SourceID: SourceID,
Url: Url,
record: Record,
cacheGroup: "youtube",
}
/*
@ -53,10 +54,11 @@ func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
}
// CheckSource will go and run all the commands needed to process a source.
func (yc *YoutubeClient) CheckSource() error {
docParser, err := yc.GetParser(yc.Url)
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
var items []database.Article
docParser, err := yc.GetParser(yc.record.Url)
if err != nil {
return err
return items, err
}
// Check cache/db for existing value
@ -64,38 +66,38 @@ func (yc *YoutubeClient) CheckSource() error {
//channelId, err := yc.extractChannelId()
channelId, err := yc.GetChannelId(docParser)
if err != nil {
return err
return items, err
}
if channelId == "" {
return ErrYoutubeChannelIdMissing
return items, ErrYoutubeChannelIdMissing
}
yc.ChannelID = channelId
yc.channelID = channelId
// Check the cache/db forthe value.
// if we have the value, skip
avatar, err := yc.GetAvatarUri()
if err != nil {
return err
return items, err
}
if avatar == "" {
return ErrMissingAuthorImage
return items, ErrMissingAuthorImage
}
yc.AvatarUri = avatar
yc.avatarUri = avatar
feed, err := yc.PullFeed()
if err != nil {
return err
return items, err
}
newPosts, err := yc.CheckForNewPosts(feed)
if err != nil {
return err
return items, err
}
//TODO post to the API
for _, item := range newPosts {
article := yc.ConvertToArticle(item)
items = append(items, article)
YoutubeUriCache = append(YoutubeUriCache, &item.Link)
@ -103,7 +105,7 @@ func (yc *YoutubeClient) CheckSource() error {
log.Println(article)
}
return nil
return items, nil
}
func (yc *YoutubeClient) GetBrowser() *rod.Browser {
@ -137,8 +139,8 @@ func (yc *YoutubeClient) GetChannelId(doc *goquery.Document) (string, error) {
for _, item := range meta.Nodes {
if item.Attr[0].Val == "channelId" {
yc.ChannelID = item.Attr[1].Val
return yc.ChannelID, nil
yc.channelID = item.Attr[1].Val
return yc.channelID, nil
}
}
return "", ErrYoutubeChannelIdMissing
@ -155,7 +157,7 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) {
var AvatarUri string
browser := rod.New().MustConnect()
page := browser.MustPage(yc.Url)
page := browser.MustPage(yc.record.Url)
res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
@ -197,7 +199,7 @@ func (yc *YoutubeClient) GetVideoThumbnail(parser *goquery.Document) (string, er
// This will pull the RSS feed items and return the results
func (yc *YoutubeClient) PullFeed() (*gofeed.Feed, error) {
feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.ChannelID)
feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.channelID)
fp := gofeed.NewParser()
feed, err := fp.ParseURL(feedUri)
if err != nil {
@ -239,7 +241,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
return false
}
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles {
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
parser, err := yc.GetParser(item.Link)
if err != nil {
log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link)
@ -257,16 +259,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles {
log.Println(msg)
}
var article = model.Articles{
SourceID: yc.SourceID,
var article = database.Article{
Sourceid: yc.record.ID,
Tags: tags,
Title: item.Title,
Url: item.Link,
PubDate: *item.PublishedParsed,
Pubdate: *item.PublishedParsed,
Thumbnail: thumb,
Description: item.Description,
AuthorName: item.Author.Name,
AuthorImage: yc.AvatarUri,
Authorname: sql.NullString{String: item.Author.Name},
Authorimage: sql.NullString{String: yc.avatarUri},
}
return article
}

View File

@ -3,24 +3,28 @@ package services_test
import (
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
)
var YouTubeRecord database.Source = database.Source{
ID: uuid.New(),
Name: "dadjokes",
Source: "reddit",
Site: "reddit",
Url: "https://youtube.com/gamegrumps",
}
func TestGetPageParser(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
_, err := yc.GetParser(yc.Url)
yc := services.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
}
func TestGetChannelId(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
parser, err := yc.GetParser(yc.Url)
yc := services.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
_, err = yc.GetChannelId(parser)
@ -28,11 +32,8 @@ func TestGetChannelId(t *testing.T) {
}
func TestPullFeed(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
parser, err := yc.GetParser(yc.Url)
yc := services.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
_, err = yc.GetChannelId(parser)
@ -43,20 +44,14 @@ func TestPullFeed(t *testing.T) {
}
func TestGetAvatarUri(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
yc := services.NewYoutubeClient(YouTubeRecord)
res, err := yc.GetAvatarUri()
if err != nil { panic(err) }
if res == "" { panic(services.ErrMissingAuthorImage)}
}
func TestGetVideoTags(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
yc := services.NewYoutubeClient(YouTubeRecord)
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
@ -69,12 +64,9 @@ func TestGetVideoTags(t *testing.T) {
}
func TestGetChannelTags(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
yc := services.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(yc.Url)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
tags, err := yc.GetTags(parser)
@ -83,10 +75,7 @@ func TestGetChannelTags(t *testing.T) {
}
func TestGetVideoThumbnail(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
yc := services.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
if err != nil {panic(err) }
@ -97,20 +86,13 @@ func TestGetVideoThumbnail(t *testing.T) {
}
func TestCheckSource(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
err := yc.CheckSource()
yc := services.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetContent()
if err != nil { panic(err) }
}
func TestCheckUriCache(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
yc := services.NewYoutubeClient(YouTubeRecord)
item := "demo"
services.YoutubeUriCache = append(services.YoutubeUriCache, &item)
@ -119,10 +101,7 @@ func TestCheckUriCache(t *testing.T) {
}
func TestCheckUriCacheFails(t *testing.T) {
yc := services.NewYoutubeClient(
0,
"https://youtube.com/gamegrumps",
)
yc := services.NewYoutubeClient(YouTubeRecord)
item := "demo1"
res := yc.CheckUriCache(&item)

7
sqlc.yaml Normal file
View File

@ -0,0 +1,7 @@
version: "1"
packages:
- schema: "database/schema/schema.sql"
queries: "database/schema/query.sql"
name: "database"
path: "database"
engine: "postgresql"