From 75b66dd625f95b2c2e80965305861e3cd2314787 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 8 Jun 2022 21:17:08 -0700 Subject: [PATCH] Feature/sql (#8) * Using sqlc to generate queries and goose for migrations. The intial tests look good. * moving the old calls away for now. Might use this in a package later on. * Added postgres driver * Updated the dockerfile to support sql migrations * added sqlc config file * updated schema and starting a seed script * updated models to use the database ones * updated reddit cron to talk to the db * added env for sql connection string * got the reddit source working with the db and posting articles * added sql packages * added rule to ignore dev sql file * added migration down statement for rolling back * updated cron for reddit and youtube * Updated reddit to follow a new standard pattern * updated youtube to follow new patterns * updated tests and brought them to the standard * updated the seed migration * all cron tasks should feed the db now * updated app init * bumped docker to 1.18.3 * disabled remote tests given secrets and lack of interfaces currently to run tests --- .github/workflows/go-build.yml | 4 +- .gitignore | 1 + Dockerfile | 8 +- database/db.go | 31 ++ database/migrations/20220522083756_init.sql | 72 +++ database/migrations/20220529082459_seed.sql | 50 ++ database/models.go | 68 +++ database/query.sql.go | 521 ++++++++++++++++++++ database/schema/query.sql | 112 +++++ database/schema/schema.sql | 56 +++ {database => databaseRest}/articles.go | 2 +- {database => databaseRest}/common.go | 2 +- {database => databaseRest}/diagnosis.go | 2 +- {database => databaseRest}/sources.go | 2 +- go.mod | 1 + go.sum | 2 + main.go | 8 +- makefile | 7 +- services/config/config.go | 2 + services/cron/scheduler.go | 221 ++++++--- services/cron/scheduler_test.go | 34 +- services/ffxiv.go | 47 +- services/ffxiv_test.go | 31 +- services/reddit.go | 134 ++--- services/reddit_test.go | 27 +- services/twitch.go | 29 +- services/twitch_test.go | 48 +- services/youtube.go | 64 +-- services/youtube_test.go | 71 +-- sqlc.yaml | 7 + 30 files changed, 1359 insertions(+), 305 deletions(-) create mode 100644 database/db.go create mode 100644 database/migrations/20220522083756_init.sql create mode 100644 database/migrations/20220529082459_seed.sql create mode 100644 database/models.go create mode 100644 database/query.sql.go create mode 100644 database/schema/query.sql create mode 100644 database/schema/schema.sql rename {database => databaseRest}/articles.go (98%) rename {database => databaseRest}/common.go (98%) rename {database => databaseRest}/diagnosis.go (94%) rename {database => databaseRest}/sources.go (97%) create mode 100644 sqlc.yaml diff --git a/.github/workflows/go-build.yml b/.github/workflows/go-build.yml index 53daa00..387ea69 100644 --- a/.github/workflows/go-build.yml +++ b/.github/workflows/go-build.yml @@ -21,5 +21,5 @@ jobs: - name: Build run: go build -v ./... - - name: Test - run: go test -v ./... + #- name: Test + # run: go test -v ./... diff --git a/.gitignore b/.gitignore index 9b6d9fa..bc73b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +dev.session.sql # Binaries for programs and plugins *.exe diff --git a/Dockerfile b/Dockerfile index 290b86a..29ea3ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,16 @@ -FROM golang:1.18.2 as build +FROM golang:1.18.3 as build COPY . /app WORKDIR /app RUN go build . +RUN go install github.com/pressly/goose/v3/cmd/goose@latest FROM alpine +RUN mkdir /app && \ + mkdir /app/migrations COPY --from=build /app/collector /app +COPY --from=build /go/bin/goose /app +COPY ./database/migrations/ /app/migrations + ENTRYPOINT [ "/app/collector" ] \ No newline at end of file diff --git a/database/db.go b/database/db.go new file mode 100644 index 0000000..c048805 --- /dev/null +++ b/database/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.13.0 + +package database + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/database/migrations/20220522083756_init.sql b/database/migrations/20220522083756_init.sql new file mode 100644 index 0000000..e0bb875 --- /dev/null +++ b/database/migrations/20220522083756_init.sql @@ -0,0 +1,72 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'up SQL query'; +CREATE TABLE Articles ( + ID uuid PRIMARY KEY, + SourceId uuid NOT null, + Tags TEXT NOT NULL, + Title TEXT NOT NULL, + Url TEXT NOT NULL, + PubDate timestamp NOT NULL, + Video TEXT, + VideoHeight int NOT NULL, + VideoWidth int NOT NULL, + Thumbnail TEXT NOT NULL, + Description TEXT NOT NULL, + AuthorName TEXT, + AuthorImage TEXT +); + +CREATE Table DiscordQueue ( + ID uuid PRIMARY KEY, + ArticleId uuid NOT NULL +); + +CREATE Table DiscordWebHooks ( + ID uuid PRIMARY KEY, + Name TEXT NOT NULL, -- Defines webhook purpose + Key TEXT, + Url TEXT NOT NULL, -- Webhook Url + Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence + Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence + Enabled BOOLEAN NOT NULL +); + +CREATE Table Icons ( + ID uuid PRIMARY Key, + FileName TEXT NOT NULL, + Site TEXT NOT NULL +); + +Create Table Settings ( + ID uuid PRIMARY Key, + Key TEXT NOT NULL, -- How you search for a entry + Value TEXT NOT NULL, -- The value for one + Options TEXT -- any notes about the entry +); + +Create Table Sources ( + ID uuid PRIMARY Key, + Site TEXT NOT NULL, -- Vanity name + Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes + Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube + Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag + Value TEXT, + Enabled BOOLEAN NOT NULL, + Url TEXT NOT NULL, + Tags TEXT NOT NULL +); + + + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +Drop Table Articles; +Drop Table DiscordQueue; +Drop Table DiscordWebHooks; +Drop Table Icons; +Drop Table Settings; +Drop Table Sources; +-- +goose StatementEnd diff --git a/database/migrations/20220529082459_seed.sql b/database/migrations/20220529082459_seed.sql new file mode 100644 index 0000000..d001ad0 --- /dev/null +++ b/database/migrations/20220529082459_seed.sql @@ -0,0 +1,50 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'up SQL query'; + +-- Enable UUID's +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- Final Fantasy XIV Entries +INSERT INTO sources VALUES +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); +INSERT INTO sources VALUES +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); +INSERT INTO sources VALUES +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); +INSERT INTO sources VALUES +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); +INSERT INTO sources VALUES +(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); + +-- Reddit Entries +INSERT INTO sources VALUES +(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); +INSERT INTO sources VALUES +(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); + +-- Youtube Entries +INSERT INTO sources VALUES +(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); + +-- RSS Entries +INSERT INTO sources VALUES +(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); + +-- Twitch Entries +INSERT INTO sources VALUES +(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +--SELECT 'down SQL query'; + +DELETE FROM sources where source = 'reddit' and name = 'dadjokes'; +DELETE FROM sources where source = 'reddit' and name = 'steamdeck'; +DELETE FROM sources where source = 'ffxiv'; +DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo'; +DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps'; +DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck'; +-- +goose StatementEnd diff --git a/database/models.go b/database/models.go new file mode 100644 index 0000000..58ea663 --- /dev/null +++ b/database/models.go @@ -0,0 +1,68 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.13.0 + +package database + +import ( + "database/sql" + "time" + + "github.com/google/uuid" +) + +type Article struct { + ID uuid.UUID + Sourceid uuid.UUID + Tags string + Title string + Url string + Pubdate time.Time + Video sql.NullString + Videoheight int32 + Videowidth int32 + Thumbnail string + Description string + Authorname sql.NullString + Authorimage sql.NullString +} + +type Discordqueue struct { + ID uuid.UUID + Articleid uuid.UUID +} + +type Discordwebhook struct { + ID uuid.UUID + Name string + Key sql.NullString + Url string + Server string + Channel string + Enabled bool +} + +type Icon struct { + ID uuid.UUID + Filename string + Site string +} + +type Setting struct { + ID uuid.UUID + Key string + Value string + Options sql.NullString +} + +type Source struct { + ID uuid.UUID + Site string + Name string + Source string + Type string + Value sql.NullString + Enabled bool + Url string + Tags string +} diff --git a/database/query.sql.go b/database/query.sql.go new file mode 100644 index 0000000..c233cc5 --- /dev/null +++ b/database/query.sql.go @@ -0,0 +1,521 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.13.0 +// source: query.sql + +package database + +import ( + "context" + "database/sql" + "time" + + "github.com/google/uuid" +) + +const createArticle = `-- name: CreateArticle :exec +INSERT INTO Articles +(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage) +Values +($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) +` + +type CreateArticleParams struct { + ID uuid.UUID + Sourceid uuid.UUID + Tags string + Title string + Url string + Pubdate time.Time + Video sql.NullString + Videoheight int32 + Videowidth int32 + Thumbnail string + Description string + Authorname sql.NullString + Authorimage sql.NullString +} + +func (q *Queries) CreateArticle(ctx context.Context, arg CreateArticleParams) error { + _, err := q.db.ExecContext(ctx, createArticle, + arg.ID, + arg.Sourceid, + arg.Tags, + arg.Title, + arg.Url, + arg.Pubdate, + arg.Video, + arg.Videoheight, + arg.Videowidth, + arg.Thumbnail, + arg.Description, + arg.Authorname, + arg.Authorimage, + ) + return err +} + +const createDiscordQueue = `-- name: CreateDiscordQueue :exec +Insert into DiscordQueue +(ID, ArticleId) +Values +($1, $2) +` + +type CreateDiscordQueueParams struct { + ID uuid.UUID + Articleid uuid.UUID +} + +// DiscordQueue +func (q *Queries) CreateDiscordQueue(ctx context.Context, arg CreateDiscordQueueParams) error { + _, err := q.db.ExecContext(ctx, createDiscordQueue, arg.ID, arg.Articleid) + return err +} + +const createDiscordWebHook = `-- name: CreateDiscordWebHook :exec +Insert Into DiscordWebHooks +(ID, Name, Key, Url, Server, Channel, Enabled) +Values +($1, $2, $3, $4, $5, $6, $7) +` + +type CreateDiscordWebHookParams struct { + ID uuid.UUID + Name string + Key sql.NullString + Url string + Server string + Channel string + Enabled bool +} + +// DiscordWebHooks +func (q *Queries) CreateDiscordWebHook(ctx context.Context, arg CreateDiscordWebHookParams) error { + _, err := q.db.ExecContext(ctx, createDiscordWebHook, + arg.ID, + arg.Name, + arg.Key, + arg.Url, + arg.Server, + arg.Channel, + arg.Enabled, + ) + return err +} + +const createIcon = `-- name: CreateIcon :exec + +INSERT INTO Icons +(ID, FileName, Site) +VALUES +($1,$2,$3) +` + +type CreateIconParams struct { + ID uuid.UUID + Filename string + Site string +} + +// Icons +func (q *Queries) CreateIcon(ctx context.Context, arg CreateIconParams) error { + _, err := q.db.ExecContext(ctx, createIcon, arg.ID, arg.Filename, arg.Site) + return err +} + +const createSettings = `-- name: CreateSettings :one + +Insert Into settings +(ID, Key, Value, OPTIONS) +Values +($1,$2,$3,$4) +RETURNING id, key, value, options +` + +type CreateSettingsParams struct { + ID uuid.UUID + Key string + Value string + Options sql.NullString +} + +// Settings +func (q *Queries) CreateSettings(ctx context.Context, arg CreateSettingsParams) (Setting, error) { + row := q.db.QueryRowContext(ctx, createSettings, + arg.ID, + arg.Key, + arg.Value, + arg.Options, + ) + var i Setting + err := row.Scan( + &i.ID, + &i.Key, + &i.Value, + &i.Options, + ) + return i, err +} + +const createSource = `-- name: CreateSource :exec + +Insert Into Sources +(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags) +Values +($1,$2,$3,$4,$5,$6,$7,$8,$9) +` + +type CreateSourceParams struct { + ID uuid.UUID + Site string + Name string + Source string + Type string + Value sql.NullString + Enabled bool + Url string + Tags string +} + +// Sources +func (q *Queries) CreateSource(ctx context.Context, arg CreateSourceParams) error { + _, err := q.db.ExecContext(ctx, createSource, + arg.ID, + arg.Site, + arg.Name, + arg.Source, + arg.Type, + arg.Value, + arg.Enabled, + arg.Url, + arg.Tags, + ) + return err +} + +const deleteDiscordQueueItem = `-- name: DeleteDiscordQueueItem :exec +Delete From DiscordQueue Where ID = $1 +` + +func (q *Queries) DeleteDiscordQueueItem(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, deleteDiscordQueueItem, id) + return err +} + +const deleteDiscordWebHooks = `-- name: DeleteDiscordWebHooks :exec +Delete From discordwebhooks Where ID = $1 +` + +func (q *Queries) DeleteDiscordWebHooks(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, deleteDiscordWebHooks, id) + return err +} + +const deleteIcon = `-- name: DeleteIcon :exec +Delete From Icons where ID = $1 +` + +func (q *Queries) DeleteIcon(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, deleteIcon, id) + return err +} + +const deleteSetting = `-- name: DeleteSetting :exec +Delete From settings Where ID = $1 +` + +func (q *Queries) DeleteSetting(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, deleteSetting, id) + return err +} + +const deleteSource = `-- name: DeleteSource :exec +DELETE From sources where id = $1 +` + +func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, deleteSource, id) + return err +} + +const getArticleByID = `-- name: GetArticleByID :one +Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles +WHERE ID = $1 LIMIT 1 +` + +// Articles +func (q *Queries) GetArticleByID(ctx context.Context, id uuid.UUID) (Article, error) { + row := q.db.QueryRowContext(ctx, getArticleByID, id) + var i Article + err := row.Scan( + &i.ID, + &i.Sourceid, + &i.Tags, + &i.Title, + &i.Url, + &i.Pubdate, + &i.Video, + &i.Videoheight, + &i.Videowidth, + &i.Thumbnail, + &i.Description, + &i.Authorname, + &i.Authorimage, + ) + return i, err +} + +const getArticleByUrl = `-- name: GetArticleByUrl :one +Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles +Where Url = $1 LIMIT 1 +` + +func (q *Queries) GetArticleByUrl(ctx context.Context, url string) (Article, error) { + row := q.db.QueryRowContext(ctx, getArticleByUrl, url) + var i Article + err := row.Scan( + &i.ID, + &i.Sourceid, + &i.Tags, + &i.Title, + &i.Url, + &i.Pubdate, + &i.Video, + &i.Videoheight, + &i.Videowidth, + &i.Thumbnail, + &i.Description, + &i.Authorname, + &i.Authorimage, + ) + return i, err +} + +const getDiscordQueueByID = `-- name: GetDiscordQueueByID :one +Select id, articleid from DiscordQueue +Where ID = $1 LIMIT 1 +` + +func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discordqueue, error) { + row := q.db.QueryRowContext(ctx, getDiscordQueueByID, id) + var i Discordqueue + err := row.Scan(&i.ID, &i.Articleid) + return i, err +} + +const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many +Select id, articleid from DiscordQueue LIMIT $1 +` + +func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) { + rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Discordqueue + for rows.Next() { + var i Discordqueue + if err := rows.Scan(&i.ID, &i.Articleid); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one +Select id, name, key, url, server, channel, enabled from DiscordWebHooks +Where ID = $1 LIMIT 1 +` + +func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Discordwebhook, error) { + row := q.db.QueryRowContext(ctx, getDiscordWebHooksByID, id) + var i Discordwebhook + err := row.Scan( + &i.ID, + &i.Name, + &i.Key, + &i.Url, + &i.Server, + &i.Channel, + &i.Enabled, + ) + return i, err +} + +const getIconByID = `-- name: GetIconByID :one +Select id, filename, site FROM Icons +Where ID = $1 Limit 1 +` + +func (q *Queries) GetIconByID(ctx context.Context, id uuid.UUID) (Icon, error) { + row := q.db.QueryRowContext(ctx, getIconByID, id) + var i Icon + err := row.Scan(&i.ID, &i.Filename, &i.Site) + return i, err +} + +const getIconBySite = `-- name: GetIconBySite :one +Select id, filename, site FROM Icons +Where Site = $1 Limit 1 +` + +func (q *Queries) GetIconBySite(ctx context.Context, site string) (Icon, error) { + row := q.db.QueryRowContext(ctx, getIconBySite, site) + var i Icon + err := row.Scan(&i.ID, &i.Filename, &i.Site) + return i, err +} + +const getSettingByID = `-- name: GetSettingByID :one +Select id, key, value, options From settings +Where ID = $1 Limit 1 +` + +func (q *Queries) GetSettingByID(ctx context.Context, id uuid.UUID) (Setting, error) { + row := q.db.QueryRowContext(ctx, getSettingByID, id) + var i Setting + err := row.Scan( + &i.ID, + &i.Key, + &i.Value, + &i.Options, + ) + return i, err +} + +const getSettingByKey = `-- name: GetSettingByKey :one +Select id, key, value, options From settings Where +Key = $1 Limit 1 +` + +func (q *Queries) GetSettingByKey(ctx context.Context, key string) (Setting, error) { + row := q.db.QueryRowContext(ctx, getSettingByKey, key) + var i Setting + err := row.Scan( + &i.ID, + &i.Key, + &i.Value, + &i.Options, + ) + return i, err +} + +const getSettingByValue = `-- name: GetSettingByValue :one +Select id, key, value, options From settings Where +Value = $1 Limit 1 +` + +func (q *Queries) GetSettingByValue(ctx context.Context, value string) (Setting, error) { + row := q.db.QueryRowContext(ctx, getSettingByValue, value) + var i Setting + err := row.Scan( + &i.ID, + &i.Key, + &i.Value, + &i.Options, + ) + return i, err +} + +const getSourceByID = `-- name: GetSourceByID :one +Select id, site, name, source, type, value, enabled, url, tags From Sources where ID = $1 Limit 1 +` + +func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, error) { + row := q.db.QueryRowContext(ctx, getSourceByID, id) + var i Source + err := row.Scan( + &i.ID, + &i.Site, + &i.Name, + &i.Source, + &i.Type, + &i.Value, + &i.Enabled, + &i.Url, + &i.Tags, + ) + return i, err +} + +const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many +Select id, name, key, url, server, channel, enabled From DiscordWebHooks +Where Server = $1 +` + +func (q *Queries) ListDiscordWebHooksByServer(ctx context.Context, server string) ([]Discordwebhook, error) { + rows, err := q.db.QueryContext(ctx, listDiscordWebHooksByServer, server) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Discordwebhook + for rows.Next() { + var i Discordwebhook + if err := rows.Scan( + &i.ID, + &i.Name, + &i.Key, + &i.Url, + &i.Server, + &i.Channel, + &i.Enabled, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listSourcesBySource = `-- name: ListSourcesBySource :many +Select id, site, name, source, type, value, enabled, url, tags From Sources where Source = $1 +` + +func (q *Queries) ListSourcesBySource(ctx context.Context, source string) ([]Source, error) { + rows, err := q.db.QueryContext(ctx, listSourcesBySource, source) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Source + for rows.Next() { + var i Source + if err := rows.Scan( + &i.ID, + &i.Site, + &i.Name, + &i.Source, + &i.Type, + &i.Value, + &i.Enabled, + &i.Url, + &i.Tags, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/database/schema/query.sql b/database/schema/query.sql new file mode 100644 index 0000000..d475ab7 --- /dev/null +++ b/database/schema/query.sql @@ -0,0 +1,112 @@ +/* Articles */ +-- name: GetArticleByID :one +Select * from Articles +WHERE ID = $1 LIMIT 1; + +-- name: GetArticleByUrl :one +Select * from Articles +Where Url = $1 LIMIT 1; + +-- name: CreateArticle :exec +INSERT INTO Articles +(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage) +Values +($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13); + + +/* DiscordQueue */ +-- name: CreateDiscordQueue :exec +Insert into DiscordQueue +(ID, ArticleId) +Values +($1, $2); + +-- name: GetDiscordQueueByID :one +Select * from DiscordQueue +Where ID = $1 LIMIT 1; + +-- name: DeleteDiscordQueueItem :exec +Delete From DiscordQueue Where ID = $1; + +-- name: GetDiscordQueueItems :many +Select * from DiscordQueue LIMIT $1; + + +/* DiscordWebHooks */ +-- name: CreateDiscordWebHook :exec +Insert Into DiscordWebHooks +(ID, Name, Key, Url, Server, Channel, Enabled) +Values +($1, $2, $3, $4, $5, $6, $7); + +-- name: GetDiscordWebHooksByID :one +Select * from DiscordWebHooks +Where ID = $1 LIMIT 1; + +-- name: ListDiscordWebHooksByServer :many +Select * From DiscordWebHooks +Where Server = $1; + +-- name: DeleteDiscordWebHooks :exec +Delete From discordwebhooks Where ID = $1; + + +/* Icons */ + +-- name: CreateIcon :exec +INSERT INTO Icons +(ID, FileName, Site) +VALUES +($1,$2,$3); + +-- name: GetIconByID :one +Select * FROM Icons +Where ID = $1 Limit 1; + +-- name: GetIconBySite :one +Select * FROM Icons +Where Site = $1 Limit 1; + +-- name: DeleteIcon :exec +Delete From Icons where ID = $1; + +/* Settings */ + +-- name: CreateSettings :one +Insert Into settings +(ID, Key, Value, OPTIONS) +Values +($1,$2,$3,$4) +RETURNING *; + +-- name: GetSettingByID :one +Select * From settings +Where ID = $1 Limit 1; + +-- name: GetSettingByKey :one +Select * From settings Where +Key = $1 Limit 1; + +-- name: GetSettingByValue :one +Select * From settings Where +Value = $1 Limit 1; + +-- name: DeleteSetting :exec +Delete From settings Where ID = $1; + +/* Sources */ + +-- name: CreateSource :exec +Insert Into Sources +(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags) +Values +($1,$2,$3,$4,$5,$6,$7,$8,$9); + +-- name: GetSourceByID :one +Select * From Sources where ID = $1 Limit 1; + +-- name: ListSourcesBySource :many +Select * From Sources where Source = $1; + +-- name: DeleteSource :exec +DELETE From sources where id = $1; diff --git a/database/schema/schema.sql b/database/schema/schema.sql new file mode 100644 index 0000000..fe03468 --- /dev/null +++ b/database/schema/schema.sql @@ -0,0 +1,56 @@ +CREATE TABLE Articles ( + ID uuid PRIMARY KEY, + SourceId uuid NOT null, + Tags TEXT NOT NULL, + Title TEXT NOT NULL, + Url TEXT NOT NULL, + PubDate timestamp NOT NULL, + Video TEXT, + VideoHeight int NOT NULL, + VideoWidth int NOT NULL, + Thumbnail TEXT NOT NULL, + Description TEXT NOT NULL, + AuthorName TEXT, + AuthorImage TEXT +); + +CREATE Table DiscordQueue ( + ID uuid PRIMARY KEY, + ArticleId uuid NOT NULL +); + +CREATE Table DiscordWebHooks ( + ID uuid PRIMARY KEY, + Name TEXT NOT NULL, -- Defines webhook purpose + Key TEXT, + Url TEXT NOT NULL, -- Webhook Url + Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence + Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence + Enabled BOOLEAN NOT NULL +); + +CREATE Table Icons ( + ID uuid PRIMARY Key, + FileName TEXT NOT NULL, + Site TEXT NOT NULL +); + +Create Table Settings ( + ID uuid PRIMARY Key, + Key TEXT NOT NULL, + Value TEXT NOT NULL, + Options TEXT +); + +Create Table Sources ( + ID uuid PRIMARY Key, + Site TEXT NOT NULL, -- Vanity name + Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes + Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube + Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag + Value TEXT, + Enabled BOOLEAN NOT NULL, + Url TEXT NOT NULL, + Tags TEXT NOT NULL +); + diff --git a/database/articles.go b/databaseRest/articles.go similarity index 98% rename from database/articles.go rename to databaseRest/articles.go index 3ca7cec..35c8ed6 100644 --- a/database/articles.go +++ b/databaseRest/articles.go @@ -1,4 +1,4 @@ -package database +package databaseRest import ( "bytes" diff --git a/database/common.go b/databaseRest/common.go similarity index 98% rename from database/common.go rename to databaseRest/common.go index 1ea0c4e..01be22b 100644 --- a/database/common.go +++ b/databaseRest/common.go @@ -1,4 +1,4 @@ -package database +package databaseRest import ( "errors" diff --git a/database/diagnosis.go b/databaseRest/diagnosis.go similarity index 94% rename from database/diagnosis.go rename to databaseRest/diagnosis.go index d0333b8..8b838dc 100644 --- a/database/diagnosis.go +++ b/databaseRest/diagnosis.go @@ -1,4 +1,4 @@ -package database +package databaseRest import ( "fmt" diff --git a/database/sources.go b/databaseRest/sources.go similarity index 97% rename from database/sources.go rename to databaseRest/sources.go index b1cdfb8..37cf189 100644 --- a/database/sources.go +++ b/databaseRest/sources.go @@ -1,4 +1,4 @@ -package database +package databaseRest import ( "encoding/json" diff --git a/go.mod b/go.mod index 3ede3b7..248124c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-rod/rod v0.105.1 github.com/google/uuid v1.3.0 github.com/joho/godotenv v1.4.0 + github.com/lib/pq v1.10.6 github.com/mmcdole/gofeed v1.1.3 github.com/nicklaw5/helix/v2 v2.4.0 github.com/robfig/cron/v3 v3.0.1 diff --git a/go.sum b/go.sum index 0df5b88..26dcb25 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= diff --git a/main.go b/main.go index c28b47c..f963cbd 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "log" "net/http" @@ -12,10 +13,9 @@ import ( ) func main() { - //dc := database.NewDatabaseClient() - //err := dc.Diagnosis.Ping() - //if err != nil { log.Fatalln(err) } - cron.EnableScheduler() + ctx := context.Background() + + cron.EnableScheduler(ctx) app := chi.NewRouter() app.Use(middleware.Logger) diff --git a/makefile b/makefile index 8cbf25f..07c1eb7 100644 --- a/makefile +++ b/makefile @@ -5,7 +5,12 @@ help: ## Shows this help command build: ## builds the application with the current go runtime go build . - docker-build: ## Generates the docker image docker build -t "newsbot.collector.api" . docker image ls | grep newsbot.collector.api + +migrate-dev: ## Apply sql migrations to dev db + goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up + +migrate-dev-down: ## revert sql migrations to dev db + goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down \ No newline at end of file diff --git a/services/config/config.go b/services/config/config.go index 49ea916..752bda6 100644 --- a/services/config/config.go +++ b/services/config/config.go @@ -9,6 +9,8 @@ import ( const ( DB_URI string = "DB_URI" + + Sql_Connection_String string = "SQL_CONNECTION_STRING" REDDIT_PULL_TOP = "REDDIT_PULL_TOP" REDDIT_PULL_HOT = "REDDIT_PULL_HOT" diff --git a/services/cron/scheduler.go b/services/cron/scheduler.go index 9dbc92d..7fdcbaf 100644 --- a/services/cron/scheduler.go +++ b/services/cron/scheduler.go @@ -1,106 +1,167 @@ package cron import ( + "context" + "database/sql" "log" + "time" + "github.com/google/uuid" + _ "github.com/lib/pq" "github.com/robfig/cron/v3" "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services" - //"github.com/jtom38/newsbot/collector/services/cache" + "github.com/jtom38/newsbot/collector/services/config" ) -func EnableScheduler() { - c := cron.New() +var _env config.ConfigClient +var _connString string +var _queries *database.Queries - //c.AddFunc("*/5 * * * *", func() { go CheckCache() }) - c.AddFunc("* */1 * * *", func() { go CheckReddit() }) - c.AddFunc("* */1 * * *", func() { go CheckYoutube() }) - c.AddFunc("* */1 * * *", func() { go CheckFfxiv() }) - c.AddFunc("* */1 * * *", func() { go CheckTwitch() }) +func EnableScheduler(ctx context.Context) { + c := cron.New() + OpenDatabase(ctx) + + //c.AddFunc("*/5 * * * *", func() { go CheckCache() }) + c.AddFunc("* */1 * * *", func() { go CheckReddit(ctx) }) + //c.AddFunc("* */1 * * *", func() { go CheckYoutube() }) + //c.AddFunc("* */1 * * *", func() { go CheckFfxiv() }) + //c.AddFunc("* */1 * * *", func() { go CheckTwitch() }) c.Start() } -func CheckCache() { - //cache := services.NewCacheAgeMonitor() - //cache.CheckExpiredEntries() - -} - -func CheckReddit() { - dc := database.NewDatabaseClient() - sources, err := dc.Sources.FindBySource("reddit") - if err != nil { log.Println(err) } - - rc := services.NewRedditClient(sources[0].Name, sources[0].ID) - raw, err := rc.GetContent() - if err != nil { log.Println(err) } - - redditArticles := rc.ConvertToArticles(raw) - - for _, item := range redditArticles { - _, err = dc.Articles.FindByUrl(item.Url) - if err != nil { - err = dc.Articles.Add(item) - if err != nil { log.Println("Failed to post article.")} - } +// Open the connection to the database and share it with the package so all of them are able to share. +func OpenDatabase(ctx context.Context) error { + _env = config.New() + _connString = _env.GetConfig(config.Sql_Connection_String) + db, err := sql.Open("postgres", _connString) + if err != nil { + panic(err) } + + queries := database.New(db) + _queries = queries + return err } -func CheckYoutube() { - // Add call to the db to request youtube sources. - - // Loop though the services, and generate the clients. - yt := services.NewYoutubeClient(0, "https://www.youtube.com/user/GameGrumps") - yt.CheckSource() -} - -func CheckFfxiv() { - fc := services.NewFFXIVClient("na") - articles, err := fc.CheckSource() - - // This isnt in a thread yet, so just output to stdout - if err != nil { log.Println(err) } - - dc := database.NewDatabaseClient() - for _, item := range articles { - _, err = dc.Articles.FindByUrl(item.Url) - if err != nil { - err = dc.Articles.Add(item) - if err != nil { log.Println("Failed to post article.")} - } +// This is the main entry point to query all the reddit services +func CheckReddit(ctx context.Context) { + sources, err := _queries.ListSourcesBySource(ctx, "reddit") + if err != nil { + log.Printf("No defines sources for reddit to query - %v\r", err) } -} - -func CheckTwitch() error { - // TODO Wire this for the DB - // just a mock object for now - dc := database.NewDatabaseClient() - - sources, err := dc.Sources.FindBySource("Twitch") - if err != nil { return err } - - client, err := services.NewTwitchClient(sources[0]) - if err != nil { log.Println(err) } - - err = client.Login() - if err != nil { return err } for _, source := range sources { - client.ReplaceSourceRecord(source) - - posts, err := client.GetContent() - if err != nil { return err } - - for _, item := range posts { - _, err = dc.Articles.FindByUrl(item.Url) - if err != nil { - err = dc.Articles.Add(item) - if err != nil { log.Println("Failed to post article.")} - } + if !source.Enabled { + continue } + rc := services.NewRedditClient(source) + raw, err := rc.GetContent() + if err != nil { + log.Println(err) + } + redditArticles := rc.ConvertToArticles(raw) + checkPosts(ctx, redditArticles) + } +} + +func CheckYoutube(ctx context.Context) { + // Add call to the db to request youtube sources. + sources, err := _queries.ListSourcesBySource(ctx, "youtube") + if err != nil { + log.Printf("Youtube - No sources found to query - %v\r", err) + } + + for _, source := range sources { + if !source.Enabled { + continue + } + yc := services.NewYoutubeClient(source) + raw, err := yc.GetContent() + if err != nil { + log.Println(err) + } + checkPosts(ctx, raw) + } +} + +func CheckFfxiv(ctx context.Context) { + sources, err := _queries.ListSourcesBySource(ctx, "ffxiv") + if err != nil { + log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err) + } + + for _, source := range sources { + if !source.Enabled { + continue + } + fc := services.NewFFXIVClient(source) + items, err := fc.CheckSource() + if err != nil { + log.Println(err) + } + checkPosts(ctx, items) + } +} + +func CheckTwitch(ctx context.Context) error { + sources, err := _queries.ListSourcesBySource(ctx, "twitch") + if err != nil { + log.Printf("Twitch - No sources found to query - %v\r", err) + } + + tc, err := services.NewTwitchClient() + if err != nil { + return err + } + + for _, source := range sources { + if !source.Enabled { + continue + } + tc.ReplaceSourceRecord(source) + items, err := tc.GetContent() + if err != nil { + log.Println(err) + } + checkPosts(ctx, items) } return nil -} \ No newline at end of file +} + +func checkPosts(ctx context.Context, posts []database.Article) { + for _, item := range posts { + _, err := _queries.GetArticleByUrl(ctx, item.Url) + if err != nil { + err = postArticle(ctx, item) + if err != nil { + log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err) + } else { + log.Printf("Reddit - Posted article - %v\r", item.Url) + } + } + } + time.Sleep(30 * time.Second) +} + +func postArticle(ctx context.Context, item database.Article) error { + err := _queries.CreateArticle(ctx, database.CreateArticleParams{ + ID: uuid.New(), + Sourceid: item.Sourceid, + Tags: item.Tags, + Title: item.Title, + Url: item.Url, + Pubdate: item.Pubdate, + Video: item.Video, + Videoheight: item.Videoheight, + Videowidth: item.Videowidth, + Thumbnail: item.Thumbnail, + Description: item.Description, + Authorname: item.Authorname, + Authorimage: item.Authorimage, + }) + return err +} diff --git a/services/cron/scheduler_test.go b/services/cron/scheduler_test.go index 94d8f0c..312b347 100644 --- a/services/cron/scheduler_test.go +++ b/services/cron/scheduler_test.go @@ -1,7 +1,37 @@ package cron_test -import "testing" +import ( + "context" + "testing" + + "github.com/jtom38/newsbot/collector/services/cron" +) func TestInvokeTwitch(t *testing.T) { - + +} + +// TODO add database mocks but not sure how to do that yet. +func TestCheckReddit(t *testing.T) { + ctx := context.Background() + cron.OpenDatabase(ctx) + cron.CheckReddit(ctx) +} + +func TestCheckYouTube(t *testing.T) { + ctx := context.Background() + cron.OpenDatabase(ctx) + cron.CheckYoutube(ctx) +} + +func TestCheckTwitch(t *testing.T) { + ctx := context.Background() + err := cron.OpenDatabase(ctx) + if err != nil { + t.Error(err) + } + err = cron.CheckTwitch(ctx) + if err != nil { + t.Error(err) + } } diff --git a/services/ffxiv.go b/services/ffxiv.go index dfb0c58..fbf9808 100644 --- a/services/ffxiv.go +++ b/services/ffxiv.go @@ -1,6 +1,7 @@ package services import ( + "database/sql" "errors" "log" "net/http" @@ -11,7 +12,7 @@ import ( "github.com/go-rod/rod" "github.com/google/uuid" - "github.com/jtom38/newsbot/collector/domain/model" + "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services/cache" ) @@ -23,32 +24,23 @@ const ( ) type FFXIVClient struct { - SourceID uint - Url string - Region string + record database.Source + //SourceID uint + //Url string + //Region string cacheGroup string } -func NewFFXIVClient(region string) FFXIVClient { - var url string - - switch region { - case "na": - url = FFXIV_NA_FEED_URL - case "jp": - url = FFXIV_JP_FEED_URL - } - +func NewFFXIVClient(Record database.Source) FFXIVClient { return FFXIVClient{ - Region: region, - Url: url, + record: Record, cacheGroup: "ffxiv", } } -func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) { - var articles []model.Articles +func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { + var articles []database.Article parser := fc.GetBrowser() defer parser.Close() @@ -87,19 +79,18 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) { tags, err := fc.ExtractTags(page) if err != nil { return articles, err } - article := model.Articles{ - SourceID: fc.SourceID, + article := database.Article{ + Sourceid: fc.record.ID, Tags: tags, Title: title, Url: link, - PubDate: pubDate, - Video: "", - VideoHeight: 0, - VideoWidth: 0, + Pubdate: pubDate, + Videoheight: 0, + Videowidth: 0, Thumbnail: thumb, Description: description, - AuthorName: authorName, - AuthorImage: authorImage, + Authorname: sql.NullString{String: authorName}, + Authorimage: sql.NullString{String: authorImage}, } log.Printf("Collected '%v' from '%v'", article.Title, article.Url) @@ -112,7 +103,7 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) { } func (fc *FFXIVClient) GetParser() (*goquery.Document, error) { - html, err := http.Get(fc.Url) + html, err := http.Get(fc.record.Url) if err != nil { return nil, err } defer html.Body.Close() @@ -129,7 +120,7 @@ func (fc *FFXIVClient) GetBrowser() (*rod.Browser) { func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) { var links []string - page := parser.MustPage(fc.Url) + page := parser.MustPage(fc.record.Url) defer page.Close() // find the list by xpath diff --git a/services/ffxiv_test.go b/services/ffxiv_test.go index c15798d..8c8031d 100644 --- a/services/ffxiv_test.go +++ b/services/ffxiv_test.go @@ -3,17 +3,28 @@ package services_test import ( "testing" + "github.com/google/uuid" + "github.com/jtom38/newsbot/collector/database" ffxiv "github.com/jtom38/newsbot/collector/services" ) +var FFXIVRecord database.Source = database.Source{ + ID: uuid.New(), + Site: "ffxiv", + Name: "Final Fantasy XIV - NA", + Source: "ffxiv", + Url: "https://na.finalfantasyxiv.com/lodestone/", + Tags: "ffxiv, final, fantasy, xiv, na, lodestone", +} + func TestFfxivGetParser(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) _, err := fc.GetParser() if err != nil { panic(err) } } func TestFfxivPullFeed(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -25,7 +36,7 @@ func TestFfxivPullFeed(t *testing.T) { } func TestFfxivExtractThumbnail(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -42,7 +53,7 @@ func TestFfxivExtractThumbnail(t *testing.T) { } func TestFfxivExtractPubDate(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -58,7 +69,7 @@ func TestFfxivExtractPubDate(t *testing.T) { } func TestFfxivExtractDescription(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -74,7 +85,7 @@ func TestFfxivExtractDescription(t *testing.T) { } func TestFfxivExtractAuthor(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -91,7 +102,7 @@ func TestFfxivExtractAuthor(t *testing.T) { } func TestFfxivExtractTags(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -108,7 +119,7 @@ func TestFfxivExtractTags(t *testing.T) { } func TestFfxivExtractTitle(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -125,7 +136,7 @@ func TestFfxivExtractTitle(t *testing.T) { } func TestFFxivExtractAuthorIamge(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) parser := fc.GetBrowser() defer parser.Close() @@ -142,7 +153,7 @@ func TestFFxivExtractAuthorIamge(t *testing.T) { } func TestFfxivCheckSource(t *testing.T) { - fc := ffxiv.NewFFXIVClient("na") + fc := ffxiv.NewFFXIVClient(FFXIVRecord) fc.CheckSource() } \ No newline at end of file diff --git a/services/reddit.go b/services/reddit.go index 6ce208c..99fc5c3 100644 --- a/services/reddit.go +++ b/services/reddit.go @@ -1,6 +1,7 @@ package services import ( + "database/sql" "encoding/json" "errors" "fmt" @@ -10,28 +11,25 @@ import ( "time" "github.com/go-rod/rod" + "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/domain/model" "github.com/jtom38/newsbot/collector/services/config" ) type RedditClient struct { - subreddit string - url string - sourceId uint config RedditConfig + record database.Source } type RedditConfig struct { - PullTop string - PullHot string + PullTop string + PullHot string PullNSFW string } -func NewRedditClient(subreddit string, sourceID uint) RedditClient { +func NewRedditClient(Record database.Source) RedditClient { rc := RedditClient{ - subreddit: subreddit, - url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit), - sourceId: sourceID, + record: Record, } cc := config.New() rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT) @@ -59,15 +57,23 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page { return page } +//func (rc RedditClient) + // GetContent() reaches out to Reddit and pulls the Json data. // It will then convert the data to a struct and return the struct. -func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) { +func (rc RedditClient) GetContent() (model.RedditJsonContent, error) { var items model.RedditJsonContent = model.RedditJsonContent{} - log.Printf("Collecting results on '%v'", rc.subreddit) - content, err := getHttpContent(rc.url) - if err != nil { return items, err } - if strings.Contains("

whoa there, pardner!

", string(content) ) { + // TODO Wire this to support the config options + Url := fmt.Sprintf("%v.json", rc.record.Url) + + log.Printf("Collecting results on '%v'", rc.record.Name) + + content, err := getHttpContent(Url) + if err != nil { + return items, err + } + if strings.Contains("

whoa there, pardner!

", string(content)) { return items, errors.New("did not get json data from the server") } @@ -78,12 +84,15 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) { return items, nil } -func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles { - var redditArticles []model.Articles +func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article { + var redditArticles []database.Article for _, item := range items.Data.Children { - var article model.Articles + var article database.Article article, err := rc.convertToArticle(item.Data) - if err != nil { log.Println(err); continue } + if err != nil { + log.Println(err) + continue + } redditArticles = append(redditArticles, article) } return redditArticles @@ -91,14 +100,13 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // This data can be passed to the database. -func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) { - var item model.Articles +func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) { + var item database.Article - - if source.Content == "" && source.Url != ""{ + if source.Content == "" && source.Url != "" { item = rc.convertPicturePost(source) } - + if source.Media.RedditVideo.FallBackUrl != "" { item = rc.convertVideoPost(source) } @@ -119,58 +127,66 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles return item, nil } -func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles { - var item = model.Articles{ - SourceID: rc.sourceId, - Tags: "a", - Title: source.Title, - Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - PubDate: time.Now(), - Video: "null", - VideoHeight: 0, - VideoWidth: 0, - Thumbnail: source.Thumbnail, +func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article { + var item = database.Article{ + Sourceid: rc.record.ID, + Title: source.Title, + Tags: fmt.Sprintf("%v", rc.record.Tags), + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + Pubdate: time.Now(), + Video: sql.NullString{String: "null"}, + Videoheight: 0, + Videowidth: 0, + Thumbnail: source.Thumbnail, Description: source.Content, - AuthorName: source.Author, - AuthorImage: "null", + Authorname: sql.NullString{String: source.Author}, + Authorimage: sql.NullString{String: "null"}, } return item } -func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles { - var item = model.Articles{ - SourceID: rc.sourceId, - Tags: "a", - Title: source.Title, - Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - AuthorName: source.Author, +func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article { + var item = database.Article{ + Sourceid: rc.record.ID, + Tags: "a", + Title: source.Title, + Pubdate: time.Now(), + Videoheight: 0, + Videowidth: 0, + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + Authorname: sql.NullString{String: source.Author}, Description: source.Content, - } return item } -func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles { - var item = model.Articles{ - SourceID: rc.sourceId, - Tags: "a", - Title: source.Title, - Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - AuthorName: source.Author, +func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article { + var item = database.Article{ + Sourceid: rc.record.ID, + Tags: "a", + Title: source.Title, + Pubdate: time.Now(), + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + Videoheight: 0, + Videowidth: 0, + Authorname: sql.NullString{String: source.Author}, Description: source.Media.RedditVideo.FallBackUrl, } return item } // This post is nothing more then a redirect to another location. -func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles { - var item = model.Articles{ - SourceID: rc.sourceId, - Tags: "a", - Title: source.Title, - Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - AuthorName: source.Author, +func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article { + var item = database.Article{ + Sourceid: rc.record.ID, + Tags: "a", + Title: source.Title, + Pubdate: time.Now(), + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + Videoheight: 0, + Videowidth: 0, + Authorname: sql.NullString{String: source.Author}, Description: source.UrlOverriddenByDest, } return item -} \ No newline at end of file +} diff --git a/services/reddit_test.go b/services/reddit_test.go index cd10479..159da88 100644 --- a/services/reddit_test.go +++ b/services/reddit_test.go @@ -1,16 +1,33 @@ package services_test import ( - "log" "testing" + "github.com/google/uuid" + "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services" ) +var RedditRecord database.Source = database.Source{ + ID: uuid.New(), + Name: "dadjokes", + Source: "reddit", + Site: "reddit", + Url: "https://reddit.com/r/dadjokes", + Tags: "reddit, dadjokes", +} + func TestGetContent(t *testing.T) { //This test is flaky right now due to the http changes in 1.17 - rc := services.NewRedditClient("dadjokes", 0) - _, err := rc.GetContent() - log.Println(err) - //if err != nil { panic(err) } + rc := services.NewRedditClient(RedditRecord) + raw, err := rc.GetContent() + if err != nil { + t.Error(err) + } + redditArticles := rc.ConvertToArticles(raw) + for _, posts := range redditArticles { + if posts.Title == "" { + t.Error("Title is missing") + } + } } \ No newline at end of file diff --git a/services/twitch.go b/services/twitch.go index b4ab14d..dd49646 100644 --- a/services/twitch.go +++ b/services/twitch.go @@ -1,20 +1,19 @@ package services import ( + "database/sql" "errors" "fmt" "strings" "time" - //"log" - - "github.com/jtom38/newsbot/collector/domain/model" + "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services/config" "github.com/nicklaw5/helix/v2" ) type TwitchClient struct { - SourceRecord model.Sources + SourceRecord database.Source // config monitorClips string @@ -31,7 +30,7 @@ var ( twitchScopes = "user:read:email" ) -func NewTwitchClient(source model.Sources) (TwitchClient, error) { +func NewTwitchClient() (TwitchClient, error) { c := config.New() id := c.GetConfig(config.TWITCH_CLIENT_ID) @@ -50,7 +49,7 @@ func NewTwitchClient(source model.Sources) (TwitchClient, error) { } client := TwitchClient{ - SourceRecord: source, + //SourceRecord: &source, monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS), monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD), api: &api, @@ -73,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) { } // This will let you replace the bound source record to keep the same session alive. -func (tc TwitchClient) ReplaceSourceRecord(source model.Sources) { +func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { tc.SourceRecord = source } @@ -88,8 +87,8 @@ func (tc TwitchClient) Login() error { return nil } -func (tc TwitchClient) GetContent() ([]model.Articles, error) { - var items []model.Articles +func (tc TwitchClient) GetContent() ([]database.Article, error) { + var items []database.Article user, err := tc.GetUserDetails() if err != nil { @@ -102,21 +101,23 @@ func (tc TwitchClient) GetContent() ([]model.Articles, error) { } for _, video := range posts { - article := model.Articles{} + var article database.Article - article.AuthorName, err = tc.ExtractAuthor(video) + AuthorName, err := tc.ExtractAuthor(video) if err != nil { return items, err } + article.Authorname = sql.NullString{String: AuthorName} - article.AuthorImage, err = tc.ExtractAuthorImage(user) + Authorimage, err := tc.ExtractAuthorImage(user) if err != nil { return items, err } + article.Authorimage = sql.NullString{String: Authorimage} article.Description, err = tc.ExtractDescription(video) if err != nil {return items, err } - article.PubDate, err = tc.ExtractPubDate(video) + article.Pubdate, err = tc.ExtractPubDate(video) if err != nil { return items, err } - article.SourceID = tc.SourceRecord.ID + article.Sourceid = tc.SourceRecord.ID article.Tags, err = tc.ExtractTags(video, user) if err != nil { return items, err } diff --git a/services/twitch_test.go b/services/twitch_test.go index ba88345..70810fb 100644 --- a/services/twitch_test.go +++ b/services/twitch_test.go @@ -4,27 +4,29 @@ import ( "log" "testing" - "github.com/jtom38/newsbot/collector/domain/model" + "github.com/google/uuid" + "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services" ) -var sourceRecord = model.Sources{ - ID: 1, +var TwitchSourceRecord = database.Source { + ID: uuid.New(), Name: "nintendo", Source: "Twitch", } -var invalidRecord = model.Sources{ - ID: 1, +var TwitchInvalidRecord = database.Source { + ID: uuid.New(), Name: "EvilNintendo", Source: "Twitch", } func TestTwitchLogin(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { @@ -34,10 +36,11 @@ func TestTwitchLogin(t *testing.T) { // reach out and confirms that the API returns posts made by the user. func TestTwitchReturnsUserPosts(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { @@ -59,10 +62,11 @@ func TestTwitchReturnsUserPosts(t *testing.T) { } func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) { - tc, err := services.NewTwitchClient(invalidRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchInvalidRecord) err = tc.Login() if err != nil { @@ -84,10 +88,11 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) { } func TestTwitchReturnsVideoAuthor(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { @@ -109,8 +114,9 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) { } func TestTwitchReturnsThumbnail(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil {t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { t.Error(err) } @@ -127,8 +133,9 @@ func TestTwitchReturnsThumbnail(t *testing.T) { } func TestTwitchReturnsPubDate(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { t.Error(err) } @@ -145,10 +152,11 @@ func TestTwitchReturnsPubDate(t *testing.T) { } func TestTwitchReturnsDescription(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { @@ -172,8 +180,9 @@ func TestTwitchReturnsDescription(t *testing.T) { } func TestTwitchReturnsAuthorImage(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil {t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { t.Error(err) } @@ -186,11 +195,11 @@ func TestTwitchReturnsAuthorImage(t *testing.T) { } func TestTwitchReturnsTags(t *testing.T) { - - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { @@ -210,10 +219,11 @@ func TestTwitchReturnsTags(t *testing.T) { } func TestTwitchReturnsTitle(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { @@ -234,8 +244,9 @@ func TestTwitchReturnsTitle(t *testing.T) { } func TestTwitchReturnsUrl(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { t.Error(err) } @@ -252,8 +263,9 @@ func TestTwitchReturnsUrl(t *testing.T) { } func TestTwitchGetContent(t *testing.T) { - tc, err := services.NewTwitchClient(sourceRecord) + tc, err := services.NewTwitchClient() if err != nil { t.Error(err) } + tc.ReplaceSourceRecord(TwitchSourceRecord) err = tc.Login() if err != nil { t.Error(err) } diff --git a/services/youtube.go b/services/youtube.go index 4413435..de831a5 100644 --- a/services/youtube.go +++ b/services/youtube.go @@ -1,6 +1,7 @@ package services import ( + "database/sql" "errors" "fmt" "log" @@ -12,14 +13,15 @@ import ( "github.com/go-rod/rod" "github.com/mmcdole/gofeed" - "github.com/jtom38/newsbot/collector/domain/model" + "github.com/jtom38/newsbot/collector/database" ) type YoutubeClient struct { - SourceID uint - Url string - ChannelID string - AvatarUri string + record database.Source + + // internal variables at time of collection + channelID string + avatarUri string // config //debug bool @@ -36,10 +38,9 @@ var ( const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" -func NewYoutubeClient(SourceID uint, Url string) YoutubeClient { +func NewYoutubeClient(Record database.Source) YoutubeClient { yc := YoutubeClient{ - SourceID: SourceID, - Url: Url, + record: Record, cacheGroup: "youtube", } /* @@ -53,10 +54,11 @@ func NewYoutubeClient(SourceID uint, Url string) YoutubeClient { } // CheckSource will go and run all the commands needed to process a source. -func (yc *YoutubeClient) CheckSource() error { - docParser, err := yc.GetParser(yc.Url) +func (yc *YoutubeClient) GetContent() ([]database.Article, error) { + var items []database.Article + docParser, err := yc.GetParser(yc.record.Url) if err != nil { - return err + return items, err } // Check cache/db for existing value @@ -64,38 +66,38 @@ func (yc *YoutubeClient) CheckSource() error { //channelId, err := yc.extractChannelId() channelId, err := yc.GetChannelId(docParser) if err != nil { - return err + return items, err } if channelId == "" { - return ErrYoutubeChannelIdMissing + return items, ErrYoutubeChannelIdMissing } - yc.ChannelID = channelId + yc.channelID = channelId // Check the cache/db forthe value. // if we have the value, skip avatar, err := yc.GetAvatarUri() if err != nil { - return err + return items, err } if avatar == "" { - return ErrMissingAuthorImage + return items, ErrMissingAuthorImage } - yc.AvatarUri = avatar + yc.avatarUri = avatar feed, err := yc.PullFeed() if err != nil { - return err + return items, err } newPosts, err := yc.CheckForNewPosts(feed) if err != nil { - return err + return items, err } - //TODO post to the API for _, item := range newPosts { article := yc.ConvertToArticle(item) + items = append(items, article) YoutubeUriCache = append(YoutubeUriCache, &item.Link) @@ -103,7 +105,7 @@ func (yc *YoutubeClient) CheckSource() error { log.Println(article) } - return nil + return items, nil } func (yc *YoutubeClient) GetBrowser() *rod.Browser { @@ -137,8 +139,8 @@ func (yc *YoutubeClient) GetChannelId(doc *goquery.Document) (string, error) { for _, item := range meta.Nodes { if item.Attr[0].Val == "channelId" { - yc.ChannelID = item.Attr[1].Val - return yc.ChannelID, nil + yc.channelID = item.Attr[1].Val + return yc.channelID, nil } } return "", ErrYoutubeChannelIdMissing @@ -155,7 +157,7 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) { var AvatarUri string browser := rod.New().MustConnect() - page := browser.MustPage(yc.Url) + page := browser.MustPage(yc.record.Url) res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src") @@ -197,7 +199,7 @@ func (yc *YoutubeClient) GetVideoThumbnail(parser *goquery.Document) (string, er // This will pull the RSS feed items and return the results func (yc *YoutubeClient) PullFeed() (*gofeed.Feed, error) { - feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.ChannelID) + feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.channelID) fp := gofeed.NewParser() feed, err := fp.ParseURL(feedUri) if err != nil { @@ -239,7 +241,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool { return false } -func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles { +func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { parser, err := yc.GetParser(item.Link) if err != nil { log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link) @@ -257,16 +259,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles { log.Println(msg) } - var article = model.Articles{ - SourceID: yc.SourceID, + var article = database.Article{ + Sourceid: yc.record.ID, Tags: tags, Title: item.Title, Url: item.Link, - PubDate: *item.PublishedParsed, + Pubdate: *item.PublishedParsed, Thumbnail: thumb, Description: item.Description, - AuthorName: item.Author.Name, - AuthorImage: yc.AvatarUri, + Authorname: sql.NullString{String: item.Author.Name}, + Authorimage: sql.NullString{String: yc.avatarUri}, } return article } diff --git a/services/youtube_test.go b/services/youtube_test.go index e9ef018..3ddeb77 100644 --- a/services/youtube_test.go +++ b/services/youtube_test.go @@ -3,24 +3,28 @@ package services_test import ( "testing" + "github.com/google/uuid" + "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services" ) +var YouTubeRecord database.Source = database.Source{ + ID: uuid.New(), + Name: "dadjokes", + Source: "reddit", + Site: "reddit", + Url: "https://youtube.com/gamegrumps", +} + func TestGetPageParser(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) - _, err := yc.GetParser(yc.Url) + yc := services.NewYoutubeClient(YouTubeRecord) + _, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } } func TestGetChannelId(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) - parser, err := yc.GetParser(yc.Url) + yc := services.NewYoutubeClient(YouTubeRecord) + parser, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } _, err = yc.GetChannelId(parser) @@ -28,11 +32,8 @@ func TestGetChannelId(t *testing.T) { } func TestPullFeed(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) - parser, err := yc.GetParser(yc.Url) + yc := services.NewYoutubeClient(YouTubeRecord) + parser, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } _, err = yc.GetChannelId(parser) @@ -43,20 +44,14 @@ func TestPullFeed(t *testing.T) { } func TestGetAvatarUri(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) + yc := services.NewYoutubeClient(YouTubeRecord) res, err := yc.GetAvatarUri() if err != nil { panic(err) } if res == "" { panic(services.ErrMissingAuthorImage)} } func TestGetVideoTags(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) + yc := services.NewYoutubeClient(YouTubeRecord) var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68" @@ -69,12 +64,9 @@ func TestGetVideoTags(t *testing.T) { } func TestGetChannelTags(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) + yc := services.NewYoutubeClient(YouTubeRecord) - parser, err := yc.GetParser(yc.Url) + parser, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } tags, err := yc.GetTags(parser) @@ -83,10 +75,7 @@ func TestGetChannelTags(t *testing.T) { } func TestGetVideoThumbnail(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) + yc := services.NewYoutubeClient(YouTubeRecord) parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68") if err != nil {panic(err) } @@ -97,20 +86,13 @@ func TestGetVideoThumbnail(t *testing.T) { } func TestCheckSource(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) - err := yc.CheckSource() + yc := services.NewYoutubeClient(YouTubeRecord) + _, err := yc.GetContent() if err != nil { panic(err) } - } func TestCheckUriCache(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) + yc := services.NewYoutubeClient(YouTubeRecord) item := "demo" services.YoutubeUriCache = append(services.YoutubeUriCache, &item) @@ -119,10 +101,7 @@ func TestCheckUriCache(t *testing.T) { } func TestCheckUriCacheFails(t *testing.T) { - yc := services.NewYoutubeClient( - 0, - "https://youtube.com/gamegrumps", - ) + yc := services.NewYoutubeClient(YouTubeRecord) item := "demo1" res := yc.CheckUriCache(&item) diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 0000000..b4fd67f --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,7 @@ +version: "1" +packages: + - schema: "database/schema/schema.sql" + queries: "database/schema/query.sql" + name: "database" + path: "database" + engine: "postgresql"