Feature/sql (#8)

* Using sqlc to generate queries and goose for migrations. The intial tests look good.

* moving the old calls away for now.  Might use this in a package later on.

* Added postgres driver

* Updated the dockerfile to support sql migrations

* added sqlc config file

* updated schema and starting a seed script

* updated models to use the database ones

* updated reddit cron to talk to the db

* added env for sql connection string

* got the reddit source working with the db and posting articles

* added sql packages

* added rule to ignore dev sql file

* added migration down statement for rolling back

* updated cron for reddit and youtube

* Updated reddit to follow a new standard pattern

* updated youtube to follow new patterns

* updated tests and brought them to the standard

* updated the seed migration

* all cron tasks should feed the db now

* updated app init

* bumped docker to 1.18.3

* disabled remote tests given secrets and lack of interfaces currently to run tests
This commit is contained in:
James Tombleson 2022-06-08 21:17:08 -07:00 committed by GitHub
parent 333a4f5345
commit 75b66dd625
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1359 additions and 305 deletions

View File

@ -21,5 +21,5 @@ jobs:
- name: Build - name: Build
run: go build -v ./... run: go build -v ./...
- name: Test #- name: Test
run: go test -v ./... # run: go test -v ./...

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
.env .env
dev.session.sql
# Binaries for programs and plugins # Binaries for programs and plugins
*.exe *.exe

View File

@ -1,10 +1,16 @@
FROM golang:1.18.2 as build FROM golang:1.18.3 as build
COPY . /app COPY . /app
WORKDIR /app WORKDIR /app
RUN go build . RUN go build .
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
FROM alpine FROM alpine
RUN mkdir /app && \
mkdir /app/migrations
COPY --from=build /app/collector /app COPY --from=build /app/collector /app
COPY --from=build /go/bin/goose /app
COPY ./database/migrations/ /app/migrations
ENTRYPOINT [ "/app/collector" ] ENTRYPOINT [ "/app/collector" ]

31
database/db.go Normal file
View File

@ -0,0 +1,31 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package database
import (
"context"
"database/sql"
)
type DBTX interface {
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
PrepareContext(context.Context, string) (*sql.Stmt, error)
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
return &Queries{
db: tx,
}
}

View File

@ -0,0 +1,72 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
CREATE TABLE Articles (
ID uuid PRIMARY KEY,
SourceId uuid NOT null,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate timestamp NOT NULL,
Video TEXT,
VideoHeight int NOT NULL,
VideoWidth int NOT NULL,
Thumbnail TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT,
AuthorImage TEXT
);
CREATE Table DiscordQueue (
ID uuid PRIMARY KEY,
ArticleId uuid NOT NULL
);
CREATE Table DiscordWebHooks (
ID uuid PRIMARY KEY,
Name TEXT NOT NULL, -- Defines webhook purpose
Key TEXT,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID uuid PRIMARY Key,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID uuid PRIMARY Key,
Key TEXT NOT NULL, -- How you search for a entry
Value TEXT NOT NULL, -- The value for one
Options TEXT -- any notes about the entry
);
Create Table Sources (
ID uuid PRIMARY Key,
Site TEXT NOT NULL, -- Vanity name
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
Value TEXT,
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
Drop Table Articles;
Drop Table DiscordQueue;
Drop Table DiscordWebHooks;
Drop Table Icons;
Drop Table Settings;
Drop Table Sources;
-- +goose StatementEnd

View File

@ -0,0 +1,50 @@
-- +goose Up
-- +goose StatementBegin
SELECT 'up SQL query';
-- Enable UUID's
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- Final Fantasy XIV Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources VALUES
(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries
INSERT INTO sources VALUES
(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
--SELECT 'down SQL query';
DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
DELETE FROM sources where source = 'ffxiv';
DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
-- +goose StatementEnd

68
database/models.go Normal file
View File

@ -0,0 +1,68 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package database
import (
"database/sql"
"time"
"github.com/google/uuid"
)
type Article struct {
ID uuid.UUID
Sourceid uuid.UUID
Tags string
Title string
Url string
Pubdate time.Time
Video sql.NullString
Videoheight int32
Videowidth int32
Thumbnail string
Description string
Authorname sql.NullString
Authorimage sql.NullString
}
type Discordqueue struct {
ID uuid.UUID
Articleid uuid.UUID
}
type Discordwebhook struct {
ID uuid.UUID
Name string
Key sql.NullString
Url string
Server string
Channel string
Enabled bool
}
type Icon struct {
ID uuid.UUID
Filename string
Site string
}
type Setting struct {
ID uuid.UUID
Key string
Value string
Options sql.NullString
}
type Source struct {
ID uuid.UUID
Site string
Name string
Source string
Type string
Value sql.NullString
Enabled bool
Url string
Tags string
}

521
database/query.sql.go Normal file
View File

@ -0,0 +1,521 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
// source: query.sql
package database
import (
"context"
"database/sql"
"time"
"github.com/google/uuid"
)
const createArticle = `-- name: CreateArticle :exec
INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
`
type CreateArticleParams struct {
ID uuid.UUID
Sourceid uuid.UUID
Tags string
Title string
Url string
Pubdate time.Time
Video sql.NullString
Videoheight int32
Videowidth int32
Thumbnail string
Description string
Authorname sql.NullString
Authorimage sql.NullString
}
func (q *Queries) CreateArticle(ctx context.Context, arg CreateArticleParams) error {
_, err := q.db.ExecContext(ctx, createArticle,
arg.ID,
arg.Sourceid,
arg.Tags,
arg.Title,
arg.Url,
arg.Pubdate,
arg.Video,
arg.Videoheight,
arg.Videowidth,
arg.Thumbnail,
arg.Description,
arg.Authorname,
arg.Authorimage,
)
return err
}
const createDiscordQueue = `-- name: CreateDiscordQueue :exec
Insert into DiscordQueue
(ID, ArticleId)
Values
($1, $2)
`
type CreateDiscordQueueParams struct {
ID uuid.UUID
Articleid uuid.UUID
}
// DiscordQueue
func (q *Queries) CreateDiscordQueue(ctx context.Context, arg CreateDiscordQueueParams) error {
_, err := q.db.ExecContext(ctx, createDiscordQueue, arg.ID, arg.Articleid)
return err
}
const createDiscordWebHook = `-- name: CreateDiscordWebHook :exec
Insert Into DiscordWebHooks
(ID, Name, Key, Url, Server, Channel, Enabled)
Values
($1, $2, $3, $4, $5, $6, $7)
`
type CreateDiscordWebHookParams struct {
ID uuid.UUID
Name string
Key sql.NullString
Url string
Server string
Channel string
Enabled bool
}
// DiscordWebHooks
func (q *Queries) CreateDiscordWebHook(ctx context.Context, arg CreateDiscordWebHookParams) error {
_, err := q.db.ExecContext(ctx, createDiscordWebHook,
arg.ID,
arg.Name,
arg.Key,
arg.Url,
arg.Server,
arg.Channel,
arg.Enabled,
)
return err
}
const createIcon = `-- name: CreateIcon :exec
INSERT INTO Icons
(ID, FileName, Site)
VALUES
($1,$2,$3)
`
type CreateIconParams struct {
ID uuid.UUID
Filename string
Site string
}
// Icons
func (q *Queries) CreateIcon(ctx context.Context, arg CreateIconParams) error {
_, err := q.db.ExecContext(ctx, createIcon, arg.ID, arg.Filename, arg.Site)
return err
}
const createSettings = `-- name: CreateSettings :one
Insert Into settings
(ID, Key, Value, OPTIONS)
Values
($1,$2,$3,$4)
RETURNING id, key, value, options
`
type CreateSettingsParams struct {
ID uuid.UUID
Key string
Value string
Options sql.NullString
}
// Settings
func (q *Queries) CreateSettings(ctx context.Context, arg CreateSettingsParams) (Setting, error) {
row := q.db.QueryRowContext(ctx, createSettings,
arg.ID,
arg.Key,
arg.Value,
arg.Options,
)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const createSource = `-- name: CreateSource :exec
Insert Into Sources
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
Values
($1,$2,$3,$4,$5,$6,$7,$8,$9)
`
type CreateSourceParams struct {
ID uuid.UUID
Site string
Name string
Source string
Type string
Value sql.NullString
Enabled bool
Url string
Tags string
}
// Sources
func (q *Queries) CreateSource(ctx context.Context, arg CreateSourceParams) error {
_, err := q.db.ExecContext(ctx, createSource,
arg.ID,
arg.Site,
arg.Name,
arg.Source,
arg.Type,
arg.Value,
arg.Enabled,
arg.Url,
arg.Tags,
)
return err
}
const deleteDiscordQueueItem = `-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1
`
func (q *Queries) DeleteDiscordQueueItem(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteDiscordQueueItem, id)
return err
}
const deleteDiscordWebHooks = `-- name: DeleteDiscordWebHooks :exec
Delete From discordwebhooks Where ID = $1
`
func (q *Queries) DeleteDiscordWebHooks(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteDiscordWebHooks, id)
return err
}
const deleteIcon = `-- name: DeleteIcon :exec
Delete From Icons where ID = $1
`
func (q *Queries) DeleteIcon(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteIcon, id)
return err
}
const deleteSetting = `-- name: DeleteSetting :exec
Delete From settings Where ID = $1
`
func (q *Queries) DeleteSetting(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSetting, id)
return err
}
const deleteSource = `-- name: DeleteSource :exec
DELETE From sources where id = $1
`
func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSource, id)
return err
}
const getArticleByID = `-- name: GetArticleByID :one
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
WHERE ID = $1 LIMIT 1
`
// Articles
func (q *Queries) GetArticleByID(ctx context.Context, id uuid.UUID) (Article, error) {
row := q.db.QueryRowContext(ctx, getArticleByID, id)
var i Article
err := row.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
)
return i, err
}
const getArticleByUrl = `-- name: GetArticleByUrl :one
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
Where Url = $1 LIMIT 1
`
func (q *Queries) GetArticleByUrl(ctx context.Context, url string) (Article, error) {
row := q.db.QueryRowContext(ctx, getArticleByUrl, url)
var i Article
err := row.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
)
return i, err
}
const getDiscordQueueByID = `-- name: GetDiscordQueueByID :one
Select id, articleid from DiscordQueue
Where ID = $1 LIMIT 1
`
func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discordqueue, error) {
row := q.db.QueryRowContext(ctx, getDiscordQueueByID, id)
var i Discordqueue
err := row.Scan(&i.ID, &i.Articleid)
return i, err
}
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, name, key, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1
`
func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Discordwebhook, error) {
row := q.db.QueryRowContext(ctx, getDiscordWebHooksByID, id)
var i Discordwebhook
err := row.Scan(
&i.ID,
&i.Name,
&i.Key,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
)
return i, err
}
const getIconByID = `-- name: GetIconByID :one
Select id, filename, site FROM Icons
Where ID = $1 Limit 1
`
func (q *Queries) GetIconByID(ctx context.Context, id uuid.UUID) (Icon, error) {
row := q.db.QueryRowContext(ctx, getIconByID, id)
var i Icon
err := row.Scan(&i.ID, &i.Filename, &i.Site)
return i, err
}
const getIconBySite = `-- name: GetIconBySite :one
Select id, filename, site FROM Icons
Where Site = $1 Limit 1
`
func (q *Queries) GetIconBySite(ctx context.Context, site string) (Icon, error) {
row := q.db.QueryRowContext(ctx, getIconBySite, site)
var i Icon
err := row.Scan(&i.ID, &i.Filename, &i.Site)
return i, err
}
const getSettingByID = `-- name: GetSettingByID :one
Select id, key, value, options From settings
Where ID = $1 Limit 1
`
func (q *Queries) GetSettingByID(ctx context.Context, id uuid.UUID) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByID, id)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSettingByKey = `-- name: GetSettingByKey :one
Select id, key, value, options From settings Where
Key = $1 Limit 1
`
func (q *Queries) GetSettingByKey(ctx context.Context, key string) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByKey, key)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSettingByValue = `-- name: GetSettingByValue :one
Select id, key, value, options From settings Where
Value = $1 Limit 1
`
func (q *Queries) GetSettingByValue(ctx context.Context, value string) (Setting, error) {
row := q.db.QueryRowContext(ctx, getSettingByValue, value)
var i Setting
err := row.Scan(
&i.ID,
&i.Key,
&i.Value,
&i.Options,
)
return i, err
}
const getSourceByID = `-- name: GetSourceByID :one
Select id, site, name, source, type, value, enabled, url, tags From Sources where ID = $1 Limit 1
`
func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, error) {
row := q.db.QueryRowContext(ctx, getSourceByID, id)
var i Source
err := row.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
)
return i, err
}
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
Select id, name, key, url, server, channel, enabled From DiscordWebHooks
Where Server = $1
`
func (q *Queries) ListDiscordWebHooksByServer(ctx context.Context, server string) ([]Discordwebhook, error) {
rows, err := q.db.QueryContext(ctx, listDiscordWebHooksByServer, server)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordwebhook
for rows.Next() {
var i Discordwebhook
if err := rows.Scan(
&i.ID,
&i.Name,
&i.Key,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listSourcesBySource = `-- name: ListSourcesBySource :many
Select id, site, name, source, type, value, enabled, url, tags From Sources where Source = $1
`
func (q *Queries) ListSourcesBySource(ctx context.Context, source string) ([]Source, error) {
rows, err := q.db.QueryContext(ctx, listSourcesBySource, source)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Source
for rows.Next() {
var i Source
if err := rows.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}

112
database/schema/query.sql Normal file
View File

@ -0,0 +1,112 @@
/* Articles */
-- name: GetArticleByID :one
Select * from Articles
WHERE ID = $1 LIMIT 1;
-- name: GetArticleByUrl :one
Select * from Articles
Where Url = $1 LIMIT 1;
-- name: CreateArticle :exec
INSERT INTO Articles
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
Values
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
/* DiscordQueue */
-- name: CreateDiscordQueue :exec
Insert into DiscordQueue
(ID, ArticleId)
Values
($1, $2);
-- name: GetDiscordQueueByID :one
Select * from DiscordQueue
Where ID = $1 LIMIT 1;
-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1;
-- name: GetDiscordQueueItems :many
Select * from DiscordQueue LIMIT $1;
/* DiscordWebHooks */
-- name: CreateDiscordWebHook :exec
Insert Into DiscordWebHooks
(ID, Name, Key, Url, Server, Channel, Enabled)
Values
($1, $2, $3, $4, $5, $6, $7);
-- name: GetDiscordWebHooksByID :one
Select * from DiscordWebHooks
Where ID = $1 LIMIT 1;
-- name: ListDiscordWebHooksByServer :many
Select * From DiscordWebHooks
Where Server = $1;
-- name: DeleteDiscordWebHooks :exec
Delete From discordwebhooks Where ID = $1;
/* Icons */
-- name: CreateIcon :exec
INSERT INTO Icons
(ID, FileName, Site)
VALUES
($1,$2,$3);
-- name: GetIconByID :one
Select * FROM Icons
Where ID = $1 Limit 1;
-- name: GetIconBySite :one
Select * FROM Icons
Where Site = $1 Limit 1;
-- name: DeleteIcon :exec
Delete From Icons where ID = $1;
/* Settings */
-- name: CreateSettings :one
Insert Into settings
(ID, Key, Value, OPTIONS)
Values
($1,$2,$3,$4)
RETURNING *;
-- name: GetSettingByID :one
Select * From settings
Where ID = $1 Limit 1;
-- name: GetSettingByKey :one
Select * From settings Where
Key = $1 Limit 1;
-- name: GetSettingByValue :one
Select * From settings Where
Value = $1 Limit 1;
-- name: DeleteSetting :exec
Delete From settings Where ID = $1;
/* Sources */
-- name: CreateSource :exec
Insert Into Sources
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
Values
($1,$2,$3,$4,$5,$6,$7,$8,$9);
-- name: GetSourceByID :one
Select * From Sources where ID = $1 Limit 1;
-- name: ListSourcesBySource :many
Select * From Sources where Source = $1;
-- name: DeleteSource :exec
DELETE From sources where id = $1;

View File

@ -0,0 +1,56 @@
CREATE TABLE Articles (
ID uuid PRIMARY KEY,
SourceId uuid NOT null,
Tags TEXT NOT NULL,
Title TEXT NOT NULL,
Url TEXT NOT NULL,
PubDate timestamp NOT NULL,
Video TEXT,
VideoHeight int NOT NULL,
VideoWidth int NOT NULL,
Thumbnail TEXT NOT NULL,
Description TEXT NOT NULL,
AuthorName TEXT,
AuthorImage TEXT
);
CREATE Table DiscordQueue (
ID uuid PRIMARY KEY,
ArticleId uuid NOT NULL
);
CREATE Table DiscordWebHooks (
ID uuid PRIMARY KEY,
Name TEXT NOT NULL, -- Defines webhook purpose
Key TEXT,
Url TEXT NOT NULL, -- Webhook Url
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
Enabled BOOLEAN NOT NULL
);
CREATE Table Icons (
ID uuid PRIMARY Key,
FileName TEXT NOT NULL,
Site TEXT NOT NULL
);
Create Table Settings (
ID uuid PRIMARY Key,
Key TEXT NOT NULL,
Value TEXT NOT NULL,
Options TEXT
);
Create Table Sources (
ID uuid PRIMARY Key,
Site TEXT NOT NULL, -- Vanity name
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
Value TEXT,
Enabled BOOLEAN NOT NULL,
Url TEXT NOT NULL,
Tags TEXT NOT NULL
);

View File

@ -1,4 +1,4 @@
package database package databaseRest
import ( import (
"bytes" "bytes"

View File

@ -1,4 +1,4 @@
package database package databaseRest
import ( import (
"errors" "errors"

View File

@ -1,4 +1,4 @@
package database package databaseRest
import ( import (
"fmt" "fmt"

View File

@ -1,4 +1,4 @@
package database package databaseRest
import ( import (
"encoding/json" "encoding/json"

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/go-rod/rod v0.105.1 github.com/go-rod/rod v0.105.1
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/joho/godotenv v1.4.0 github.com/joho/godotenv v1.4.0
github.com/lib/pq v1.10.6
github.com/mmcdole/gofeed v1.1.3 github.com/mmcdole/gofeed v1.1.3
github.com/nicklaw5/helix/v2 v2.4.0 github.com/nicklaw5/helix/v2 v2.4.0
github.com/robfig/cron/v3 v3.0.1 github.com/robfig/cron/v3 v3.0.1

2
go.sum
View File

@ -22,6 +22,8 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=

View File

@ -1,6 +1,7 @@
package main package main
import ( import (
"context"
"log" "log"
"net/http" "net/http"
@ -12,10 +13,9 @@ import (
) )
func main() { func main() {
//dc := database.NewDatabaseClient() ctx := context.Background()
//err := dc.Diagnosis.Ping()
//if err != nil { log.Fatalln(err) } cron.EnableScheduler(ctx)
cron.EnableScheduler()
app := chi.NewRouter() app := chi.NewRouter()
app.Use(middleware.Logger) app.Use(middleware.Logger)

View File

@ -5,7 +5,12 @@ help: ## Shows this help command
build: ## builds the application with the current go runtime build: ## builds the application with the current go runtime
go build . go build .
docker-build: ## Generates the docker image docker-build: ## Generates the docker image
docker build -t "newsbot.collector.api" . docker build -t "newsbot.collector.api" .
docker image ls | grep newsbot.collector.api docker image ls | grep newsbot.collector.api
migrate-dev: ## Apply sql migrations to dev db
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down

View File

@ -10,6 +10,8 @@ import (
const ( const (
DB_URI string = "DB_URI" DB_URI string = "DB_URI"
Sql_Connection_String string = "SQL_CONNECTION_STRING"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP" REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT" REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW" REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"

View File

@ -1,106 +1,167 @@
package cron package cron
import ( import (
"context"
"database/sql"
"log" "log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services"
//"github.com/jtom38/newsbot/collector/services/cache" "github.com/jtom38/newsbot/collector/services/config"
) )
func EnableScheduler() { var _env config.ConfigClient
var _connString string
var _queries *database.Queries
func EnableScheduler(ctx context.Context) {
c := cron.New() c := cron.New()
OpenDatabase(ctx)
//c.AddFunc("*/5 * * * *", func() { go CheckCache() }) //c.AddFunc("*/5 * * * *", func() { go CheckCache() })
c.AddFunc("* */1 * * *", func() { go CheckReddit() }) c.AddFunc("* */1 * * *", func() { go CheckReddit(ctx) })
c.AddFunc("* */1 * * *", func() { go CheckYoutube() }) //c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
c.AddFunc("* */1 * * *", func() { go CheckFfxiv() }) //c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
c.AddFunc("* */1 * * *", func() { go CheckTwitch() }) //c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
c.Start() c.Start()
} }
func CheckCache() { // Open the connection to the database and share it with the package so all of them are able to share.
//cache := services.NewCacheAgeMonitor() func OpenDatabase(ctx context.Context) error {
//cache.CheckExpiredEntries() _env = config.New()
_connString = _env.GetConfig(config.Sql_Connection_String)
} db, err := sql.Open("postgres", _connString)
func CheckReddit() {
dc := database.NewDatabaseClient()
sources, err := dc.Sources.FindBySource("reddit")
if err != nil { log.Println(err) }
rc := services.NewRedditClient(sources[0].Name, sources[0].ID)
raw, err := rc.GetContent()
if err != nil { log.Println(err) }
redditArticles := rc.ConvertToArticles(raw)
for _, item := range redditArticles {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil { if err != nil {
err = dc.Articles.Add(item) panic(err)
if err != nil { log.Println("Failed to post article.")}
}
} }
queries := database.New(db)
_queries = queries
return err
} }
func CheckYoutube() { // This is the main entry point to query all the reddit services
// Add call to the db to request youtube sources. func CheckReddit(ctx context.Context) {
sources, err := _queries.ListSourcesBySource(ctx, "reddit")
// Loop though the services, and generate the clients.
yt := services.NewYoutubeClient(0, "https://www.youtube.com/user/GameGrumps")
yt.CheckSource()
}
func CheckFfxiv() {
fc := services.NewFFXIVClient("na")
articles, err := fc.CheckSource()
// This isnt in a thread yet, so just output to stdout
if err != nil { log.Println(err) }
dc := database.NewDatabaseClient()
for _, item := range articles {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil { if err != nil {
err = dc.Articles.Add(item) log.Printf("No defines sources for reddit to query - %v\r", err)
if err != nil { log.Println("Failed to post article.")}
} }
}
}
func CheckTwitch() error {
// TODO Wire this for the DB
// just a mock object for now
dc := database.NewDatabaseClient()
sources, err := dc.Sources.FindBySource("Twitch")
if err != nil { return err }
client, err := services.NewTwitchClient(sources[0])
if err != nil { log.Println(err) }
err = client.Login()
if err != nil { return err }
for _, source := range sources { for _, source := range sources {
client.ReplaceSourceRecord(source) if !source.Enabled {
continue
posts, err := client.GetContent() }
if err != nil { return err } rc := services.NewRedditClient(source)
raw, err := rc.GetContent()
for _, item := range posts {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil { if err != nil {
err = dc.Articles.Add(item) log.Println(err)
if err != nil { log.Println("Failed to post article.")}
} }
redditArticles := rc.ConvertToArticles(raw)
checkPosts(ctx, redditArticles)
} }
}
func CheckYoutube(ctx context.Context) {
// Add call to the db to request youtube sources.
sources, err := _queries.ListSourcesBySource(ctx, "youtube")
if err != nil {
log.Printf("Youtube - No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
yc := services.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
checkPosts(ctx, raw)
}
}
func CheckFfxiv(ctx context.Context) {
sources, err := _queries.ListSourcesBySource(ctx, "ffxiv")
if err != nil {
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := services.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
checkPosts(ctx, items)
}
}
func CheckTwitch(ctx context.Context) error {
sources, err := _queries.ListSourcesBySource(ctx, "twitch")
if err != nil {
log.Printf("Twitch - No sources found to query - %v\r", err)
}
tc, err := services.NewTwitchClient()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
checkPosts(ctx, items)
} }
return nil return nil
} }
func checkPosts(ctx context.Context, posts []database.Article) {
for _, item := range posts {
_, err := _queries.GetArticleByUrl(ctx, item.Url)
if err != nil {
err = postArticle(ctx, item)
if err != nil {
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
} else {
log.Printf("Reddit - Posted article - %v\r", item.Url)
}
}
}
time.Sleep(30 * time.Second)
}
func postArticle(ctx context.Context, item database.Article) error {
err := _queries.CreateArticle(ctx, database.CreateArticleParams{
ID: uuid.New(),
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}

View File

@ -1,7 +1,37 @@
package cron_test package cron_test
import "testing" import (
"context"
"testing"
"github.com/jtom38/newsbot/collector/services/cron"
)
func TestInvokeTwitch(t *testing.T) { func TestInvokeTwitch(t *testing.T) {
} }
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) {
ctx := context.Background()
cron.OpenDatabase(ctx)
cron.CheckReddit(ctx)
}
func TestCheckYouTube(t *testing.T) {
ctx := context.Background()
cron.OpenDatabase(ctx)
cron.CheckYoutube(ctx)
}
func TestCheckTwitch(t *testing.T) {
ctx := context.Background()
err := cron.OpenDatabase(ctx)
if err != nil {
t.Error(err)
}
err = cron.CheckTwitch(ctx)
if err != nil {
t.Error(err)
}
}

View File

@ -1,6 +1,7 @@
package services package services
import ( import (
"database/sql"
"errors" "errors"
"log" "log"
"net/http" "net/http"
@ -11,7 +12,7 @@ import (
"github.com/go-rod/rod" "github.com/go-rod/rod"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/domain/model" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/cache" "github.com/jtom38/newsbot/collector/services/cache"
) )
@ -23,32 +24,23 @@ const (
) )
type FFXIVClient struct { type FFXIVClient struct {
SourceID uint record database.Source
Url string //SourceID uint
Region string //Url string
//Region string
cacheGroup string cacheGroup string
} }
func NewFFXIVClient(region string) FFXIVClient { func NewFFXIVClient(Record database.Source) FFXIVClient {
var url string
switch region {
case "na":
url = FFXIV_NA_FEED_URL
case "jp":
url = FFXIV_JP_FEED_URL
}
return FFXIVClient{ return FFXIVClient{
Region: region, record: Record,
Url: url,
cacheGroup: "ffxiv", cacheGroup: "ffxiv",
} }
} }
func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) { func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
var articles []model.Articles var articles []database.Article
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -87,19 +79,18 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
tags, err := fc.ExtractTags(page) tags, err := fc.ExtractTags(page)
if err != nil { return articles, err } if err != nil { return articles, err }
article := model.Articles{ article := database.Article{
SourceID: fc.SourceID, Sourceid: fc.record.ID,
Tags: tags, Tags: tags,
Title: title, Title: title,
Url: link, Url: link,
PubDate: pubDate, Pubdate: pubDate,
Video: "", Videoheight: 0,
VideoHeight: 0, Videowidth: 0,
VideoWidth: 0,
Thumbnail: thumb, Thumbnail: thumb,
Description: description, Description: description,
AuthorName: authorName, Authorname: sql.NullString{String: authorName},
AuthorImage: authorImage, Authorimage: sql.NullString{String: authorImage},
} }
log.Printf("Collected '%v' from '%v'", article.Title, article.Url) log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
@ -112,7 +103,7 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
} }
func (fc *FFXIVClient) GetParser() (*goquery.Document, error) { func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
html, err := http.Get(fc.Url) html, err := http.Get(fc.record.Url)
if err != nil { return nil, err } if err != nil { return nil, err }
defer html.Body.Close() defer html.Body.Close()
@ -129,7 +120,7 @@ func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) { func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) {
var links []string var links []string
page := parser.MustPage(fc.Url) page := parser.MustPage(fc.record.Url)
defer page.Close() defer page.Close()
// find the list by xpath // find the list by xpath

View File

@ -3,17 +3,28 @@ package services_test
import ( import (
"testing" "testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services" ffxiv "github.com/jtom38/newsbot/collector/services"
) )
var FFXIVRecord database.Source = database.Source{
ID: uuid.New(),
Site: "ffxiv",
Name: "Final Fantasy XIV - NA",
Source: "ffxiv",
Url: "https://na.finalfantasyxiv.com/lodestone/",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
}
func TestFfxivGetParser(t *testing.T) { func TestFfxivGetParser(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
_, err := fc.GetParser() _, err := fc.GetParser()
if err != nil { panic(err) } if err != nil { panic(err) }
} }
func TestFfxivPullFeed(t *testing.T) { func TestFfxivPullFeed(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -25,7 +36,7 @@ func TestFfxivPullFeed(t *testing.T) {
} }
func TestFfxivExtractThumbnail(t *testing.T) { func TestFfxivExtractThumbnail(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -42,7 +53,7 @@ func TestFfxivExtractThumbnail(t *testing.T) {
} }
func TestFfxivExtractPubDate(t *testing.T) { func TestFfxivExtractPubDate(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -58,7 +69,7 @@ func TestFfxivExtractPubDate(t *testing.T) {
} }
func TestFfxivExtractDescription(t *testing.T) { func TestFfxivExtractDescription(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -74,7 +85,7 @@ func TestFfxivExtractDescription(t *testing.T) {
} }
func TestFfxivExtractAuthor(t *testing.T) { func TestFfxivExtractAuthor(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -91,7 +102,7 @@ func TestFfxivExtractAuthor(t *testing.T) {
} }
func TestFfxivExtractTags(t *testing.T) { func TestFfxivExtractTags(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -108,7 +119,7 @@ func TestFfxivExtractTags(t *testing.T) {
} }
func TestFfxivExtractTitle(t *testing.T) { func TestFfxivExtractTitle(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -125,7 +136,7 @@ func TestFfxivExtractTitle(t *testing.T) {
} }
func TestFFxivExtractAuthorIamge(t *testing.T) { func TestFFxivExtractAuthorIamge(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -142,7 +153,7 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
} }
func TestFfxivCheckSource(t *testing.T) { func TestFfxivCheckSource(t *testing.T) {
fc := ffxiv.NewFFXIVClient("na") fc := ffxiv.NewFFXIVClient(FFXIVRecord)
fc.CheckSource() fc.CheckSource()
} }

View File

@ -1,6 +1,7 @@
package services package services
import ( import (
"database/sql"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -10,15 +11,14 @@ import (
"time" "time"
"github.com/go-rod/rod" "github.com/go-rod/rod"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/model" "github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/config" "github.com/jtom38/newsbot/collector/services/config"
) )
type RedditClient struct { type RedditClient struct {
subreddit string
url string
sourceId uint
config RedditConfig config RedditConfig
record database.Source
} }
type RedditConfig struct { type RedditConfig struct {
@ -27,11 +27,9 @@ type RedditConfig struct {
PullNSFW string PullNSFW string
} }
func NewRedditClient(subreddit string, sourceID uint) RedditClient { func NewRedditClient(Record database.Source) RedditClient {
rc := RedditClient{ rc := RedditClient{
subreddit: subreddit, record: Record,
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
sourceId: sourceID,
} }
cc := config.New() cc := config.New()
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT) rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
@ -59,15 +57,23 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
return page return page
} }
//func (rc RedditClient)
// GetContent() reaches out to Reddit and pulls the Json data. // GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct. // It will then convert the data to a struct and return the struct.
func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) { func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
var items model.RedditJsonContent = model.RedditJsonContent{} var items model.RedditJsonContent = model.RedditJsonContent{}
log.Printf("Collecting results on '%v'", rc.subreddit) // TODO Wire this to support the config options
content, err := getHttpContent(rc.url) Url := fmt.Sprintf("%v.json", rc.record.Url)
if err != nil { return items, err }
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content) ) { log.Printf("Collecting results on '%v'", rc.record.Name)
content, err := getHttpContent(Url)
if err != nil {
return items, err
}
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
return items, errors.New("did not get json data from the server") return items, errors.New("did not get json data from the server")
} }
@ -78,12 +84,15 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
return items, nil return items, nil
} }
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles { func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
var redditArticles []model.Articles var redditArticles []database.Article
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article model.Articles var article database.Article
article, err := rc.convertToArticle(item.Data) article, err := rc.convertToArticle(item.Data)
if err != nil { log.Println(err); continue } if err != nil {
log.Println(err)
continue
}
redditArticles = append(redditArticles, article) redditArticles = append(redditArticles, article)
} }
return redditArticles return redditArticles
@ -91,11 +100,10 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) { func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
var item model.Articles var item database.Article
if source.Content == "" && source.Url != "" {
if source.Content == "" && source.Url != ""{
item = rc.convertPicturePost(source) item = rc.convertPicturePost(source)
} }
@ -119,57 +127,65 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles
return item, nil return item, nil
} }
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles { func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
var item = model.Articles{ var item = database.Article{
SourceID: rc.sourceId, Sourceid: rc.record.ID,
Tags: "a",
Title: source.Title, Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
PubDate: time.Now(), Pubdate: time.Now(),
Video: "null", Video: sql.NullString{String: "null"},
VideoHeight: 0, Videoheight: 0,
VideoWidth: 0, Videowidth: 0,
Thumbnail: source.Thumbnail, Thumbnail: source.Thumbnail,
Description: source.Content, Description: source.Content,
AuthorName: source.Author, Authorname: sql.NullString{String: source.Author},
AuthorImage: "null", Authorimage: sql.NullString{String: "null"},
} }
return item return item
} }
func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles { func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
var item = model.Articles{ var item = database.Article{
SourceID: rc.sourceId, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, Authorname: sql.NullString{String: source.Author},
Description: source.Content, Description: source.Content,
} }
return item return item
} }
func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles { func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
var item = model.Articles{ var item = database.Article{
SourceID: rc.sourceId, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl, Description: source.Media.RedditVideo.FallBackUrl,
} }
return item return item
} }
// This post is nothing more then a redirect to another location. // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles { func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
var item = model.Articles{ var item = database.Article{
SourceID: rc.sourceId, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest, Description: source.UrlOverriddenByDest,
} }
return item return item

View File

@ -1,16 +1,33 @@
package services_test package services_test
import ( import (
"log"
"testing" "testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services"
) )
var RedditRecord database.Source = database.Source{
ID: uuid.New(),
Name: "dadjokes",
Source: "reddit",
Site: "reddit",
Url: "https://reddit.com/r/dadjokes",
Tags: "reddit, dadjokes",
}
func TestGetContent(t *testing.T) { func TestGetContent(t *testing.T) {
//This test is flaky right now due to the http changes in 1.17 //This test is flaky right now due to the http changes in 1.17
rc := services.NewRedditClient("dadjokes", 0) rc := services.NewRedditClient(RedditRecord)
_, err := rc.GetContent() raw, err := rc.GetContent()
log.Println(err) if err != nil {
//if err != nil { panic(err) } t.Error(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, posts := range redditArticles {
if posts.Title == "" {
t.Error("Title is missing")
}
}
} }

View File

@ -1,20 +1,19 @@
package services package services
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"time" "time"
//"log" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/config" "github.com/jtom38/newsbot/collector/services/config"
"github.com/nicklaw5/helix/v2" "github.com/nicklaw5/helix/v2"
) )
type TwitchClient struct { type TwitchClient struct {
SourceRecord model.Sources SourceRecord database.Source
// config // config
monitorClips string monitorClips string
@ -31,7 +30,7 @@ var (
twitchScopes = "user:read:email" twitchScopes = "user:read:email"
) )
func NewTwitchClient(source model.Sources) (TwitchClient, error) { func NewTwitchClient() (TwitchClient, error) {
c := config.New() c := config.New()
id := c.GetConfig(config.TWITCH_CLIENT_ID) id := c.GetConfig(config.TWITCH_CLIENT_ID)
@ -50,7 +49,7 @@ func NewTwitchClient(source model.Sources) (TwitchClient, error) {
} }
client := TwitchClient{ client := TwitchClient{
SourceRecord: source, //SourceRecord: &source,
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS), monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD), monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
api: &api, api: &api,
@ -73,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
} }
// This will let you replace the bound source record to keep the same session alive. // This will let you replace the bound source record to keep the same session alive.
func (tc TwitchClient) ReplaceSourceRecord(source model.Sources) { func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
tc.SourceRecord = source tc.SourceRecord = source
} }
@ -88,8 +87,8 @@ func (tc TwitchClient) Login() error {
return nil return nil
} }
func (tc TwitchClient) GetContent() ([]model.Articles, error) { func (tc TwitchClient) GetContent() ([]database.Article, error) {
var items []model.Articles var items []database.Article
user, err := tc.GetUserDetails() user, err := tc.GetUserDetails()
if err != nil { if err != nil {
@ -102,21 +101,23 @@ func (tc TwitchClient) GetContent() ([]model.Articles, error) {
} }
for _, video := range posts { for _, video := range posts {
article := model.Articles{} var article database.Article
article.AuthorName, err = tc.ExtractAuthor(video) AuthorName, err := tc.ExtractAuthor(video)
if err != nil { return items, err } if err != nil { return items, err }
article.Authorname = sql.NullString{String: AuthorName}
article.AuthorImage, err = tc.ExtractAuthorImage(user) Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil { return items, err } if err != nil { return items, err }
article.Authorimage = sql.NullString{String: Authorimage}
article.Description, err = tc.ExtractDescription(video) article.Description, err = tc.ExtractDescription(video)
if err != nil {return items, err } if err != nil {return items, err }
article.PubDate, err = tc.ExtractPubDate(video) article.Pubdate, err = tc.ExtractPubDate(video)
if err != nil { return items, err } if err != nil { return items, err }
article.SourceID = tc.SourceRecord.ID article.Sourceid = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user) article.Tags, err = tc.ExtractTags(video, user)
if err != nil { return items, err } if err != nil { return items, err }

View File

@ -4,27 +4,29 @@ import (
"log" "log"
"testing" "testing"
"github.com/jtom38/newsbot/collector/domain/model" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services"
) )
var sourceRecord = model.Sources{ var TwitchSourceRecord = database.Source {
ID: 1, ID: uuid.New(),
Name: "nintendo", Name: "nintendo",
Source: "Twitch", Source: "Twitch",
} }
var invalidRecord = model.Sources{ var TwitchInvalidRecord = database.Source {
ID: 1, ID: uuid.New(),
Name: "EvilNintendo", Name: "EvilNintendo",
Source: "Twitch", Source: "Twitch",
} }
func TestTwitchLogin(t *testing.T) { func TestTwitchLogin(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -34,10 +36,11 @@ func TestTwitchLogin(t *testing.T) {
// reach out and confirms that the API returns posts made by the user. // reach out and confirms that the API returns posts made by the user.
func TestTwitchReturnsUserPosts(t *testing.T) { func TestTwitchReturnsUserPosts(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -59,10 +62,11 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
} }
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) { func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
tc, err := services.NewTwitchClient(invalidRecord) tc, err := services.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchInvalidRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -84,10 +88,11 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
} }
func TestTwitchReturnsVideoAuthor(t *testing.T) { func TestTwitchReturnsVideoAuthor(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -109,8 +114,9 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
} }
func TestTwitchReturnsThumbnail(t *testing.T) { func TestTwitchReturnsThumbnail(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil {t.Error(err) } if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
@ -127,8 +133,9 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
} }
func TestTwitchReturnsPubDate(t *testing.T) { func TestTwitchReturnsPubDate(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
@ -145,10 +152,11 @@ func TestTwitchReturnsPubDate(t *testing.T) {
} }
func TestTwitchReturnsDescription(t *testing.T) { func TestTwitchReturnsDescription(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -172,8 +180,9 @@ func TestTwitchReturnsDescription(t *testing.T) {
} }
func TestTwitchReturnsAuthorImage(t *testing.T) { func TestTwitchReturnsAuthorImage(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil {t.Error(err) } if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
@ -186,11 +195,11 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
} }
func TestTwitchReturnsTags(t *testing.T) { func TestTwitchReturnsTags(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := services.NewTwitchClient(sourceRecord)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -210,10 +219,11 @@ func TestTwitchReturnsTags(t *testing.T) {
} }
func TestTwitchReturnsTitle(t *testing.T) { func TestTwitchReturnsTitle(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { if err != nil {
@ -234,8 +244,9 @@ func TestTwitchReturnsTitle(t *testing.T) {
} }
func TestTwitchReturnsUrl(t *testing.T) { func TestTwitchReturnsUrl(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
@ -252,8 +263,9 @@ func TestTwitchReturnsUrl(t *testing.T) {
} }
func TestTwitchGetContent(t *testing.T) { func TestTwitchGetContent(t *testing.T) {
tc, err := services.NewTwitchClient(sourceRecord) tc, err := services.NewTwitchClient()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
err = tc.Login() err = tc.Login()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }

View File

@ -1,6 +1,7 @@
package services package services
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@ -12,14 +13,15 @@ import (
"github.com/go-rod/rod" "github.com/go-rod/rod"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"github.com/jtom38/newsbot/collector/domain/model" "github.com/jtom38/newsbot/collector/database"
) )
type YoutubeClient struct { type YoutubeClient struct {
SourceID uint record database.Source
Url string
ChannelID string // internal variables at time of collection
AvatarUri string channelID string
avatarUri string
// config // config
//debug bool //debug bool
@ -36,10 +38,9 @@ var (
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
func NewYoutubeClient(SourceID uint, Url string) YoutubeClient { func NewYoutubeClient(Record database.Source) YoutubeClient {
yc := YoutubeClient{ yc := YoutubeClient{
SourceID: SourceID, record: Record,
Url: Url,
cacheGroup: "youtube", cacheGroup: "youtube",
} }
/* /*
@ -53,10 +54,11 @@ func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
} }
// CheckSource will go and run all the commands needed to process a source. // CheckSource will go and run all the commands needed to process a source.
func (yc *YoutubeClient) CheckSource() error { func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
docParser, err := yc.GetParser(yc.Url) var items []database.Article
docParser, err := yc.GetParser(yc.record.Url)
if err != nil { if err != nil {
return err return items, err
} }
// Check cache/db for existing value // Check cache/db for existing value
@ -64,38 +66,38 @@ func (yc *YoutubeClient) CheckSource() error {
//channelId, err := yc.extractChannelId() //channelId, err := yc.extractChannelId()
channelId, err := yc.GetChannelId(docParser) channelId, err := yc.GetChannelId(docParser)
if err != nil { if err != nil {
return err return items, err
} }
if channelId == "" { if channelId == "" {
return ErrYoutubeChannelIdMissing return items, ErrYoutubeChannelIdMissing
} }
yc.ChannelID = channelId yc.channelID = channelId
// Check the cache/db forthe value. // Check the cache/db forthe value.
// if we have the value, skip // if we have the value, skip
avatar, err := yc.GetAvatarUri() avatar, err := yc.GetAvatarUri()
if err != nil { if err != nil {
return err return items, err
} }
if avatar == "" { if avatar == "" {
return ErrMissingAuthorImage return items, ErrMissingAuthorImage
} }
yc.AvatarUri = avatar yc.avatarUri = avatar
feed, err := yc.PullFeed() feed, err := yc.PullFeed()
if err != nil { if err != nil {
return err return items, err
} }
newPosts, err := yc.CheckForNewPosts(feed) newPosts, err := yc.CheckForNewPosts(feed)
if err != nil { if err != nil {
return err return items, err
} }
//TODO post to the API
for _, item := range newPosts { for _, item := range newPosts {
article := yc.ConvertToArticle(item) article := yc.ConvertToArticle(item)
items = append(items, article)
YoutubeUriCache = append(YoutubeUriCache, &item.Link) YoutubeUriCache = append(YoutubeUriCache, &item.Link)
@ -103,7 +105,7 @@ func (yc *YoutubeClient) CheckSource() error {
log.Println(article) log.Println(article)
} }
return nil return items, nil
} }
func (yc *YoutubeClient) GetBrowser() *rod.Browser { func (yc *YoutubeClient) GetBrowser() *rod.Browser {
@ -137,8 +139,8 @@ func (yc *YoutubeClient) GetChannelId(doc *goquery.Document) (string, error) {
for _, item := range meta.Nodes { for _, item := range meta.Nodes {
if item.Attr[0].Val == "channelId" { if item.Attr[0].Val == "channelId" {
yc.ChannelID = item.Attr[1].Val yc.channelID = item.Attr[1].Val
return yc.ChannelID, nil return yc.channelID, nil
} }
} }
return "", ErrYoutubeChannelIdMissing return "", ErrYoutubeChannelIdMissing
@ -155,7 +157,7 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) {
var AvatarUri string var AvatarUri string
browser := rod.New().MustConnect() browser := rod.New().MustConnect()
page := browser.MustPage(yc.Url) page := browser.MustPage(yc.record.Url)
res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src") res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
@ -197,7 +199,7 @@ func (yc *YoutubeClient) GetVideoThumbnail(parser *goquery.Document) (string, er
// This will pull the RSS feed items and return the results // This will pull the RSS feed items and return the results
func (yc *YoutubeClient) PullFeed() (*gofeed.Feed, error) { func (yc *YoutubeClient) PullFeed() (*gofeed.Feed, error) {
feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.ChannelID) feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.channelID)
fp := gofeed.NewParser() fp := gofeed.NewParser()
feed, err := fp.ParseURL(feedUri) feed, err := fp.ParseURL(feedUri)
if err != nil { if err != nil {
@ -239,7 +241,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
return false return false
} }
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
parser, err := yc.GetParser(item.Link) parser, err := yc.GetParser(item.Link)
if err != nil { if err != nil {
log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link) log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link)
@ -257,16 +259,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles {
log.Println(msg) log.Println(msg)
} }
var article = model.Articles{ var article = database.Article{
SourceID: yc.SourceID, Sourceid: yc.record.ID,
Tags: tags, Tags: tags,
Title: item.Title, Title: item.Title,
Url: item.Link, Url: item.Link,
PubDate: *item.PublishedParsed, Pubdate: *item.PublishedParsed,
Thumbnail: thumb, Thumbnail: thumb,
Description: item.Description, Description: item.Description,
AuthorName: item.Author.Name, Authorname: sql.NullString{String: item.Author.Name},
AuthorImage: yc.AvatarUri, Authorimage: sql.NullString{String: yc.avatarUri},
} }
return article return article
} }

View File

@ -3,24 +3,28 @@ package services_test
import ( import (
"testing" "testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services"
) )
var YouTubeRecord database.Source = database.Source{
ID: uuid.New(),
Name: "dadjokes",
Source: "reddit",
Site: "reddit",
Url: "https://youtube.com/gamegrumps",
}
func TestGetPageParser(t *testing.T) { func TestGetPageParser(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0, _, err := yc.GetParser(YouTubeRecord.Url)
"https://youtube.com/gamegrumps",
)
_, err := yc.GetParser(yc.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
} }
func TestGetChannelId(t *testing.T) { func TestGetChannelId(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0, parser, err := yc.GetParser(YouTubeRecord.Url)
"https://youtube.com/gamegrumps",
)
parser, err := yc.GetParser(yc.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
_, err = yc.GetChannelId(parser) _, err = yc.GetChannelId(parser)
@ -28,11 +32,8 @@ func TestGetChannelId(t *testing.T) {
} }
func TestPullFeed(t *testing.T) { func TestPullFeed(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0, parser, err := yc.GetParser(YouTubeRecord.Url)
"https://youtube.com/gamegrumps",
)
parser, err := yc.GetParser(yc.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
_, err = yc.GetChannelId(parser) _, err = yc.GetChannelId(parser)
@ -43,20 +44,14 @@ func TestPullFeed(t *testing.T) {
} }
func TestGetAvatarUri(t *testing.T) { func TestGetAvatarUri(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0,
"https://youtube.com/gamegrumps",
)
res, err := yc.GetAvatarUri() res, err := yc.GetAvatarUri()
if err != nil { panic(err) } if err != nil { panic(err) }
if res == "" { panic(services.ErrMissingAuthorImage)} if res == "" { panic(services.ErrMissingAuthorImage)}
} }
func TestGetVideoTags(t *testing.T) { func TestGetVideoTags(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0,
"https://youtube.com/gamegrumps",
)
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68" var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
@ -69,12 +64,9 @@ func TestGetVideoTags(t *testing.T) {
} }
func TestGetChannelTags(t *testing.T) { func TestGetChannelTags(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0,
"https://youtube.com/gamegrumps",
)
parser, err := yc.GetParser(yc.Url) parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
tags, err := yc.GetTags(parser) tags, err := yc.GetTags(parser)
@ -83,10 +75,7 @@ func TestGetChannelTags(t *testing.T) {
} }
func TestGetVideoThumbnail(t *testing.T) { func TestGetVideoThumbnail(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0,
"https://youtube.com/gamegrumps",
)
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68") parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
if err != nil {panic(err) } if err != nil {panic(err) }
@ -97,20 +86,13 @@ func TestGetVideoThumbnail(t *testing.T) {
} }
func TestCheckSource(t *testing.T) { func TestCheckSource(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0, _, err := yc.GetContent()
"https://youtube.com/gamegrumps",
)
err := yc.CheckSource()
if err != nil { panic(err) } if err != nil { panic(err) }
} }
func TestCheckUriCache(t *testing.T) { func TestCheckUriCache(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0,
"https://youtube.com/gamegrumps",
)
item := "demo" item := "demo"
services.YoutubeUriCache = append(services.YoutubeUriCache, &item) services.YoutubeUriCache = append(services.YoutubeUriCache, &item)
@ -119,10 +101,7 @@ func TestCheckUriCache(t *testing.T) {
} }
func TestCheckUriCacheFails(t *testing.T) { func TestCheckUriCacheFails(t *testing.T) {
yc := services.NewYoutubeClient( yc := services.NewYoutubeClient(YouTubeRecord)
0,
"https://youtube.com/gamegrumps",
)
item := "demo1" item := "demo1"
res := yc.CheckUriCache(&item) res := yc.CheckUriCache(&item)

7
sqlc.yaml Normal file
View File

@ -0,0 +1,7 @@
version: "1"
packages:
- schema: "database/schema/schema.sql"
queries: "database/schema/query.sql"
name: "database"
path: "database"
engine: "postgresql"