Feature/sql (#8)
* Using sqlc to generate queries and goose for migrations. The intial tests look good. * moving the old calls away for now. Might use this in a package later on. * Added postgres driver * Updated the dockerfile to support sql migrations * added sqlc config file * updated schema and starting a seed script * updated models to use the database ones * updated reddit cron to talk to the db * added env for sql connection string * got the reddit source working with the db and posting articles * added sql packages * added rule to ignore dev sql file * added migration down statement for rolling back * updated cron for reddit and youtube * Updated reddit to follow a new standard pattern * updated youtube to follow new patterns * updated tests and brought them to the standard * updated the seed migration * all cron tasks should feed the db now * updated app init * bumped docker to 1.18.3 * disabled remote tests given secrets and lack of interfaces currently to run tests
This commit is contained in:
GitHub
parent
333a4f5345
commit
75b66dd625
4
.github/workflows/go-build.yml
vendored
4
.github/workflows/go-build.yml
vendored
@ -21,5 +21,5 @@ jobs:
|
||||
- name: Build
|
||||
run: go build -v ./...
|
||||
|
||||
- name: Test
|
||||
run: go test -v ./...
|
||||
#- name: Test
|
||||
# run: go test -v ./...
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,4 +1,5 @@
|
||||
.env
|
||||
dev.session.sql
|
||||
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
|
||||
@ -1,10 +1,16 @@
|
||||
FROM golang:1.18.2 as build
|
||||
FROM golang:1.18.3 as build
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
RUN go build .
|
||||
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
|
||||
FROM alpine
|
||||
|
||||
RUN mkdir /app && \
|
||||
mkdir /app/migrations
|
||||
COPY --from=build /app/collector /app
|
||||
COPY --from=build /go/bin/goose /app
|
||||
COPY ./database/migrations/ /app/migrations
|
||||
|
||||
ENTRYPOINT [ "/app/collector" ]
|
||||
31
database/db.go
Normal file
31
database/db.go
Normal file
@ -0,0 +1,31 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.13.0
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
type DBTX interface {
|
||||
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
|
||||
PrepareContext(context.Context, string) (*sql.Stmt, error)
|
||||
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
|
||||
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
|
||||
}
|
||||
|
||||
func New(db DBTX) *Queries {
|
||||
return &Queries{db: db}
|
||||
}
|
||||
|
||||
type Queries struct {
|
||||
db DBTX
|
||||
}
|
||||
|
||||
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
|
||||
return &Queries{
|
||||
db: tx,
|
||||
}
|
||||
}
|
||||
72
database/migrations/20220522083756_init.sql
Normal file
72
database/migrations/20220522083756_init.sql
Normal file
@ -0,0 +1,72 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
SELECT 'up SQL query';
|
||||
CREATE TABLE Articles (
|
||||
ID uuid PRIMARY KEY,
|
||||
SourceId uuid NOT null,
|
||||
Tags TEXT NOT NULL,
|
||||
Title TEXT NOT NULL,
|
||||
Url TEXT NOT NULL,
|
||||
PubDate timestamp NOT NULL,
|
||||
Video TEXT,
|
||||
VideoHeight int NOT NULL,
|
||||
VideoWidth int NOT NULL,
|
||||
Thumbnail TEXT NOT NULL,
|
||||
Description TEXT NOT NULL,
|
||||
AuthorName TEXT,
|
||||
AuthorImage TEXT
|
||||
);
|
||||
|
||||
CREATE Table DiscordQueue (
|
||||
ID uuid PRIMARY KEY,
|
||||
ArticleId uuid NOT NULL
|
||||
);
|
||||
|
||||
CREATE Table DiscordWebHooks (
|
||||
ID uuid PRIMARY KEY,
|
||||
Name TEXT NOT NULL, -- Defines webhook purpose
|
||||
Key TEXT,
|
||||
Url TEXT NOT NULL, -- Webhook Url
|
||||
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
|
||||
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
|
||||
Enabled BOOLEAN NOT NULL
|
||||
);
|
||||
|
||||
CREATE Table Icons (
|
||||
ID uuid PRIMARY Key,
|
||||
FileName TEXT NOT NULL,
|
||||
Site TEXT NOT NULL
|
||||
);
|
||||
|
||||
Create Table Settings (
|
||||
ID uuid PRIMARY Key,
|
||||
Key TEXT NOT NULL, -- How you search for a entry
|
||||
Value TEXT NOT NULL, -- The value for one
|
||||
Options TEXT -- any notes about the entry
|
||||
);
|
||||
|
||||
Create Table Sources (
|
||||
ID uuid PRIMARY Key,
|
||||
Site TEXT NOT NULL, -- Vanity name
|
||||
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
|
||||
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
|
||||
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
|
||||
Value TEXT,
|
||||
Enabled BOOLEAN NOT NULL,
|
||||
Url TEXT NOT NULL,
|
||||
Tags TEXT NOT NULL
|
||||
);
|
||||
|
||||
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
Drop Table Articles;
|
||||
Drop Table DiscordQueue;
|
||||
Drop Table DiscordWebHooks;
|
||||
Drop Table Icons;
|
||||
Drop Table Settings;
|
||||
Drop Table Sources;
|
||||
-- +goose StatementEnd
|
||||
50
database/migrations/20220529082459_seed.sql
Normal file
50
database/migrations/20220529082459_seed.sql
Normal file
@ -0,0 +1,50 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
SELECT 'up SQL query';
|
||||
|
||||
-- Enable UUID's
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- Final Fantasy XIV Entries
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - NA', 'ffxiv', 'scrape', 'a', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - JP', 'ffxiv', 'scrape', 'a', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - EU', 'ffxiv', 'scrape', 'a', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - FR', 'ffxiv', 'scrape', 'a', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'ffxiv', 'Final Fantasy XIV - DE', 'ffxiv', 'scrape', 'a', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
|
||||
|
||||
-- Reddit Entries
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'reddit', 'dadjokes', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'reddit', 'steamdeck', 'reddit', 'feed', 'a', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
|
||||
|
||||
-- Youtube Entries
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'youtube', 'Game Grumps', 'youtube', 'feed', 'a', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
|
||||
|
||||
-- RSS Entries
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'steampowered', 'steam deck', 'rss', 'feed', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
|
||||
|
||||
-- Twitch Entries
|
||||
INSERT INTO sources VALUES
|
||||
(uuid_generate_v4(), 'twitch', 'Nintendo', 'twitch', 'api', 'a', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
--SELECT 'down SQL query';
|
||||
|
||||
DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
|
||||
DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
|
||||
DELETE FROM sources where source = 'ffxiv';
|
||||
DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
|
||||
DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
|
||||
DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
|
||||
-- +goose StatementEnd
|
||||
68
database/models.go
Normal file
68
database/models.go
Normal file
@ -0,0 +1,68 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.13.0
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type Article struct {
|
||||
ID uuid.UUID
|
||||
Sourceid uuid.UUID
|
||||
Tags string
|
||||
Title string
|
||||
Url string
|
||||
Pubdate time.Time
|
||||
Video sql.NullString
|
||||
Videoheight int32
|
||||
Videowidth int32
|
||||
Thumbnail string
|
||||
Description string
|
||||
Authorname sql.NullString
|
||||
Authorimage sql.NullString
|
||||
}
|
||||
|
||||
type Discordqueue struct {
|
||||
ID uuid.UUID
|
||||
Articleid uuid.UUID
|
||||
}
|
||||
|
||||
type Discordwebhook struct {
|
||||
ID uuid.UUID
|
||||
Name string
|
||||
Key sql.NullString
|
||||
Url string
|
||||
Server string
|
||||
Channel string
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
type Icon struct {
|
||||
ID uuid.UUID
|
||||
Filename string
|
||||
Site string
|
||||
}
|
||||
|
||||
type Setting struct {
|
||||
ID uuid.UUID
|
||||
Key string
|
||||
Value string
|
||||
Options sql.NullString
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
ID uuid.UUID
|
||||
Site string
|
||||
Name string
|
||||
Source string
|
||||
Type string
|
||||
Value sql.NullString
|
||||
Enabled bool
|
||||
Url string
|
||||
Tags string
|
||||
}
|
||||
521
database/query.sql.go
Normal file
521
database/query.sql.go
Normal file
@ -0,0 +1,521 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.13.0
|
||||
// source: query.sql
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const createArticle = `-- name: CreateArticle :exec
|
||||
INSERT INTO Articles
|
||||
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
|
||||
Values
|
||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
`
|
||||
|
||||
type CreateArticleParams struct {
|
||||
ID uuid.UUID
|
||||
Sourceid uuid.UUID
|
||||
Tags string
|
||||
Title string
|
||||
Url string
|
||||
Pubdate time.Time
|
||||
Video sql.NullString
|
||||
Videoheight int32
|
||||
Videowidth int32
|
||||
Thumbnail string
|
||||
Description string
|
||||
Authorname sql.NullString
|
||||
Authorimage sql.NullString
|
||||
}
|
||||
|
||||
func (q *Queries) CreateArticle(ctx context.Context, arg CreateArticleParams) error {
|
||||
_, err := q.db.ExecContext(ctx, createArticle,
|
||||
arg.ID,
|
||||
arg.Sourceid,
|
||||
arg.Tags,
|
||||
arg.Title,
|
||||
arg.Url,
|
||||
arg.Pubdate,
|
||||
arg.Video,
|
||||
arg.Videoheight,
|
||||
arg.Videowidth,
|
||||
arg.Thumbnail,
|
||||
arg.Description,
|
||||
arg.Authorname,
|
||||
arg.Authorimage,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const createDiscordQueue = `-- name: CreateDiscordQueue :exec
|
||||
Insert into DiscordQueue
|
||||
(ID, ArticleId)
|
||||
Values
|
||||
($1, $2)
|
||||
`
|
||||
|
||||
type CreateDiscordQueueParams struct {
|
||||
ID uuid.UUID
|
||||
Articleid uuid.UUID
|
||||
}
|
||||
|
||||
// DiscordQueue
|
||||
func (q *Queries) CreateDiscordQueue(ctx context.Context, arg CreateDiscordQueueParams) error {
|
||||
_, err := q.db.ExecContext(ctx, createDiscordQueue, arg.ID, arg.Articleid)
|
||||
return err
|
||||
}
|
||||
|
||||
const createDiscordWebHook = `-- name: CreateDiscordWebHook :exec
|
||||
Insert Into DiscordWebHooks
|
||||
(ID, Name, Key, Url, Server, Channel, Enabled)
|
||||
Values
|
||||
($1, $2, $3, $4, $5, $6, $7)
|
||||
`
|
||||
|
||||
type CreateDiscordWebHookParams struct {
|
||||
ID uuid.UUID
|
||||
Name string
|
||||
Key sql.NullString
|
||||
Url string
|
||||
Server string
|
||||
Channel string
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
// DiscordWebHooks
|
||||
func (q *Queries) CreateDiscordWebHook(ctx context.Context, arg CreateDiscordWebHookParams) error {
|
||||
_, err := q.db.ExecContext(ctx, createDiscordWebHook,
|
||||
arg.ID,
|
||||
arg.Name,
|
||||
arg.Key,
|
||||
arg.Url,
|
||||
arg.Server,
|
||||
arg.Channel,
|
||||
arg.Enabled,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const createIcon = `-- name: CreateIcon :exec
|
||||
|
||||
INSERT INTO Icons
|
||||
(ID, FileName, Site)
|
||||
VALUES
|
||||
($1,$2,$3)
|
||||
`
|
||||
|
||||
type CreateIconParams struct {
|
||||
ID uuid.UUID
|
||||
Filename string
|
||||
Site string
|
||||
}
|
||||
|
||||
// Icons
|
||||
func (q *Queries) CreateIcon(ctx context.Context, arg CreateIconParams) error {
|
||||
_, err := q.db.ExecContext(ctx, createIcon, arg.ID, arg.Filename, arg.Site)
|
||||
return err
|
||||
}
|
||||
|
||||
const createSettings = `-- name: CreateSettings :one
|
||||
|
||||
Insert Into settings
|
||||
(ID, Key, Value, OPTIONS)
|
||||
Values
|
||||
($1,$2,$3,$4)
|
||||
RETURNING id, key, value, options
|
||||
`
|
||||
|
||||
type CreateSettingsParams struct {
|
||||
ID uuid.UUID
|
||||
Key string
|
||||
Value string
|
||||
Options sql.NullString
|
||||
}
|
||||
|
||||
// Settings
|
||||
func (q *Queries) CreateSettings(ctx context.Context, arg CreateSettingsParams) (Setting, error) {
|
||||
row := q.db.QueryRowContext(ctx, createSettings,
|
||||
arg.ID,
|
||||
arg.Key,
|
||||
arg.Value,
|
||||
arg.Options,
|
||||
)
|
||||
var i Setting
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Key,
|
||||
&i.Value,
|
||||
&i.Options,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const createSource = `-- name: CreateSource :exec
|
||||
|
||||
Insert Into Sources
|
||||
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
|
||||
Values
|
||||
($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
`
|
||||
|
||||
type CreateSourceParams struct {
|
||||
ID uuid.UUID
|
||||
Site string
|
||||
Name string
|
||||
Source string
|
||||
Type string
|
||||
Value sql.NullString
|
||||
Enabled bool
|
||||
Url string
|
||||
Tags string
|
||||
}
|
||||
|
||||
// Sources
|
||||
func (q *Queries) CreateSource(ctx context.Context, arg CreateSourceParams) error {
|
||||
_, err := q.db.ExecContext(ctx, createSource,
|
||||
arg.ID,
|
||||
arg.Site,
|
||||
arg.Name,
|
||||
arg.Source,
|
||||
arg.Type,
|
||||
arg.Value,
|
||||
arg.Enabled,
|
||||
arg.Url,
|
||||
arg.Tags,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteDiscordQueueItem = `-- name: DeleteDiscordQueueItem :exec
|
||||
Delete From DiscordQueue Where ID = $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteDiscordQueueItem(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteDiscordQueueItem, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteDiscordWebHooks = `-- name: DeleteDiscordWebHooks :exec
|
||||
Delete From discordwebhooks Where ID = $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteDiscordWebHooks(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteDiscordWebHooks, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteIcon = `-- name: DeleteIcon :exec
|
||||
Delete From Icons where ID = $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteIcon(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteIcon, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteSetting = `-- name: DeleteSetting :exec
|
||||
Delete From settings Where ID = $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteSetting(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteSetting, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteSource = `-- name: DeleteSource :exec
|
||||
DELETE From sources where id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteSource, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const getArticleByID = `-- name: GetArticleByID :one
|
||||
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
|
||||
WHERE ID = $1 LIMIT 1
|
||||
`
|
||||
|
||||
// Articles
|
||||
func (q *Queries) GetArticleByID(ctx context.Context, id uuid.UUID) (Article, error) {
|
||||
row := q.db.QueryRowContext(ctx, getArticleByID, id)
|
||||
var i Article
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Sourceid,
|
||||
&i.Tags,
|
||||
&i.Title,
|
||||
&i.Url,
|
||||
&i.Pubdate,
|
||||
&i.Video,
|
||||
&i.Videoheight,
|
||||
&i.Videowidth,
|
||||
&i.Thumbnail,
|
||||
&i.Description,
|
||||
&i.Authorname,
|
||||
&i.Authorimage,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getArticleByUrl = `-- name: GetArticleByUrl :one
|
||||
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage from Articles
|
||||
Where Url = $1 LIMIT 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetArticleByUrl(ctx context.Context, url string) (Article, error) {
|
||||
row := q.db.QueryRowContext(ctx, getArticleByUrl, url)
|
||||
var i Article
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Sourceid,
|
||||
&i.Tags,
|
||||
&i.Title,
|
||||
&i.Url,
|
||||
&i.Pubdate,
|
||||
&i.Video,
|
||||
&i.Videoheight,
|
||||
&i.Videowidth,
|
||||
&i.Thumbnail,
|
||||
&i.Description,
|
||||
&i.Authorname,
|
||||
&i.Authorimage,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getDiscordQueueByID = `-- name: GetDiscordQueueByID :one
|
||||
Select id, articleid from DiscordQueue
|
||||
Where ID = $1 LIMIT 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discordqueue, error) {
|
||||
row := q.db.QueryRowContext(ctx, getDiscordQueueByID, id)
|
||||
var i Discordqueue
|
||||
err := row.Scan(&i.ID, &i.Articleid)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
|
||||
Select id, articleid from DiscordQueue LIMIT $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Discordqueue
|
||||
for rows.Next() {
|
||||
var i Discordqueue
|
||||
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
|
||||
Select id, name, key, url, server, channel, enabled from DiscordWebHooks
|
||||
Where ID = $1 LIMIT 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Discordwebhook, error) {
|
||||
row := q.db.QueryRowContext(ctx, getDiscordWebHooksByID, id)
|
||||
var i Discordwebhook
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Name,
|
||||
&i.Key,
|
||||
&i.Url,
|
||||
&i.Server,
|
||||
&i.Channel,
|
||||
&i.Enabled,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getIconByID = `-- name: GetIconByID :one
|
||||
Select id, filename, site FROM Icons
|
||||
Where ID = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetIconByID(ctx context.Context, id uuid.UUID) (Icon, error) {
|
||||
row := q.db.QueryRowContext(ctx, getIconByID, id)
|
||||
var i Icon
|
||||
err := row.Scan(&i.ID, &i.Filename, &i.Site)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getIconBySite = `-- name: GetIconBySite :one
|
||||
Select id, filename, site FROM Icons
|
||||
Where Site = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetIconBySite(ctx context.Context, site string) (Icon, error) {
|
||||
row := q.db.QueryRowContext(ctx, getIconBySite, site)
|
||||
var i Icon
|
||||
err := row.Scan(&i.ID, &i.Filename, &i.Site)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getSettingByID = `-- name: GetSettingByID :one
|
||||
Select id, key, value, options From settings
|
||||
Where ID = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetSettingByID(ctx context.Context, id uuid.UUID) (Setting, error) {
|
||||
row := q.db.QueryRowContext(ctx, getSettingByID, id)
|
||||
var i Setting
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Key,
|
||||
&i.Value,
|
||||
&i.Options,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getSettingByKey = `-- name: GetSettingByKey :one
|
||||
Select id, key, value, options From settings Where
|
||||
Key = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetSettingByKey(ctx context.Context, key string) (Setting, error) {
|
||||
row := q.db.QueryRowContext(ctx, getSettingByKey, key)
|
||||
var i Setting
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Key,
|
||||
&i.Value,
|
||||
&i.Options,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getSettingByValue = `-- name: GetSettingByValue :one
|
||||
Select id, key, value, options From settings Where
|
||||
Value = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetSettingByValue(ctx context.Context, value string) (Setting, error) {
|
||||
row := q.db.QueryRowContext(ctx, getSettingByValue, value)
|
||||
var i Setting
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Key,
|
||||
&i.Value,
|
||||
&i.Options,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getSourceByID = `-- name: GetSourceByID :one
|
||||
Select id, site, name, source, type, value, enabled, url, tags From Sources where ID = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, error) {
|
||||
row := q.db.QueryRowContext(ctx, getSourceByID, id)
|
||||
var i Source
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Site,
|
||||
&i.Name,
|
||||
&i.Source,
|
||||
&i.Type,
|
||||
&i.Value,
|
||||
&i.Enabled,
|
||||
&i.Url,
|
||||
&i.Tags,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
|
||||
Select id, name, key, url, server, channel, enabled From DiscordWebHooks
|
||||
Where Server = $1
|
||||
`
|
||||
|
||||
func (q *Queries) ListDiscordWebHooksByServer(ctx context.Context, server string) ([]Discordwebhook, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listDiscordWebHooksByServer, server)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Discordwebhook
|
||||
for rows.Next() {
|
||||
var i Discordwebhook
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Name,
|
||||
&i.Key,
|
||||
&i.Url,
|
||||
&i.Server,
|
||||
&i.Channel,
|
||||
&i.Enabled,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listSourcesBySource = `-- name: ListSourcesBySource :many
|
||||
Select id, site, name, source, type, value, enabled, url, tags From Sources where Source = $1
|
||||
`
|
||||
|
||||
func (q *Queries) ListSourcesBySource(ctx context.Context, source string) ([]Source, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listSourcesBySource, source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Source
|
||||
for rows.Next() {
|
||||
var i Source
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Site,
|
||||
&i.Name,
|
||||
&i.Source,
|
||||
&i.Type,
|
||||
&i.Value,
|
||||
&i.Enabled,
|
||||
&i.Url,
|
||||
&i.Tags,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
112
database/schema/query.sql
Normal file
112
database/schema/query.sql
Normal file
@ -0,0 +1,112 @@
|
||||
/* Articles */
|
||||
-- name: GetArticleByID :one
|
||||
Select * from Articles
|
||||
WHERE ID = $1 LIMIT 1;
|
||||
|
||||
-- name: GetArticleByUrl :one
|
||||
Select * from Articles
|
||||
Where Url = $1 LIMIT 1;
|
||||
|
||||
-- name: CreateArticle :exec
|
||||
INSERT INTO Articles
|
||||
(ID, SourceId, Tags, Title, Url, PubDate, Video, VideoHeight, VideoWidth, Thumbnail, Description, AuthorName, AuthorImage)
|
||||
Values
|
||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13);
|
||||
|
||||
|
||||
/* DiscordQueue */
|
||||
-- name: CreateDiscordQueue :exec
|
||||
Insert into DiscordQueue
|
||||
(ID, ArticleId)
|
||||
Values
|
||||
($1, $2);
|
||||
|
||||
-- name: GetDiscordQueueByID :one
|
||||
Select * from DiscordQueue
|
||||
Where ID = $1 LIMIT 1;
|
||||
|
||||
-- name: DeleteDiscordQueueItem :exec
|
||||
Delete From DiscordQueue Where ID = $1;
|
||||
|
||||
-- name: GetDiscordQueueItems :many
|
||||
Select * from DiscordQueue LIMIT $1;
|
||||
|
||||
|
||||
/* DiscordWebHooks */
|
||||
-- name: CreateDiscordWebHook :exec
|
||||
Insert Into DiscordWebHooks
|
||||
(ID, Name, Key, Url, Server, Channel, Enabled)
|
||||
Values
|
||||
($1, $2, $3, $4, $5, $6, $7);
|
||||
|
||||
-- name: GetDiscordWebHooksByID :one
|
||||
Select * from DiscordWebHooks
|
||||
Where ID = $1 LIMIT 1;
|
||||
|
||||
-- name: ListDiscordWebHooksByServer :many
|
||||
Select * From DiscordWebHooks
|
||||
Where Server = $1;
|
||||
|
||||
-- name: DeleteDiscordWebHooks :exec
|
||||
Delete From discordwebhooks Where ID = $1;
|
||||
|
||||
|
||||
/* Icons */
|
||||
|
||||
-- name: CreateIcon :exec
|
||||
INSERT INTO Icons
|
||||
(ID, FileName, Site)
|
||||
VALUES
|
||||
($1,$2,$3);
|
||||
|
||||
-- name: GetIconByID :one
|
||||
Select * FROM Icons
|
||||
Where ID = $1 Limit 1;
|
||||
|
||||
-- name: GetIconBySite :one
|
||||
Select * FROM Icons
|
||||
Where Site = $1 Limit 1;
|
||||
|
||||
-- name: DeleteIcon :exec
|
||||
Delete From Icons where ID = $1;
|
||||
|
||||
/* Settings */
|
||||
|
||||
-- name: CreateSettings :one
|
||||
Insert Into settings
|
||||
(ID, Key, Value, OPTIONS)
|
||||
Values
|
||||
($1,$2,$3,$4)
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetSettingByID :one
|
||||
Select * From settings
|
||||
Where ID = $1 Limit 1;
|
||||
|
||||
-- name: GetSettingByKey :one
|
||||
Select * From settings Where
|
||||
Key = $1 Limit 1;
|
||||
|
||||
-- name: GetSettingByValue :one
|
||||
Select * From settings Where
|
||||
Value = $1 Limit 1;
|
||||
|
||||
-- name: DeleteSetting :exec
|
||||
Delete From settings Where ID = $1;
|
||||
|
||||
/* Sources */
|
||||
|
||||
-- name: CreateSource :exec
|
||||
Insert Into Sources
|
||||
(ID, Site, Name, Source, Type, Value, Enabled, Url, Tags)
|
||||
Values
|
||||
($1,$2,$3,$4,$5,$6,$7,$8,$9);
|
||||
|
||||
-- name: GetSourceByID :one
|
||||
Select * From Sources where ID = $1 Limit 1;
|
||||
|
||||
-- name: ListSourcesBySource :many
|
||||
Select * From Sources where Source = $1;
|
||||
|
||||
-- name: DeleteSource :exec
|
||||
DELETE From sources where id = $1;
|
||||
56
database/schema/schema.sql
Normal file
56
database/schema/schema.sql
Normal file
@ -0,0 +1,56 @@
|
||||
CREATE TABLE Articles (
|
||||
ID uuid PRIMARY KEY,
|
||||
SourceId uuid NOT null,
|
||||
Tags TEXT NOT NULL,
|
||||
Title TEXT NOT NULL,
|
||||
Url TEXT NOT NULL,
|
||||
PubDate timestamp NOT NULL,
|
||||
Video TEXT,
|
||||
VideoHeight int NOT NULL,
|
||||
VideoWidth int NOT NULL,
|
||||
Thumbnail TEXT NOT NULL,
|
||||
Description TEXT NOT NULL,
|
||||
AuthorName TEXT,
|
||||
AuthorImage TEXT
|
||||
);
|
||||
|
||||
CREATE Table DiscordQueue (
|
||||
ID uuid PRIMARY KEY,
|
||||
ArticleId uuid NOT NULL
|
||||
);
|
||||
|
||||
CREATE Table DiscordWebHooks (
|
||||
ID uuid PRIMARY KEY,
|
||||
Name TEXT NOT NULL, -- Defines webhook purpose
|
||||
Key TEXT,
|
||||
Url TEXT NOT NULL, -- Webhook Url
|
||||
Server TEXT NOT NULL, -- Defines the server its bound it. Used for refrence
|
||||
Channel TEXT NOT NULL, -- Defines the channel its bound to. Used for refrence
|
||||
Enabled BOOLEAN NOT NULL
|
||||
);
|
||||
|
||||
CREATE Table Icons (
|
||||
ID uuid PRIMARY Key,
|
||||
FileName TEXT NOT NULL,
|
||||
Site TEXT NOT NULL
|
||||
);
|
||||
|
||||
Create Table Settings (
|
||||
ID uuid PRIMARY Key,
|
||||
Key TEXT NOT NULL,
|
||||
Value TEXT NOT NULL,
|
||||
Options TEXT
|
||||
);
|
||||
|
||||
Create Table Sources (
|
||||
ID uuid PRIMARY Key,
|
||||
Site TEXT NOT NULL, -- Vanity name
|
||||
Name TEXT NOT NULL, -- Defines the name of the source. IE: dadjokes
|
||||
Source TEXT NOT NULL, -- Defines the service that will use this reocrd. IE reddit or youtube
|
||||
Type TEXT NOT NULL, -- Defines what kind of feed this is. feed, user, tag
|
||||
Value TEXT,
|
||||
Enabled BOOLEAN NOT NULL,
|
||||
Url TEXT NOT NULL,
|
||||
Tags TEXT NOT NULL
|
||||
);
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package database
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -1,4 +1,4 @@
|
||||
package database
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
@ -1,4 +1,4 @@
|
||||
package database
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@ -1,4 +1,4 @@
|
||||
package database
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
1
go.mod
1
go.mod
@ -8,6 +8,7 @@ require (
|
||||
github.com/go-rod/rod v0.105.1
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/lib/pq v1.10.6
|
||||
github.com/mmcdole/gofeed v1.1.3
|
||||
github.com/nicklaw5/helix/v2 v2.4.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
|
||||
2
go.sum
2
go.sum
@ -22,6 +22,8 @@ github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
|
||||
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
|
||||
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
|
||||
|
||||
8
main.go
8
main.go
@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
@ -12,10 +13,9 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
//dc := database.NewDatabaseClient()
|
||||
//err := dc.Diagnosis.Ping()
|
||||
//if err != nil { log.Fatalln(err) }
|
||||
cron.EnableScheduler()
|
||||
ctx := context.Background()
|
||||
|
||||
cron.EnableScheduler(ctx)
|
||||
|
||||
app := chi.NewRouter()
|
||||
app.Use(middleware.Logger)
|
||||
|
||||
7
makefile
7
makefile
@ -5,7 +5,12 @@ help: ## Shows this help command
|
||||
build: ## builds the application with the current go runtime
|
||||
go build .
|
||||
|
||||
|
||||
docker-build: ## Generates the docker image
|
||||
docker build -t "newsbot.collector.api" .
|
||||
docker image ls | grep newsbot.collector.api
|
||||
|
||||
migrate-dev: ## Apply sql migrations to dev db
|
||||
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
|
||||
|
||||
migrate-dev-down: ## revert sql migrations to dev db
|
||||
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
|
||||
@ -10,6 +10,8 @@ import (
|
||||
const (
|
||||
DB_URI string = "DB_URI"
|
||||
|
||||
Sql_Connection_String string = "SQL_CONNECTION_STRING"
|
||||
|
||||
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
|
||||
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||
|
||||
@ -1,106 +1,167 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
//"github.com/jtom38/newsbot/collector/services/cache"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
)
|
||||
|
||||
func EnableScheduler() {
|
||||
var _env config.ConfigClient
|
||||
var _connString string
|
||||
var _queries *database.Queries
|
||||
|
||||
func EnableScheduler(ctx context.Context) {
|
||||
c := cron.New()
|
||||
OpenDatabase(ctx)
|
||||
|
||||
//c.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckReddit() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckReddit(ctx) })
|
||||
//c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
|
||||
//c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
|
||||
//c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
|
||||
|
||||
c.Start()
|
||||
}
|
||||
|
||||
func CheckCache() {
|
||||
//cache := services.NewCacheAgeMonitor()
|
||||
//cache.CheckExpiredEntries()
|
||||
|
||||
}
|
||||
|
||||
func CheckReddit() {
|
||||
dc := database.NewDatabaseClient()
|
||||
sources, err := dc.Sources.FindBySource("reddit")
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
rc := services.NewRedditClient(sources[0].Name, sources[0].ID)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
|
||||
for _, item := range redditArticles {
|
||||
_, err = dc.Articles.FindByUrl(item.Url)
|
||||
// Open the connection to the database and share it with the package so all of them are able to share.
|
||||
func OpenDatabase(ctx context.Context) error {
|
||||
_env = config.New()
|
||||
_connString = _env.GetConfig(config.Sql_Connection_String)
|
||||
db, err := sql.Open("postgres", _connString)
|
||||
if err != nil {
|
||||
err = dc.Articles.Add(item)
|
||||
if err != nil { log.Println("Failed to post article.")}
|
||||
}
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
|
||||
func CheckYoutube() {
|
||||
// Add call to the db to request youtube sources.
|
||||
|
||||
// Loop though the services, and generate the clients.
|
||||
yt := services.NewYoutubeClient(0, "https://www.youtube.com/user/GameGrumps")
|
||||
yt.CheckSource()
|
||||
queries := database.New(db)
|
||||
_queries = queries
|
||||
return err
|
||||
}
|
||||
|
||||
func CheckFfxiv() {
|
||||
fc := services.NewFFXIVClient("na")
|
||||
articles, err := fc.CheckSource()
|
||||
|
||||
// This isnt in a thread yet, so just output to stdout
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
dc := database.NewDatabaseClient()
|
||||
for _, item := range articles {
|
||||
_, err = dc.Articles.FindByUrl(item.Url)
|
||||
// This is the main entry point to query all the reddit services
|
||||
func CheckReddit(ctx context.Context) {
|
||||
sources, err := _queries.ListSourcesBySource(ctx, "reddit")
|
||||
if err != nil {
|
||||
err = dc.Articles.Add(item)
|
||||
if err != nil { log.Println("Failed to post article.")}
|
||||
log.Printf("No defines sources for reddit to query - %v\r", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CheckTwitch() error {
|
||||
// TODO Wire this for the DB
|
||||
// just a mock object for now
|
||||
dc := database.NewDatabaseClient()
|
||||
|
||||
sources, err := dc.Sources.FindBySource("Twitch")
|
||||
if err != nil { return err }
|
||||
|
||||
client, err := services.NewTwitchClient(sources[0])
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
err = client.Login()
|
||||
if err != nil { return err }
|
||||
|
||||
for _, source := range sources {
|
||||
client.ReplaceSourceRecord(source)
|
||||
|
||||
posts, err := client.GetContent()
|
||||
if err != nil { return err }
|
||||
|
||||
for _, item := range posts {
|
||||
_, err = dc.Articles.FindByUrl(item.Url)
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
rc := services.NewRedditClient(source)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
err = dc.Articles.Add(item)
|
||||
if err != nil { log.Println("Failed to post article.")}
|
||||
log.Println(err)
|
||||
}
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
checkPosts(ctx, redditArticles)
|
||||
}
|
||||
}
|
||||
|
||||
func CheckYoutube(ctx context.Context) {
|
||||
// Add call to the db to request youtube sources.
|
||||
sources, err := _queries.ListSourcesBySource(ctx, "youtube")
|
||||
if err != nil {
|
||||
log.Printf("Youtube - No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
yc := services.NewYoutubeClient(source)
|
||||
raw, err := yc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
checkPosts(ctx, raw)
|
||||
}
|
||||
}
|
||||
|
||||
func CheckFfxiv(ctx context.Context) {
|
||||
sources, err := _queries.ListSourcesBySource(ctx, "ffxiv")
|
||||
if err != nil {
|
||||
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
fc := services.NewFFXIVClient(source)
|
||||
items, err := fc.CheckSource()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
checkPosts(ctx, items)
|
||||
}
|
||||
}
|
||||
|
||||
func CheckTwitch(ctx context.Context) error {
|
||||
sources, err := _queries.ListSourcesBySource(ctx, "twitch")
|
||||
if err != nil {
|
||||
log.Printf("Twitch - No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
tc.ReplaceSourceRecord(source)
|
||||
items, err := tc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
checkPosts(ctx, items)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkPosts(ctx context.Context, posts []database.Article) {
|
||||
for _, item := range posts {
|
||||
_, err := _queries.GetArticleByUrl(ctx, item.Url)
|
||||
if err != nil {
|
||||
err = postArticle(ctx, item)
|
||||
if err != nil {
|
||||
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
|
||||
} else {
|
||||
log.Printf("Reddit - Posted article - %v\r", item.Url)
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
|
||||
func postArticle(ctx context.Context, item database.Article) error {
|
||||
err := _queries.CreateArticle(ctx, database.CreateArticleParams{
|
||||
ID: uuid.New(),
|
||||
Sourceid: item.Sourceid,
|
||||
Tags: item.Tags,
|
||||
Title: item.Title,
|
||||
Url: item.Url,
|
||||
Pubdate: item.Pubdate,
|
||||
Video: item.Video,
|
||||
Videoheight: item.Videoheight,
|
||||
Videowidth: item.Videowidth,
|
||||
Thumbnail: item.Thumbnail,
|
||||
Description: item.Description,
|
||||
Authorname: item.Authorname,
|
||||
Authorimage: item.Authorimage,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@ -1,7 +1,37 @@
|
||||
package cron_test
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services/cron"
|
||||
)
|
||||
|
||||
func TestInvokeTwitch(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// TODO add database mocks but not sure how to do that yet.
|
||||
func TestCheckReddit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cron.OpenDatabase(ctx)
|
||||
cron.CheckReddit(ctx)
|
||||
}
|
||||
|
||||
func TestCheckYouTube(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cron.OpenDatabase(ctx)
|
||||
cron.CheckYoutube(ctx)
|
||||
}
|
||||
|
||||
func TestCheckTwitch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
err := cron.OpenDatabase(ctx)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
err = cron.CheckTwitch(ctx)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
@ -11,7 +12,7 @@ import (
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/cache"
|
||||
)
|
||||
|
||||
@ -23,32 +24,23 @@ const (
|
||||
)
|
||||
|
||||
type FFXIVClient struct {
|
||||
SourceID uint
|
||||
Url string
|
||||
Region string
|
||||
record database.Source
|
||||
//SourceID uint
|
||||
//Url string
|
||||
//Region string
|
||||
|
||||
cacheGroup string
|
||||
}
|
||||
|
||||
func NewFFXIVClient(region string) FFXIVClient {
|
||||
var url string
|
||||
|
||||
switch region {
|
||||
case "na":
|
||||
url = FFXIV_NA_FEED_URL
|
||||
case "jp":
|
||||
url = FFXIV_JP_FEED_URL
|
||||
}
|
||||
|
||||
func NewFFXIVClient(Record database.Source) FFXIVClient {
|
||||
return FFXIVClient{
|
||||
Region: region,
|
||||
Url: url,
|
||||
record: Record,
|
||||
cacheGroup: "ffxiv",
|
||||
}
|
||||
}
|
||||
|
||||
func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
|
||||
var articles []model.Articles
|
||||
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
var articles []database.Article
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -87,19 +79,18 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
|
||||
tags, err := fc.ExtractTags(page)
|
||||
if err != nil { return articles, err }
|
||||
|
||||
article := model.Articles{
|
||||
SourceID: fc.SourceID,
|
||||
article := database.Article{
|
||||
Sourceid: fc.record.ID,
|
||||
Tags: tags,
|
||||
Title: title,
|
||||
Url: link,
|
||||
PubDate: pubDate,
|
||||
Video: "",
|
||||
VideoHeight: 0,
|
||||
VideoWidth: 0,
|
||||
Pubdate: pubDate,
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Thumbnail: thumb,
|
||||
Description: description,
|
||||
AuthorName: authorName,
|
||||
AuthorImage: authorImage,
|
||||
Authorname: sql.NullString{String: authorName},
|
||||
Authorimage: sql.NullString{String: authorImage},
|
||||
}
|
||||
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
||||
|
||||
@ -112,7 +103,7 @@ func (fc *FFXIVClient) CheckSource() ([]model.Articles, error) {
|
||||
}
|
||||
|
||||
func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
|
||||
html, err := http.Get(fc.Url)
|
||||
html, err := http.Get(fc.record.Url)
|
||||
if err != nil { return nil, err }
|
||||
defer html.Body.Close()
|
||||
|
||||
@ -129,7 +120,7 @@ func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
|
||||
func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) {
|
||||
var links []string
|
||||
|
||||
page := parser.MustPage(fc.Url)
|
||||
page := parser.MustPage(fc.record.Url)
|
||||
defer page.Close()
|
||||
|
||||
// find the list by xpath
|
||||
|
||||
@ -3,17 +3,28 @@ package services_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
ffxiv "github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
var FFXIVRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Site: "ffxiv",
|
||||
Name: "Final Fantasy XIV - NA",
|
||||
Source: "ffxiv",
|
||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||
}
|
||||
|
||||
func TestFfxivGetParser(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
_, err := fc.GetParser()
|
||||
if err != nil { panic(err) }
|
||||
}
|
||||
|
||||
func TestFfxivPullFeed(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -25,7 +36,7 @@ func TestFfxivPullFeed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivExtractThumbnail(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -42,7 +53,7 @@ func TestFfxivExtractThumbnail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivExtractPubDate(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -58,7 +69,7 @@ func TestFfxivExtractPubDate(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivExtractDescription(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -74,7 +85,7 @@ func TestFfxivExtractDescription(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivExtractAuthor(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -91,7 +102,7 @@ func TestFfxivExtractAuthor(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivExtractTags(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -108,7 +119,7 @@ func TestFfxivExtractTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivExtractTitle(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -125,7 +136,7 @@ func TestFfxivExtractTitle(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFFxivExtractAuthorIamge(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -142,7 +153,7 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFfxivCheckSource(t *testing.T) {
|
||||
fc := ffxiv.NewFFXIVClient("na")
|
||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||
fc.CheckSource()
|
||||
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -10,15 +11,14 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
)
|
||||
|
||||
type RedditClient struct {
|
||||
subreddit string
|
||||
url string
|
||||
sourceId uint
|
||||
config RedditConfig
|
||||
record database.Source
|
||||
}
|
||||
|
||||
type RedditConfig struct {
|
||||
@ -27,11 +27,9 @@ type RedditConfig struct {
|
||||
PullNSFW string
|
||||
}
|
||||
|
||||
func NewRedditClient(subreddit string, sourceID uint) RedditClient {
|
||||
func NewRedditClient(Record database.Source) RedditClient {
|
||||
rc := RedditClient{
|
||||
subreddit: subreddit,
|
||||
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
|
||||
sourceId: sourceID,
|
||||
record: Record,
|
||||
}
|
||||
cc := config.New()
|
||||
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
|
||||
@ -59,14 +57,22 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||
return page
|
||||
}
|
||||
|
||||
//func (rc RedditClient)
|
||||
|
||||
// GetContent() reaches out to Reddit and pulls the Json data.
|
||||
// It will then convert the data to a struct and return the struct.
|
||||
func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
|
||||
var items model.RedditJsonContent = model.RedditJsonContent{}
|
||||
|
||||
log.Printf("Collecting results on '%v'", rc.subreddit)
|
||||
content, err := getHttpContent(rc.url)
|
||||
if err != nil { return items, err }
|
||||
// TODO Wire this to support the config options
|
||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||
|
||||
log.Printf("Collecting results on '%v'", rc.record.Name)
|
||||
|
||||
content, err := getHttpContent(Url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
if strings.Contains("<h1>whoa there, pardner!</h1>", string(content)) {
|
||||
return items, errors.New("did not get json data from the server")
|
||||
}
|
||||
@ -78,12 +84,15 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles {
|
||||
var redditArticles []model.Articles
|
||||
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
|
||||
var redditArticles []database.Article
|
||||
for _, item := range items.Data.Children {
|
||||
var article model.Articles
|
||||
var article database.Article
|
||||
article, err := rc.convertToArticle(item.Data)
|
||||
if err != nil { log.Println(err); continue }
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
redditArticles = append(redditArticles, article)
|
||||
}
|
||||
return redditArticles
|
||||
@ -91,9 +100,8 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) {
|
||||
var item model.Articles
|
||||
|
||||
func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
|
||||
var item database.Article
|
||||
|
||||
if source.Content == "" && source.Url != "" {
|
||||
item = rc.convertPicturePost(source)
|
||||
@ -119,57 +127,65 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
Tags: "a",
|
||||
func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Title: source.Title,
|
||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
PubDate: time.Now(),
|
||||
Video: "null",
|
||||
VideoHeight: 0,
|
||||
VideoWidth: 0,
|
||||
Pubdate: time.Now(),
|
||||
Video: sql.NullString{String: "null"},
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
AuthorName: source.Author,
|
||||
AuthorImage: "null",
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Authorimage: sql.NullString{String: "null"},
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
AuthorName: source.Author,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Description: source.Content,
|
||||
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
AuthorName: source.Author,
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Description: source.Media.RedditVideo.FallBackUrl,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// This post is nothing more then a redirect to another location.
|
||||
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
AuthorName: source.Author,
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Description: source.UrlOverriddenByDest,
|
||||
}
|
||||
return item
|
||||
|
||||
@ -1,16 +1,33 @@
|
||||
package services_test
|
||||
|
||||
import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
var RedditRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
Url: "https://reddit.com/r/dadjokes",
|
||||
Tags: "reddit, dadjokes",
|
||||
}
|
||||
|
||||
func TestGetContent(t *testing.T) {
|
||||
//This test is flaky right now due to the http changes in 1.17
|
||||
rc := services.NewRedditClient("dadjokes", 0)
|
||||
_, err := rc.GetContent()
|
||||
log.Println(err)
|
||||
//if err != nil { panic(err) }
|
||||
rc := services.NewRedditClient(RedditRecord)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
for _, posts := range redditArticles {
|
||||
if posts.Title == "" {
|
||||
t.Error("Title is missing")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,20 +1,19 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
//"log"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"github.com/nicklaw5/helix/v2"
|
||||
)
|
||||
|
||||
type TwitchClient struct {
|
||||
SourceRecord model.Sources
|
||||
SourceRecord database.Source
|
||||
|
||||
// config
|
||||
monitorClips string
|
||||
@ -31,7 +30,7 @@ var (
|
||||
twitchScopes = "user:read:email"
|
||||
)
|
||||
|
||||
func NewTwitchClient(source model.Sources) (TwitchClient, error) {
|
||||
func NewTwitchClient() (TwitchClient, error) {
|
||||
c := config.New()
|
||||
|
||||
id := c.GetConfig(config.TWITCH_CLIENT_ID)
|
||||
@ -50,7 +49,7 @@ func NewTwitchClient(source model.Sources) (TwitchClient, error) {
|
||||
}
|
||||
|
||||
client := TwitchClient{
|
||||
SourceRecord: source,
|
||||
//SourceRecord: &source,
|
||||
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
|
||||
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
|
||||
api: &api,
|
||||
@ -73,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
||||
}
|
||||
|
||||
// This will let you replace the bound source record to keep the same session alive.
|
||||
func (tc TwitchClient) ReplaceSourceRecord(source model.Sources) {
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
||||
tc.SourceRecord = source
|
||||
}
|
||||
|
||||
@ -88,8 +87,8 @@ func (tc TwitchClient) Login() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) GetContent() ([]model.Articles, error) {
|
||||
var items []model.Articles
|
||||
func (tc TwitchClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
@ -102,21 +101,23 @@ func (tc TwitchClient) GetContent() ([]model.Articles, error) {
|
||||
}
|
||||
|
||||
for _, video := range posts {
|
||||
article := model.Articles{}
|
||||
var article database.Article
|
||||
|
||||
article.AuthorName, err = tc.ExtractAuthor(video)
|
||||
AuthorName, err := tc.ExtractAuthor(video)
|
||||
if err != nil { return items, err }
|
||||
article.Authorname = sql.NullString{String: AuthorName}
|
||||
|
||||
article.AuthorImage, err = tc.ExtractAuthorImage(user)
|
||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||
if err != nil { return items, err }
|
||||
article.Authorimage = sql.NullString{String: Authorimage}
|
||||
|
||||
article.Description, err = tc.ExtractDescription(video)
|
||||
if err != nil {return items, err }
|
||||
|
||||
article.PubDate, err = tc.ExtractPubDate(video)
|
||||
article.Pubdate, err = tc.ExtractPubDate(video)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.SourceID = tc.SourceRecord.ID
|
||||
article.Sourceid = tc.SourceRecord.ID
|
||||
article.Tags, err = tc.ExtractTags(video, user)
|
||||
if err != nil { return items, err }
|
||||
|
||||
|
||||
@ -4,27 +4,29 @@ import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
var sourceRecord = model.Sources{
|
||||
ID: 1,
|
||||
var TwitchSourceRecord = database.Source {
|
||||
ID: uuid.New(),
|
||||
Name: "nintendo",
|
||||
Source: "Twitch",
|
||||
}
|
||||
|
||||
var invalidRecord = model.Sources{
|
||||
ID: 1,
|
||||
var TwitchInvalidRecord = database.Source {
|
||||
ID: uuid.New(),
|
||||
Name: "EvilNintendo",
|
||||
Source: "Twitch",
|
||||
}
|
||||
|
||||
func TestTwitchLogin(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -34,10 +36,11 @@ func TestTwitchLogin(t *testing.T) {
|
||||
|
||||
// reach out and confirms that the API returns posts made by the user.
|
||||
func TestTwitchReturnsUserPosts(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -59,10 +62,11 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(invalidRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchInvalidRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -84,10 +88,11 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsVideoAuthor(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -109,8 +114,9 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsThumbnail(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
@ -127,8 +133,9 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsPubDate(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil { t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
@ -145,10 +152,11 @@ func TestTwitchReturnsPubDate(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsDescription(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -172,8 +180,9 @@ func TestTwitchReturnsDescription(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsAuthorImage(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
@ -186,11 +195,11 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsTags(t *testing.T) {
|
||||
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -210,10 +219,11 @@ func TestTwitchReturnsTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsTitle(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
@ -234,8 +244,9 @@ func TestTwitchReturnsTitle(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsUrl(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil { t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
@ -252,8 +263,9 @@ func TestTwitchReturnsUrl(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchGetContent(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
tc, err := services.NewTwitchClient()
|
||||
if err != nil { t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -12,14 +13,15 @@ import (
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
)
|
||||
|
||||
type YoutubeClient struct {
|
||||
SourceID uint
|
||||
Url string
|
||||
ChannelID string
|
||||
AvatarUri string
|
||||
record database.Source
|
||||
|
||||
// internal variables at time of collection
|
||||
channelID string
|
||||
avatarUri string
|
||||
|
||||
// config
|
||||
//debug bool
|
||||
@ -36,10 +38,9 @@ var (
|
||||
|
||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||
|
||||
func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
|
||||
func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
yc := YoutubeClient{
|
||||
SourceID: SourceID,
|
||||
Url: Url,
|
||||
record: Record,
|
||||
cacheGroup: "youtube",
|
||||
}
|
||||
/*
|
||||
@ -53,10 +54,11 @@ func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
|
||||
}
|
||||
|
||||
// CheckSource will go and run all the commands needed to process a source.
|
||||
func (yc *YoutubeClient) CheckSource() error {
|
||||
docParser, err := yc.GetParser(yc.Url)
|
||||
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
docParser, err := yc.GetParser(yc.record.Url)
|
||||
if err != nil {
|
||||
return err
|
||||
return items, err
|
||||
}
|
||||
|
||||
// Check cache/db for existing value
|
||||
@ -64,38 +66,38 @@ func (yc *YoutubeClient) CheckSource() error {
|
||||
//channelId, err := yc.extractChannelId()
|
||||
channelId, err := yc.GetChannelId(docParser)
|
||||
if err != nil {
|
||||
return err
|
||||
return items, err
|
||||
}
|
||||
if channelId == "" {
|
||||
return ErrYoutubeChannelIdMissing
|
||||
return items, ErrYoutubeChannelIdMissing
|
||||
}
|
||||
yc.ChannelID = channelId
|
||||
yc.channelID = channelId
|
||||
|
||||
// Check the cache/db forthe value.
|
||||
// if we have the value, skip
|
||||
avatar, err := yc.GetAvatarUri()
|
||||
if err != nil {
|
||||
return err
|
||||
return items, err
|
||||
}
|
||||
if avatar == "" {
|
||||
return ErrMissingAuthorImage
|
||||
return items, ErrMissingAuthorImage
|
||||
}
|
||||
yc.AvatarUri = avatar
|
||||
yc.avatarUri = avatar
|
||||
|
||||
feed, err := yc.PullFeed()
|
||||
if err != nil {
|
||||
return err
|
||||
return items, err
|
||||
}
|
||||
|
||||
newPosts, err := yc.CheckForNewPosts(feed)
|
||||
if err != nil {
|
||||
return err
|
||||
return items, err
|
||||
}
|
||||
|
||||
//TODO post to the API
|
||||
for _, item := range newPosts {
|
||||
|
||||
article := yc.ConvertToArticle(item)
|
||||
items = append(items, article)
|
||||
|
||||
YoutubeUriCache = append(YoutubeUriCache, &item.Link)
|
||||
|
||||
@ -103,7 +105,7 @@ func (yc *YoutubeClient) CheckSource() error {
|
||||
log.Println(article)
|
||||
}
|
||||
|
||||
return nil
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (yc *YoutubeClient) GetBrowser() *rod.Browser {
|
||||
@ -137,8 +139,8 @@ func (yc *YoutubeClient) GetChannelId(doc *goquery.Document) (string, error) {
|
||||
for _, item := range meta.Nodes {
|
||||
|
||||
if item.Attr[0].Val == "channelId" {
|
||||
yc.ChannelID = item.Attr[1].Val
|
||||
return yc.ChannelID, nil
|
||||
yc.channelID = item.Attr[1].Val
|
||||
return yc.channelID, nil
|
||||
}
|
||||
}
|
||||
return "", ErrYoutubeChannelIdMissing
|
||||
@ -155,7 +157,7 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) {
|
||||
var AvatarUri string
|
||||
|
||||
browser := rod.New().MustConnect()
|
||||
page := browser.MustPage(yc.Url)
|
||||
page := browser.MustPage(yc.record.Url)
|
||||
|
||||
res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
|
||||
|
||||
@ -197,7 +199,7 @@ func (yc *YoutubeClient) GetVideoThumbnail(parser *goquery.Document) (string, er
|
||||
|
||||
// This will pull the RSS feed items and return the results
|
||||
func (yc *YoutubeClient) PullFeed() (*gofeed.Feed, error) {
|
||||
feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.ChannelID)
|
||||
feedUri := fmt.Sprintf("%v%v", YOUTUBE_FEED_URL, yc.channelID)
|
||||
fp := gofeed.NewParser()
|
||||
feed, err := fp.ParseURL(feedUri)
|
||||
if err != nil {
|
||||
@ -239,7 +241,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles {
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
parser, err := yc.GetParser(item.Link)
|
||||
if err != nil {
|
||||
log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||
@ -257,16 +259,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles {
|
||||
log.Println(msg)
|
||||
}
|
||||
|
||||
var article = model.Articles{
|
||||
SourceID: yc.SourceID,
|
||||
var article = database.Article{
|
||||
Sourceid: yc.record.ID,
|
||||
Tags: tags,
|
||||
Title: item.Title,
|
||||
Url: item.Link,
|
||||
PubDate: *item.PublishedParsed,
|
||||
Pubdate: *item.PublishedParsed,
|
||||
Thumbnail: thumb,
|
||||
Description: item.Description,
|
||||
AuthorName: item.Author.Name,
|
||||
AuthorImage: yc.AvatarUri,
|
||||
Authorname: sql.NullString{String: item.Author.Name},
|
||||
Authorimage: sql.NullString{String: yc.avatarUri},
|
||||
}
|
||||
return article
|
||||
}
|
||||
|
||||
@ -3,24 +3,28 @@ package services_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
var YouTubeRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
Url: "https://youtube.com/gamegrumps",
|
||||
}
|
||||
|
||||
func TestGetPageParser(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
_, err := yc.GetParser(yc.Url)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
_, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
}
|
||||
|
||||
func TestGetChannelId(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
parser, err := yc.GetParser(yc.Url)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
|
||||
_, err = yc.GetChannelId(parser)
|
||||
@ -28,11 +32,8 @@ func TestGetChannelId(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPullFeed(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
parser, err := yc.GetParser(yc.Url)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
|
||||
_, err = yc.GetChannelId(parser)
|
||||
@ -43,20 +44,14 @@ func TestPullFeed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetAvatarUri(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
res, err := yc.GetAvatarUri()
|
||||
if err != nil { panic(err) }
|
||||
if res == "" { panic(services.ErrMissingAuthorImage)}
|
||||
}
|
||||
|
||||
func TestGetVideoTags(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
|
||||
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
|
||||
|
||||
@ -69,12 +64,9 @@ func TestGetVideoTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetChannelTags(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
|
||||
parser, err := yc.GetParser(yc.Url)
|
||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
|
||||
tags, err := yc.GetTags(parser)
|
||||
@ -83,10 +75,7 @@ func TestGetChannelTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetVideoThumbnail(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
|
||||
if err != nil {panic(err) }
|
||||
|
||||
@ -97,20 +86,13 @@ func TestGetVideoThumbnail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckSource(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
err := yc.CheckSource()
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
_, err := yc.GetContent()
|
||||
if err != nil { panic(err) }
|
||||
|
||||
}
|
||||
|
||||
func TestCheckUriCache(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
item := "demo"
|
||||
|
||||
services.YoutubeUriCache = append(services.YoutubeUriCache, &item)
|
||||
@ -119,10 +101,7 @@ func TestCheckUriCache(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckUriCacheFails(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(
|
||||
0,
|
||||
"https://youtube.com/gamegrumps",
|
||||
)
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
item := "demo1"
|
||||
|
||||
res := yc.CheckUriCache(&item)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user