Feature Flags (#11)
* added feature flags around background workers * background workers are moved to a new package as outputs are starting to get added * package name was updated * updated refs to the new input package * query and sql updates on routes * moved the services and starting to add discord web hook * query update
This commit is contained in:
parent
713205bb03
commit
0e0058506a
@ -534,33 +534,6 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
|
||||
Select id, articleid from DiscordQueue LIMIT $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Discordqueue
|
||||
for rows.Next() {
|
||||
var i Discordqueue
|
||||
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
|
||||
Select id, url, server, channel, enabled from DiscordWebHooks
|
||||
Where ID = $1 LIMIT 1
|
||||
@ -770,6 +743,33 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
|
||||
Select id, articleid from DiscordQueue LIMIT $1
|
||||
`
|
||||
|
||||
func (q *Queries) ListDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listDiscordQueueItems, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Discordqueue
|
||||
for rows.Next() {
|
||||
var i Discordqueue
|
||||
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
|
||||
Select id, url, server, channel, enabled From DiscordWebHooks
|
||||
Where Server = $1
|
||||
@ -938,6 +938,33 @@ func (q *Queries) ListSubscriptions(ctx context.Context, limit int32) ([]Subscri
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listSubscriptionsBySourceId = `-- name: ListSubscriptionsBySourceId :many
|
||||
Select id, discordwebhookid, sourceid From subscriptions where sourceid = $1
|
||||
`
|
||||
|
||||
func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid.UUID) ([]Subscription, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listSubscriptionsBySourceId, sourceid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Subscription
|
||||
for rows.Next() {
|
||||
var i Subscription
|
||||
if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const querySubscriptions = `-- name: QuerySubscriptions :many
|
||||
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2
|
||||
`
|
||||
|
@ -48,7 +48,7 @@ Where ID = $1 LIMIT 1;
|
||||
-- name: DeleteDiscordQueueItem :exec
|
||||
Delete From DiscordQueue Where ID = $1;
|
||||
|
||||
-- name: GetDiscordQueueItems :many
|
||||
-- name: ListDiscordQueueItems :many
|
||||
Select * from DiscordQueue LIMIT $1;
|
||||
|
||||
/* DiscordWebHooks */
|
||||
@ -151,6 +151,9 @@ Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3);
|
||||
-- name: ListSubscriptions :many
|
||||
Select * From subscriptions Limit $1;
|
||||
|
||||
-- name: ListSubscriptionsBySourceId :many
|
||||
Select * From subscriptions where sourceid = $1;
|
||||
|
||||
-- name: QuerySubscriptions :many
|
||||
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2;
|
||||
|
||||
|
@ -28,7 +28,7 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/articles/by/sourceid/{id}": {
|
||||
"/articles/by/sourceid": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -42,7 +42,7 @@ const docTemplate = `{
|
||||
"type": "string",
|
||||
"description": "Source ID UUID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
|
@ -19,7 +19,7 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/articles/by/sourceid/{id}": {
|
||||
"/articles/by/sourceid": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -33,7 +33,7 @@
|
||||
"type": "string",
|
||||
"description": "Source ID UUID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
|
@ -26,11 +26,11 @@ paths:
|
||||
summary: Returns an article based on defined ID.
|
||||
tags:
|
||||
- articles
|
||||
/articles/by/sourceid/{id}:
|
||||
/articles/by/sourceid:
|
||||
get:
|
||||
parameters:
|
||||
- description: Source ID UUID
|
||||
in: path
|
||||
in: query
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
|
@ -65,15 +65,18 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
|
||||
// TODO add page support
|
||||
// GetArticlesBySourceID
|
||||
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
|
||||
// @Param id path string true "Source ID UUID"
|
||||
// @Param id query string true "Source ID UUID"
|
||||
// @Produce application/json
|
||||
// @Tags articles
|
||||
// @Router /articles/by/sourceid/{id} [get]
|
||||
// @Router /articles/by/sourceid [get]
|
||||
func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
id := chi.URLParam(r, "ID")
|
||||
uuid, err := uuid.Parse(id)
|
||||
r.URL.Query()
|
||||
query := r.URL.Query()
|
||||
_id := query["id"][0]
|
||||
|
||||
uuid, err := uuid.Parse(_id)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
|
@ -13,7 +13,7 @@ import (
|
||||
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
res, err := s.Db.GetDiscordQueueItems(*s.ctx, 100)
|
||||
res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
|
@ -1,8 +1,11 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
@ -12,16 +15,22 @@ const (
|
||||
|
||||
Sql_Connection_String string = "SQL_CONNECTION_STRING"
|
||||
|
||||
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
|
||||
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
|
||||
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||
|
||||
FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
|
||||
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
|
||||
|
||||
FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
|
||||
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
|
||||
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
|
||||
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
|
||||
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
|
||||
|
||||
FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
|
||||
|
||||
)
|
||||
|
||||
type ConfigClient struct {}
|
||||
@ -43,6 +52,22 @@ func (cc *ConfigClient) GetConfig(key string) string {
|
||||
return res
|
||||
}
|
||||
|
||||
func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
|
||||
cc.RefreshEnv()
|
||||
|
||||
res, filled := os.LookupEnv(flag)
|
||||
if !filled {
|
||||
errorMessage := fmt.Sprintf("'%v' was not found", flag)
|
||||
return false, errors.New(errorMessage)
|
||||
}
|
||||
|
||||
b, err := strconv.ParseBool(res)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
|
||||
func (cc *ConfigClient) RefreshEnv() {
|
||||
loadEnvFile()
|
||||
|
@ -11,13 +11,14 @@ import (
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"github.com/jtom38/newsbot/collector/services/output"
|
||||
)
|
||||
|
||||
type Cron struct {
|
||||
Db *database.Queries
|
||||
ctx *context.Context
|
||||
Db *database.Queries
|
||||
ctx *context.Context
|
||||
timer *cron.Cron
|
||||
}
|
||||
|
||||
@ -35,7 +36,7 @@ func openDatabase() (*database.Queries, error) {
|
||||
|
||||
func New(ctx context.Context) *Cron {
|
||||
c := &Cron{
|
||||
ctx: &ctx,
|
||||
ctx: &ctx,
|
||||
}
|
||||
|
||||
timer := cron.New()
|
||||
@ -46,10 +47,33 @@ func New(ctx context.Context) *Cron {
|
||||
c.Db = queries
|
||||
|
||||
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||
//timer.AddFunc("* */30 * * *", func() { go c.CheckReddit(ctx) })
|
||||
//timer.AddFunc("* */1 * * *", func() { go CheckYoutube() })
|
||||
//timer.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
|
||||
//timer.AddFunc("* */1 * * *", func() { go CheckTwitch() })
|
||||
features := config.New()
|
||||
|
||||
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() })
|
||||
log.Print("Reddit backend was enabled")
|
||||
//go c.CheckReddit()
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() })
|
||||
log.Print("YouTube backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() })
|
||||
log.Print("FFXIV backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() })
|
||||
log.Print("Twitch backend was enabled")
|
||||
}
|
||||
|
||||
c.timer = timer
|
||||
return c
|
||||
}
|
||||
@ -63,72 +87,77 @@ func (c *Cron) Stop() {
|
||||
}
|
||||
|
||||
// This is the main entry point to query all the reddit services
|
||||
func (c *Cron) CheckReddit(ctx context.Context) {
|
||||
func (c *Cron) CheckReddit() {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
|
||||
if err != nil {
|
||||
log.Printf("No defines sources for reddit to query - %v\r", err)
|
||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
rc := services.NewRedditClient(source)
|
||||
log.Printf("[Reddit] Checking '%v'...", source.Name)
|
||||
rc := input.NewRedditClient(source)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
c.checkPosts(*c.ctx, redditArticles)
|
||||
c.checkPosts(redditArticles, "Reddit")
|
||||
}
|
||||
log.Print("[Reddit] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckYoutube(ctx context.Context) {
|
||||
func (c *Cron) CheckYoutube() {
|
||||
// Add call to the db to request youtube sources.
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
|
||||
if err != nil {
|
||||
log.Printf("Youtube - No sources found to query - %v\r", err)
|
||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
yc := services.NewYoutubeClient(source)
|
||||
log.Printf("[YouTube] Checking '%v'...", source.Name)
|
||||
yc := input.NewYoutubeClient(source)
|
||||
raw, err := yc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(*c.ctx, raw)
|
||||
c.checkPosts(raw, "YouTube")
|
||||
}
|
||||
log.Print("[YouTube] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckFfxiv(ctx context.Context) {
|
||||
func (c *Cron) CheckFfxiv() {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
|
||||
if err != nil {
|
||||
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
|
||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
fc := services.NewFFXIVClient(source)
|
||||
fc := input.NewFFXIVClient(source)
|
||||
items, err := fc.CheckSource()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(*c.ctx, items)
|
||||
c.checkPosts(items, "FFXIV")
|
||||
}
|
||||
log.Printf("[FFXIV Done!]")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckTwitch(ctx context.Context) error {
|
||||
func (c *Cron) CheckTwitch() error {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
|
||||
if err != nil {
|
||||
log.Printf("Twitch - No sources found to query - %v\r", err)
|
||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
tc, err := services.NewTwitchClient()
|
||||
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -137,33 +166,96 @@ func (c *Cron) CheckTwitch(ctx context.Context) error {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[Twitch] Checking '%v'...", source.Name)
|
||||
tc.ReplaceSourceRecord(source)
|
||||
items, err := tc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(*c.ctx, items)
|
||||
c.checkPosts(items, "Twitch")
|
||||
}
|
||||
|
||||
log.Print("[Twitch] Done!")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) CheckDiscordQueue() error {
|
||||
// Get items from the table
|
||||
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, queue := range(queueItems) {
|
||||
// Get the articleByID
|
||||
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get the SourceByID
|
||||
//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
|
||||
var endpoints []string
|
||||
// List Subscription by SourceID
|
||||
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get the webhhooks to send to
|
||||
for _, sub := range(subs) {
|
||||
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// store them in an array
|
||||
endpoints = append(endpoints, webhook.Url)
|
||||
}
|
||||
|
||||
// Create Discord Message
|
||||
dwh := output.NewDiscordWebHookMessage(endpoints, article)
|
||||
err = dwh.GeneratePayload()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send Message
|
||||
err = dwh.SendPayload()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove the item from the queue, given we sent our notification.
|
||||
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) checkPosts(ctx context.Context, posts []database.Article) {
|
||||
func (c *Cron) checkPosts(posts []database.Article, sourceName string) {
|
||||
for _, item := range posts {
|
||||
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
|
||||
if err != nil {
|
||||
err = c.postArticle(ctx, item)
|
||||
err = c.postArticle(item)
|
||||
if err != nil {
|
||||
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
|
||||
log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
||||
} else {
|
||||
log.Printf("Reddit - Posted article - %v\r", item.Url)
|
||||
log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
|
||||
func (c *Cron) postArticle(ctx context.Context, item database.Article) error {
|
||||
func (c *Cron) postArticle(item database.Article) error {
|
||||
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
|
||||
ID: uuid.New(),
|
||||
Sourceid: item.Sourceid,
|
||||
|
@ -14,20 +14,20 @@ func TestInvokeTwitch(t *testing.T) {
|
||||
// TODO add database mocks but not sure how to do that yet.
|
||||
func TestCheckReddit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.Cron{}
|
||||
c.CheckReddit(ctx)
|
||||
c := cron.New(ctx)
|
||||
c.CheckReddit()
|
||||
}
|
||||
|
||||
func TestCheckYouTube(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.Cron{}
|
||||
c.CheckYoutube(ctx)
|
||||
c := cron.New(ctx)
|
||||
c.CheckYoutube()
|
||||
}
|
||||
|
||||
func TestCheckTwitch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.Cron{}
|
||||
err := c.CheckTwitch(ctx)
|
||||
c := cron.New(ctx)
|
||||
err := c.CheckTwitch()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
package services
|
||||
package input
|
||||
|
||||
import "errors"
|
||||
|
@ -1,4 +1,4 @@
|
||||
package services
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
@ -1,11 +1,11 @@
|
||||
package services_test
|
||||
package input_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
ffxiv "github.com/jtom38/newsbot/collector/services"
|
||||
ffxiv "github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var FFXIVRecord database.Source = database.Source{
|
@ -1,4 +1,4 @@
|
||||
package services
|
||||
package input
|
||||
|
||||
import (
|
||||
"net/http"
|
@ -1,4 +1,4 @@
|
||||
package services
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
@ -1,11 +1,11 @@
|
||||
package services_test
|
||||
package input_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var RedditRecord database.Source = database.Source{
|
||||
@ -19,7 +19,7 @@ var RedditRecord database.Source = database.Source{
|
||||
|
||||
func TestGetContent(t *testing.T) {
|
||||
//This test is flaky right now due to the http changes in 1.17
|
||||
rc := services.NewRedditClient(RedditRecord)
|
||||
rc := input.NewRedditClient(RedditRecord)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
t.Error(err)
|
55
services/input/rss.go
Normal file
55
services/input/rss.go
Normal file
@ -0,0 +1,55 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/services/cache"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
type rssClient struct {
|
||||
SourceRecord model.Sources
|
||||
}
|
||||
|
||||
func NewRssClient(sourceRecord model.Sources) rssClient {
|
||||
client := rssClient{
|
||||
SourceRecord: sourceRecord,
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
|
||||
//rc.SourceRecord = source
|
||||
//}
|
||||
|
||||
func (rc rssClient) getCacheGroup() string {
|
||||
return fmt.Sprintf("rss-%v", rc.SourceRecord.Name)
|
||||
}
|
||||
|
||||
func (rc rssClient) GetContent() error {
|
||||
feed, err := rc.PullFeed()
|
||||
if err != nil { return err }
|
||||
|
||||
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
|
||||
|
||||
for _, item := range feed.Items {
|
||||
log.Println(item)
|
||||
|
||||
cacheClient.FindByValue(item.Link)
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
|
||||
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
|
||||
fp := gofeed.NewParser()
|
||||
feed, err := fp.ParseURL(feedUri)
|
||||
if err != nil { return nil, err }
|
||||
|
||||
return feed, nil
|
||||
}
|
26
services/input/rss_test.go
Normal file
26
services/input/rss_test.go
Normal file
@ -0,0 +1,26 @@
|
||||
package input_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var rssRecord = model.Sources {
|
||||
ID: 1,
|
||||
Name: "ArsTechnica",
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||
}
|
||||
|
||||
func TestRssClientConstructor(t *testing.T) {
|
||||
input.NewRssClient(rssRecord)
|
||||
}
|
||||
|
||||
func TestRssGetFeed(t *testing.T) {
|
||||
client := input.NewRssClient(rssRecord)
|
||||
feed, err := client.PullFeed()
|
||||
if err != nil { t.Error(err) }
|
||||
if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")}
|
||||
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package services
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
@ -1,4 +1,4 @@
|
||||
package services_test
|
||||
package input_test
|
||||
|
||||
import (
|
||||
"log"
|
||||
@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var TwitchSourceRecord = database.Source {
|
||||
@ -22,7 +22,7 @@ var TwitchInvalidRecord = database.Source {
|
||||
}
|
||||
|
||||
func TestTwitchLogin(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -36,7 +36,7 @@ func TestTwitchLogin(t *testing.T) {
|
||||
|
||||
// reach out and confirms that the API returns posts made by the user.
|
||||
func TestTwitchReturnsUserPosts(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -62,7 +62,7 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -88,7 +88,7 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsVideoAuthor(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -114,7 +114,7 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsThumbnail(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
@ -133,7 +133,7 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsPubDate(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil { t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
@ -152,7 +152,7 @@ func TestTwitchReturnsPubDate(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsDescription(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -180,7 +180,7 @@ func TestTwitchReturnsDescription(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsAuthorImage(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
@ -195,7 +195,7 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsTags(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -219,7 +219,7 @@ func TestTwitchReturnsTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsTitle(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -244,7 +244,7 @@ func TestTwitchReturnsTitle(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchReturnsUrl(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil { t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
||||
@ -263,7 +263,7 @@ func TestTwitchReturnsUrl(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTwitchGetContent(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient()
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil { t.Error(err) }
|
||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||
|
@ -1,4 +1,4 @@
|
||||
package services
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
@ -102,7 +102,7 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
||||
YoutubeUriCache = append(YoutubeUriCache, &item.Link)
|
||||
|
||||
// Add the post to local cache
|
||||
log.Println(article)
|
||||
//log.Println(article)
|
||||
}
|
||||
|
||||
return items, nil
|
@ -1,11 +1,11 @@
|
||||
package services_test
|
||||
package input_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var YouTubeRecord database.Source = database.Source{
|
||||
@ -17,13 +17,13 @@ var YouTubeRecord database.Source = database.Source{
|
||||
}
|
||||
|
||||
func TestGetPageParser(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
_, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
}
|
||||
|
||||
func TestGetChannelId(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
|
||||
@ -32,7 +32,7 @@ func TestGetChannelId(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPullFeed(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
|
||||
@ -44,14 +44,14 @@ func TestPullFeed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetAvatarUri(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
res, err := yc.GetAvatarUri()
|
||||
if err != nil { panic(err) }
|
||||
if res == "" { panic(services.ErrMissingAuthorImage)}
|
||||
if res == "" { panic(input.ErrMissingAuthorImage)}
|
||||
}
|
||||
|
||||
func TestGetVideoTags(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
|
||||
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
|
||||
|
||||
@ -64,7 +64,7 @@ func TestGetVideoTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetChannelTags(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
|
||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||
if err != nil { panic(err) }
|
||||
@ -75,7 +75,7 @@ func TestGetChannelTags(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetVideoThumbnail(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
|
||||
if err != nil {panic(err) }
|
||||
|
||||
@ -86,22 +86,22 @@ func TestGetVideoThumbnail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckSource(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
_, err := yc.GetContent()
|
||||
if err != nil { panic(err) }
|
||||
}
|
||||
|
||||
func TestCheckUriCache(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
item := "demo"
|
||||
|
||||
services.YoutubeUriCache = append(services.YoutubeUriCache, &item)
|
||||
input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
|
||||
res := yc.CheckUriCache(&item)
|
||||
if res == false { panic("expected a value to come back")}
|
||||
}
|
||||
|
||||
func TestCheckUriCacheFails(t *testing.T) {
|
||||
yc := services.NewYoutubeClient(YouTubeRecord)
|
||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||
item := "demo1"
|
||||
|
||||
res := yc.CheckUriCache(&item)
|
110
services/output/discordwebhook.go
Normal file
110
services/output/discordwebhook.go
Normal file
@ -0,0 +1,110 @@
|
||||
package output
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
)
|
||||
|
||||
type discordField struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Value string `json:"value,omitempty"`
|
||||
Inline bool `json:"inline,omitempty"`
|
||||
}
|
||||
|
||||
type discordAuthor struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Url string `json:"url,omitempty"`
|
||||
IconUrl string `json:"icon_url,omitempty"`
|
||||
}
|
||||
|
||||
type discordImage struct {
|
||||
Url string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
type discordEmbed struct {
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Url string `json:"url,omitempty"`
|
||||
Color int32 `json:"color,omitempty"`
|
||||
Timestamp time.Time `json:"timestamp,omitempty"`
|
||||
Fields []discordField `json:"fields,omitempty"`
|
||||
Author discordAuthor `json:"author,omitempty"`
|
||||
Image discordImage `json:"image,omitempty"`
|
||||
Thumbnail discordImage `json:"thumbnail,omitempty"`
|
||||
}
|
||||
|
||||
// Root object for Discord Webhook messages
|
||||
type discordMessage struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
Embeds []discordEmbed `json:"embeds,omitempty"`
|
||||
}
|
||||
|
||||
type Discord struct {
|
||||
Subscriptions []string
|
||||
article database.Article
|
||||
Message discordMessage
|
||||
}
|
||||
|
||||
func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord {
|
||||
return Discord{
|
||||
Subscriptions: Subscriptions,
|
||||
article: Article,
|
||||
Message: discordMessage{
|
||||
Embeds: []discordEmbed{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (dwh Discord) GeneratePayload() error {
|
||||
// Convert the message
|
||||
embed := discordEmbed {
|
||||
Title: dwh.article.Title,
|
||||
Description: dwh.convertFromHtml(dwh.article.Description),
|
||||
Url: dwh.article.Url,
|
||||
Thumbnail: discordImage{
|
||||
Url: dwh.article.Thumbnail,
|
||||
},
|
||||
}
|
||||
var arr []discordEmbed
|
||||
|
||||
arr = append(arr, embed)
|
||||
dwh.Message.Embeds = arr
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dwh Discord) SendPayload() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dwh Discord) convertFromHtml(body string) string {
|
||||
clean := body
|
||||
clean = strings.ReplaceAll(clean, "<h2>", "**")
|
||||
clean = strings.ReplaceAll(clean, "</h2>", "**")
|
||||
clean = strings.ReplaceAll(clean, "<h3>", "**")
|
||||
clean = strings.ReplaceAll(clean, "</h3>", "**\r\n")
|
||||
clean = strings.ReplaceAll(clean, "<strong>", "**")
|
||||
clean = strings.ReplaceAll(clean, "</strong>", "**\r\n")
|
||||
clean = strings.ReplaceAll(clean, "<ul>", "\r\n")
|
||||
clean = strings.ReplaceAll(clean, "</ul>", "")
|
||||
clean = strings.ReplaceAll(clean, "</li>", "\r\n")
|
||||
clean = strings.ReplaceAll(clean, "<li>", "> ")
|
||||
clean = strings.ReplaceAll(clean, "“", "\"")
|
||||
clean = strings.ReplaceAll(clean, "”", "\"")
|
||||
clean = strings.ReplaceAll(clean, "…", "...")
|
||||
clean = strings.ReplaceAll(clean, "<b>", "**")
|
||||
clean = strings.ReplaceAll(clean, "</b>", "**")
|
||||
clean = strings.ReplaceAll(clean, "<br>", "\r\n")
|
||||
clean = strings.ReplaceAll(clean, "<br/>", "\r\n")
|
||||
clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*")
|
||||
clean = strings.ReplaceAll(clean, "\xa0", "\r\n")
|
||||
clean = strings.ReplaceAll(clean, "<p>", "")
|
||||
clean = strings.ReplaceAll(clean, "</p>", "\r\n")
|
||||
return clean
|
||||
}
|
||||
|
||||
func (dwh Discord) convertLinks(body string) string {
|
||||
//items := regexp.MustCompile("<a(.*?)a>")
|
||||
return ""
|
||||
}
|
73
services/output/discordwebhook_test.go
Normal file
73
services/output/discordwebhook_test.go
Normal file
@ -0,0 +1,73 @@
|
||||
package output_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/output"
|
||||
)
|
||||
|
||||
var article database.Article = database.Article{
|
||||
ID: uuid.New(),
|
||||
Sourceid: uuid.New(),
|
||||
Tags: "unit, testing",
|
||||
Title: "Demo",
|
||||
Url: "https://github.com/jtom38/newsbot.collector.api",
|
||||
Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Description: "Hello World",
|
||||
}
|
||||
|
||||
func getWebhook() ([]string, error){
|
||||
var endpoints []string
|
||||
|
||||
_, err := os.Open(".env")
|
||||
if err != nil {
|
||||
return endpoints, err
|
||||
}
|
||||
|
||||
err = godotenv.Load()
|
||||
if err != nil {
|
||||
return endpoints, err
|
||||
}
|
||||
|
||||
res := os.Getenv("TESTS_DISCORD_WEBHOOK")
|
||||
if res == "" {
|
||||
return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing")
|
||||
}
|
||||
endpoints = strings.Split(res, "")
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) {
|
||||
hook, err := getWebhook()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
d := output.NewDiscordWebHookMessage(hook, article)
|
||||
if len(d.Subscriptions) == 0 {
|
||||
t.Error("no subscriptions found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscordMessageContainsTitle(t *testing.T) {
|
||||
hook, err := getWebhook()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
d := output.NewDiscordWebHookMessage(hook, article)
|
||||
err = d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if d.Message.Embeds[0].Title == "" {
|
||||
t.Error("no title was found ")
|
||||
}
|
||||
}
|
6
services/output/interface.go
Normal file
6
services/output/interface.go
Normal file
@ -0,0 +1,6 @@
|
||||
package output
|
||||
|
||||
type Output interface {
|
||||
GeneratePayload() error
|
||||
SendPayload() error
|
||||
}
|
Loading…
Reference in New Issue
Block a user