Feature Flags (#11)

* added feature flags around background workers

* background workers are moved to a new package as outputs are starting to get added

* package name was updated

* updated refs to the new input package

* query and sql updates on routes

* moved the services and starting to add discord web hook

* query update
This commit is contained in:
James Tombleson 2022-06-30 14:54:58 -07:00 committed by GitHub
parent 713205bb03
commit 0e0058506a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 536 additions and 116 deletions

View File

@ -534,33 +534,6 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
return i, err return i, err
} }
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, url, server, channel, enabled from DiscordWebHooks Select id, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1 Where ID = $1 LIMIT 1
@ -770,6 +743,33 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
return items, nil return items, nil
} }
const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) ListDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, listDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
Select id, url, server, channel, enabled From DiscordWebHooks Select id, url, server, channel, enabled From DiscordWebHooks
Where Server = $1 Where Server = $1
@ -938,6 +938,33 @@ func (q *Queries) ListSubscriptions(ctx context.Context, limit int32) ([]Subscri
return items, nil return items, nil
} }
const listSubscriptionsBySourceId = `-- name: ListSubscriptionsBySourceId :many
Select id, discordwebhookid, sourceid From subscriptions where sourceid = $1
`
func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid.UUID) ([]Subscription, error) {
rows, err := q.db.QueryContext(ctx, listSubscriptionsBySourceId, sourceid)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Subscription
for rows.Next() {
var i Subscription
if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const querySubscriptions = `-- name: QuerySubscriptions :many const querySubscriptions = `-- name: QuerySubscriptions :many
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2
` `

View File

@ -48,7 +48,7 @@ Where ID = $1 LIMIT 1;
-- name: DeleteDiscordQueueItem :exec -- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1; Delete From DiscordQueue Where ID = $1;
-- name: GetDiscordQueueItems :many -- name: ListDiscordQueueItems :many
Select * from DiscordQueue LIMIT $1; Select * from DiscordQueue LIMIT $1;
/* DiscordWebHooks */ /* DiscordWebHooks */
@ -151,6 +151,9 @@ Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3);
-- name: ListSubscriptions :many -- name: ListSubscriptions :many
Select * From subscriptions Limit $1; Select * From subscriptions Limit $1;
-- name: ListSubscriptionsBySourceId :many
Select * From subscriptions where sourceid = $1;
-- name: QuerySubscriptions :many -- name: QuerySubscriptions :many
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2; Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2;

View File

@ -28,7 +28,7 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/articles/by/sourceid/{id}": { "/articles/by/sourceid": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -42,7 +42,7 @@ const docTemplate = `{
"type": "string", "type": "string",
"description": "Source ID UUID", "description": "Source ID UUID",
"name": "id", "name": "id",
"in": "path", "in": "query",
"required": true "required": true
} }
], ],

View File

@ -19,7 +19,7 @@
"responses": {} "responses": {}
} }
}, },
"/articles/by/sourceid/{id}": { "/articles/by/sourceid": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -33,7 +33,7 @@
"type": "string", "type": "string",
"description": "Source ID UUID", "description": "Source ID UUID",
"name": "id", "name": "id",
"in": "path", "in": "query",
"required": true "required": true
} }
], ],

View File

@ -26,11 +26,11 @@ paths:
summary: Returns an article based on defined ID. summary: Returns an article based on defined ID.
tags: tags:
- articles - articles
/articles/by/sourceid/{id}: /articles/by/sourceid:
get: get:
parameters: parameters:
- description: Source ID UUID - description: Source ID UUID
in: path in: query
name: id name: id
required: true required: true
type: string type: string

View File

@ -65,15 +65,18 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
// TODO add page support // TODO add page support
// GetArticlesBySourceID // GetArticlesBySourceID
// @Summary Finds the articles based on the SourceID provided. Returns the top 50. // @Summary Finds the articles based on the SourceID provided. Returns the top 50.
// @Param id path string true "Source ID UUID" // @Param id query string true "Source ID UUID"
// @Produce application/json // @Produce application/json
// @Tags articles // @Tags articles
// @Router /articles/by/sourceid/{id} [get] // @Router /articles/by/sourceid [get]
func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) { func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
id := chi.URLParam(r, "ID") r.URL.Query()
uuid, err := uuid.Parse(id) query := r.URL.Query()
_id := query["id"][0]
uuid, err := uuid.Parse(_id)
if err != nil { if err != nil {
w.Write([]byte(err.Error())) w.Write([]byte(err.Error()))
panic(err) panic(err)

View File

@ -13,7 +13,7 @@ import (
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) { func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
res, err := s.Db.GetDiscordQueueItems(*s.ctx, 100) res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
if err != nil { if err != nil {
w.Write([]byte(err.Error())) w.Write([]byte(err.Error()))
panic(err) panic(err)

View File

@ -1,8 +1,11 @@
package config package config
import ( import (
"os" "errors"
"fmt"
"log" "log"
"os"
"strconv"
"github.com/joho/godotenv" "github.com/joho/godotenv"
) )
@ -12,16 +15,22 @@ const (
Sql_Connection_String string = "SQL_CONNECTION_STRING" Sql_Connection_String string = "SQL_CONNECTION_STRING"
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP" REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT" REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW" REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
YOUTUBE_DEBUG = "YOUTUBE_DEBUG" YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID" TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET" TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS" TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD" TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
) )
type ConfigClient struct {} type ConfigClient struct {}
@ -43,6 +52,22 @@ func (cc *ConfigClient) GetConfig(key string) string {
return res return res
} }
func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
cc.RefreshEnv()
res, filled := os.LookupEnv(flag)
if !filled {
errorMessage := fmt.Sprintf("'%v' was not found", flag)
return false, errors.New(errorMessage)
}
b, err := strconv.ParseBool(res)
if err != nil {
return false, err
}
return b, nil
}
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes. // Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
func (cc *ConfigClient) RefreshEnv() { func (cc *ConfigClient) RefreshEnv() {
loadEnvFile() loadEnvFile()

View File

@ -11,8 +11,9 @@ import (
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/config" "github.com/jtom38/newsbot/collector/services/config"
"github.com/jtom38/newsbot/collector/services/output"
) )
type Cron struct { type Cron struct {
@ -46,10 +47,33 @@ func New(ctx context.Context) *Cron {
c.Db = queries c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//timer.AddFunc("* */30 * * *", func() { go c.CheckReddit(ctx) }) features := config.New()
//timer.AddFunc("* */1 * * *", func() { go CheckYoutube() })
//timer.AddFunc("* */1 * * *", func() { go CheckFfxiv() }) res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
//timer.AddFunc("* */1 * * *", func() { go CheckTwitch() }) if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() })
log.Print("Reddit backend was enabled")
//go c.CheckReddit()
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() })
log.Print("YouTube backend was enabled")
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() })
log.Print("FFXIV backend was enabled")
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() })
log.Print("Twitch backend was enabled")
}
c.timer = timer c.timer = timer
return c return c
} }
@ -63,72 +87,77 @@ func (c *Cron) Stop() {
} }
// This is the main entry point to query all the reddit services // This is the main entry point to query all the reddit services
func (c *Cron) CheckReddit(ctx context.Context) { func (c *Cron) CheckReddit() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit") sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
if err != nil { if err != nil {
log.Printf("No defines sources for reddit to query - %v\r", err) log.Printf("[Reddit] No sources found to query - %v\r", err)
} }
for _, source := range sources { for _, source := range sources {
if !source.Enabled { if !source.Enabled {
continue continue
} }
rc := services.NewRedditClient(source) log.Printf("[Reddit] Checking '%v'...", source.Name)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent() raw, err := rc.GetContent()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
redditArticles := rc.ConvertToArticles(raw) redditArticles := rc.ConvertToArticles(raw)
c.checkPosts(*c.ctx, redditArticles) c.checkPosts(redditArticles, "Reddit")
} }
log.Print("[Reddit] Done!")
} }
func (c *Cron) CheckYoutube(ctx context.Context) { func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources. // Add call to the db to request youtube sources.
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube") sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
if err != nil { if err != nil {
log.Printf("Youtube - No sources found to query - %v\r", err) log.Printf("[Youtube] No sources found to query - %v\r", err)
} }
for _, source := range sources { for _, source := range sources {
if !source.Enabled { if !source.Enabled {
continue continue
} }
yc := services.NewYoutubeClient(source) log.Printf("[YouTube] Checking '%v'...", source.Name)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent() raw, err := yc.GetContent()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
c.checkPosts(*c.ctx, raw) c.checkPosts(raw, "YouTube")
} }
log.Print("[YouTube] Done!")
} }
func (c *Cron) CheckFfxiv(ctx context.Context) { func (c *Cron) CheckFfxiv() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv") sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
if err != nil { if err != nil {
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err) log.Printf("[FFXIV] No sources found to query - %v\r", err)
} }
for _, source := range sources { for _, source := range sources {
if !source.Enabled { if !source.Enabled {
continue continue
} }
fc := services.NewFFXIVClient(source) fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource() items, err := fc.CheckSource()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
c.checkPosts(*c.ctx, items) c.checkPosts(items, "FFXIV")
} }
log.Printf("[FFXIV Done!]")
} }
func (c *Cron) CheckTwitch(ctx context.Context) error { func (c *Cron) CheckTwitch() error {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch") sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
if err != nil { if err != nil {
log.Printf("Twitch - No sources found to query - %v\r", err) log.Printf("[Twitch] No sources found to query - %v\r", err)
} }
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
return err return err
} }
@ -137,33 +166,96 @@ func (c *Cron) CheckTwitch(ctx context.Context) error {
if !source.Enabled { if !source.Enabled {
continue continue
} }
log.Printf("[Twitch] Checking '%v'...", source.Name)
tc.ReplaceSourceRecord(source) tc.ReplaceSourceRecord(source)
items, err := tc.GetContent() items, err := tc.GetContent()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
c.checkPosts(*c.ctx, items) c.checkPosts(items, "Twitch")
}
log.Print("[Twitch] Done!")
return nil
}
func (c *Cron) CheckDiscordQueue() error {
// Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
if err != nil {
return err
}
for _, queue := range(queueItems) {
// Get the articleByID
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
if err != nil {
return err
}
// Get the SourceByID
//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
//if err != nil {
// return err
//}
var endpoints []string
// List Subscription by SourceID
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
if err != nil {
return err
}
// Get the webhhooks to send to
for _, sub := range(subs) {
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
if err != nil {
return err
}
// store them in an array
endpoints = append(endpoints, webhook.Url)
}
// Create Discord Message
dwh := output.NewDiscordWebHookMessage(endpoints, article)
err = dwh.GeneratePayload()
if err != nil {
return err
}
// Send Message
err = dwh.SendPayload()
if err != nil {
return err
}
// Remove the item from the queue, given we sent our notification.
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
if err != nil {
return err
}
} }
return nil return nil
} }
func (c *Cron) checkPosts(ctx context.Context, posts []database.Article) { func (c *Cron) checkPosts(posts []database.Article, sourceName string) {
for _, item := range posts { for _, item := range posts {
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) _, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
if err != nil { if err != nil {
err = c.postArticle(ctx, item) err = c.postArticle(item)
if err != nil { if err != nil {
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err) log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
} else { } else {
log.Printf("Reddit - Posted article - %v\r", item.Url) log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
} }
} }
} }
time.Sleep(30 * time.Second) time.Sleep(30 * time.Second)
} }
func (c *Cron) postArticle(ctx context.Context, item database.Article) error { func (c *Cron) postArticle(item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{ err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: uuid.New(), ID: uuid.New(),
Sourceid: item.Sourceid, Sourceid: item.Sourceid,

View File

@ -14,20 +14,20 @@ func TestInvokeTwitch(t *testing.T) {
// TODO add database mocks but not sure how to do that yet. // TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) { func TestCheckReddit(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.Cron{} c := cron.New(ctx)
c.CheckReddit(ctx) c.CheckReddit()
} }
func TestCheckYouTube(t *testing.T) { func TestCheckYouTube(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.Cron{} c := cron.New(ctx)
c.CheckYoutube(ctx) c.CheckYoutube()
} }
func TestCheckTwitch(t *testing.T) { func TestCheckTwitch(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.Cron{} c := cron.New(ctx)
err := c.CheckTwitch(ctx) err := c.CheckTwitch()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }

View File

@ -1,4 +1,4 @@
package services package input
import "errors" import "errors"

View File

@ -1,4 +1,4 @@
package services package input
import ( import (
"database/sql" "database/sql"

View File

@ -1,11 +1,11 @@
package services_test package input_test
import ( import (
"testing" "testing"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services" ffxiv "github.com/jtom38/newsbot/collector/services/input"
) )
var FFXIVRecord database.Source = database.Source{ var FFXIVRecord database.Source = database.Source{

View File

@ -1,4 +1,4 @@
package services package input
import ( import (
"net/http" "net/http"

View File

@ -1,4 +1,4 @@
package services package input
import ( import (
"database/sql" "database/sql"

View File

@ -1,11 +1,11 @@
package services_test package input_test
import ( import (
"testing" "testing"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services/input"
) )
var RedditRecord database.Source = database.Source{ var RedditRecord database.Source = database.Source{
@ -19,7 +19,7 @@ var RedditRecord database.Source = database.Source{
func TestGetContent(t *testing.T) { func TestGetContent(t *testing.T) {
//This test is flaky right now due to the http changes in 1.17 //This test is flaky right now due to the http changes in 1.17
rc := services.NewRedditClient(RedditRecord) rc := input.NewRedditClient(RedditRecord)
raw, err := rc.GetContent() raw, err := rc.GetContent()
if err != nil { if err != nil {
t.Error(err) t.Error(err)

55
services/input/rss.go Normal file
View File

@ -0,0 +1,55 @@
package input
import (
"fmt"
"log"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/cache"
"github.com/mmcdole/gofeed"
)
type rssClient struct {
SourceRecord model.Sources
}
func NewRssClient(sourceRecord model.Sources) rssClient {
client := rssClient{
SourceRecord: sourceRecord,
}
return client
}
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
//rc.SourceRecord = source
//}
func (rc rssClient) getCacheGroup() string {
return fmt.Sprintf("rss-%v", rc.SourceRecord.Name)
}
func (rc rssClient) GetContent() error {
feed, err := rc.PullFeed()
if err != nil { return err }
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
for _, item := range feed.Items {
log.Println(item)
cacheClient.FindByValue(item.Link)
}
return nil
}
func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
fp := gofeed.NewParser()
feed, err := fp.ParseURL(feedUri)
if err != nil { return nil, err }
return feed, nil
}

View File

@ -0,0 +1,26 @@
package input_test
import (
"testing"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/input"
)
var rssRecord = model.Sources {
ID: 1,
Name: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index",
}
func TestRssClientConstructor(t *testing.T) {
input.NewRssClient(rssRecord)
}
func TestRssGetFeed(t *testing.T) {
client := input.NewRssClient(rssRecord)
feed, err := client.PullFeed()
if err != nil { t.Error(err) }
if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")}
}

View File

@ -1,4 +1,4 @@
package services package input
import ( import (
"database/sql" "database/sql"

View File

@ -1,4 +1,4 @@
package services_test package input_test
import ( import (
"log" "log"
@ -6,7 +6,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services/input"
) )
var TwitchSourceRecord = database.Source { var TwitchSourceRecord = database.Source {
@ -22,7 +22,7 @@ var TwitchInvalidRecord = database.Source {
} }
func TestTwitchLogin(t *testing.T) { func TestTwitchLogin(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -36,7 +36,7 @@ func TestTwitchLogin(t *testing.T) {
// reach out and confirms that the API returns posts made by the user. // reach out and confirms that the API returns posts made by the user.
func TestTwitchReturnsUserPosts(t *testing.T) { func TestTwitchReturnsUserPosts(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -62,7 +62,7 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
} }
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) { func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -88,7 +88,7 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
} }
func TestTwitchReturnsVideoAuthor(t *testing.T) { func TestTwitchReturnsVideoAuthor(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -114,7 +114,7 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
} }
func TestTwitchReturnsThumbnail(t *testing.T) { func TestTwitchReturnsThumbnail(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil {t.Error(err) } if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord) tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -133,7 +133,7 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
} }
func TestTwitchReturnsPubDate(t *testing.T) { func TestTwitchReturnsPubDate(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord) tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -152,7 +152,7 @@ func TestTwitchReturnsPubDate(t *testing.T) {
} }
func TestTwitchReturnsDescription(t *testing.T) { func TestTwitchReturnsDescription(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -180,7 +180,7 @@ func TestTwitchReturnsDescription(t *testing.T) {
} }
func TestTwitchReturnsAuthorImage(t *testing.T) { func TestTwitchReturnsAuthorImage(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil {t.Error(err) } if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord) tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -195,7 +195,7 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
} }
func TestTwitchReturnsTags(t *testing.T) { func TestTwitchReturnsTags(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -219,7 +219,7 @@ func TestTwitchReturnsTags(t *testing.T) {
} }
func TestTwitchReturnsTitle(t *testing.T) { func TestTwitchReturnsTitle(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -244,7 +244,7 @@ func TestTwitchReturnsTitle(t *testing.T) {
} }
func TestTwitchReturnsUrl(t *testing.T) { func TestTwitchReturnsUrl(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord) tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -263,7 +263,7 @@ func TestTwitchReturnsUrl(t *testing.T) {
} }
func TestTwitchGetContent(t *testing.T) { func TestTwitchGetContent(t *testing.T) {
tc, err := services.NewTwitchClient() tc, err := input.NewTwitchClient()
if err != nil { t.Error(err) } if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord) tc.ReplaceSourceRecord(TwitchSourceRecord)

View File

@ -1,4 +1,4 @@
package services package input
import ( import (
"database/sql" "database/sql"
@ -102,7 +102,7 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
YoutubeUriCache = append(YoutubeUriCache, &item.Link) YoutubeUriCache = append(YoutubeUriCache, &item.Link)
// Add the post to local cache // Add the post to local cache
log.Println(article) //log.Println(article)
} }
return items, nil return items, nil

View File

@ -1,11 +1,11 @@
package services_test package input_test
import ( import (
"testing" "testing"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services/input"
) )
var YouTubeRecord database.Source = database.Source{ var YouTubeRecord database.Source = database.Source{
@ -17,13 +17,13 @@ var YouTubeRecord database.Source = database.Source{
} }
func TestGetPageParser(t *testing.T) { func TestGetPageParser(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetParser(YouTubeRecord.Url) _, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
} }
func TestGetChannelId(t *testing.T) { func TestGetChannelId(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url) parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
@ -32,7 +32,7 @@ func TestGetChannelId(t *testing.T) {
} }
func TestPullFeed(t *testing.T) { func TestPullFeed(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url) parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
@ -44,14 +44,14 @@ func TestPullFeed(t *testing.T) {
} }
func TestGetAvatarUri(t *testing.T) { func TestGetAvatarUri(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
res, err := yc.GetAvatarUri() res, err := yc.GetAvatarUri()
if err != nil { panic(err) } if err != nil { panic(err) }
if res == "" { panic(services.ErrMissingAuthorImage)} if res == "" { panic(input.ErrMissingAuthorImage)}
} }
func TestGetVideoTags(t *testing.T) { func TestGetVideoTags(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68" var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
@ -64,7 +64,7 @@ func TestGetVideoTags(t *testing.T) {
} }
func TestGetChannelTags(t *testing.T) { func TestGetChannelTags(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url) parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) } if err != nil { panic(err) }
@ -75,7 +75,7 @@ func TestGetChannelTags(t *testing.T) {
} }
func TestGetVideoThumbnail(t *testing.T) { func TestGetVideoThumbnail(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68") parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
if err != nil {panic(err) } if err != nil {panic(err) }
@ -86,22 +86,22 @@ func TestGetVideoThumbnail(t *testing.T) {
} }
func TestCheckSource(t *testing.T) { func TestCheckSource(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetContent() _, err := yc.GetContent()
if err != nil { panic(err) } if err != nil { panic(err) }
} }
func TestCheckUriCache(t *testing.T) { func TestCheckUriCache(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
item := "demo" item := "demo"
services.YoutubeUriCache = append(services.YoutubeUriCache, &item) input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
res := yc.CheckUriCache(&item) res := yc.CheckUriCache(&item)
if res == false { panic("expected a value to come back")} if res == false { panic("expected a value to come back")}
} }
func TestCheckUriCacheFails(t *testing.T) { func TestCheckUriCacheFails(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord) yc := input.NewYoutubeClient(YouTubeRecord)
item := "demo1" item := "demo1"
res := yc.CheckUriCache(&item) res := yc.CheckUriCache(&item)

View File

@ -0,0 +1,110 @@
package output
import (
"strings"
"time"
"github.com/jtom38/newsbot/collector/database"
)
type discordField struct {
Name string `json:"name,omitempty"`
Value string `json:"value,omitempty"`
Inline bool `json:"inline,omitempty"`
}
type discordAuthor struct {
Name string `json:"name,omitempty"`
Url string `json:"url,omitempty"`
IconUrl string `json:"icon_url,omitempty"`
}
type discordImage struct {
Url string `json:"url,omitempty"`
}
type discordEmbed struct {
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Url string `json:"url,omitempty"`
Color int32 `json:"color,omitempty"`
Timestamp time.Time `json:"timestamp,omitempty"`
Fields []discordField `json:"fields,omitempty"`
Author discordAuthor `json:"author,omitempty"`
Image discordImage `json:"image,omitempty"`
Thumbnail discordImage `json:"thumbnail,omitempty"`
}
// Root object for Discord Webhook messages
type discordMessage struct {
Content string `json:"content,omitempty"`
Embeds []discordEmbed `json:"embeds,omitempty"`
}
type Discord struct {
Subscriptions []string
article database.Article
Message discordMessage
}
func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord {
return Discord{
Subscriptions: Subscriptions,
article: Article,
Message: discordMessage{
Embeds: []discordEmbed{},
},
}
}
func (dwh Discord) GeneratePayload() error {
// Convert the message
embed := discordEmbed {
Title: dwh.article.Title,
Description: dwh.convertFromHtml(dwh.article.Description),
Url: dwh.article.Url,
Thumbnail: discordImage{
Url: dwh.article.Thumbnail,
},
}
var arr []discordEmbed
arr = append(arr, embed)
dwh.Message.Embeds = arr
return nil
}
func (dwh Discord) SendPayload() error {
return nil
}
func (dwh Discord) convertFromHtml(body string) string {
clean := body
clean = strings.ReplaceAll(clean, "<h2>", "**")
clean = strings.ReplaceAll(clean, "</h2>", "**")
clean = strings.ReplaceAll(clean, "<h3>", "**")
clean = strings.ReplaceAll(clean, "</h3>", "**\r\n")
clean = strings.ReplaceAll(clean, "<strong>", "**")
clean = strings.ReplaceAll(clean, "</strong>", "**\r\n")
clean = strings.ReplaceAll(clean, "<ul>", "\r\n")
clean = strings.ReplaceAll(clean, "</ul>", "")
clean = strings.ReplaceAll(clean, "</li>", "\r\n")
clean = strings.ReplaceAll(clean, "<li>", "> ")
clean = strings.ReplaceAll(clean, "&#8220;", "\"")
clean = strings.ReplaceAll(clean, "&#8221;", "\"")
clean = strings.ReplaceAll(clean, "&#8230;", "...")
clean = strings.ReplaceAll(clean, "<b>", "**")
clean = strings.ReplaceAll(clean, "</b>", "**")
clean = strings.ReplaceAll(clean, "<br>", "\r\n")
clean = strings.ReplaceAll(clean, "<br/>", "\r\n")
clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*")
clean = strings.ReplaceAll(clean, "\xa0", "\r\n")
clean = strings.ReplaceAll(clean, "<p>", "")
clean = strings.ReplaceAll(clean, "</p>", "\r\n")
return clean
}
func (dwh Discord) convertLinks(body string) string {
//items := regexp.MustCompile("<a(.*?)a>")
return ""
}

View File

@ -0,0 +1,73 @@
package output_test
import (
"errors"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/joho/godotenv"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/output"
)
var article database.Article = database.Article{
ID: uuid.New(),
Sourceid: uuid.New(),
Tags: "unit, testing",
Title: "Demo",
Url: "https://github.com/jtom38/newsbot.collector.api",
Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Description: "Hello World",
}
func getWebhook() ([]string, error){
var endpoints []string
_, err := os.Open(".env")
if err != nil {
return endpoints, err
}
err = godotenv.Load()
if err != nil {
return endpoints, err
}
res := os.Getenv("TESTS_DISCORD_WEBHOOK")
if res == "" {
return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing")
}
endpoints = strings.Split(res, "")
return endpoints, nil
}
func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) {
hook, err := getWebhook()
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(hook, article)
if len(d.Subscriptions) == 0 {
t.Error("no subscriptions found")
}
}
func TestDiscordMessageContainsTitle(t *testing.T) {
hook, err := getWebhook()
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(hook, article)
err = d.GeneratePayload()
if err != nil {
t.Error(err)
}
if d.Message.Embeds[0].Title == "" {
t.Error("no title was found ")
}
}

View File

@ -0,0 +1,6 @@
package output
type Output interface {
GeneratePayload() error
SendPayload() error
}