From 0e0058506a19d244a10f123e76b10d0e392d4875 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 30 Jun 2022 14:54:58 -0700 Subject: [PATCH] Feature Flags (#11) * added feature flags around background workers * background workers are moved to a new package as outputs are starting to get added * package name was updated * updated refs to the new input package * query and sql updates on routes * moved the services and starting to add discord web hook * query update --- database/query.sql.go | 81 ++++++++----- database/schema/query.sql | 5 +- docs/docs.go | 4 +- docs/swagger.json | 4 +- docs/swagger.yaml | 4 +- routes/articles.go | 11 +- routes/discordQueue.go | 2 +- services/config/config.go | 27 ++++- services/cron/scheduler.go | 152 ++++++++++++++++++++----- services/cron/scheduler_test.go | 12 +- services/{ => input}/common.go | 2 +- services/{ => input}/ffxiv.go | 2 +- services/{ => input}/ffxiv_test.go | 4 +- services/{ => input}/httpClient.go | 2 +- services/{ => input}/reddit.go | 2 +- services/{ => input}/reddit_test.go | 6 +- services/input/rss.go | 55 +++++++++ services/input/rss_test.go | 26 +++++ services/{ => input}/twitch.go | 2 +- services/{ => input}/twitch_test.go | 28 ++--- services/{ => input}/youtube.go | 4 +- services/{ => input}/youtube_test.go | 28 ++--- services/output/discordwebhook.go | 110 ++++++++++++++++++ services/output/discordwebhook_test.go | 73 ++++++++++++ services/output/interface.go | 6 + 25 files changed, 536 insertions(+), 116 deletions(-) rename services/{ => input}/common.go (97%) rename services/{ => input}/ffxiv.go (99%) rename services/{ => input}/ffxiv_test.go (97%) rename services/{ => input}/httpClient.go (97%) rename services/{ => input}/reddit.go (99%) rename services/{ => input}/reddit_test.go (83%) create mode 100644 services/input/rss.go create mode 100644 services/input/rss_test.go rename services/{ => input}/twitch.go (99%) rename services/{ => input}/twitch_test.go (90%) rename services/{ => input}/youtube.go (99%) rename services/{ => input}/youtube_test.go (76%) create mode 100644 services/output/discordwebhook.go create mode 100644 services/output/discordwebhook_test.go create mode 100644 services/output/interface.go diff --git a/database/query.sql.go b/database/query.sql.go index d0a0095..e1318d2 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -534,33 +534,6 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor return i, err } -const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many -Select id, articleid from DiscordQueue LIMIT $1 -` - -func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) { - rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit) - if err != nil { - return nil, err - } - defer rows.Close() - var items []Discordqueue - for rows.Next() { - var i Discordqueue - if err := rows.Scan(&i.ID, &i.Articleid); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one Select id, url, server, channel, enabled from DiscordWebHooks Where ID = $1 LIMIT 1 @@ -770,6 +743,33 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err return items, nil } +const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many +Select id, articleid from DiscordQueue LIMIT $1 +` + +func (q *Queries) ListDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) { + rows, err := q.db.QueryContext(ctx, listDiscordQueueItems, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Discordqueue + for rows.Next() { + var i Discordqueue + if err := rows.Scan(&i.ID, &i.Articleid); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many Select id, url, server, channel, enabled From DiscordWebHooks Where Server = $1 @@ -938,6 +938,33 @@ func (q *Queries) ListSubscriptions(ctx context.Context, limit int32) ([]Subscri return items, nil } +const listSubscriptionsBySourceId = `-- name: ListSubscriptionsBySourceId :many +Select id, discordwebhookid, sourceid From subscriptions where sourceid = $1 +` + +func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid.UUID) ([]Subscription, error) { + rows, err := q.db.QueryContext(ctx, listSubscriptionsBySourceId, sourceid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Subscription + for rows.Next() { + var i Subscription + if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const querySubscriptions = `-- name: QuerySubscriptions :many Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 ` diff --git a/database/schema/query.sql b/database/schema/query.sql index c17e231..13ff152 100644 --- a/database/schema/query.sql +++ b/database/schema/query.sql @@ -48,7 +48,7 @@ Where ID = $1 LIMIT 1; -- name: DeleteDiscordQueueItem :exec Delete From DiscordQueue Where ID = $1; --- name: GetDiscordQueueItems :many +-- name: ListDiscordQueueItems :many Select * from DiscordQueue LIMIT $1; /* DiscordWebHooks */ @@ -151,6 +151,9 @@ Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3); -- name: ListSubscriptions :many Select * From subscriptions Limit $1; +-- name: ListSubscriptionsBySourceId :many +Select * From subscriptions where sourceid = $1; + -- name: QuerySubscriptions :many Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2; diff --git a/docs/docs.go b/docs/docs.go index 44f8639..c332165 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -28,7 +28,7 @@ const docTemplate = `{ "responses": {} } }, - "/articles/by/sourceid/{id}": { + "/articles/by/sourceid": { "get": { "produces": [ "application/json" @@ -42,7 +42,7 @@ const docTemplate = `{ "type": "string", "description": "Source ID UUID", "name": "id", - "in": "path", + "in": "query", "required": true } ], diff --git a/docs/swagger.json b/docs/swagger.json index 3d9aa36..327f8a0 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -19,7 +19,7 @@ "responses": {} } }, - "/articles/by/sourceid/{id}": { + "/articles/by/sourceid": { "get": { "produces": [ "application/json" @@ -33,7 +33,7 @@ "type": "string", "description": "Source ID UUID", "name": "id", - "in": "path", + "in": "query", "required": true } ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 40a85bb..fe7d7cf 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -26,11 +26,11 @@ paths: summary: Returns an article based on defined ID. tags: - articles - /articles/by/sourceid/{id}: + /articles/by/sourceid: get: parameters: - description: Source ID UUID - in: path + in: query name: id required: true type: string diff --git a/routes/articles.go b/routes/articles.go index 504d658..d2ee736 100644 --- a/routes/articles.go +++ b/routes/articles.go @@ -65,15 +65,18 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) { // TODO add page support // GetArticlesBySourceID // @Summary Finds the articles based on the SourceID provided. Returns the top 50. -// @Param id path string true "Source ID UUID" +// @Param id query string true "Source ID UUID" // @Produce application/json // @Tags articles -// @Router /articles/by/sourceid/{id} [get] +// @Router /articles/by/sourceid [get] func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - id := chi.URLParam(r, "ID") - uuid, err := uuid.Parse(id) + r.URL.Query() + query := r.URL.Query() + _id := query["id"][0] + + uuid, err := uuid.Parse(_id) if err != nil { w.Write([]byte(err.Error())) panic(err) diff --git a/routes/discordQueue.go b/routes/discordQueue.go index efbdb71..246e5bb 100644 --- a/routes/discordQueue.go +++ b/routes/discordQueue.go @@ -13,7 +13,7 @@ import ( func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - res, err := s.Db.GetDiscordQueueItems(*s.ctx, 100) + res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100) if err != nil { w.Write([]byte(err.Error())) panic(err) diff --git a/services/config/config.go b/services/config/config.go index 752bda6..240aff0 100644 --- a/services/config/config.go +++ b/services/config/config.go @@ -1,8 +1,11 @@ package config import ( - "os" + "errors" + "fmt" "log" + "os" + "strconv" "github.com/joho/godotenv" ) @@ -12,16 +15,22 @@ const ( Sql_Connection_String string = "SQL_CONNECTION_STRING" + FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND" REDDIT_PULL_TOP = "REDDIT_PULL_TOP" REDDIT_PULL_HOT = "REDDIT_PULL_HOT" REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW" + FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND" YOUTUBE_DEBUG = "YOUTUBE_DEBUG" + FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND" TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID" TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET" TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS" TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD" + + FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND" + ) type ConfigClient struct {} @@ -43,6 +52,22 @@ func (cc *ConfigClient) GetConfig(key string) string { return res } +func (cc *ConfigClient) GetFeature(flag string) (bool, error) { + cc.RefreshEnv() + + res, filled := os.LookupEnv(flag) + if !filled { + errorMessage := fmt.Sprintf("'%v' was not found", flag) + return false, errors.New(errorMessage) + } + + b, err := strconv.ParseBool(res) + if err != nil { + return false, err + } + return b, nil +} + // Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes. func (cc *ConfigClient) RefreshEnv() { loadEnvFile() diff --git a/services/cron/scheduler.go b/services/cron/scheduler.go index b94f3d1..ea04d70 100644 --- a/services/cron/scheduler.go +++ b/services/cron/scheduler.go @@ -11,13 +11,14 @@ import ( "github.com/robfig/cron/v3" "github.com/jtom38/newsbot/collector/database" - "github.com/jtom38/newsbot/collector/services" + "github.com/jtom38/newsbot/collector/services/input" "github.com/jtom38/newsbot/collector/services/config" + "github.com/jtom38/newsbot/collector/services/output" ) type Cron struct { - Db *database.Queries - ctx *context.Context + Db *database.Queries + ctx *context.Context timer *cron.Cron } @@ -35,7 +36,7 @@ func openDatabase() (*database.Queries, error) { func New(ctx context.Context) *Cron { c := &Cron{ - ctx: &ctx, + ctx: &ctx, } timer := cron.New() @@ -46,10 +47,33 @@ func New(ctx context.Context) *Cron { c.Db = queries //timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) - //timer.AddFunc("* */30 * * *", func() { go c.CheckReddit(ctx) }) - //timer.AddFunc("* */1 * * *", func() { go CheckYoutube() }) - //timer.AddFunc("* */1 * * *", func() { go CheckFfxiv() }) - //timer.AddFunc("* */1 * * *", func() { go CheckTwitch() }) + features := config.New() + + res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND) + if res { + timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() }) + log.Print("Reddit backend was enabled") + //go c.CheckReddit() + } + + res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND) + if res { + timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() }) + log.Print("YouTube backend was enabled") + } + + res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND) + if res { + timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() }) + log.Print("FFXIV backend was enabled") + } + + res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND) + if res { + timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() }) + log.Print("Twitch backend was enabled") + } + c.timer = timer return c } @@ -63,72 +87,77 @@ func (c *Cron) Stop() { } // This is the main entry point to query all the reddit services -func (c *Cron) CheckReddit(ctx context.Context) { +func (c *Cron) CheckReddit() { sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit") if err != nil { - log.Printf("No defines sources for reddit to query - %v\r", err) + log.Printf("[Reddit] No sources found to query - %v\r", err) } for _, source := range sources { if !source.Enabled { continue } - rc := services.NewRedditClient(source) + log.Printf("[Reddit] Checking '%v'...", source.Name) + rc := input.NewRedditClient(source) raw, err := rc.GetContent() if err != nil { log.Println(err) } redditArticles := rc.ConvertToArticles(raw) - c.checkPosts(*c.ctx, redditArticles) + c.checkPosts(redditArticles, "Reddit") } + log.Print("[Reddit] Done!") } -func (c *Cron) CheckYoutube(ctx context.Context) { +func (c *Cron) CheckYoutube() { // Add call to the db to request youtube sources. sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube") if err != nil { - log.Printf("Youtube - No sources found to query - %v\r", err) + log.Printf("[Youtube] No sources found to query - %v\r", err) } for _, source := range sources { if !source.Enabled { continue } - yc := services.NewYoutubeClient(source) + log.Printf("[YouTube] Checking '%v'...", source.Name) + yc := input.NewYoutubeClient(source) raw, err := yc.GetContent() if err != nil { log.Println(err) } - c.checkPosts(*c.ctx, raw) + c.checkPosts(raw, "YouTube") } + log.Print("[YouTube] Done!") } -func (c *Cron) CheckFfxiv(ctx context.Context) { +func (c *Cron) CheckFfxiv() { sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv") if err != nil { - log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err) + log.Printf("[FFXIV] No sources found to query - %v\r", err) } for _, source := range sources { if !source.Enabled { continue } - fc := services.NewFFXIVClient(source) + fc := input.NewFFXIVClient(source) items, err := fc.CheckSource() if err != nil { log.Println(err) } - c.checkPosts(*c.ctx, items) + c.checkPosts(items, "FFXIV") } + log.Printf("[FFXIV Done!]") } -func (c *Cron) CheckTwitch(ctx context.Context) error { +func (c *Cron) CheckTwitch() error { sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch") if err != nil { - log.Printf("Twitch - No sources found to query - %v\r", err) + log.Printf("[Twitch] No sources found to query - %v\r", err) } - - tc, err := services.NewTwitchClient() + + tc, err := input.NewTwitchClient() if err != nil { return err } @@ -137,33 +166,96 @@ func (c *Cron) CheckTwitch(ctx context.Context) error { if !source.Enabled { continue } + log.Printf("[Twitch] Checking '%v'...", source.Name) tc.ReplaceSourceRecord(source) items, err := tc.GetContent() if err != nil { log.Println(err) } - c.checkPosts(*c.ctx, items) + c.checkPosts(items, "Twitch") + } + + log.Print("[Twitch] Done!") + return nil +} + +func (c *Cron) CheckDiscordQueue() error { + // Get items from the table + queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50) + if err != nil { + return err + } + + for _, queue := range(queueItems) { + // Get the articleByID + article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid) + if err != nil { + return err + } + + // Get the SourceByID + //source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid) + //if err != nil { + // return err + //} + + var endpoints []string + // List Subscription by SourceID + subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid) + if err != nil { + return err + } + + // Get the webhhooks to send to + for _, sub := range(subs) { + webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid) + if err != nil { + return err + } + + // store them in an array + endpoints = append(endpoints, webhook.Url) + } + + // Create Discord Message + dwh := output.NewDiscordWebHookMessage(endpoints, article) + err = dwh.GeneratePayload() + if err != nil { + return err + } + + // Send Message + err = dwh.SendPayload() + if err != nil { + return err + } + + // Remove the item from the queue, given we sent our notification. + err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID) + if err != nil { + return err + } } return nil } -func (c *Cron) checkPosts(ctx context.Context, posts []database.Article) { +func (c *Cron) checkPosts(posts []database.Article, sourceName string) { for _, item := range posts { _, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) if err != nil { - err = c.postArticle(ctx, item) + err = c.postArticle(item) if err != nil { - log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err) + log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) } else { - log.Printf("Reddit - Posted article - %v\r", item.Url) + log.Printf("[%v] Posted article - %v\r", sourceName, item.Url) } } } time.Sleep(30 * time.Second) } -func (c *Cron) postArticle(ctx context.Context, item database.Article) error { +func (c *Cron) postArticle(item database.Article) error { err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{ ID: uuid.New(), Sourceid: item.Sourceid, diff --git a/services/cron/scheduler_test.go b/services/cron/scheduler_test.go index d2e84bf..74c4d44 100644 --- a/services/cron/scheduler_test.go +++ b/services/cron/scheduler_test.go @@ -14,20 +14,20 @@ func TestInvokeTwitch(t *testing.T) { // TODO add database mocks but not sure how to do that yet. func TestCheckReddit(t *testing.T) { ctx := context.Background() - c := cron.Cron{} - c.CheckReddit(ctx) + c := cron.New(ctx) + c.CheckReddit() } func TestCheckYouTube(t *testing.T) { ctx := context.Background() - c := cron.Cron{} - c.CheckYoutube(ctx) + c := cron.New(ctx) + c.CheckYoutube() } func TestCheckTwitch(t *testing.T) { ctx := context.Background() - c := cron.Cron{} - err := c.CheckTwitch(ctx) + c := cron.New(ctx) + err := c.CheckTwitch() if err != nil { t.Error(err) } diff --git a/services/common.go b/services/input/common.go similarity index 97% rename from services/common.go rename to services/input/common.go index 7ba2868..6cae09a 100644 --- a/services/common.go +++ b/services/input/common.go @@ -1,4 +1,4 @@ -package services +package input import "errors" diff --git a/services/ffxiv.go b/services/input/ffxiv.go similarity index 99% rename from services/ffxiv.go rename to services/input/ffxiv.go index fbf9808..0881ef3 100644 --- a/services/ffxiv.go +++ b/services/input/ffxiv.go @@ -1,4 +1,4 @@ -package services +package input import ( "database/sql" diff --git a/services/ffxiv_test.go b/services/input/ffxiv_test.go similarity index 97% rename from services/ffxiv_test.go rename to services/input/ffxiv_test.go index 8c8031d..8aaaf9a 100644 --- a/services/ffxiv_test.go +++ b/services/input/ffxiv_test.go @@ -1,11 +1,11 @@ -package services_test +package input_test import ( "testing" "github.com/google/uuid" "github.com/jtom38/newsbot/collector/database" - ffxiv "github.com/jtom38/newsbot/collector/services" + ffxiv "github.com/jtom38/newsbot/collector/services/input" ) var FFXIVRecord database.Source = database.Source{ diff --git a/services/httpClient.go b/services/input/httpClient.go similarity index 97% rename from services/httpClient.go rename to services/input/httpClient.go index cd59fdc..3039497 100644 --- a/services/httpClient.go +++ b/services/input/httpClient.go @@ -1,4 +1,4 @@ -package services +package input import ( "net/http" diff --git a/services/reddit.go b/services/input/reddit.go similarity index 99% rename from services/reddit.go rename to services/input/reddit.go index 99fc5c3..56183df 100644 --- a/services/reddit.go +++ b/services/input/reddit.go @@ -1,4 +1,4 @@ -package services +package input import ( "database/sql" diff --git a/services/reddit_test.go b/services/input/reddit_test.go similarity index 83% rename from services/reddit_test.go rename to services/input/reddit_test.go index 159da88..b57f9c9 100644 --- a/services/reddit_test.go +++ b/services/input/reddit_test.go @@ -1,11 +1,11 @@ -package services_test +package input_test import ( "testing" "github.com/google/uuid" "github.com/jtom38/newsbot/collector/database" - "github.com/jtom38/newsbot/collector/services" + "github.com/jtom38/newsbot/collector/services/input" ) var RedditRecord database.Source = database.Source{ @@ -19,7 +19,7 @@ var RedditRecord database.Source = database.Source{ func TestGetContent(t *testing.T) { //This test is flaky right now due to the http changes in 1.17 - rc := services.NewRedditClient(RedditRecord) + rc := input.NewRedditClient(RedditRecord) raw, err := rc.GetContent() if err != nil { t.Error(err) diff --git a/services/input/rss.go b/services/input/rss.go new file mode 100644 index 0000000..8f84c5b --- /dev/null +++ b/services/input/rss.go @@ -0,0 +1,55 @@ +package input + +import ( + "fmt" + "log" + + "github.com/jtom38/newsbot/collector/domain/model" + "github.com/jtom38/newsbot/collector/services/cache" + "github.com/mmcdole/gofeed" +) + +type rssClient struct { + SourceRecord model.Sources +} + +func NewRssClient(sourceRecord model.Sources) rssClient { + client := rssClient{ + SourceRecord: sourceRecord, + } + + return client +} + +//func (rc rssClient) ReplaceSourceRecord(source model.Sources) { + //rc.SourceRecord = source +//} + +func (rc rssClient) getCacheGroup() string { + return fmt.Sprintf("rss-%v", rc.SourceRecord.Name) +} + +func (rc rssClient) GetContent() error { + feed, err := rc.PullFeed() + if err != nil { return err } + + cacheClient := cache.NewCacheClient(rc.getCacheGroup()) + + for _, item := range feed.Items { + log.Println(item) + + cacheClient.FindByValue(item.Link) + + } + + return nil +} + +func (rc rssClient) PullFeed() (*gofeed.Feed, error) { + feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url) + fp := gofeed.NewParser() + feed, err := fp.ParseURL(feedUri) + if err != nil { return nil, err } + + return feed, nil +} \ No newline at end of file diff --git a/services/input/rss_test.go b/services/input/rss_test.go new file mode 100644 index 0000000..d92cfc5 --- /dev/null +++ b/services/input/rss_test.go @@ -0,0 +1,26 @@ +package input_test + +import ( + "testing" + + "github.com/jtom38/newsbot/collector/domain/model" + "github.com/jtom38/newsbot/collector/services/input" +) + +var rssRecord = model.Sources { + ID: 1, + Name: "ArsTechnica", + Url: "https://feeds.arstechnica.com/arstechnica/index", +} + +func TestRssClientConstructor(t *testing.T) { + input.NewRssClient(rssRecord) +} + +func TestRssGetFeed(t *testing.T) { + client := input.NewRssClient(rssRecord) + feed, err := client.PullFeed() + if err != nil { t.Error(err) } + if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")} + +} \ No newline at end of file diff --git a/services/twitch.go b/services/input/twitch.go similarity index 99% rename from services/twitch.go rename to services/input/twitch.go index dd49646..e251ef9 100644 --- a/services/twitch.go +++ b/services/input/twitch.go @@ -1,4 +1,4 @@ -package services +package input import ( "database/sql" diff --git a/services/twitch_test.go b/services/input/twitch_test.go similarity index 90% rename from services/twitch_test.go rename to services/input/twitch_test.go index 70810fb..c574304 100644 --- a/services/twitch_test.go +++ b/services/input/twitch_test.go @@ -1,4 +1,4 @@ -package services_test +package input_test import ( "log" @@ -6,7 +6,7 @@ import ( "github.com/google/uuid" "github.com/jtom38/newsbot/collector/database" - "github.com/jtom38/newsbot/collector/services" + "github.com/jtom38/newsbot/collector/services/input" ) var TwitchSourceRecord = database.Source { @@ -22,7 +22,7 @@ var TwitchInvalidRecord = database.Source { } func TestTwitchLogin(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -36,7 +36,7 @@ func TestTwitchLogin(t *testing.T) { // reach out and confirms that the API returns posts made by the user. func TestTwitchReturnsUserPosts(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -62,7 +62,7 @@ func TestTwitchReturnsUserPosts(t *testing.T) { } func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -88,7 +88,7 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) { } func TestTwitchReturnsVideoAuthor(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -114,7 +114,7 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) { } func TestTwitchReturnsThumbnail(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil {t.Error(err) } tc.ReplaceSourceRecord(TwitchSourceRecord) @@ -133,7 +133,7 @@ func TestTwitchReturnsThumbnail(t *testing.T) { } func TestTwitchReturnsPubDate(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } tc.ReplaceSourceRecord(TwitchSourceRecord) @@ -152,7 +152,7 @@ func TestTwitchReturnsPubDate(t *testing.T) { } func TestTwitchReturnsDescription(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -180,7 +180,7 @@ func TestTwitchReturnsDescription(t *testing.T) { } func TestTwitchReturnsAuthorImage(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil {t.Error(err) } tc.ReplaceSourceRecord(TwitchSourceRecord) @@ -195,7 +195,7 @@ func TestTwitchReturnsAuthorImage(t *testing.T) { } func TestTwitchReturnsTags(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -219,7 +219,7 @@ func TestTwitchReturnsTags(t *testing.T) { } func TestTwitchReturnsTitle(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } @@ -244,7 +244,7 @@ func TestTwitchReturnsTitle(t *testing.T) { } func TestTwitchReturnsUrl(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } tc.ReplaceSourceRecord(TwitchSourceRecord) @@ -263,7 +263,7 @@ func TestTwitchReturnsUrl(t *testing.T) { } func TestTwitchGetContent(t *testing.T) { - tc, err := services.NewTwitchClient() + tc, err := input.NewTwitchClient() if err != nil { t.Error(err) } tc.ReplaceSourceRecord(TwitchSourceRecord) diff --git a/services/youtube.go b/services/input/youtube.go similarity index 99% rename from services/youtube.go rename to services/input/youtube.go index de831a5..1f7057d 100644 --- a/services/youtube.go +++ b/services/input/youtube.go @@ -1,4 +1,4 @@ -package services +package input import ( "database/sql" @@ -102,7 +102,7 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) { YoutubeUriCache = append(YoutubeUriCache, &item.Link) // Add the post to local cache - log.Println(article) + //log.Println(article) } return items, nil diff --git a/services/youtube_test.go b/services/input/youtube_test.go similarity index 76% rename from services/youtube_test.go rename to services/input/youtube_test.go index 3ddeb77..5163a70 100644 --- a/services/youtube_test.go +++ b/services/input/youtube_test.go @@ -1,11 +1,11 @@ -package services_test +package input_test import ( "testing" "github.com/google/uuid" "github.com/jtom38/newsbot/collector/database" - "github.com/jtom38/newsbot/collector/services" + "github.com/jtom38/newsbot/collector/services/input" ) var YouTubeRecord database.Source = database.Source{ @@ -17,13 +17,13 @@ var YouTubeRecord database.Source = database.Source{ } func TestGetPageParser(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) _, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } } func TestGetChannelId(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) parser, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } @@ -32,7 +32,7 @@ func TestGetChannelId(t *testing.T) { } func TestPullFeed(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) parser, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } @@ -44,14 +44,14 @@ func TestPullFeed(t *testing.T) { } func TestGetAvatarUri(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) res, err := yc.GetAvatarUri() if err != nil { panic(err) } - if res == "" { panic(services.ErrMissingAuthorImage)} + if res == "" { panic(input.ErrMissingAuthorImage)} } func TestGetVideoTags(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68" @@ -64,7 +64,7 @@ func TestGetVideoTags(t *testing.T) { } func TestGetChannelTags(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) parser, err := yc.GetParser(YouTubeRecord.Url) if err != nil { panic(err) } @@ -75,7 +75,7 @@ func TestGetChannelTags(t *testing.T) { } func TestGetVideoThumbnail(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68") if err != nil {panic(err) } @@ -86,22 +86,22 @@ func TestGetVideoThumbnail(t *testing.T) { } func TestCheckSource(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) _, err := yc.GetContent() if err != nil { panic(err) } } func TestCheckUriCache(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) item := "demo" - services.YoutubeUriCache = append(services.YoutubeUriCache, &item) + input.YoutubeUriCache = append(input.YoutubeUriCache, &item) res := yc.CheckUriCache(&item) if res == false { panic("expected a value to come back")} } func TestCheckUriCacheFails(t *testing.T) { - yc := services.NewYoutubeClient(YouTubeRecord) + yc := input.NewYoutubeClient(YouTubeRecord) item := "demo1" res := yc.CheckUriCache(&item) diff --git a/services/output/discordwebhook.go b/services/output/discordwebhook.go new file mode 100644 index 0000000..438a2fc --- /dev/null +++ b/services/output/discordwebhook.go @@ -0,0 +1,110 @@ +package output + +import ( + "strings" + "time" + + "github.com/jtom38/newsbot/collector/database" +) + +type discordField struct { + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` + Inline bool `json:"inline,omitempty"` +} + +type discordAuthor struct { + Name string `json:"name,omitempty"` + Url string `json:"url,omitempty"` + IconUrl string `json:"icon_url,omitempty"` +} + +type discordImage struct { + Url string `json:"url,omitempty"` +} + +type discordEmbed struct { + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Url string `json:"url,omitempty"` + Color int32 `json:"color,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty"` + Fields []discordField `json:"fields,omitempty"` + Author discordAuthor `json:"author,omitempty"` + Image discordImage `json:"image,omitempty"` + Thumbnail discordImage `json:"thumbnail,omitempty"` +} + +// Root object for Discord Webhook messages +type discordMessage struct { + Content string `json:"content,omitempty"` + Embeds []discordEmbed `json:"embeds,omitempty"` +} + +type Discord struct { + Subscriptions []string + article database.Article + Message discordMessage +} + +func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord { + return Discord{ + Subscriptions: Subscriptions, + article: Article, + Message: discordMessage{ + Embeds: []discordEmbed{}, + }, + } +} + +func (dwh Discord) GeneratePayload() error { + // Convert the message + embed := discordEmbed { + Title: dwh.article.Title, + Description: dwh.convertFromHtml(dwh.article.Description), + Url: dwh.article.Url, + Thumbnail: discordImage{ + Url: dwh.article.Thumbnail, + }, + } + var arr []discordEmbed + + arr = append(arr, embed) + dwh.Message.Embeds = arr + return nil +} + +func (dwh Discord) SendPayload() error { + return nil +} + +func (dwh Discord) convertFromHtml(body string) string { + clean := body + clean = strings.ReplaceAll(clean, "

", "**") + clean = strings.ReplaceAll(clean, "

", "**") + clean = strings.ReplaceAll(clean, "

", "**") + clean = strings.ReplaceAll(clean, "

", "**\r\n") + clean = strings.ReplaceAll(clean, "", "**") + clean = strings.ReplaceAll(clean, "", "**\r\n") + clean = strings.ReplaceAll(clean, "", "") + clean = strings.ReplaceAll(clean, "", "\r\n") + clean = strings.ReplaceAll(clean, "
  • ", "> ") + clean = strings.ReplaceAll(clean, "“", "\"") + clean = strings.ReplaceAll(clean, "”", "\"") + clean = strings.ReplaceAll(clean, "…", "...") + clean = strings.ReplaceAll(clean, "", "**") + clean = strings.ReplaceAll(clean, "", "**") + clean = strings.ReplaceAll(clean, "
    ", "\r\n") + clean = strings.ReplaceAll(clean, "
    ", "\r\n") + clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*") + clean = strings.ReplaceAll(clean, "\xa0", "\r\n") + clean = strings.ReplaceAll(clean, "

    ", "") + clean = strings.ReplaceAll(clean, "

    ", "\r\n") + return clean +} + +func (dwh Discord) convertLinks(body string) string { + //items := regexp.MustCompile("") + return "" +} diff --git a/services/output/discordwebhook_test.go b/services/output/discordwebhook_test.go new file mode 100644 index 0000000..61affa4 --- /dev/null +++ b/services/output/discordwebhook_test.go @@ -0,0 +1,73 @@ +package output_test + +import ( + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/joho/godotenv" + "github.com/jtom38/newsbot/collector/database" + "github.com/jtom38/newsbot/collector/services/output" +) + +var article database.Article = database.Article{ + ID: uuid.New(), + Sourceid: uuid.New(), + Tags: "unit, testing", + Title: "Demo", + Url: "https://github.com/jtom38/newsbot.collector.api", + Pubdate: time.Now(), + Videoheight: 0, + Videowidth: 0, + Description: "Hello World", +} + +func getWebhook() ([]string, error){ + var endpoints []string + + _, err := os.Open(".env") + if err != nil { + return endpoints, err + } + + err = godotenv.Load() + if err != nil { + return endpoints, err + } + + res := os.Getenv("TESTS_DISCORD_WEBHOOK") + if res == "" { + return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing") + } + endpoints = strings.Split(res, "") + return endpoints, nil +} + +func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) { + hook, err := getWebhook() + if err != nil { + t.Error(err) + } + d := output.NewDiscordWebHookMessage(hook, article) + if len(d.Subscriptions) == 0 { + t.Error("no subscriptions found") + } +} + +func TestDiscordMessageContainsTitle(t *testing.T) { + hook, err := getWebhook() + if err != nil { + t.Error(err) + } + d := output.NewDiscordWebHookMessage(hook, article) + err = d.GeneratePayload() + if err != nil { + t.Error(err) + } + if d.Message.Embeds[0].Title == "" { + t.Error("no title was found ") + } +} \ No newline at end of file diff --git a/services/output/interface.go b/services/output/interface.go new file mode 100644 index 0000000..979c3c7 --- /dev/null +++ b/services/output/interface.go @@ -0,0 +1,6 @@ +package output + +type Output interface { + GeneratePayload() error + SendPayload() error +} \ No newline at end of file