Feature Flags (#11)

* added feature flags around background workers

* background workers are moved to a new package as outputs are starting to get added

* package name was updated

* updated refs to the new input package

* query and sql updates on routes

* moved the services and starting to add discord web hook

* query update

database/query.sql.go

@@ -534,33 +534,6 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
 	return i, err
 }
 
-const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
-Select id, articleid from DiscordQueue LIMIT $1
-`
-
-func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
-	rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []Discordqueue
-	for rows.Next() {
-		var i Discordqueue
-		if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
 Select id, url, server, channel, enabled from DiscordWebHooks
 Where ID = $1 LIMIT 1
@@ -770,6 +743,33 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
 	return items, nil
 }
 
+const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
+Select id, articleid from DiscordQueue LIMIT $1
+`
+
+func (q *Queries) ListDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
+	rows, err := q.db.QueryContext(ctx, listDiscordQueueItems, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []Discordqueue
+	for rows.Next() {
+		var i Discordqueue
+		if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
 Select id, url, server, channel, enabled From DiscordWebHooks
 Where Server = $1
@@ -938,6 +938,33 @@ func (q *Queries) ListSubscriptions(ctx context.Context, limit int32) ([]Subscri
 	return items, nil
 }
 
+const listSubscriptionsBySourceId = `-- name: ListSubscriptionsBySourceId :many
+Select id, discordwebhookid, sourceid From subscriptions where sourceid = $1
+`
+
+func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid.UUID) ([]Subscription, error) {
+	rows, err := q.db.QueryContext(ctx, listSubscriptionsBySourceId, sourceid)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []Subscription
+	for rows.Next() {
+		var i Subscription
+		if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const querySubscriptions = `-- name: QuerySubscriptions :many
 Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2
 `

database/schema/query.sql

@@ -48,7 +48,7 @@ Where ID = $1 LIMIT 1;
 -- name: DeleteDiscordQueueItem :exec
 Delete From DiscordQueue Where ID = $1;
 
--- name: GetDiscordQueueItems :many
+-- name: ListDiscordQueueItems :many
 Select * from DiscordQueue LIMIT $1;
 
 /* DiscordWebHooks */
@@ -151,6 +151,9 @@ Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3);
 -- name: ListSubscriptions :many
 Select * From subscriptions Limit $1;
 
+-- name: ListSubscriptionsBySourceId :many
+Select * From subscriptions where sourceid = $1;
+
 -- name: QuerySubscriptions :many
 Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2;
 

docs/docs.go

@@ -28,7 +28,7 @@ const docTemplate = `{
                 "responses": {}
             }
         },
-        "/articles/by/sourceid/{id}": {
+        "/articles/by/sourceid": {
             "get": {
                 "produces": [
                     "application/json"
@@ -42,7 +42,7 @@ const docTemplate = `{
                         "type": "string",
                         "description": "Source ID UUID",
                         "name": "id",
-                        "in": "path",
+                        "in": "query",
                         "required": true
                     }
                 ],

docs/swagger.json

@@ -19,7 +19,7 @@
                 "responses": {}
             }
         },
-        "/articles/by/sourceid/{id}": {
+        "/articles/by/sourceid": {
             "get": {
                 "produces": [
                     "application/json"
@@ -33,7 +33,7 @@
                         "type": "string",
                         "description": "Source ID UUID",
                         "name": "id",
-                        "in": "path",
+                        "in": "query",
                         "required": true
                     }
                 ],

docs/swagger.yaml

@@ -26,11 +26,11 @@ paths:
       summary: Returns an article based on defined ID.
       tags:
       - articles
-  /articles/by/sourceid/{id}:
+  /articles/by/sourceid:
     get:
       parameters:
       - description: Source ID UUID
-        in: path
+        in: query
         name: id
         required: true
         type: string

routes/articles.go

@@ -65,15 +65,18 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
 // TODO add page support
 // GetArticlesBySourceID
 // @Summary  Finds the articles based on the SourceID provided.  Returns the top 50.
-// @Param    id  path  string  true  "Source ID UUID"
+// @Param    id  query  string  true  "Source ID UUID"
 // @Produce  application/json
 // @Tags     articles
-// @Router   /articles/by/sourceid/{id} [get]
+// @Router   /articles/by/sourceid [get]
 func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
-	id := chi.URLParam(r, "ID")
-	uuid, err := uuid.Parse(id)
+	r.URL.Query()
+	query := r.URL.Query()
+	_id := query["id"][0]
+
+	uuid, err := uuid.Parse(_id)
 	if err != nil {
 		w.Write([]byte(err.Error()))
 		panic(err)

routes/discordQueue.go

@@ -13,7 +13,7 @@ import (
 func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	
-	res, err := s.Db.GetDiscordQueueItems(*s.ctx, 100)
+	res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
 	if err != nil {
 		w.Write([]byte(err.Error()))
 		panic(err)

services/config/config.go

@@ -1,8 +1,11 @@
 package config
 
 import (
-	"os"
+	"errors"
+	"fmt"
 	"log"
+	"os"
+	"strconv"
 
 	"github.com/joho/godotenv"
 )
@@ -12,16 +15,22 @@ const (
 	
 	Sql_Connection_String string = "SQL_CONNECTION_STRING"
 
+	FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
 	REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
 	REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
 	REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
 
+	FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
 	YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
 
+	FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
 	TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
 	TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
 	TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
 	TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
+
+	FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
+
 )
 
 type ConfigClient struct {}
@@ -43,6 +52,22 @@ func (cc *ConfigClient) GetConfig(key string) string {
 	return res
 }
 
+func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
+	cc.RefreshEnv()
+
+	res, filled := os.LookupEnv(flag)
+	if !filled {
+		errorMessage := fmt.Sprintf("'%v' was not found", flag)
+		return false, errors.New(errorMessage)
+	}
+
+	b, err := strconv.ParseBool(res)
+	if err != nil {
+		return false, err
+	}
+	return b, nil
+}
+
 // Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
 func (cc *ConfigClient) RefreshEnv() {
 	loadEnvFile()

services/cron/scheduler.go

@@ -11,8 +11,9 @@ import (
 	"github.com/robfig/cron/v3"
 
 	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services"
+	"github.com/jtom38/newsbot/collector/services/input"
 	"github.com/jtom38/newsbot/collector/services/config"
+	"github.com/jtom38/newsbot/collector/services/output"
 )
 
 type Cron struct {
@@ -46,10 +47,33 @@ func New(ctx context.Context) *Cron {
 	c.Db = queries
 
 	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
-	//timer.AddFunc("* */30 * * *", func() { go c.CheckReddit(ctx) })
-	//timer.AddFunc("* */1 * * *", func() { go CheckYoutube() })
-	//timer.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
-	//timer.AddFunc("* */1 * * *", func() { go CheckTwitch() })
+	features := config.New()
+
+	res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
+	if res {
+		timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() })
+		log.Print("Reddit backend was enabled")
+		//go c.CheckReddit()
+	}
+
+	res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
+	if res {
+		timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() })
+		log.Print("YouTube backend was enabled")
+	}
+
+	res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
+	if res {
+		timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() })
+		log.Print("FFXIV backend was enabled")
+	}
+
+	res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
+	if res {
+		timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() })
+		log.Print("Twitch backend was enabled")
+	}
+	
 	c.timer = timer
 	return c
 }
@@ -63,72 +87,77 @@ func (c *Cron) Stop() {
 }
 
 // This is the main entry point to query all the reddit services
-func (c *Cron) CheckReddit(ctx context.Context) {
+func (c *Cron) CheckReddit() {
 	sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
 	if err != nil {
-		log.Printf("No defines sources for reddit to query - %v\r", err)
+		log.Printf("[Reddit] No sources found to query - %v\r", err)
 	}
 
 	for _, source := range sources {
 		if !source.Enabled {
 			continue
 		}
-		rc := services.NewRedditClient(source)
+		log.Printf("[Reddit] Checking '%v'...", source.Name)
+		rc := input.NewRedditClient(source)
 		raw, err := rc.GetContent()
 		if err != nil {
 			log.Println(err)
 		}
 		redditArticles := rc.ConvertToArticles(raw)
-		c.checkPosts(*c.ctx, redditArticles)
+		c.checkPosts(redditArticles, "Reddit")
 	}
+	log.Print("[Reddit] Done!")
 }
 
-func (c *Cron) CheckYoutube(ctx context.Context) {
+func (c *Cron) CheckYoutube() {
 	// Add call to the db to request youtube sources.
 	sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
 	if err != nil {
-		log.Printf("Youtube - No sources found to query - %v\r", err)
+		log.Printf("[Youtube] No sources found to query - %v\r", err)
 	}
 
 	for _, source := range sources {
 		if !source.Enabled {
 			continue
 		}
-		yc := services.NewYoutubeClient(source)
+		log.Printf("[YouTube] Checking '%v'...", source.Name)
+		yc := input.NewYoutubeClient(source)
 		raw, err := yc.GetContent()
 		if err != nil {
 			log.Println(err)
 		}
-		c.checkPosts(*c.ctx, raw)
+		c.checkPosts(raw, "YouTube")
 	}
+	log.Print("[YouTube] Done!")
 }
 
-func (c *Cron) CheckFfxiv(ctx context.Context) {
+func (c *Cron) CheckFfxiv() {
 	sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
 	if err != nil {
-		log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
+		log.Printf("[FFXIV] No sources found to query - %v\r", err)
 	}
 
 	for _, source := range sources {
 		if !source.Enabled {
 			continue
 		}
-		fc := services.NewFFXIVClient(source)
+		fc := input.NewFFXIVClient(source)
 		items, err := fc.CheckSource()
 		if err != nil {
 			log.Println(err)
 		}
-		c.checkPosts(*c.ctx, items)
+		c.checkPosts(items, "FFXIV")
 	}
+	log.Printf("[FFXIV Done!]")
 }
 
-func (c *Cron) CheckTwitch(ctx context.Context) error {
+func (c *Cron) CheckTwitch() error {
 	sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
 	if err != nil {
-		log.Printf("Twitch - No sources found to query - %v\r", err)
+		log.Printf("[Twitch] No sources found to query - %v\r", err)
 	}
 
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		return err
 	}
@@ -137,33 +166,96 @@ func (c *Cron) CheckTwitch(ctx context.Context) error {
 		if !source.Enabled {
 			continue
 		}
+		log.Printf("[Twitch] Checking '%v'...", source.Name)
 		tc.ReplaceSourceRecord(source)
 		items, err := tc.GetContent()
 		if err != nil {
 			log.Println(err)
 		}
-		c.checkPosts(*c.ctx, items)
+		c.checkPosts(items, "Twitch")
+	}
+
+	log.Print("[Twitch] Done!")
+	return nil
+}
+
+func (c *Cron) CheckDiscordQueue() error {
+	// Get items from the table
+	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
+	if err != nil {
+		return err
+	}
+
+	for _, queue := range(queueItems) {
+		// Get the articleByID
+		article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
+		if err != nil {
+			return err
+		}
+
+		// Get the SourceByID
+		//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
+		//if err != nil {
+		//	return err
+		//}
+
+		var endpoints []string
+		// List Subscription by SourceID
+		subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
+		if err != nil {
+			return err
+		}
+
+		// Get the webhhooks to send to
+		for _, sub := range(subs) {
+			webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
+			if err != nil {
+				return err
+			}
+
+			// store them in an array
+			endpoints = append(endpoints, webhook.Url)
+		}
+
+		// Create Discord Message
+		dwh := output.NewDiscordWebHookMessage(endpoints, article)
+		err = dwh.GeneratePayload()
+		if err != nil {
+			return err
+		}
+		
+		// Send Message
+		err = dwh.SendPayload()
+		if err != nil {
+			return err
+		}
+
+		// Remove the item from the queue, given we sent our notification.
+		err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
-func (c *Cron) checkPosts(ctx context.Context, posts []database.Article) {
+func (c *Cron) checkPosts(posts []database.Article, sourceName string) {
 	for _, item := range posts {
 		_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
 		if err != nil {
-			err = c.postArticle(ctx, item)
+			err = c.postArticle(item)
 			if err != nil {
-				log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
+				log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
 			} else {
-				log.Printf("Reddit - Posted article - %v\r", item.Url)
+				log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
 			}
 		}
 	}
 	time.Sleep(30 * time.Second)
 }
 
-func (c *Cron) postArticle(ctx context.Context, item database.Article) error {
+func (c *Cron) postArticle(item database.Article) error {
 	err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
 		ID:          uuid.New(),
 		Sourceid:    item.Sourceid,

services/cron/scheduler_test.go

@@ -14,20 +14,20 @@ func TestInvokeTwitch(t *testing.T) {
 // TODO add database mocks but not sure how to do that yet.
 func TestCheckReddit(t *testing.T) {
 	ctx := context.Background()
-	c := cron.Cron{}
-	c.CheckReddit(ctx)
+	c := cron.New(ctx)
+	c.CheckReddit()
 }
 
 func TestCheckYouTube(t *testing.T) {
 	ctx := context.Background()
-	c := cron.Cron{}
-	c.CheckYoutube(ctx)
+	c := cron.New(ctx)
+	c.CheckYoutube()
 }
 
 func TestCheckTwitch(t *testing.T) {
 	ctx := context.Background()
-	c := cron.Cron{}
-	err := c.CheckTwitch(ctx)
+	c := cron.New(ctx)
+	err := c.CheckTwitch()
 	if err != nil {
 		t.Error(err)
 	}

services/input/common.go

@@ -1,4 +1,4 @@
-package services
+package input
 
 import "errors"
 

services/input/ffxiv.go

@@ -1,4 +1,4 @@
-package services
+package input
 
 import (
 	"database/sql"

services/input/ffxiv_test.go

@@ -1,11 +1,11 @@
-package services_test
+package input_test
 
 import (
 	"testing"
 
 	"github.com/google/uuid"
 	"github.com/jtom38/newsbot/collector/database"
-	ffxiv "github.com/jtom38/newsbot/collector/services"
+	ffxiv "github.com/jtom38/newsbot/collector/services/input"
 )
 
 var FFXIVRecord database.Source = database.Source{

services/input/httpClient.go

@@ -1,4 +1,4 @@
-package services
+package input
 
 import (
 	"net/http"

services/input/reddit.go

@@ -1,4 +1,4 @@
-package services
+package input
 
 import (
 	"database/sql"

services/input/reddit_test.go

@@ -1,11 +1,11 @@
-package services_test
+package input_test
 
 import (
 	"testing"
 
 	"github.com/google/uuid"
 	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services"
+	"github.com/jtom38/newsbot/collector/services/input"
 )
 
 var RedditRecord database.Source = database.Source{
@@ -19,7 +19,7 @@ var RedditRecord database.Source = database.Source{
 
 func TestGetContent(t *testing.T) {
 	//This test is flaky right now due to the http changes in 1.17
-	rc := services.NewRedditClient(RedditRecord)
+	rc := input.NewRedditClient(RedditRecord)
 	raw, err := rc.GetContent()
 	if err != nil {
 		t.Error(err)

services/input/rss.go

@@ -0,0 +1,55 @@
+package input
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/jtom38/newsbot/collector/domain/model"
+	"github.com/jtom38/newsbot/collector/services/cache"
+	"github.com/mmcdole/gofeed"
+)
+
+type rssClient struct {
+	SourceRecord model.Sources
+}
+
+func NewRssClient(sourceRecord model.Sources) rssClient {
+	client := rssClient{
+		SourceRecord: sourceRecord,
+	}
+
+	return client
+}
+
+//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
+	//rc.SourceRecord = source
+//}
+
+func (rc rssClient) getCacheGroup() string {
+	return fmt.Sprintf("rss-%v", rc.SourceRecord.Name)
+}
+
+func (rc rssClient) GetContent() error {
+	feed, err := rc.PullFeed()
+	if err != nil { return err }
+
+	cacheClient := cache.NewCacheClient(rc.getCacheGroup())
+
+	for _, item := range feed.Items {
+		log.Println(item)
+
+		cacheClient.FindByValue(item.Link)
+
+	}
+
+	return nil
+}
+
+func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
+	feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
+	fp := gofeed.NewParser()
+	feed, err := fp.ParseURL(feedUri)
+	if err != nil { return nil, err }
+
+	return feed, nil
+}

services/input/rss_test.go

@@ -0,0 +1,26 @@
+package input_test
+
+import (
+	"testing"
+
+	"github.com/jtom38/newsbot/collector/domain/model"
+	"github.com/jtom38/newsbot/collector/services/input"
+)
+
+var rssRecord = model.Sources {
+	ID: 1,
+	Name: "ArsTechnica",
+	Url: "https://feeds.arstechnica.com/arstechnica/index",
+}
+
+func TestRssClientConstructor(t *testing.T) {
+	input.NewRssClient(rssRecord)
+}
+
+func TestRssGetFeed(t *testing.T) {
+	client := input.NewRssClient(rssRecord)
+	feed, err := client.PullFeed()
+	if err != nil { t.Error(err) }
+	if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")}
+
+}

services/input/twitch.go

@@ -1,4 +1,4 @@
-package services
+package input
 
 import (
 	"database/sql"

services/input/twitch_test.go

@@ -1,4 +1,4 @@
-package services_test
+package input_test
 
 import (
 	"log"
@@ -6,7 +6,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services"
+	"github.com/jtom38/newsbot/collector/services/input"
 )
 
 var TwitchSourceRecord = database.Source {
@@ -22,7 +22,7 @@ var TwitchInvalidRecord = database.Source {
 }
 
 func TestTwitchLogin(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -36,7 +36,7 @@ func TestTwitchLogin(t *testing.T) {
 
 // reach out and confirms that the API returns posts made by the user.
 func TestTwitchReturnsUserPosts(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -62,7 +62,7 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
 }
 
 func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -88,7 +88,7 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
 }
 
 func TestTwitchReturnsVideoAuthor(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -114,7 +114,7 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
 }
 
 func TestTwitchReturnsThumbnail(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {t.Error(err) }
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
@@ -133,7 +133,7 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
 }
 
 func TestTwitchReturnsPubDate(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil { t.Error(err) }
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
@@ -152,7 +152,7 @@ func TestTwitchReturnsPubDate(t *testing.T) {
 }
 
 func TestTwitchReturnsDescription(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -180,7 +180,7 @@ func TestTwitchReturnsDescription(t *testing.T) {
 }
 
 func TestTwitchReturnsAuthorImage(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {t.Error(err) }
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
@@ -195,7 +195,7 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
 }
 
 func TestTwitchReturnsTags(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -219,7 +219,7 @@ func TestTwitchReturnsTags(t *testing.T) {
 }
 
 func TestTwitchReturnsTitle(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil {
 		t.Error(err)
 	}
@@ -244,7 +244,7 @@ func TestTwitchReturnsTitle(t *testing.T) {
 }
 
 func TestTwitchReturnsUrl(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil { t.Error(err) }
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 
@@ -263,7 +263,7 @@ func TestTwitchReturnsUrl(t *testing.T) {
 }
 
 func TestTwitchGetContent(t *testing.T) {
-	tc, err := services.NewTwitchClient()
+	tc, err := input.NewTwitchClient()
 	if err != nil { t.Error(err) }
 	tc.ReplaceSourceRecord(TwitchSourceRecord)
 

services/input/youtube.go

@@ -1,4 +1,4 @@
-package services
+package input
 
 import (
 	"database/sql"
@@ -102,7 +102,7 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
 		YoutubeUriCache = append(YoutubeUriCache, &item.Link)
 
 		// Add the post to local cache
-		log.Println(article)
+		//log.Println(article)
 	}
 
 	return items, nil

services/input/youtube_test.go

@@ -1,11 +1,11 @@
-package services_test
+package input_test
 
 import (
 	"testing"
 
 	"github.com/google/uuid"
 	"github.com/jtom38/newsbot/collector/database"
-	"github.com/jtom38/newsbot/collector/services"
+	"github.com/jtom38/newsbot/collector/services/input"
 )
 
 var YouTubeRecord database.Source = database.Source{
@@ -17,13 +17,13 @@ var YouTubeRecord database.Source = database.Source{
 }
 
 func TestGetPageParser(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	_, err := yc.GetParser(YouTubeRecord.Url)
 	if err != nil { panic(err) }
 }
 
 func TestGetChannelId(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	parser, err := yc.GetParser(YouTubeRecord.Url)
 	if err != nil { panic(err) }
 
@@ -32,7 +32,7 @@ func TestGetChannelId(t *testing.T) {
 }
 
 func TestPullFeed(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	parser, err := yc.GetParser(YouTubeRecord.Url)
 	if err != nil { panic(err) }
 
@@ -44,14 +44,14 @@ func TestPullFeed(t *testing.T) {
 }
 
 func TestGetAvatarUri(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	res, err := yc.GetAvatarUri()
 	if err != nil { panic(err) }
-	if res == "" { panic(services.ErrMissingAuthorImage)}
+	if res == "" { panic(input.ErrMissingAuthorImage)}
 }
 
 func TestGetVideoTags(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 
 	var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
 
@@ -64,7 +64,7 @@ func TestGetVideoTags(t *testing.T) {
 }
 
 func TestGetChannelTags(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 
 	parser, err := yc.GetParser(YouTubeRecord.Url)
 	if err != nil { panic(err) }
@@ -75,7 +75,7 @@ func TestGetChannelTags(t *testing.T) {
 }
 
 func TestGetVideoThumbnail(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
 	if err != nil {panic(err) }
 
@@ -86,22 +86,22 @@ func TestGetVideoThumbnail(t *testing.T) {
 }
 
 func TestCheckSource(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	_, err := yc.GetContent()
 	if err != nil { panic(err) }
 }
 
 func TestCheckUriCache(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	item := "demo"
 
-	services.YoutubeUriCache = append(services.YoutubeUriCache, &item)
+	input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
 	res := yc.CheckUriCache(&item)
 	if res == false { panic("expected a value to come back")}
 }
 
 func TestCheckUriCacheFails(t *testing.T) {
-	yc := services.NewYoutubeClient(YouTubeRecord)
+	yc := input.NewYoutubeClient(YouTubeRecord)
 	item := "demo1"
 	
 	res := yc.CheckUriCache(&item)

services/output/discordwebhook.go

@@ -0,0 +1,110 @@
+package output
+
+import (
+	"strings"
+	"time"
+
+	"github.com/jtom38/newsbot/collector/database"
+)
+
+type discordField struct {
+	Name string `json:"name,omitempty"`
+	Value string `json:"value,omitempty"`
+	Inline bool `json:"inline,omitempty"`
+}
+
+type discordAuthor struct {
+	Name string `json:"name,omitempty"`
+	Url string `json:"url,omitempty"`
+	IconUrl string `json:"icon_url,omitempty"`
+}
+
+type discordImage struct {
+	Url string `json:"url,omitempty"`
+}
+
+type discordEmbed struct {
+	Title string `json:"title,omitempty"`
+	Description string `json:"description,omitempty"`
+	Url string `json:"url,omitempty"`
+	Color int32 `json:"color,omitempty"`
+	Timestamp time.Time `json:"timestamp,omitempty"`
+	Fields []discordField `json:"fields,omitempty"`
+	Author discordAuthor `json:"author,omitempty"`
+	Image discordImage `json:"image,omitempty"`
+	Thumbnail discordImage `json:"thumbnail,omitempty"`
+}
+
+// Root object for Discord Webhook messages
+type discordMessage struct {
+	Content string `json:"content,omitempty"`
+	Embeds []discordEmbed `json:"embeds,omitempty"`
+}
+
+type Discord struct {
+	Subscriptions []string
+	article database.Article
+	Message discordMessage
+}
+
+func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord {
+	return Discord{
+		Subscriptions: Subscriptions,
+		article: Article,
+		Message: discordMessage{
+			Embeds: []discordEmbed{},
+		},
+	}
+}
+
+func (dwh Discord) GeneratePayload() error {
+	// Convert the message 
+	embed := discordEmbed {
+		Title: dwh.article.Title,
+		Description: dwh.convertFromHtml(dwh.article.Description),
+		Url: dwh.article.Url,
+		Thumbnail: discordImage{
+			Url: dwh.article.Thumbnail,
+		},
+	}
+	var arr []discordEmbed
+
+	arr = append(arr, embed)
+	dwh.Message.Embeds = arr
+	return nil
+}
+
+func (dwh Discord) SendPayload() error {
+	return nil
+}
+
+func (dwh Discord) convertFromHtml(body string) string {
+	clean := body
+	clean = strings.ReplaceAll(clean, "<h2>", "**")
+	clean = strings.ReplaceAll(clean, "</h2>", "**")
+	clean = strings.ReplaceAll(clean, "<h3>", "**")
+	clean = strings.ReplaceAll(clean, "</h3>", "**\r\n")
+	clean = strings.ReplaceAll(clean, "<strong>", "**")
+	clean = strings.ReplaceAll(clean, "</strong>", "**\r\n")
+	clean = strings.ReplaceAll(clean, "<ul>", "\r\n")
+	clean = strings.ReplaceAll(clean, "</ul>", "")
+	clean = strings.ReplaceAll(clean, "</li>", "\r\n")
+	clean = strings.ReplaceAll(clean, "<li>", "> ")
+	clean = strings.ReplaceAll(clean, "&#8220;", "\"")
+	clean = strings.ReplaceAll(clean, "&#8221;", "\"")
+	clean = strings.ReplaceAll(clean, "&#8230;", "...")
+	clean = strings.ReplaceAll(clean, "<b>", "**")
+	clean = strings.ReplaceAll(clean, "</b>", "**")
+	clean = strings.ReplaceAll(clean, "<br>", "\r\n")
+	clean = strings.ReplaceAll(clean, "<br/>", "\r\n")
+	clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*")
+	clean = strings.ReplaceAll(clean, "\xa0", "\r\n")
+	clean = strings.ReplaceAll(clean, "<p>", "")
+	clean = strings.ReplaceAll(clean, "</p>", "\r\n")
+	return clean
+}
+
+func (dwh Discord) convertLinks(body string) string {
+	//items := regexp.MustCompile("<a(.*?)a>")
+	return ""
+} 

services/output/discordwebhook_test.go

@@ -0,0 +1,73 @@
+package output_test
+
+import (
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/joho/godotenv"
+	"github.com/jtom38/newsbot/collector/database"
+	"github.com/jtom38/newsbot/collector/services/output"
+)
+
+var article database.Article = database.Article{
+	ID: uuid.New(),
+	Sourceid: uuid.New(),
+	Tags: "unit, testing",
+	Title: "Demo",
+	Url: "https://github.com/jtom38/newsbot.collector.api",
+	Pubdate: time.Now(),
+	Videoheight: 0,
+	Videowidth: 0,
+	Description: "Hello World",
+}
+
+func getWebhook() ([]string, error){
+	var endpoints []string
+
+	_, err := os.Open(".env")
+	if err != nil {
+		return endpoints, err
+	}
+		
+	err = godotenv.Load()
+	if err != nil {
+		return endpoints, err
+	}
+
+	res := os.Getenv("TESTS_DISCORD_WEBHOOK")
+	if res == "" {
+		return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing")
+	}
+	endpoints = strings.Split(res, "")
+	return endpoints, nil
+}
+
+func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) {
+	hook, err := getWebhook()
+	if err != nil {
+		t.Error(err)
+	}
+	d := output.NewDiscordWebHookMessage(hook, article)
+	if len(d.Subscriptions) == 0 {
+		t.Error("no subscriptions found")
+	}
+}
+
+func TestDiscordMessageContainsTitle(t *testing.T) {
+	hook, err := getWebhook()
+	if err != nil {
+		t.Error(err)
+	}
+	d := output.NewDiscordWebHookMessage(hook, article)
+	err = d.GeneratePayload()
+	if err != nil {
+		t.Error(err)
+	}
+	if d.Message.Embeds[0].Title == "" {
+		t.Error("no title was found ")
+	}
+}

services/output/interface.go

@@ -0,0 +1,6 @@
+package output
+
+type Output interface {
+	GeneratePayload() error
+	SendPayload() error
+}