jtom38
/
newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature Flags (#11) 

* added feature flags around background workers

* background workers are moved to a new package as outputs are starting to get added

* package name was updated

* updated refs to the new input package

* query and sql updates on routes

* moved the services and starting to add discord web hook

* query update
This commit is contained in:
James Tombleson 2022-06-30 14:54:58 -07:00 committed by GitHub
parent 713205bb03
commit 0e0058506a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 536 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
database/query.sql.go
View File

@ -534,33 +534,6 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
return i, err
}
const getDiscordQueueItems = `-- name: GetDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) GetDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, getDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1
@ -770,6 +743,33 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
return items, nil
}
const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
func (q *Queries) ListDiscordQueueItems(ctx context.Context, limit int32) ([]Discordqueue, error) {
rows, err := q.db.QueryContext(ctx, listDiscordQueueItems, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordqueue
for rows.Next() {
var i Discordqueue
if err := rows.Scan(&i.ID, &i.Articleid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listDiscordWebHooksByServer = `-- name: ListDiscordWebHooksByServer :many
Select id, url, server, channel, enabled From DiscordWebHooks
Where Server = $1
@ -938,6 +938,33 @@ func (q *Queries) ListSubscriptions(ctx context.Context, limit int32) ([]Subscri
return items, nil
}
const listSubscriptionsBySourceId = `-- name: ListSubscriptionsBySourceId :many
Select id, discordwebhookid, sourceid From subscriptions where sourceid = $1
`
func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid.UUID) ([]Subscription, error) {
rows, err := q.db.QueryContext(ctx, listSubscriptionsBySourceId, sourceid)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Subscription
for rows.Next() {
var i Subscription
if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const querySubscriptions = `-- name: QuerySubscriptions :many
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2
`

5
database/schema/query.sql
View File

@ -48,7 +48,7 @@ Where ID = $1 LIMIT 1;
-- name: DeleteDiscordQueueItem :exec
Delete From DiscordQueue Where ID = $1;
-- name: GetDiscordQueueItems :many
-- name: ListDiscordQueueItems :many
Select * from DiscordQueue LIMIT $1;
/* DiscordWebHooks */
@ -151,6 +151,9 @@ Insert Into subscriptions (ID, DiscordWebHookId, SourceId) Values ($1, $2, $3);
-- name: ListSubscriptions :many
Select * From subscriptions Limit $1;
-- name: ListSubscriptionsBySourceId :many
Select * From subscriptions where sourceid = $1;
-- name: QuerySubscriptions :many
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2;

4
docs/docs.go
View File

@ -28,7 +28,7 @@ const docTemplate = `{
"responses": {}
}
},
"/articles/by/sourceid/{id}": {
"/articles/by/sourceid": {
"get": {
"produces": [
"application/json"
@ -42,7 +42,7 @@ const docTemplate = `{
"type": "string",
"description": "Source ID UUID",
"name": "id",
"in": "path",
"in": "query",
"required": true
}
],

4
docs/swagger.json
View File

@ -19,7 +19,7 @@
"responses": {}
}
},
"/articles/by/sourceid/{id}": {
"/articles/by/sourceid": {
"get": {
"produces": [
"application/json"
@ -33,7 +33,7 @@
"type": "string",
"description": "Source ID UUID",
"name": "id",
"in": "path",
"in": "query",
"required": true
}
],

4
docs/swagger.yaml
View File

@ -26,11 +26,11 @@ paths:
summary: Returns an article based on defined ID.
tags:
- articles
/articles/by/sourceid/{id}:
/articles/by/sourceid:
get:
parameters:
- description: Source ID UUID
in: path
in: query
name: id
required: true
type: string

11
routes/articles.go
View File

@ -65,15 +65,18 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
// TODO add page support
// GetArticlesBySourceID
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
// @Param id path string true "Source ID UUID"
// @Param id query string true "Source ID UUID"
// @Produce application/json
// @Tags articles
// @Router /articles/by/sourceid/{id} [get]
// @Router /articles/by/sourceid [get]
func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
id := chi.URLParam(r, "ID")
uuid, err := uuid.Parse(id)
r.URL.Query()
query := r.URL.Query()
_id := query["id"][0]
uuid, err := uuid.Parse(_id)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)

2
routes/discordQueue.go
View File

@ -13,7 +13,7 @@ import (
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
res, err := s.Db.GetDiscordQueueItems(*s.ctx, 100)
res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)

27
services/config/config.go
View File

@ -1,8 +1,11 @@
package config
import (
"os"
"errors"
"fmt"
"log"
"os"
"strconv"
"github.com/joho/godotenv"
)
@ -12,16 +15,22 @@ const (
Sql_Connection_String string = "SQL_CONNECTION_STRING"
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
)
type ConfigClient struct {}
@ -43,6 +52,22 @@ func (cc *ConfigClient) GetConfig(key string) string {
return res
}
func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
cc.RefreshEnv()
res, filled := os.LookupEnv(flag)
if !filled {
errorMessage := fmt.Sprintf("'%v' was not found", flag)
return false, errors.New(errorMessage)
}
b, err := strconv.ParseBool(res)
if err != nil {
return false, err
}
return b, nil
}
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
func (cc *ConfigClient) RefreshEnv() {
loadEnvFile()

152
services/cron/scheduler.go
View File

@ -11,13 +11,14 @@ import (
"github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
"github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/jtom38/newsbot/collector/services/output"
)
type Cron struct {
Db *database.Queries
ctx *context.Context
Db *database.Queries
ctx *context.Context
timer *cron.Cron
}
@ -35,7 +36,7 @@ func openDatabase() (*database.Queries, error) {
func New(ctx context.Context) *Cron {
c := &Cron{
ctx: &ctx,
ctx: &ctx,
}
timer := cron.New()
@ -46,10 +47,33 @@ func New(ctx context.Context) *Cron {
c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//timer.AddFunc("* */30 * * *", func() { go c.CheckReddit(ctx) })
//timer.AddFunc("* */1 * * *", func() { go CheckYoutube() })
//timer.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
//timer.AddFunc("* */1 * * *", func() { go CheckTwitch() })
features := config.New()
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() })
log.Print("Reddit backend was enabled")
//go c.CheckReddit()
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() })
log.Print("YouTube backend was enabled")
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() })
log.Print("FFXIV backend was enabled")
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() })
log.Print("Twitch backend was enabled")
}
c.timer = timer
return c
}
@ -63,72 +87,77 @@ func (c *Cron) Stop() {
}
// This is the main entry point to query all the reddit services
func (c *Cron) CheckReddit(ctx context.Context) {
func (c *Cron) CheckReddit() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
if err != nil {
log.Printf("No defines sources for reddit to query - %v\r", err)
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
rc := services.NewRedditClient(source)
log.Printf("[Reddit] Checking '%v'...", source.Name)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
c.checkPosts(*c.ctx, redditArticles)
c.checkPosts(redditArticles, "Reddit")
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CheckYoutube(ctx context.Context) {
func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources.
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
if err != nil {
log.Printf("Youtube - No sources found to query - %v\r", err)
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
yc := services.NewYoutubeClient(source)
log.Printf("[YouTube] Checking '%v'...", source.Name)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(*c.ctx, raw)
c.checkPosts(raw, "YouTube")
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CheckFfxiv(ctx context.Context) {
func (c *Cron) CheckFfxiv() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
if err != nil {
log.Printf("Final Fantasy XIV - No sources found to query - %v\r", err)
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := services.NewFFXIVClient(source)
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
c.checkPosts(*c.ctx, items)
c.checkPosts(items, "FFXIV")
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CheckTwitch(ctx context.Context) error {
func (c *Cron) CheckTwitch() error {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
if err != nil {
log.Printf("Twitch - No sources found to query - %v\r", err)
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
return err
}
@ -137,33 +166,96 @@ func (c *Cron) CheckTwitch(ctx context.Context) error {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.Name)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(*c.ctx, items)
c.checkPosts(items, "Twitch")
}
log.Print("[Twitch] Done!")
return nil
}
func (c *Cron) CheckDiscordQueue() error {
// Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
if err != nil {
return err
}
for _, queue := range(queueItems) {
// Get the articleByID
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
if err != nil {
return err
}
// Get the SourceByID
//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
//if err != nil {
// return err
//}
var endpoints []string
// List Subscription by SourceID
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
if err != nil {
return err
}
// Get the webhhooks to send to
for _, sub := range(subs) {
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
if err != nil {
return err
}
// store them in an array
endpoints = append(endpoints, webhook.Url)
}
// Create Discord Message
dwh := output.NewDiscordWebHookMessage(endpoints, article)
err = dwh.GeneratePayload()
if err != nil {
return err
}
// Send Message
err = dwh.SendPayload()
if err != nil {
return err
}
// Remove the item from the queue, given we sent our notification.
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
if err != nil {
return err
}
}
return nil
}
func (c *Cron) checkPosts(ctx context.Context, posts []database.Article) {
func (c *Cron) checkPosts(posts []database.Article, sourceName string) {
for _, item := range posts {
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
if err != nil {
err = c.postArticle(ctx, item)
err = c.postArticle(item)
if err != nil {
log.Printf("Reddit - Failed to post article - %v - %v.\r", item.Url, err)
log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
} else {
log.Printf("Reddit - Posted article - %v\r", item.Url)
log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
}
}
}
time.Sleep(30 * time.Second)
}
func (c *Cron) postArticle(ctx context.Context, item database.Article) error {
func (c *Cron) postArticle(item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: uuid.New(),
Sourceid: item.Sourceid,

12
services/cron/scheduler_test.go
View File

@ -14,20 +14,20 @@ func TestInvokeTwitch(t *testing.T) {
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) {
ctx := context.Background()
c := cron.Cron{}
c.CheckReddit(ctx)
c := cron.New(ctx)
c.CheckReddit()
}
func TestCheckYouTube(t *testing.T) {
ctx := context.Background()
c := cron.Cron{}
c.CheckYoutube(ctx)
c := cron.New(ctx)
c.CheckYoutube()
}
func TestCheckTwitch(t *testing.T) {
ctx := context.Background()
c := cron.Cron{}
err := c.CheckTwitch(ctx)
c := cron.New(ctx)
err := c.CheckTwitch()
if err != nil {
t.Error(err)
}

2
services/common.go → services/input/common.go
View File

@ -1,4 +1,4 @@
package services
package input
import "errors"

2
services/ffxiv.go → services/input/ffxiv.go
View File

@ -1,4 +1,4 @@
package services
package input
import (
"database/sql"

4
services/ffxiv_test.go → services/input/ffxiv_test.go
View File

@ -1,11 +1,11 @@
package services_test
package input_test
import (
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services"
ffxiv "github.com/jtom38/newsbot/collector/services/input"
)
var FFXIVRecord database.Source = database.Source{

2
services/httpClient.go → services/input/httpClient.go
View File

@ -1,4 +1,4 @@
package services
package input
import (
"net/http"

2
services/reddit.go → services/input/reddit.go
View File

@ -1,4 +1,4 @@
package services
package input
import (
"database/sql"

6
services/reddit_test.go → services/input/reddit_test.go
View File

@ -1,11 +1,11 @@
package services_test
package input_test
import (
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
"github.com/jtom38/newsbot/collector/services/input"
)
var RedditRecord database.Source = database.Source{
@ -19,7 +19,7 @@ var RedditRecord database.Source = database.Source{
func TestGetContent(t *testing.T) {
//This test is flaky right now due to the http changes in 1.17
rc := services.NewRedditClient(RedditRecord)
rc := input.NewRedditClient(RedditRecord)
raw, err := rc.GetContent()
if err != nil {
t.Error(err)

55
services/input/rss.go Normal file
View File

@ -0,0 +1,55 @@
package input
import (
"fmt"
"log"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/cache"
"github.com/mmcdole/gofeed"
)
type rssClient struct {
SourceRecord model.Sources
}
func NewRssClient(sourceRecord model.Sources) rssClient {
client := rssClient{
SourceRecord: sourceRecord,
}
return client
}
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
//rc.SourceRecord = source
//}
func (rc rssClient) getCacheGroup() string {
return fmt.Sprintf("rss-%v", rc.SourceRecord.Name)
}
func (rc rssClient) GetContent() error {
feed, err := rc.PullFeed()
if err != nil { return err }
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
for _, item := range feed.Items {
log.Println(item)
cacheClient.FindByValue(item.Link)
}
return nil
}
func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
fp := gofeed.NewParser()
feed, err := fp.ParseURL(feedUri)
if err != nil { return nil, err }
return feed, nil
}

26
services/input/rss_test.go Normal file
View File

@ -0,0 +1,26 @@
package input_test
import (
"testing"
"github.com/jtom38/newsbot/collector/domain/model"
"github.com/jtom38/newsbot/collector/services/input"
)
var rssRecord = model.Sources {
ID: 1,
Name: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index",
}
func TestRssClientConstructor(t *testing.T) {
input.NewRssClient(rssRecord)
}
func TestRssGetFeed(t *testing.T) {
client := input.NewRssClient(rssRecord)
feed, err := client.PullFeed()
if err != nil { t.Error(err) }
if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")}
}

2
services/twitch.go → services/input/twitch.go
View File

@ -1,4 +1,4 @@
package services
package input
import (
"database/sql"

28
services/twitch_test.go → services/input/twitch_test.go
View File

@ -1,4 +1,4 @@
package services_test
package input_test
import (
"log"
@ -6,7 +6,7 @@ import (
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
"github.com/jtom38/newsbot/collector/services/input"
)
var TwitchSourceRecord = database.Source {
@ -22,7 +22,7 @@ var TwitchInvalidRecord = database.Source {
}
func TestTwitchLogin(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -36,7 +36,7 @@ func TestTwitchLogin(t *testing.T) {
// reach out and confirms that the API returns posts made by the user.
func TestTwitchReturnsUserPosts(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -62,7 +62,7 @@ func TestTwitchReturnsUserPosts(t *testing.T) {
}
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -88,7 +88,7 @@ func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
}
func TestTwitchReturnsVideoAuthor(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -114,7 +114,7 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
}
func TestTwitchReturnsThumbnail(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -133,7 +133,7 @@ func TestTwitchReturnsThumbnail(t *testing.T) {
}
func TestTwitchReturnsPubDate(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -152,7 +152,7 @@ func TestTwitchReturnsPubDate(t *testing.T) {
}
func TestTwitchReturnsDescription(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -180,7 +180,7 @@ func TestTwitchReturnsDescription(t *testing.T) {
}
func TestTwitchReturnsAuthorImage(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -195,7 +195,7 @@ func TestTwitchReturnsAuthorImage(t *testing.T) {
}
func TestTwitchReturnsTags(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -219,7 +219,7 @@ func TestTwitchReturnsTags(t *testing.T) {
}
func TestTwitchReturnsTitle(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil {
t.Error(err)
}
@ -244,7 +244,7 @@ func TestTwitchReturnsTitle(t *testing.T) {
}
func TestTwitchReturnsUrl(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)
@ -263,7 +263,7 @@ func TestTwitchReturnsUrl(t *testing.T) {
}
func TestTwitchGetContent(t *testing.T) {
tc, err := services.NewTwitchClient()
tc, err := input.NewTwitchClient()
if err != nil { t.Error(err) }
tc.ReplaceSourceRecord(TwitchSourceRecord)

4
services/youtube.go → services/input/youtube.go
View File

@ -1,4 +1,4 @@
package services
package input
import (
"database/sql"
@ -102,7 +102,7 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
YoutubeUriCache = append(YoutubeUriCache, &item.Link)
// Add the post to local cache
log.Println(article)
//log.Println(article)
}
return items, nil

28
services/youtube_test.go → services/input/youtube_test.go
View File

@ -1,11 +1,11 @@
package services_test
package input_test
import (
"testing"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services"
"github.com/jtom38/newsbot/collector/services/input"
)
var YouTubeRecord database.Source = database.Source{
@ -17,13 +17,13 @@ var YouTubeRecord database.Source = database.Source{
}
func TestGetPageParser(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
}
func TestGetChannelId(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
@ -32,7 +32,7 @@ func TestGetChannelId(t *testing.T) {
}
func TestPullFeed(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
@ -44,14 +44,14 @@ func TestPullFeed(t *testing.T) {
}
func TestGetAvatarUri(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
res, err := yc.GetAvatarUri()
if err != nil { panic(err) }
if res == "" { panic(services.ErrMissingAuthorImage)}
if res == "" { panic(input.ErrMissingAuthorImage)}
}
func TestGetVideoTags(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
@ -64,7 +64,7 @@ func TestGetVideoTags(t *testing.T) {
}
func TestGetChannelTags(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser(YouTubeRecord.Url)
if err != nil { panic(err) }
@ -75,7 +75,7 @@ func TestGetChannelTags(t *testing.T) {
}
func TestGetVideoThumbnail(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
if err != nil {panic(err) }
@ -86,22 +86,22 @@ func TestGetVideoThumbnail(t *testing.T) {
}
func TestCheckSource(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
_, err := yc.GetContent()
if err != nil { panic(err) }
}
func TestCheckUriCache(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
item := "demo"
services.YoutubeUriCache = append(services.YoutubeUriCache, &item)
input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
res := yc.CheckUriCache(&item)
if res == false { panic("expected a value to come back")}
}
func TestCheckUriCacheFails(t *testing.T) {
yc := services.NewYoutubeClient(YouTubeRecord)
yc := input.NewYoutubeClient(YouTubeRecord)
item := "demo1"
res := yc.CheckUriCache(&item)

110
services/output/discordwebhook.go Normal file
View File

@ -0,0 +1,110 @@
package output
import (
"strings"
"time"
"github.com/jtom38/newsbot/collector/database"
)
type discordField struct {
Name string `json:"name,omitempty"`
Value string `json:"value,omitempty"`
Inline bool `json:"inline,omitempty"`
}
type discordAuthor struct {
Name string `json:"name,omitempty"`
Url string `json:"url,omitempty"`
IconUrl string `json:"icon_url,omitempty"`
}
type discordImage struct {
Url string `json:"url,omitempty"`
}
type discordEmbed struct {
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Url string `json:"url,omitempty"`
Color int32 `json:"color,omitempty"`
Timestamp time.Time `json:"timestamp,omitempty"`
Fields []discordField `json:"fields,omitempty"`
Author discordAuthor `json:"author,omitempty"`
Image discordImage `json:"image,omitempty"`
Thumbnail discordImage `json:"thumbnail,omitempty"`
}
// Root object for Discord Webhook messages
type discordMessage struct {
Content string `json:"content,omitempty"`
Embeds []discordEmbed `json:"embeds,omitempty"`
}
type Discord struct {
Subscriptions []string
article database.Article
Message discordMessage
}
func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord {
return Discord{
Subscriptions: Subscriptions,
article: Article,
Message: discordMessage{
Embeds: []discordEmbed{},
},
}
}
func (dwh Discord) GeneratePayload() error {
// Convert the message
embed := discordEmbed {
Title: dwh.article.Title,
Description: dwh.convertFromHtml(dwh.article.Description),
Url: dwh.article.Url,
Thumbnail: discordImage{
Url: dwh.article.Thumbnail,
},
}
var arr []discordEmbed
arr = append(arr, embed)
dwh.Message.Embeds = arr
return nil
}
func (dwh Discord) SendPayload() error {
return nil
}
func (dwh Discord) convertFromHtml(body string) string {
clean := body
clean = strings.ReplaceAll(clean, "<h2>", "**")
clean = strings.ReplaceAll(clean, "</h2>", "**")
clean = strings.ReplaceAll(clean, "<h3>", "**")
clean = strings.ReplaceAll(clean, "</h3>", "**\r\n")
clean = strings.ReplaceAll(clean, "<strong>", "**")
clean = strings.ReplaceAll(clean, "</strong>", "**\r\n")
clean = strings.ReplaceAll(clean, "<ul>", "\r\n")
clean = strings.ReplaceAll(clean, "</ul>", "")
clean = strings.ReplaceAll(clean, "</li>", "\r\n")
clean = strings.ReplaceAll(clean, "<li>", "> ")
clean = strings.ReplaceAll(clean, "&#8220;", "\"")
clean = strings.ReplaceAll(clean, "&#8221;", "\"")
clean = strings.ReplaceAll(clean, "&#8230;", "...")
clean = strings.ReplaceAll(clean, "<b>", "**")
clean = strings.ReplaceAll(clean, "</b>", "**")
clean = strings.ReplaceAll(clean, "<br>", "\r\n")
clean = strings.ReplaceAll(clean, "<br/>", "\r\n")
clean = strings.ReplaceAll(clean, "\xe2\x96\xa0", "*")
clean = strings.ReplaceAll(clean, "\xa0", "\r\n")
clean = strings.ReplaceAll(clean, "<p>", "")
clean = strings.ReplaceAll(clean, "</p>", "\r\n")
return clean
}
func (dwh Discord) convertLinks(body string) string {
//items := regexp.MustCompile("<a(.*?)a>")
return ""
}

73
services/output/discordwebhook_test.go Normal file
View File

@ -0,0 +1,73 @@
package output_test
import (
"errors"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/joho/godotenv"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/output"
)
var article database.Article = database.Article{
ID: uuid.New(),
Sourceid: uuid.New(),
Tags: "unit, testing",
Title: "Demo",
Url: "https://github.com/jtom38/newsbot.collector.api",
Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Description: "Hello World",
}
func getWebhook() ([]string, error){
var endpoints []string
_, err := os.Open(".env")
if err != nil {
return endpoints, err
}
err = godotenv.Load()
if err != nil {
return endpoints, err
}
res := os.Getenv("TESTS_DISCORD_WEBHOOK")
if res == "" {
return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing")
}
endpoints = strings.Split(res, "")
return endpoints, nil
}
func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) {
hook, err := getWebhook()
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(hook, article)
if len(d.Subscriptions) == 0 {
t.Error("no subscriptions found")
}
}
func TestDiscordMessageContainsTitle(t *testing.T) {
hook, err := getWebhook()
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(hook, article)
err = d.GeneratePayload()
if err != nil {
t.Error(err)
}
if d.Message.Embeds[0].Title == "" {
t.Error("no title was found ")
}
}

6
services/output/interface.go Normal file
View File

@ -0,0 +1,6 @@
package output
type Output interface {
GeneratePayload() error
SendPayload() error
}