.gitignore

@@ -11,7 +11,6 @@ server
 *.so
 *.dylib
 collector
-newsbot.db
 
 # Test binary, built with `go test -c`
 *.test

cmd/server.go

@@ -11,7 +11,7 @@ import (
 
 	"git.jamestombleson.com/jtom38/newsbot-api/docs"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
-	v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
 )
@@ -44,7 +44,7 @@ func main() {
 
 	queries := database.New(db)
 
-	c := cron.NewScheduler(ctx, db)
+	c := cron.NewScheduler(ctx)
 	c.Start()
 
 	server := v1.NewServer(ctx, queries, configs, db)

internal/services/cron/collectors.go

@@ -1,220 +0,0 @@
-package cron
-
-import (
-	"log"
-	"time"
-
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
-)
-
-func (c *Cron) CollectRssPosts() {
-	log.Println("Starting ")
-	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
-	if err != nil {
-		log.Println(err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		rssClient := input.NewRssClient(source)
-		articles, err := rssClient.GetArticles()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range articles {
-			_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
-			if err != nil {
-				log.Println(err)
-			}
-			if rowsCreated != 1 {
-				log.Println("Got back the wrong number of rows")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-}
-
-func (c *Cron) CollectRedditPosts() {
-	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit)
-	if err != nil {
-		log.Printf("[Reddit] No sources found to query - %v\r", err)
-	}
-
-	for _, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
-		rc := input.NewRedditClient(source)
-		raw, err := rc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		redditArticles := rc.ConvertToArticles(raw)
-		for _, article := range redditArticles {
-			_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new reddit article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-	}
-	log.Print("[Reddit] Done!")
-}
-
-func (c *Cron) CollectYoutubePosts() {
-	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube)
-	if err != nil {
-		log.Printf("[Youtube] No sources found to query - %v\r", err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
-		yc := input.NewYoutubeClient(source)
-		raw, err := yc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range raw {
-			_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new youtube article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-	log.Print("[YouTube] Done!")
-}
-
-func (c *Cron) CollectFfxivPosts() {
-	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
-	if err != nil {
-		log.Printf("[FFXIV] No sources found to query - %v\r", err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		fc := input.NewFFXIVClient(source)
-		items, err := fc.CheckSource()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range items {
-			_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new FFXIV article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-	log.Printf("[FFXIV Done!]")
-}
-
-func (c *Cron) CollectTwitchPosts() {
-	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch)
-	if err != nil {
-		log.Printf("[Twitch] No sources found to query - %v\r", err)
-	}
-
-	tc, err := input.NewTwitchClient()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	err = tc.Login()
-	if err != nil {
-		log.Println(err)
-	}
-
-	for sourceIndex, source := range sources {
-		if !source.Enabled {
-			continue
-		}
-
-		log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
-		tc.ReplaceSourceRecord(source)
-		items, err := tc.GetContent()
-		if err != nil {
-			log.Println(err)
-		}
-
-		for _, article := range items {
-			_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
-			if err == nil {
-				continue
-			}
-
-			rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
-			if err != nil {
-				log.Printf("Failed to add a new Twitch article to the database: %s", err)
-			}
-
-			if rowsAdded != 1 {
-				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
-			}
-		}
-
-		if sourceIndex != len(sources) {
-			time.Sleep(time.Second * 30)
-		}
-	}
-
-	log.Print("[Twitch] Done!")
-}

internal/services/cron/rss.go

@@ -0,0 +1,59 @@
+package cron
+
+import (
+	"log"
+	"time"
+
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+)
+
+func (c *Cron) CheckRssSources() {
+	log.Println("Starting ")
+	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
+	if err != nil {
+		log.Println(err)
+	}
+
+	for sourceIndex, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+
+		rssClient := input.NewRssClient(source)
+		articles, err := rssClient.GetArticles()
+		if err != nil {
+			log.Println(err)
+		}
+
+		for _, article := range articles {
+			_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
+			if err == nil {
+				continue
+			}
+
+			rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
+			if err != nil {
+				log.Println(err)
+			}
+			if rowsCreated != 1 {
+				log.Println("Got back the wrong number of rows")
+			}
+		}
+
+		if sourceIndex != len(sources) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+}
+
+func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
+	var records []domain.SourceEntity
+
+	sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
+	if err != nil {
+		return records, err
+	}
+
+	return sources, nil
+}

internal/services/cron/rss_test.go

@@ -19,7 +19,7 @@ func TestRssPullsCorrectly(t *testing.T) {
 
 	ctx := context.Background()
 	db := services.NewRepositoryService(conn)
-	rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
+	rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api",true)
 	if err != nil {
 		t.Error(err)
 		t.FailNow()
@@ -31,7 +31,7 @@ func TestRssPullsCorrectly(t *testing.T) {
 	}
 
 	client := cron.NewScheduler(ctx, conn)
-	client.CollectRssPosts()
+	client.CheckRssSources()
 
 	articles, err := db.Articles.ListByPage(ctx, 0, 100)
 	if err != nil {

internal/services/cron/scheduler.go

@@ -3,37 +3,102 @@ package cron
 import (
 	"context"
 	"database/sql"
+	"fmt"
+	"log"
+	"time"
 
+	"github.com/google/uuid"
 	_ "github.com/lib/pq"
 	"github.com/robfig/cron/v3"
 
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
 )
 
 type Cron struct {
 	Db    *database.Queries
 	ctx   *context.Context
 	timer *cron.Cron
-	repo  services.RepositoryService
 }
 
-func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
+func openDatabase() (*database.Queries, error) {
+	_env := services.NewConfig()
+	connString := _env.GetConfig(services.Sql_Connection_String)
+	if connString == "" {
+		panic("Connection String is null!")
+	}
+	db, err := sql.Open("postgres", connString)
+	if err != nil {
+		panic(err)
+	}
+
+	queries := database.New(db)
+	return queries, err
+}
+
+func NewScheduler(ctx context.Context) *Cron {
 	c := &Cron{
-		ctx:  &ctx,
+		ctx: &ctx,
-		repo: services.NewRepositoryService(conn),
 	}
 	timer := cron.New()
+	queries, err := openDatabase()
+	if err != nil {
+		panic(err)
+	}
+	c.Db = queries
 
 	//timer.AddFunc("*/5 * * * *", func()  { go CheckCache() })
 	//features := services.GetEnvConfig()
+	features := services.NewConfig()
 
-	timer.AddFunc("5 * * * *", c.CollectRssPosts)
+	timer.AddFunc("5 * * * *", c.CheckRssSources)
-	//timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts)
+
-	//timer.AddFunc("10 1-23 * * *", c.CheckYoutube)
+	//if features.RedditEnabled {
-	//timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv)
+	//	timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
-	//timer.AddFunc("15 1-23 * * *", c.CheckTwitch)
+	//	log.Print("[Input] Reddit backend was enabled")
-	//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
+	//	//go c.CheckReddit()
+	//}
+	res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
+	if res {
+		timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
+		log.Print("[Input] Reddit backend was enabled")
+		//go c.CheckReddit()
+	}
+
+	//if features.YoutubeEnabled {
+	//	timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
+	//	log.Print("[Input] YouTube backend was enabled")
+	//}
+	res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
+	if res {
+		timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
+		log.Print("[Input] YouTube backend was enabled")
+	}
+
+	//if features.FfxivEnabled {
+	//	timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
+	//	log.Print("[Input] FFXIV backend was enabled")
+	//}
+	res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
+	if res {
+		timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
+		log.Print("[Input] FFXIV backend was enabled")
+	}
+
+	//if features.TwitchEnabled {
+	//	timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
+	//	log.Print("[Input] Twitch backend was enabled")
+	//}
+	res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
+	if res {
+		timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
+		log.Print("[Input] Twitch backend was enabled")
+	}
+
+	timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
+	log.Print("[Output] Discord Output was enabled")
 
 	c.timer = timer
 	return c
@@ -47,8 +112,105 @@ func (c *Cron) Stop() {
 	c.timer.Stop()
 }
 
-/*
+// This is the main entry point to query all the reddit services
-func (c *Cron) CheckDiscordQueue() {
+func (c *Cron) CheckReddit() {
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
+	if err != nil {
+		log.Printf("[Reddit] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		log.Printf("[Reddit] Checking '%v'...", source.Name)
+		rc := input.NewRedditClient(source)
+		raw, err := rc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		redditArticles := rc.ConvertToArticles(raw)
+		c.checkPosts(redditArticles, "Reddit")
+	}
+	log.Print("[Reddit] Done!")
+}
+
+func (c *Cron) CheckYoutube() {
+	// Add call to the db to request youtube sources.
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
+	if err != nil {
+		log.Printf("[Youtube] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		log.Printf("[YouTube] Checking '%v'...", source.Name)
+		yc := input.NewYoutubeClient(source)
+		raw, err := yc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		c.checkPosts(raw, "YouTube")
+	}
+	log.Print("[YouTube] Done!")
+}
+
+func (c *Cron) CheckFfxiv() {
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
+	if err != nil {
+		log.Printf("[FFXIV] No sources found to query - %v\r", err)
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		fc := input.NewFFXIVClient(source)
+		items, err := fc.CheckSource()
+		if err != nil {
+			log.Println(err)
+		}
+		c.checkPosts(items, "FFXIV")
+	}
+	log.Printf("[FFXIV Done!]")
+}
+
+func (c *Cron) CheckTwitch() error {
+	sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
+	if err != nil {
+		log.Printf("[Twitch] No sources found to query - %v\r", err)
+	}
+
+	tc, err := input.NewTwitchClient()
+	if err != nil {
+		return err
+	}
+
+	err = tc.Login()
+	if err != nil {
+		return err
+	}
+
+	for _, source := range sources {
+		if !source.Enabled {
+			continue
+		}
+		log.Printf("[Twitch] Checking '%v'...", source.Name)
+		tc.ReplaceSourceRecord(source)
+		items, err := tc.GetContent()
+		if err != nil {
+			log.Println(err)
+		}
+		c.checkPosts(items, "Twitch")
+	}
+
+	log.Print("[Twitch] Done!")
+	return nil
+}
+
+func (c *Cron) CheckDiscordQueue() error {
 	// Get items from the table
 	queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
 	if err != nil {
@@ -117,15 +279,55 @@ func (c *Cron) CheckDiscordQueue() {
 
 	return nil
 }
-*/
 
-//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
+func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
-//	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
+	for _, item := range posts {
-//		ID:        uuid.New(),
+		_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
-//		Articleid: Id,
+		if err != nil {
-//	})
+			id := uuid.New()
-//	if err != nil {
+
-//		return err
+			err := c.postArticle(id, item)
-//	}
+			if err != nil {
-//	return nil
+				return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
-//}
+			}
+
+			err = c.addToDiscordQueue(id)
+			if err != nil {
+				return err
+			}
+
+		}
+	}
+	time.Sleep(30 * time.Second)
+	return nil
+}
+
+func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
+	err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
+		ID:          id,
+		Sourceid:    item.Sourceid,
+		Tags:        item.Tags,
+		Title:       item.Title,
+		Url:         item.Url,
+		Pubdate:     item.Pubdate,
+		Video:       item.Video,
+		Videoheight: item.Videoheight,
+		Videowidth:  item.Videowidth,
+		Thumbnail:   item.Thumbnail,
+		Description: item.Description,
+		Authorname:  item.Authorname,
+		Authorimage: item.Authorimage,
+	})
+	return err
+}
+
+func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
+	err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
+		ID:        uuid.New(),
+		Articleid: Id,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/services/cron/scheduler_test.go

@@ -1,12 +1,12 @@
 package cron_test
 
 import (
-	"database/sql"
+	"context"
+	"testing"
 
-	"github.com/pressly/goose/v3"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
 )
 
-/*
 func TestInvokeTwitch(t *testing.T) {
 
 }
@@ -15,7 +15,7 @@ func TestInvokeTwitch(t *testing.T) {
 func TestCheckReddit(t *testing.T) {
 	ctx := context.Background()
 	c := cron.NewScheduler(ctx)
-	c.Col()
+	c.CheckReddit()
 }
 
 func TestCheckYouTube(t *testing.T) {
@@ -32,7 +32,6 @@ func TestCheckTwitch(t *testing.T) {
 		t.Error(err)
 	}
 }
-*/
 
 func setupInMemoryDb() (*sql.DB, error) {
 	db, err := sql.Open("sqlite", ":memory:")

internal/services/input/ffxiv.go

@@ -1,6 +1,7 @@
 package input
 
 import (
+	"database/sql"
 	"errors"
 	"log"
 	"net/http"
@@ -12,7 +13,7 @@ import (
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/google/uuid"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
 )
 
@@ -24,7 +25,7 @@ const (
 )
 
 type FFXIVClient struct {
-	record domain.SourceEntity
+	record database.Source
 	//SourceID uint
 	//Url string
 	//Region string
@@ -32,15 +33,15 @@ type FFXIVClient struct {
 	cacheGroup string
 }
 
-func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
+func NewFFXIVClient(Record database.Source) FFXIVClient {
 	return FFXIVClient{
 		record:     Record,
 		cacheGroup: "ffxiv",
 	}
 }
 
-func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
+func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
-	var articles []domain.ArticleEntity
+	var articles []database.Article
 
 	parser := fc.GetBrowser()
 	defer parser.Close()
@@ -96,16 +97,18 @@ func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
 			return articles, err
 		}
 
-		article := domain.ArticleEntity{
+		article := database.Article{
-			SourceID:       fc.record.ID,
+			Sourceid:    fc.record.ID,
-			Tags:           tags,
+			Tags:        tags,
-			Title:          title,
+			Title:       title,
-			Url:            link,
+			Url:         link,
-			PubDate:        pubDate,
+			Pubdate:     pubDate,
-			Thumbnail:      thumb,
+			Videoheight: 0,
-			Description:    description,
+			Videowidth:  0,
-			AuthorName:     authorName,
+			Thumbnail:   thumb,
-			AuthorImageUrl: authorImage,
+			Description: description,
+			Authorname:  sql.NullString{String: authorName},
+			Authorimage: sql.NullString{String: authorImage},
 		}
 		log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
 

internal/services/input/ffxiv_test.go

@@ -3,16 +3,18 @@ package input_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"github.com/google/uuid"
 )
 
-var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
+var FFXIVRecord database.Source = database.Source{
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "Final Fantasy XIV - NA",
+	Site:   "ffxiv",
-	Source:      domain.SourceCollectorFfxiv,
+	Name:   "Final Fantasy XIV - NA",
-	Url:         "https://na.finalfantasyxiv.com/lodestone/",
+	Source: "ffxiv",
-	Tags:        "ffxiv, final, fantasy, xiv, na, lodestone",
+	Url:    "https://na.finalfantasyxiv.com/lodestone/",
+	Tags:   "ffxiv, final, fantasy, xiv, na, lodestone",
 }
 
 func TestFfxivGetParser(t *testing.T) {

internal/services/input/reddit.go

@@ -1,6 +1,7 @@
 package input
 
 import (
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -8,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
 	"github.com/go-rod/rod"
@@ -16,7 +18,7 @@ import (
 
 type RedditClient struct {
 	config RedditConfig
-	record domain.SourceEntity
+	record database.Source
 }
 
 type RedditConfig struct {
@@ -25,7 +27,7 @@ type RedditConfig struct {
 	PullNSFW string
 }
 
-func NewRedditClient(Record domain.SourceEntity) *RedditClient {
+func NewRedditClient(Record database.Source) *RedditClient {
 	rc := RedditClient{
 		record: Record,
 	}
@@ -69,7 +71,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
 	// TODO Wire this to support the config options
 	Url := fmt.Sprintf("%v.json", rc.record.Url)
 
-	log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
+	log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
 
 	content, err := getHttpContent(Url)
 	if err != nil {
@@ -86,10 +88,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
 	return items, nil
 }
 
-func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
+func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
-	var redditArticles []domain.ArticleEntity
+	var redditArticles []database.Article
 	for _, item := range items.Data.Children {
-		var article domain.ArticleEntity
+		var article database.Article
 		article, err := rc.convertToArticle(item.Data)
 		if err != nil {
 			log.Printf("[Reddit] %v", err)
@@ -102,8 +104,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []doma
 
 // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
 // This data can be passed to the database.
-func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
+func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
-	var item domain.ArticleEntity
+	var item database.Article
 
 	if source.Content == "" && source.Url != "" {
 		item = rc.convertPicturePost(source)
@@ -129,57 +131,65 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.Artic
 	return item, nil
 }
 
-func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
+func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
-	var item = domain.ArticleEntity{
+	var item = database.Article{
-		SourceID:       rc.record.ID,
+		Sourceid:    rc.record.ID,
-		Title:          source.Title,
+		Title:       source.Title,
-		Tags:           fmt.Sprintf("%v", rc.record.Tags),
+		Tags:        fmt.Sprintf("%v", rc.record.Tags),
-		Url:            fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		PubDate:        time.Now(),
+		Pubdate:     time.Now(),
-		IsVideo:        false,
+		Video:       sql.NullString{String: "null"},
-		Thumbnail:      source.Thumbnail,
+		Videoheight: 0,
-		Description:    source.Content,
+		Videowidth:  0,
-		AuthorName:     source.Author,
+		Thumbnail:   source.Thumbnail,
-		AuthorImageUrl: "null",
+		Description: source.Content,
+		Authorname:  sql.NullString{String: source.Author},
+		Authorimage: sql.NullString{String: "null"},
 	}
 	return item
 }
 
-func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
+func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
-	var item = domain.ArticleEntity{
+	var item = database.Article{
-		SourceID:    rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		PubDate:     time.Now(),
+		Pubdate:     time.Now(),
+		Videoheight: 0,
+		Videowidth:  0,
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
+		Authorname:  sql.NullString{String: source.Author},
 		Description: source.Content,
 	}
 	return item
 }
 
-func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
+func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
-	var item = domain.ArticleEntity{
+	var item = database.Article{
-		SourceID:    rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		PubDate:     time.Now(),
+		Pubdate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
+		Videoheight: 0,
+		Videowidth:  0,
+		Authorname:  sql.NullString{String: source.Author},
 		Description: source.Media.RedditVideo.FallBackUrl,
 	}
 	return item
 }
 
 // This post is nothing more then a redirect to another location.
-func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
+func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
-	var item = domain.ArticleEntity{
+	var item = database.Article{
-		SourceID:    rc.record.ID,
+		Sourceid:    rc.record.ID,
 		Tags:        "a",
 		Title:       source.Title,
-		PubDate:     time.Now(),
+		Pubdate:     time.Now(),
 		Url:         fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
-		AuthorName:  source.Author,
+		Videoheight: 0,
+		Videowidth:  0,
+		Authorname:  sql.NullString{String: source.Author},
 		Description: source.UrlOverriddenByDest,
 	}
 	return item

internal/services/input/reddit_test.go

@@ -3,16 +3,18 @@ package input_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"github.com/google/uuid"
 )
 
-var RedditRecord domain.SourceEntity = domain.SourceEntity{
+var RedditRecord database.Source = database.Source{
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "dadjokes",
+	Name:   "dadjokes",
-	Source:      domain.SourceCollectorRss,
+	Source: "reddit",
-	Url:         "https://reddit.com/r/dadjokes",
+	Site:   "reddit",
-	Tags:        "reddit, dadjokes",
+	Url:    "https://reddit.com/r/dadjokes",
+	Tags:   "reddit, dadjokes",
 }
 
 func TestGetContent(t *testing.T) {

internal/services/input/rss_test.go

@@ -8,10 +8,9 @@ import (
 )
 
 var rssRecord = domain.SourceEntity{
-	ID:          1,
+	ID:   1,
 	DisplayName: "ArsTechnica",
-	Url:         "https://feeds.arstechnica.com/arstechnica/index",
+	Url:  "https://feeds.arstechnica.com/arstechnica/index",
-	Source:      domain.SourceCollectorRss,
 }
 
 func TestRssClientConstructor(t *testing.T) {

internal/services/input/twitch.go

@@ -1,18 +1,19 @@
 package input
 
 import (
+	"database/sql"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
 	"github.com/nicklaw5/helix/v2"
 )
 
 type TwitchClient struct {
-	SourceRecord domain.SourceEntity
+	SourceRecord database.Source
 
 	// config
 	monitorClips string
@@ -71,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
 }
 
 // This will let you replace the bound source record to keep the same session alive.
-func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
+func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
 	tc.SourceRecord = source
 }
 
@@ -86,8 +87,8 @@ func (tc *TwitchClient) Login() error {
 	return nil
 }
 
-func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
+func (tc *TwitchClient) GetContent() ([]database.Article, error) {
-	var items []domain.ArticleEntity
+	var items []database.Article
 
 	user, err := tc.GetUserDetails()
 	if err != nil {
@@ -100,31 +101,31 @@ func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
 	}
 
 	for _, video := range posts {
-		var article domain.ArticleEntity
+		var article database.Article
 
 		AuthorName, err := tc.ExtractAuthor(video)
 		if err != nil {
 			return items, err
 		}
-		article.AuthorName = AuthorName
+		article.Authorname = sql.NullString{String: AuthorName}
 
 		Authorimage, err := tc.ExtractAuthorImage(user)
 		if err != nil {
 			return items, err
 		}
-		article.AuthorImageUrl = Authorimage
+		article.Authorimage = sql.NullString{String: Authorimage}
 
 		article.Description, err = tc.ExtractDescription(video)
 		if err != nil {
 			return items, err
 		}
 
-		article.PubDate, err = tc.ExtractPubDate(video)
+		article.Pubdate, err = tc.ExtractPubDate(video)
 		if err != nil {
 			return items, err
 		}
 
-		article.SourceID = tc.SourceRecord.ID
+		article.Sourceid = tc.SourceRecord.ID
 		article.Tags, err = tc.ExtractTags(video, user)
 		if err != nil {
 			return items, err
@@ -155,7 +156,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
 	var blank helix.User
 
 	users, err := tc.api.GetUsers(&helix.UsersParams{
-		Logins: []string{tc.SourceRecord.DisplayName},
+		Logins: []string{tc.SourceRecord.Name},
 	})
 	if err != nil {
 		return blank, err

internal/services/input/twitch_test.go

@@ -4,20 +4,21 @@ import (
 	"log"
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"github.com/google/uuid"
 )
 
-var TwitchSourceRecord = domain.SourceEntity{
+var TwitchSourceRecord = database.Source{
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "nintendo",
+	Name:   "nintendo",
-	Source:      domain.SourceCollectorTwitch,
+	Source: "Twitch",
 }
 
-var TwitchInvalidRecord = domain.SourceEntity{
+var TwitchInvalidRecord = database.Source{
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "EvilNintendo",
+	Name:   "EvilNintendo",
-	Source:      domain.SourceCollectorTwitch,
+	Source: "Twitch",
 }
 
 func TestTwitchLogin(t *testing.T) {

internal/services/input/youtube.go

@@ -1,6 +1,7 @@
 package input
 
 import (
+	"database/sql"
 	"errors"
 	"fmt"
 	"log"
@@ -11,11 +12,11 @@ import (
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/mmcdole/gofeed"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 )
 
 type YoutubeClient struct {
-	record domain.SourceEntity
+	record database.Source
 
 	// internal variables at time of collection
 	channelID string
@@ -36,7 +37,7 @@ var (
 
 const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
 
-func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
+func NewYoutubeClient(Record database.Source) YoutubeClient {
 	yc := YoutubeClient{
 		record:     Record,
 		cacheGroup: "youtube",
@@ -45,8 +46,8 @@ func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
 }
 
 // CheckSource will go and run all the commands needed to process a source.
-func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
+func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
-	var items []domain.ArticleEntity
+	var items []database.Article
 	docParser, err := yc.GetParser(yc.record.Url)
 	if err != nil {
 		return items, err
@@ -246,7 +247,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
 	return false
 }
 
-func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
+func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
 	parser, err := yc.GetParser(item.Link)
 	if err != nil {
 		log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
@@ -264,16 +265,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntit
 		log.Printf("[YouTube] %v", msg)
 	}
 
-	var article = domain.ArticleEntity{
+	var article = database.Article{
-		SourceID:       yc.record.ID,
+		Sourceid:    yc.record.ID,
-		Tags:           tags,
+		Tags:        tags,
-		Title:          item.Title,
+		Title:       item.Title,
-		Url:            item.Link,
+		Url:         item.Link,
-		PubDate:        *item.PublishedParsed,
+		Pubdate:     *item.PublishedParsed,
-		Thumbnail:      thumb,
+		Thumbnail:   thumb,
-		Description:    item.Description,
+		Description: item.Description,
-		AuthorName:     item.Author.Name,
+		Authorname:  sql.NullString{String: item.Author.Name},
-		AuthorImageUrl: yc.avatarUri,
+		Authorimage: sql.NullString{String: yc.avatarUri},
 	}
 	return article
 }

internal/services/input/youtube_test.go

@@ -3,15 +3,17 @@ package input_test
 import (
 	"testing"
 
-	"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
+	"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
 	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
+	"github.com/google/uuid"
 )
 
-var YouTubeRecord = domain.SourceEntity{
+var YouTubeRecord database.Source = database.Source{
-	ID:          9999,
+	ID:     uuid.New(),
-	DisplayName: "dadjokes",
+	Name:   "dadjokes",
-	Source:      domain.SourceCollectorReddit,
+	Source: "reddit",
-	Url:         "https://youtube.com/gamegrumps",
+	Site:   "reddit",
+	Url:    "https://youtube.com/gamegrumps",
 }
 
 func TestGetPageParser(t *testing.T) {

makefile

@@ -4,8 +4,8 @@ help: ## Shows this help command
 
 build: ## builds the application with the current go runtime
 	~/go/bin/swag f
-	~/go/bin/swag init -g cmd/server.go
+	~/go/bin/swag i
-	go build cmd/server.go
+	go build .
 	
 docker-build: ## Generates the docker image
 	docker build -t "newsbot.collector.api" .