newsbot-api/internal/services/cron/collectors.go

package cron

import (
	"log"
	"time"

	"git.jamestombleson.com/jtom38/newsbot-api/domain"
	"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)

func (c *Cron) CollectRssPosts() {
	log.Println("Starting ")
	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
	if err != nil {
		log.Println(err)
	}

	for sourceIndex, source := range sources {
		if !source.Enabled {
			continue
		}

		rssClient := input.NewRssClient(source)
		articles, err := rssClient.GetArticles()
		if err != nil {
			log.Println(err)
		}

		for _, article := range articles {
			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
			if err == nil {
				continue
			}

			rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
			if err != nil {
				log.Println(err)
			}
			if rowsCreated != 1 {
				log.Println("Got back the wrong number of rows")
			}
		}

		if sourceIndex != len(sources) {
			time.Sleep(time.Second * 30)
		}
	}
}

func (c *Cron) CollectRedditPosts() {
	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
	if err != nil {
		log.Printf("[Reddit] No sources found to query - %v\r", err)
	}

	for _, source := range sources {
		if !source.Enabled {
			continue
		}

		log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
		rc := input.NewRedditClient(source)
		raw, err := rc.GetContent()
		if err != nil {
			log.Println(err)
		}

		redditArticles := rc.ConvertToArticles(raw)
		for _, article := range redditArticles {
			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
			if err == nil {
				continue
			}

			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
			if err != nil {
				log.Printf("Failed to add a new reddit article to the database: %s", err)
			}

			if rowsAdded != 1 {
				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
			}
		}
	}
	log.Print("[Reddit] Done!")
}

func (c *Cron) CollectYoutubePosts() {
	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
	if err != nil {
		log.Printf("[Youtube] No sources found to query - %v\r", err)
	}

	for sourceIndex, source := range sources {
		if !source.Enabled {
			continue
		}

		log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
		yc := input.NewYoutubeClient(source)
		raw, err := yc.GetContent()
		if err != nil {
			log.Println(err)
		}

		for _, article := range raw {
			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
			if err == nil {
				continue
			}

			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
			if err != nil {
				log.Printf("Failed to add a new youtube article to the database: %s", err)
			}

			if rowsAdded != 1 {
				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
			}
		}

		if sourceIndex != len(sources) {
			time.Sleep(time.Second * 30)
		}
	}
	log.Print("[YouTube] Done!")
}

func (c *Cron) CollectFfxivPosts() {
	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
	if err != nil {
		log.Printf("[FFXIV] No sources found to query - %v\r", err)
	}

	for sourceIndex, source := range sources {
		if !source.Enabled {
			continue
		}

		fc := input.NewFFXIVClient(source)
		items, err := fc.CheckSource()
		if err != nil {
			log.Println(err)
		}

		for _, article := range items {
			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
			if err == nil {
				continue
			}

			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
			if err != nil {
				log.Printf("Failed to add a new FFXIV article to the database: %s", err)
			}

			if rowsAdded != 1 {
				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
			}
		}

		if sourceIndex != len(sources) {
			time.Sleep(time.Second * 30)
		}
	}
	log.Printf("[FFXIV Done!]")
}

func (c *Cron) CollectTwitchPosts() {
	sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
	if err != nil {
		log.Printf("[Twitch] No sources found to query - %v\r", err)
	}

	tc, err := input.NewTwitchClient()
	if err != nil {
		log.Println(err)
		return
	}

	err = tc.Login()
	if err != nil {
		log.Println(err)
	}

	for sourceIndex, source := range sources {
		if !source.Enabled {
			continue
		}

		log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
		tc.ReplaceSourceRecord(source)
		items, err := tc.GetContent()
		if err != nil {
			log.Println(err)
		}

		for _, article := range items {
			_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
			if err == nil {
				continue
			}

			rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
			if err != nil {
				log.Printf("Failed to add a new Twitch article to the database: %s", err)
			}

			if rowsAdded != 1 {
				log.Printf("no error came back when data was added to the database but the expected row count is wrong")
			}
		}

		if sourceIndex != len(sources) {
			time.Sleep(time.Second * 30)
		}
	}

	log.Print("[Twitch] Done!")
}