package cron import ( "log" "time" "git.jamestombleson.com/jtom38/newsbot-api/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ) func (c *Cron) CollectRssPosts() { log.Println("Starting ") sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss) if err != nil { log.Println(err) } for sourceIndex, source := range sources { if !source.Enabled { continue } rssClient := input.NewRssClient(source) articles, err := rssClient.GetArticles() if err != nil { log.Println(err) } for _, article := range articles { _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Println(err) } if rowsCreated != 1 { log.Println("Got back the wrong number of rows") } } if sourceIndex != len(sources) { time.Sleep(time.Second * 30) } } } func (c *Cron) CollectRedditPosts() { sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit) if err != nil { log.Printf("[Reddit] No sources found to query - %v\r", err) } for _, source := range sources { if !source.Enabled { continue } log.Printf("[Reddit] Checking '%v'...", source.DisplayName) rc := input.NewRedditClient(source) raw, err := rc.GetContent() if err != nil { log.Println(err) } redditArticles := rc.ConvertToArticles(raw) for _, article := range redditArticles { _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new reddit article to the database: %s", err) } if rowsAdded != 1 { log.Printf("no error came back when data was added to the database but the expected row count is wrong") } } } log.Print("[Reddit] Done!") } func (c *Cron) CollectYoutubePosts() { sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube) if err != nil { log.Printf("[Youtube] No sources found to query - %v\r", err) } for sourceIndex, source := range sources { if !source.Enabled { continue } log.Printf("[YouTube] Checking '%v'...", source.DisplayName) yc := input.NewYoutubeClient(source) raw, err := yc.GetContent() if err != nil { log.Println(err) } for _, article := range raw { _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new youtube article to the database: %s", err) } if rowsAdded != 1 { log.Printf("no error came back when data was added to the database but the expected row count is wrong") } } if sourceIndex != len(sources) { time.Sleep(time.Second * 30) } } log.Print("[YouTube] Done!") } func (c *Cron) CollectFfxivPosts() { sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv) if err != nil { log.Printf("[FFXIV] No sources found to query - %v\r", err) } for sourceIndex, source := range sources { if !source.Enabled { continue } fc := input.NewFFXIVClient(source) items, err := fc.CheckSource() if err != nil { log.Println(err) } for _, article := range items { _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new FFXIV article to the database: %s", err) } if rowsAdded != 1 { log.Printf("no error came back when data was added to the database but the expected row count is wrong") } } if sourceIndex != len(sources) { time.Sleep(time.Second * 30) } } log.Printf("[FFXIV Done!]") } func (c *Cron) CollectTwitchPosts() { sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch) if err != nil { log.Printf("[Twitch] No sources found to query - %v\r", err) } tc, err := input.NewTwitchClient() if err != nil { log.Println(err) return } err = tc.Login() if err != nil { log.Println(err) } for sourceIndex, source := range sources { if !source.Enabled { continue } log.Printf("[Twitch] Checking '%v'...", source.DisplayName) tc.ReplaceSourceRecord(source) items, err := tc.GetContent() if err != nil { log.Println(err) } for _, article := range items { _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new Twitch article to the database: %s", err) } if rowsAdded != 1 { log.Printf("no error came back when data was added to the database but the expected row count is wrong") } } if sourceIndex != len(sources) { time.Sleep(time.Second * 30) } } log.Print("[Twitch] Done!") }