jtom38/newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

features/rss-job #6

Merged
jtom38 merged 13 commits from features/rss-job into main 2024-05-02 17:41:12 -07:00
Conversation 0 Commits 13 Files Changed 41 +22 -22
2 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 852db161c9 - Show all commits

30
internal/services/cron/collectors.go
View File

@ -10,7 +10,7 @@ import (
func (c *Cron) CollectRssPosts() { func (c *Cron) CollectRssPosts() {
log.Println("Starting ") log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss) sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
@ -27,12 +27,12 @@ func (c *Cron) CollectRssPosts() {
} }
for _, article := range articles { for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil { if err == nil {
continue continue
} }
rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
@ -48,7 +48,7 @@ func (c *Cron) CollectRssPosts() {
} }
func (c *Cron) CollectRedditPosts() { func (c *Cron) CollectRedditPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit) sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
if err != nil { if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err) log.Printf("[Reddit] No sources found to query - %v\r", err)
} }
@ -67,12 +67,12 @@ func (c *Cron) CollectRedditPosts() {
redditArticles := rc.ConvertToArticles(raw) redditArticles := rc.ConvertToArticles(raw)
for _, article := range redditArticles { for _, article := range redditArticles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil { if err == nil {
continue continue
} }
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil { if err != nil {
log.Printf("Failed to add a new reddit article to the database: %s", err) log.Printf("Failed to add a new reddit article to the database: %s", err)
} }
@ -86,7 +86,7 @@ func (c *Cron) CollectRedditPosts() {
} }
func (c *Cron) CollectYoutubePosts() { func (c *Cron) CollectYoutubePosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube) sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
if err != nil { if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err) log.Printf("[Youtube] No sources found to query - %v\r", err)
} }
@ -104,12 +104,12 @@ func (c *Cron) CollectYoutubePosts() {
} }
for _, article := range raw { for _, article := range raw {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil { if err == nil {
continue continue
} }
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil { if err != nil {
log.Printf("Failed to add a new youtube article to the database: %s", err) log.Printf("Failed to add a new youtube article to the database: %s", err)
} }
@ -127,7 +127,7 @@ func (c *Cron) CollectYoutubePosts() {
} }
func (c *Cron) CollectFfxivPosts() { func (c *Cron) CollectFfxivPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv) sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
if err != nil { if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err) log.Printf("[FFXIV] No sources found to query - %v\r", err)
} }
@ -144,12 +144,12 @@ func (c *Cron) CollectFfxivPosts() {
} }
for _, article := range items { for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil { if err == nil {
continue continue
} }
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil { if err != nil {
log.Printf("Failed to add a new FFXIV article to the database: %s", err) log.Printf("Failed to add a new FFXIV article to the database: %s", err)
} }
@ -167,7 +167,7 @@ func (c *Cron) CollectFfxivPosts() {
} }
func (c *Cron) CollectTwitchPosts() { func (c *Cron) CollectTwitchPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch) sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
if err != nil { if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err) log.Printf("[Twitch] No sources found to query - %v\r", err)
} }
@ -196,12 +196,12 @@ func (c *Cron) CollectTwitchPosts() {
} }
for _, article := range items { for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil { if err == nil {
continue continue
} }
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil { if err != nil {
log.Printf("Failed to add a new Twitch article to the database: %s", err) log.Printf("Failed to add a new Twitch article to the database: %s", err)
} }

14
internal/services/cron/scheduler.go
View File

@ -13,14 +13,14 @@ import (
type Cron struct { type Cron struct {
Db *database.Queries Db *database.Queries
ctx *context.Context ctx context.Context
timer *cron.Cron timer *cron.Cron
repo services.RepositoryService repo services.RepositoryService
} }
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron { func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
c := &Cron{ c := &Cron{
ctx: &ctx, ctx: ctx,
repo: services.NewRepositoryService(conn), repo: services.NewRepositoryService(conn),
} }
timer := cron.New() timer := cron.New()
@ -28,11 +28,11 @@ func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//features := services.GetEnvConfig() //features := services.GetEnvConfig()
timer.AddFunc("5 * * * *", c.CollectRssPosts) timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
//timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts) //timer.AddFunc("10 * * * *", c.CollectRedditPosts)
//timer.AddFunc("10 1-23 * * *", c.CheckYoutube) //timer.AddFunc("15 * * * *", c.CheckYoutube)
//timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv) //timer.AddFunc("20 * * * *", c.CheckFfxiv)
//timer.AddFunc("15 1-23 * * *", c.CheckTwitch) //timer.AddFunc("25 * * * *", c.CheckTwitch)
//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue) //timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
c.timer = timer c.timer = timer