jtom38/newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

features/rss-job #6

Merged
jtom38 merged 13 commits from features/rss-job into main 2024-05-02 17:41:12 -07:00
Conversation 0 Commits 13 Files Changed 41 +22 -22
2 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 852db161c9 - Show all commits

30
internal/services/cron/collectors.go
View File

@ -10,7 +10,7 @@ import (
func (c *Cron) CollectRssPosts() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
@ -27,12 +27,12 @@ func (c *Cron) CollectRssPosts() {
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Println(err)
}
@ -48,7 +48,7 @@ func (c *Cron) CollectRssPosts() {
}
func (c *Cron) CollectRedditPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit)
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
@ -67,12 +67,12 @@ func (c *Cron) CollectRedditPosts() {
redditArticles := rc.ConvertToArticles(raw)
for _, article := range redditArticles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new reddit article to the database: %s", err)
}
@ -86,7 +86,7 @@ func (c *Cron) CollectRedditPosts() {
}
func (c *Cron) CollectYoutubePosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube)
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
@ -104,12 +104,12 @@ func (c *Cron) CollectYoutubePosts() {
}
for _, article := range raw {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new youtube article to the database: %s", err)
}
@ -127,7 +127,7 @@ func (c *Cron) CollectYoutubePosts() {
}
func (c *Cron) CollectFfxivPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
@ -144,12 +144,12 @@ func (c *Cron) CollectFfxivPosts() {
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new FFXIV article to the database: %s", err)
}
@ -167,7 +167,7 @@ func (c *Cron) CollectFfxivPosts() {
}
func (c *Cron) CollectTwitchPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch)
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
@ -196,12 +196,12 @@ func (c *Cron) CollectTwitchPosts() {
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new Twitch article to the database: %s", err)
}

14
internal/services/cron/scheduler.go
View File

@ -13,14 +13,14 @@ import (
type Cron struct {
Db *database.Queries
ctx *context.Context
ctx context.Context
timer *cron.Cron
repo services.RepositoryService
}
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
c := &Cron{
ctx: &ctx,
ctx: ctx,
repo: services.NewRepositoryService(conn),
}
timer := cron.New()
@ -28,11 +28,11 @@ func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//features := services.GetEnvConfig()
timer.AddFunc("5 * * * *", c.CollectRssPosts)
//timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts)
//timer.AddFunc("10 1-23 * * *", c.CheckYoutube)
//timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv)
//timer.AddFunc("15 1-23 * * *", c.CheckTwitch)
timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
//timer.AddFunc("10 * * * *", c.CollectRedditPosts)
//timer.AddFunc("15 * * * *", c.CheckYoutube)
//timer.AddFunc("20 * * * *", c.CheckFfxiv)
//timer.AddFunc("25 * * * *", c.CheckTwitch)
//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
c.timer = timer