features/rss-job #6

Merged
jtom38 merged 13 commits from features/rss-job into main 2024-05-02 17:41:12 -07:00
5 changed files with 250 additions and 290 deletions
Showing only changes of commit ddf50077b5 - Show all commits

View File

@ -0,0 +1,220 @@
package cron
import (
"log"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CollectRssPosts() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
rssClient := input.NewRssClient(source)
articles, err := rssClient.GetArticles()
if err != nil {
log.Println(err)
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Println(err)
}
if rowsCreated != 1 {
log.Println("Got back the wrong number of rows")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
}
func (c *Cron) CollectRedditPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit)
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, article := range redditArticles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new reddit article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CollectYoutubePosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube)
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range raw {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new youtube article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CollectFfxivPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new FFXIV article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CollectTwitchPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch)
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
log.Println(err)
return
}
err = tc.Login()
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new Twitch article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[Twitch] Done!")
}

View File

@ -19,7 +19,7 @@ func TestRssPullsCorrectly(t *testing.T) {
ctx := context.Background() ctx := context.Background()
db := services.NewRepositoryService(conn) db := services.NewRepositoryService(conn)
rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api",true) rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
t.FailNow() t.FailNow()
@ -31,7 +31,7 @@ func TestRssPullsCorrectly(t *testing.T) {
} }
client := cron.NewScheduler(ctx, conn) client := cron.NewScheduler(ctx, conn)
client.CheckRssSources() client.CollectRssPosts()
articles, err := db.Articles.ListByPage(ctx, 0, 100) articles, err := db.Articles.ListByPage(ctx, 0, 100)
if err != nil { if err != nil {

View File

@ -1,59 +0,0 @@
package cron
import (
"log"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CheckRssSources() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
rssClient := input.NewRssClient(source)
articles, err := rssClient.GetArticles()
if err != nil {
log.Println(err)
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Println(err)
}
if rowsCreated != 1 {
log.Println("Got back the wrong number of rows")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
}
func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
var records []domain.SourceEntity
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
if err != nil {
return records, err
}
return sources, nil
}

View File

@ -3,102 +3,37 @@ package cron
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
"log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
) )
type Cron struct { type Cron struct {
Db *database.Queries Db *database.Queries
ctx *context.Context ctx *context.Context
timer *cron.Cron timer *cron.Cron
repo services.RepositoryService
} }
func openDatabase() (*database.Queries, error) { func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
_env := services.NewConfig()
connString := _env.GetConfig(services.Sql_Connection_String)
if connString == "" {
panic("Connection String is null!")
}
db, err := sql.Open("postgres", connString)
if err != nil {
panic(err)
}
queries := database.New(db)
return queries, err
}
func NewScheduler(ctx context.Context) *Cron {
c := &Cron{ c := &Cron{
ctx: &ctx, ctx: &ctx,
repo: services.NewRepositoryService(conn),
} }
timer := cron.New() timer := cron.New()
queries, err := openDatabase()
if err != nil {
panic(err)
}
c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//features := services.GetEnvConfig() //features := services.GetEnvConfig()
features := services.NewConfig()
timer.AddFunc("5 * * * *", c.CheckRssSources) timer.AddFunc("5 * * * *", c.CollectRssPosts)
//timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts)
//if features.RedditEnabled { //timer.AddFunc("10 1-23 * * *", c.CheckYoutube)
// timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) //timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv)
// log.Print("[Input] Reddit backend was enabled") //timer.AddFunc("15 1-23 * * *", c.CheckTwitch)
// //go c.CheckReddit() //timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
//}
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
if res {
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit()
}
//if features.YoutubeEnabled {
// timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
// log.Print("[Input] YouTube backend was enabled")
//}
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled")
}
//if features.FfxivEnabled {
// timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
// log.Print("[Input] FFXIV backend was enabled")
//}
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled")
}
//if features.TwitchEnabled {
// timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
// log.Print("[Input] Twitch backend was enabled")
//}
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled")
}
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
log.Print("[Output] Discord Output was enabled")
c.timer = timer c.timer = timer
return c return c
@ -112,105 +47,8 @@ func (c *Cron) Stop() {
c.timer.Stop() c.timer.Stop()
} }
// This is the main entry point to query all the reddit services /*
func (c *Cron) CheckReddit() { func (c *Cron) CheckDiscordQueue() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.Name)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
c.checkPosts(redditArticles, "Reddit")
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources.
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.Name)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(raw, "YouTube")
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CheckFfxiv() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "FFXIV")
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CheckTwitch() error {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
return err
}
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.Name)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "Twitch")
}
log.Print("[Twitch] Done!")
return nil
}
func (c *Cron) CheckDiscordQueue() error {
// Get items from the table // Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50) queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
if err != nil { if err != nil {
@ -279,55 +117,15 @@ func (c *Cron) CheckDiscordQueue() error {
return nil return nil
} }
*/
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error { //func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
for _, item := range posts { // err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) // ID: uuid.New(),
if err != nil { // Articleid: Id,
id := uuid.New() // })
// if err != nil {
err := c.postArticle(id, item) // return err
if err != nil { // }
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) // return nil
} //}
err = c.addToDiscordQueue(id)
if err != nil {
return err
}
}
}
time.Sleep(30 * time.Second)
return nil
}
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: id,
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
ID: uuid.New(),
Articleid: Id,
})
if err != nil {
return err
}
return nil
}

View File

@ -1,12 +1,12 @@
package cron_test package cron_test
import ( import (
"context" "database/sql"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" "github.com/pressly/goose/v3"
) )
/*
func TestInvokeTwitch(t *testing.T) { func TestInvokeTwitch(t *testing.T) {
} }
@ -15,7 +15,7 @@ func TestInvokeTwitch(t *testing.T) {
func TestCheckReddit(t *testing.T) { func TestCheckReddit(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx)
c.CheckReddit() c.Col()
} }
func TestCheckYouTube(t *testing.T) { func TestCheckYouTube(t *testing.T) {
@ -32,6 +32,7 @@ func TestCheckTwitch(t *testing.T) {
t.Error(err) t.Error(err)
} }
} }
*/
func setupInMemoryDb() (*sql.DB, error) { func setupInMemoryDb() (*sql.DB, error) {
db, err := sql.Open("sqlite", ":memory:") db, err := sql.Open("sqlite", ":memory:")