Compare commits

..

No commits in common. "62fa7224c8512b8b8071316aa7eb9aedcf8d7515" and "bfa0f1023d88c8603180836fc9fa8cc6c0258b8a" have entirely different histories.

17 changed files with 424 additions and 364 deletions

1
.gitignore vendored
View File

@ -11,7 +11,6 @@ server
*.so *.so
*.dylib *.dylib
collector collector
newsbot.db
# Test binary, built with `go test -c` # Test binary, built with `go test -c`
*.test *.test

View File

@ -11,7 +11,7 @@ import (
"git.jamestombleson.com/jtom38/newsbot-api/docs" "git.jamestombleson.com/jtom38/newsbot-api/docs"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1" "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
) )
@ -44,7 +44,7 @@ func main() {
queries := database.New(db) queries := database.New(db)
c := cron.NewScheduler(ctx, db) c := cron.NewScheduler(ctx)
c.Start() c.Start()
server := v1.NewServer(ctx, queries, configs, db) server := v1.NewServer(ctx, queries, configs, db)

View File

@ -1,220 +0,0 @@
package cron
import (
"log"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CollectRssPosts() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
rssClient := input.NewRssClient(source)
articles, err := rssClient.GetArticles()
if err != nil {
log.Println(err)
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Println(err)
}
if rowsCreated != 1 {
log.Println("Got back the wrong number of rows")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
}
func (c *Cron) CollectRedditPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit)
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, article := range redditArticles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new reddit article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CollectYoutubePosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube)
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range raw {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new youtube article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CollectFfxivPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new FFXIV article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CollectTwitchPosts() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch)
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
log.Println(err)
return
}
err = tc.Login()
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Printf("Failed to add a new Twitch article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[Twitch] Done!")
}

View File

@ -0,0 +1,59 @@
package cron
import (
"log"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CheckRssSources() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
rssClient := input.NewRssClient(source)
articles, err := rssClient.GetArticles()
if err != nil {
log.Println(err)
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article)
if err != nil {
log.Println(err)
}
if rowsCreated != 1 {
log.Println("Got back the wrong number of rows")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
}
func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
var records []domain.SourceEntity
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
if err != nil {
return records, err
}
return sources, nil
}

View File

@ -19,7 +19,7 @@ func TestRssPullsCorrectly(t *testing.T) {
ctx := context.Background() ctx := context.Background()
db := services.NewRepositoryService(conn) db := services.NewRepositoryService(conn)
rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true) rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api",true)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
t.FailNow() t.FailNow()
@ -31,7 +31,7 @@ func TestRssPullsCorrectly(t *testing.T) {
} }
client := cron.NewScheduler(ctx, conn) client := cron.NewScheduler(ctx, conn)
client.CollectRssPosts() client.CheckRssSources()
articles, err := db.Articles.ListByPage(ctx, 0, 100) articles, err := db.Articles.ListByPage(ctx, 0, 100)
if err != nil { if err != nil {

View File

@ -3,37 +3,102 @@ package cron
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
"log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
) )
type Cron struct { type Cron struct {
Db *database.Queries Db *database.Queries
ctx *context.Context ctx *context.Context
timer *cron.Cron timer *cron.Cron
repo services.RepositoryService
} }
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron { func openDatabase() (*database.Queries, error) {
_env := services.NewConfig()
connString := _env.GetConfig(services.Sql_Connection_String)
if connString == "" {
panic("Connection String is null!")
}
db, err := sql.Open("postgres", connString)
if err != nil {
panic(err)
}
queries := database.New(db)
return queries, err
}
func NewScheduler(ctx context.Context) *Cron {
c := &Cron{ c := &Cron{
ctx: &ctx, ctx: &ctx,
repo: services.NewRepositoryService(conn),
} }
timer := cron.New() timer := cron.New()
queries, err := openDatabase()
if err != nil {
panic(err)
}
c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
//features := services.GetEnvConfig() //features := services.GetEnvConfig()
features := services.NewConfig()
timer.AddFunc("5 * * * *", c.CollectRssPosts) timer.AddFunc("5 * * * *", c.CheckRssSources)
//timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts)
//timer.AddFunc("10 1-23 * * *", c.CheckYoutube) //if features.RedditEnabled {
//timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv) // timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
//timer.AddFunc("15 1-23 * * *", c.CheckTwitch) // log.Print("[Input] Reddit backend was enabled")
//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue) // //go c.CheckReddit()
//}
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
if res {
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit()
}
//if features.YoutubeEnabled {
// timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
// log.Print("[Input] YouTube backend was enabled")
//}
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled")
}
//if features.FfxivEnabled {
// timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
// log.Print("[Input] FFXIV backend was enabled")
//}
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled")
}
//if features.TwitchEnabled {
// timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
// log.Print("[Input] Twitch backend was enabled")
//}
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled")
}
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
log.Print("[Output] Discord Output was enabled")
c.timer = timer c.timer = timer
return c return c
@ -47,8 +112,105 @@ func (c *Cron) Stop() {
c.timer.Stop() c.timer.Stop()
} }
/* // This is the main entry point to query all the reddit services
func (c *Cron) CheckDiscordQueue() { func (c *Cron) CheckReddit() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.Name)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
c.checkPosts(redditArticles, "Reddit")
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources.
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.Name)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(raw, "YouTube")
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CheckFfxiv() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "FFXIV")
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CheckTwitch() error {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
return err
}
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.Name)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "Twitch")
}
log.Print("[Twitch] Done!")
return nil
}
func (c *Cron) CheckDiscordQueue() error {
// Get items from the table // Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50) queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
if err != nil { if err != nil {
@ -117,15 +279,55 @@ func (c *Cron) CheckDiscordQueue() {
return nil return nil
} }
*/
//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error { func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
// err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{ for _, item := range posts {
// ID: uuid.New(), _, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
// Articleid: Id, if err != nil {
// }) id := uuid.New()
// if err != nil {
// return err err := c.postArticle(id, item)
// } if err != nil {
// return nil return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
//} }
err = c.addToDiscordQueue(id)
if err != nil {
return err
}
}
}
time.Sleep(30 * time.Second)
return nil
}
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: id,
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
ID: uuid.New(),
Articleid: Id,
})
if err != nil {
return err
}
return nil
}

View File

@ -1,12 +1,12 @@
package cron_test package cron_test
import ( import (
"database/sql" "context"
"testing"
"github.com/pressly/goose/v3" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
) )
/*
func TestInvokeTwitch(t *testing.T) { func TestInvokeTwitch(t *testing.T) {
} }
@ -15,7 +15,7 @@ func TestInvokeTwitch(t *testing.T) {
func TestCheckReddit(t *testing.T) { func TestCheckReddit(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx)
c.Col() c.CheckReddit()
} }
func TestCheckYouTube(t *testing.T) { func TestCheckYouTube(t *testing.T) {
@ -32,7 +32,6 @@ func TestCheckTwitch(t *testing.T) {
t.Error(err) t.Error(err)
} }
} }
*/
func setupInMemoryDb() (*sql.DB, error) { func setupInMemoryDb() (*sql.DB, error) {
db, err := sql.Open("sqlite", ":memory:") db, err := sql.Open("sqlite", ":memory:")

View File

@ -1,6 +1,7 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"log" "log"
"net/http" "net/http"
@ -12,7 +13,7 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid" "github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
@ -24,7 +25,7 @@ const (
) )
type FFXIVClient struct { type FFXIVClient struct {
record domain.SourceEntity record database.Source
//SourceID uint //SourceID uint
//Url string //Url string
//Region string //Region string
@ -32,15 +33,15 @@ type FFXIVClient struct {
cacheGroup string cacheGroup string
} }
func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient { func NewFFXIVClient(Record database.Source) FFXIVClient {
return FFXIVClient{ return FFXIVClient{
record: Record, record: Record,
cacheGroup: "ffxiv", cacheGroup: "ffxiv",
} }
} }
func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) { func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
var articles []domain.ArticleEntity var articles []database.Article
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -96,16 +97,18 @@ func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
return articles, err return articles, err
} }
article := domain.ArticleEntity{ article := database.Article{
SourceID: fc.record.ID, Sourceid: fc.record.ID,
Tags: tags, Tags: tags,
Title: title, Title: title,
Url: link, Url: link,
PubDate: pubDate, Pubdate: pubDate,
Videoheight: 0,
Videowidth: 0,
Thumbnail: thumb, Thumbnail: thumb,
Description: description, Description: description,
AuthorName: authorName, Authorname: sql.NullString{String: authorName},
AuthorImageUrl: authorImage, Authorimage: sql.NullString{String: authorImage},
} }
log.Printf("Collected '%v' from '%v'", article.Title, article.Url) log.Printf("Collected '%v' from '%v'", article.Title, article.Url)

View File

@ -3,14 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var FFXIVRecord domain.SourceEntity = domain.SourceEntity{ var FFXIVRecord database.Source = database.Source{
ID: 9999, ID: uuid.New(),
DisplayName: "Final Fantasy XIV - NA", Site: "ffxiv",
Source: domain.SourceCollectorFfxiv, Name: "Final Fantasy XIV - NA",
Source: "ffxiv",
Url: "https://na.finalfantasyxiv.com/lodestone/", Url: "https://na.finalfantasyxiv.com/lodestone/",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone", Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
} }

View File

@ -1,6 +1,7 @@
package input package input
import ( import (
"database/sql"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -8,6 +9,7 @@ import (
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod" "github.com/go-rod/rod"
@ -16,7 +18,7 @@ import (
type RedditClient struct { type RedditClient struct {
config RedditConfig config RedditConfig
record domain.SourceEntity record database.Source
} }
type RedditConfig struct { type RedditConfig struct {
@ -25,7 +27,7 @@ type RedditConfig struct {
PullNSFW string PullNSFW string
} }
func NewRedditClient(Record domain.SourceEntity) *RedditClient { func NewRedditClient(Record database.Source) *RedditClient {
rc := RedditClient{ rc := RedditClient{
record: Record, record: Record,
} }
@ -69,7 +71,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
// TODO Wire this to support the config options // TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url) Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName) log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
content, err := getHttpContent(Url) content, err := getHttpContent(Url)
if err != nil { if err != nil {
@ -86,10 +88,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
return items, nil return items, nil
} }
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity { func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
var redditArticles []domain.ArticleEntity var redditArticles []database.Article
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article domain.ArticleEntity var article database.Article
article, err := rc.convertToArticle(item.Data) article, err := rc.convertToArticle(item.Data)
if err != nil { if err != nil {
log.Printf("[Reddit] %v", err) log.Printf("[Reddit] %v", err)
@ -102,8 +104,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []doma
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) { func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
var item domain.ArticleEntity var item database.Article
if source.Content == "" && source.Url != "" { if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source) item = rc.convertPicturePost(source)
@ -129,57 +131,65 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.Artic
return item, nil return item, nil
} }
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity { func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
var item = domain.ArticleEntity{ var item = database.Article{
SourceID: rc.record.ID, Sourceid: rc.record.ID,
Title: source.Title, Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags), Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
PubDate: time.Now(), Pubdate: time.Now(),
IsVideo: false, Video: sql.NullString{String: "null"},
Videoheight: 0,
Videowidth: 0,
Thumbnail: source.Thumbnail, Thumbnail: source.Thumbnail,
Description: source.Content, Description: source.Content,
AuthorName: source.Author, Authorname: sql.NullString{String: source.Author},
AuthorImageUrl: "null", Authorimage: sql.NullString{String: "null"},
} }
return item return item
} }
func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity { func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
var item = domain.ArticleEntity{ var item = database.Article{
SourceID: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
PubDate: time.Now(), Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, Authorname: sql.NullString{String: source.Author},
Description: source.Content, Description: source.Content,
} }
return item return item
} }
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity { func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
var item = domain.ArticleEntity{ var item = database.Article{
SourceID: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
PubDate: time.Now(), Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl, Description: source.Media.RedditVideo.FallBackUrl,
} }
return item return item
} }
// This post is nothing more then a redirect to another location. // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity { func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
var item = domain.ArticleEntity{ var item = database.Article{
SourceID: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
PubDate: time.Now(), Pubdate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, Videoheight: 0,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest, Description: source.UrlOverriddenByDest,
} }
return item return item

View File

@ -3,14 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var RedditRecord domain.SourceEntity = domain.SourceEntity{ var RedditRecord database.Source = database.Source{
ID: 9999, ID: uuid.New(),
DisplayName: "dadjokes", Name: "dadjokes",
Source: domain.SourceCollectorRss, Source: "reddit",
Site: "reddit",
Url: "https://reddit.com/r/dadjokes", Url: "https://reddit.com/r/dadjokes",
Tags: "reddit, dadjokes", Tags: "reddit, dadjokes",
} }

View File

@ -11,7 +11,6 @@ var rssRecord = domain.SourceEntity{
ID: 1, ID: 1,
DisplayName: "ArsTechnica", DisplayName: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index", Url: "https://feeds.arstechnica.com/arstechnica/index",
Source: domain.SourceCollectorRss,
} }
func TestRssClientConstructor(t *testing.T) { func TestRssClientConstructor(t *testing.T) {

View File

@ -1,18 +1,19 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/nicklaw5/helix/v2" "github.com/nicklaw5/helix/v2"
) )
type TwitchClient struct { type TwitchClient struct {
SourceRecord domain.SourceEntity SourceRecord database.Source
// config // config
monitorClips string monitorClips string
@ -71,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
} }
// This will let you replace the bound source record to keep the same session alive. // This will let you replace the bound source record to keep the same session alive.
func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) { func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
tc.SourceRecord = source tc.SourceRecord = source
} }
@ -86,8 +87,8 @@ func (tc *TwitchClient) Login() error {
return nil return nil
} }
func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) { func (tc *TwitchClient) GetContent() ([]database.Article, error) {
var items []domain.ArticleEntity var items []database.Article
user, err := tc.GetUserDetails() user, err := tc.GetUserDetails()
if err != nil { if err != nil {
@ -100,31 +101,31 @@ func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
} }
for _, video := range posts { for _, video := range posts {
var article domain.ArticleEntity var article database.Article
AuthorName, err := tc.ExtractAuthor(video) AuthorName, err := tc.ExtractAuthor(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.AuthorName = AuthorName article.Authorname = sql.NullString{String: AuthorName}
Authorimage, err := tc.ExtractAuthorImage(user) Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil { if err != nil {
return items, err return items, err
} }
article.AuthorImageUrl = Authorimage article.Authorimage = sql.NullString{String: Authorimage}
article.Description, err = tc.ExtractDescription(video) article.Description, err = tc.ExtractDescription(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.PubDate, err = tc.ExtractPubDate(video) article.Pubdate, err = tc.ExtractPubDate(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.SourceID = tc.SourceRecord.ID article.Sourceid = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user) article.Tags, err = tc.ExtractTags(video, user)
if err != nil { if err != nil {
return items, err return items, err
@ -155,7 +156,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{ users, err := tc.api.GetUsers(&helix.UsersParams{
Logins: []string{tc.SourceRecord.DisplayName}, Logins: []string{tc.SourceRecord.Name},
}) })
if err != nil { if err != nil {
return blank, err return blank, err

View File

@ -4,20 +4,21 @@ import (
"log" "log"
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var TwitchSourceRecord = domain.SourceEntity{ var TwitchSourceRecord = database.Source{
ID: 9999, ID: uuid.New(),
DisplayName: "nintendo", Name: "nintendo",
Source: domain.SourceCollectorTwitch, Source: "Twitch",
} }
var TwitchInvalidRecord = domain.SourceEntity{ var TwitchInvalidRecord = database.Source{
ID: 9999, ID: uuid.New(),
DisplayName: "EvilNintendo", Name: "EvilNintendo",
Source: domain.SourceCollectorTwitch, Source: "Twitch",
} }
func TestTwitchLogin(t *testing.T) { func TestTwitchLogin(t *testing.T) {

View File

@ -1,6 +1,7 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@ -11,11 +12,11 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
) )
type YoutubeClient struct { type YoutubeClient struct {
record domain.SourceEntity record database.Source
// internal variables at time of collection // internal variables at time of collection
channelID string channelID string
@ -36,7 +37,7 @@ var (
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient { func NewYoutubeClient(Record database.Source) YoutubeClient {
yc := YoutubeClient{ yc := YoutubeClient{
record: Record, record: Record,
cacheGroup: "youtube", cacheGroup: "youtube",
@ -45,8 +46,8 @@ func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
} }
// CheckSource will go and run all the commands needed to process a source. // CheckSource will go and run all the commands needed to process a source.
func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) { func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
var items []domain.ArticleEntity var items []database.Article
docParser, err := yc.GetParser(yc.record.Url) docParser, err := yc.GetParser(yc.record.Url)
if err != nil { if err != nil {
return items, err return items, err
@ -246,7 +247,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
return false return false
} }
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
parser, err := yc.GetParser(item.Link) parser, err := yc.GetParser(item.Link)
if err != nil { if err != nil {
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
@ -264,16 +265,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntit
log.Printf("[YouTube] %v", msg) log.Printf("[YouTube] %v", msg)
} }
var article = domain.ArticleEntity{ var article = database.Article{
SourceID: yc.record.ID, Sourceid: yc.record.ID,
Tags: tags, Tags: tags,
Title: item.Title, Title: item.Title,
Url: item.Link, Url: item.Link,
PubDate: *item.PublishedParsed, Pubdate: *item.PublishedParsed,
Thumbnail: thumb, Thumbnail: thumb,
Description: item.Description, Description: item.Description,
AuthorName: item.Author.Name, Authorname: sql.NullString{String: item.Author.Name},
AuthorImageUrl: yc.avatarUri, Authorimage: sql.NullString{String: yc.avatarUri},
} }
return article return article
} }

View File

@ -3,14 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var YouTubeRecord = domain.SourceEntity{ var YouTubeRecord database.Source = database.Source{
ID: 9999, ID: uuid.New(),
DisplayName: "dadjokes", Name: "dadjokes",
Source: domain.SourceCollectorReddit, Source: "reddit",
Site: "reddit",
Url: "https://youtube.com/gamegrumps", Url: "https://youtube.com/gamegrumps",
} }

View File

@ -4,8 +4,8 @@ help: ## Shows this help command
build: ## builds the application with the current go runtime build: ## builds the application with the current go runtime
~/go/bin/swag f ~/go/bin/swag f
~/go/bin/swag init -g cmd/server.go ~/go/bin/swag i
go build cmd/server.go go build .
docker-build: ## Generates the docker image docker-build: ## Generates the docker image
docker build -t "newsbot.collector.api" . docker build -t "newsbot.collector.api" .