From e48b64bbaa611cc376efb9e1f8f26118ce4287ab Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Mon, 29 Apr 2024 16:29:37 -0700 Subject: [PATCH] input services now use the new entity models --- internal/services/input/ffxiv.go | 33 ++++----- internal/services/input/ffxiv_test.go | 16 ++--- internal/services/input/reddit.go | 93 +++++++++++-------------- internal/services/input/reddit_test.go | 16 ++--- internal/services/input/rss_test.go | 5 +- internal/services/input/twitch.go | 23 +++--- internal/services/input/twitch_test.go | 19 +++-- internal/services/input/youtube.go | 33 +++++---- internal/services/input/youtube_test.go | 14 ++-- 9 files changed, 116 insertions(+), 136 deletions(-) diff --git a/internal/services/input/ffxiv.go b/internal/services/input/ffxiv.go index 581eeb9..7072548 100644 --- a/internal/services/input/ffxiv.go +++ b/internal/services/input/ffxiv.go @@ -1,7 +1,6 @@ package input import ( - "database/sql" "errors" "log" "net/http" @@ -13,7 +12,7 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/google/uuid" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" ) @@ -25,7 +24,7 @@ const ( ) type FFXIVClient struct { - record database.Source + record domain.SourceEntity //SourceID uint //Url string //Region string @@ -33,15 +32,15 @@ type FFXIVClient struct { cacheGroup string } -func NewFFXIVClient(Record database.Source) FFXIVClient { +func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient { return FFXIVClient{ record: Record, cacheGroup: "ffxiv", } } -func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { - var articles []database.Article +func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) { + var articles []domain.ArticleEntity parser := fc.GetBrowser() defer parser.Close() @@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { return articles, err } - article := database.Article{ - Sourceid: fc.record.ID, - Tags: tags, - Title: title, - Url: link, - Pubdate: pubDate, - Videoheight: 0, - Videowidth: 0, - Thumbnail: thumb, - Description: description, - Authorname: sql.NullString{String: authorName}, - Authorimage: sql.NullString{String: authorImage}, + article := domain.ArticleEntity{ + SourceID: fc.record.ID, + Tags: tags, + Title: title, + Url: link, + PubDate: pubDate, + Thumbnail: thumb, + Description: description, + AuthorName: authorName, + AuthorImageUrl: authorImage, } log.Printf("Collected '%v' from '%v'", article.Title, article.Url) diff --git a/internal/services/input/ffxiv_test.go b/internal/services/input/ffxiv_test.go index bb29542..2d47770 100644 --- a/internal/services/input/ffxiv_test.go +++ b/internal/services/input/ffxiv_test.go @@ -3,18 +3,16 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var FFXIVRecord database.Source = database.Source{ - ID: uuid.New(), - Site: "ffxiv", - Name: "Final Fantasy XIV - NA", - Source: "ffxiv", - Url: "https://na.finalfantasyxiv.com/lodestone/", - Tags: "ffxiv, final, fantasy, xiv, na, lodestone", +var FFXIVRecord domain.SourceEntity = domain.SourceEntity{ + ID: 997, + DisplayName: "Final Fantasy XIV - NA", + Source: domain.SourceCollectorFfxiv, + Url: "https://na.finalfantasyxiv.com/lodestone/", + Tags: "ffxiv, final, fantasy, xiv, na, lodestone", } func TestFfxivGetParser(t *testing.T) { diff --git a/internal/services/input/reddit.go b/internal/services/input/reddit.go index 9f6c455..8663128 100644 --- a/internal/services/input/reddit.go +++ b/internal/services/input/reddit.go @@ -1,7 +1,6 @@ package input import ( - "database/sql" "encoding/json" "errors" "fmt" @@ -9,7 +8,6 @@ import ( "strings" "time" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "github.com/go-rod/rod" @@ -17,8 +15,8 @@ import ( ) type RedditClient struct { - config RedditConfig - record database.Source + config services.Configs + record domain.SourceEntity } type RedditConfig struct { @@ -27,14 +25,11 @@ type RedditConfig struct { PullNSFW string } -func NewRedditClient(Record database.Source) *RedditClient { +func NewRedditClient(record domain.SourceEntity) *RedditClient { rc := RedditClient{ - record: Record, + record: record, + config: services.GetEnvConfig(), } - cc := services.NewConfig() - rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT) - rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW) - rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP) //rc.disableHttp2Client() @@ -71,7 +66,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) { // TODO Wire this to support the config options Url := fmt.Sprintf("%v.json", rc.record.Url) - log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name) + log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName) content, err := getHttpContent(Url) if err != nil { @@ -88,24 +83,28 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) { return items, nil } -func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article { - var redditArticles []database.Article +func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity { + var redditArticles []domain.ArticleEntity + for _, item := range items.Data.Children { - var article database.Article + var article domain.ArticleEntity article, err := rc.convertToArticle(item.Data) + if err != nil { log.Printf("[Reddit] %v", err) continue } + redditArticles = append(redditArticles, article) } + return redditArticles } // ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // This data can be passed to the database. -func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) { - var item database.Article +func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) { + var item domain.ArticleEntity if source.Content == "" && source.Url != "" { item = rc.convertPicturePost(source) @@ -131,65 +130,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art return item, nil } -func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, - Title: source.Title, - Tags: fmt.Sprintf("%v", rc.record.Tags), - Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Pubdate: time.Now(), - Video: sql.NullString{String: "null"}, - Videoheight: 0, - Videowidth: 0, - Thumbnail: source.Thumbnail, - Description: source.Content, - Authorname: sql.NullString{String: source.Author}, - Authorimage: sql.NullString{String: "null"}, +func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, + Title: source.Title, + Tags: fmt.Sprintf("%v", rc.record.Tags), + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + PubDate: time.Now(), + IsVideo: false, + Thumbnail: source.Thumbnail, + Description: source.Content, + AuthorName: source.Author, + AuthorImageUrl: "", } return item } -func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, +func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, Tags: "a", Title: source.Title, - Pubdate: time.Now(), - Videoheight: 0, - Videowidth: 0, + PubDate: time.Now(), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Authorname: sql.NullString{String: source.Author}, + AuthorName: source.Author, Description: source.Content, } return item } -func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, +func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, Tags: "a", Title: source.Title, - Pubdate: time.Now(), + PubDate: time.Now(), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Videoheight: 0, - Videowidth: 0, - Authorname: sql.NullString{String: source.Author}, + AuthorName: source.Author, Description: source.Media.RedditVideo.FallBackUrl, } return item } // This post is nothing more then a redirect to another location. -func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, +func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, Tags: "a", Title: source.Title, - Pubdate: time.Now(), + PubDate: time.Now(), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Videoheight: 0, - Videowidth: 0, - Authorname: sql.NullString{String: source.Author}, + AuthorName: source.Author, Description: source.UrlOverriddenByDest, } return item diff --git a/internal/services/input/reddit_test.go b/internal/services/input/reddit_test.go index a87af6d..1ec70e0 100644 --- a/internal/services/input/reddit_test.go +++ b/internal/services/input/reddit_test.go @@ -3,18 +3,16 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var RedditRecord database.Source = database.Source{ - ID: uuid.New(), - Name: "dadjokes", - Source: "reddit", - Site: "reddit", - Url: "https://reddit.com/r/dadjokes", - Tags: "reddit, dadjokes", +var RedditRecord domain.SourceEntity = domain.SourceEntity{ + ID: 999, + DisplayName: "dadjokes", + Source: domain.SourceCollectorReddit, + Url: "https://reddit.com/r/dadjokes", + Tags: "reddit, dadjokes", } func TestGetContent(t *testing.T) { diff --git a/internal/services/input/rss_test.go b/internal/services/input/rss_test.go index 728bd85..8fc2ea8 100644 --- a/internal/services/input/rss_test.go +++ b/internal/services/input/rss_test.go @@ -8,9 +8,10 @@ import ( ) var rssRecord = domain.SourceEntity{ - ID: 1, + ID: 1, DisplayName: "ArsTechnica", - Url: "https://feeds.arstechnica.com/arstechnica/index", + Source: domain.SourceCollectorRss, + Url: "https://feeds.arstechnica.com/arstechnica/index", } func TestRssClientConstructor(t *testing.T) { diff --git a/internal/services/input/twitch.go b/internal/services/input/twitch.go index 566b8c4..317db56 100644 --- a/internal/services/input/twitch.go +++ b/internal/services/input/twitch.go @@ -1,19 +1,18 @@ package input import ( - "database/sql" "errors" "fmt" "strings" "time" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "github.com/nicklaw5/helix/v2" ) type TwitchClient struct { - SourceRecord database.Source + SourceRecord domain.SourceEntity // config monitorClips string @@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) { } // This will let you replace the bound source record to keep the same session alive. -func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { +func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) { tc.SourceRecord = source } @@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error { return nil } -func (tc *TwitchClient) GetContent() ([]database.Article, error) { - var items []database.Article +func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) { + var items []domain.ArticleEntity user, err := tc.GetUserDetails() if err != nil { @@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) { } for _, video := range posts { - var article database.Article + var article domain.ArticleEntity AuthorName, err := tc.ExtractAuthor(video) if err != nil { return items, err } - article.Authorname = sql.NullString{String: AuthorName} + article.AuthorName = AuthorName Authorimage, err := tc.ExtractAuthorImage(user) if err != nil { return items, err } - article.Authorimage = sql.NullString{String: Authorimage} + article.AuthorImageUrl = Authorimage article.Description, err = tc.ExtractDescription(video) if err != nil { return items, err } - article.Pubdate, err = tc.ExtractPubDate(video) + article.PubDate, err = tc.ExtractPubDate(video) if err != nil { return items, err } - article.Sourceid = tc.SourceRecord.ID + article.SourceID = tc.SourceRecord.ID article.Tags, err = tc.ExtractTags(video, user) if err != nil { return items, err @@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) { var blank helix.User users, err := tc.api.GetUsers(&helix.UsersParams{ - Logins: []string{tc.SourceRecord.Name}, + Logins: []string{tc.SourceRecord.DisplayName}, }) if err != nil { return blank, err diff --git a/internal/services/input/twitch_test.go b/internal/services/input/twitch_test.go index 887bb1e..df06ad3 100644 --- a/internal/services/input/twitch_test.go +++ b/internal/services/input/twitch_test.go @@ -4,21 +4,20 @@ import ( "log" "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var TwitchSourceRecord = database.Source{ - ID: uuid.New(), - Name: "nintendo", - Source: "Twitch", +var TwitchSourceRecord = domain.SourceEntity{ + ID: 9999, + DisplayName: "nintendo", + Source: domain.SourceCollectorTwitch, } -var TwitchInvalidRecord = database.Source{ - ID: uuid.New(), - Name: "EvilNintendo", - Source: "Twitch", +var TwitchInvalidRecord = domain.SourceEntity{ + ID: 9999, + DisplayName: "EvilNintendo", + Source: domain.SourceCollectorTwitch, } func TestTwitchLogin(t *testing.T) { diff --git a/internal/services/input/youtube.go b/internal/services/input/youtube.go index 379e7b9..1bfdfe2 100644 --- a/internal/services/input/youtube.go +++ b/internal/services/input/youtube.go @@ -1,7 +1,6 @@ package input import ( - "database/sql" "errors" "fmt" "log" @@ -12,11 +11,11 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/mmcdole/gofeed" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" ) type YoutubeClient struct { - record database.Source + record domain.SourceEntity // internal variables at time of collection channelID string @@ -37,7 +36,7 @@ var ( const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" -func NewYoutubeClient(Record database.Source) YoutubeClient { +func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient { yc := YoutubeClient{ record: Record, cacheGroup: "youtube", @@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient { } // CheckSource will go and run all the commands needed to process a source. -func (yc *YoutubeClient) GetContent() ([]database.Article, error) { - var items []database.Article +func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) { + var items []domain.ArticleEntity docParser, err := yc.GetParser(yc.record.Url) if err != nil { return items, err @@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool { return false } -func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { +func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity { parser, err := yc.GetParser(item.Link) if err != nil { log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) @@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { log.Printf("[YouTube] %v", msg) } - var article = database.Article{ - Sourceid: yc.record.ID, - Tags: tags, - Title: item.Title, - Url: item.Link, - Pubdate: *item.PublishedParsed, - Thumbnail: thumb, - Description: item.Description, - Authorname: sql.NullString{String: item.Author.Name}, - Authorimage: sql.NullString{String: yc.avatarUri}, + var article = domain.ArticleEntity{ + SourceID: yc.record.ID, + Tags: tags, + Title: item.Title, + Url: item.Link, + PubDate: *item.PublishedParsed, + Thumbnail: thumb, + Description: item.Description, + AuthorName: item.Author.Name, + AuthorImageUrl: yc.avatarUri, } return article } diff --git a/internal/services/input/youtube_test.go b/internal/services/input/youtube_test.go index 4005f96..dd83996 100644 --- a/internal/services/input/youtube_test.go +++ b/internal/services/input/youtube_test.go @@ -3,17 +3,15 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var YouTubeRecord database.Source = database.Source{ - ID: uuid.New(), - Name: "dadjokes", - Source: "reddit", - Site: "reddit", - Url: "https://youtube.com/gamegrumps", +var YouTubeRecord domain.SourceEntity = domain.SourceEntity{ + ID: 999, + DisplayName: "dadjokes", + Source: domain.SourceCollectorYoutube, + Url: "https://youtube.com/gamegrumps", } func TestGetPageParser(t *testing.T) {