jtom38/newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

input services now use the new entity models

Browse Source
This commit is contained in:
James Tombleson 2024-04-29 16:29:37 -07:00
parent 84d108f2dd
commit e48b64bbaa
9 changed files with 116 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
internal/services/input/ffxiv.go
View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"log" "log"
"net/http" "net/http"
@ -13,7 +12,7 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid" "github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
@ -25,7 +24,7 @@ const (
) )
type FFXIVClient struct { type FFXIVClient struct {
record database.Source record domain.SourceEntity
//SourceID uint //SourceID uint
//Url string //Url string
//Region string //Region string
@ -33,15 +32,15 @@ type FFXIVClient struct {
cacheGroup string cacheGroup string
} }
func NewFFXIVClient(Record database.Source) FFXIVClient { func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
return FFXIVClient{ return FFXIVClient{
record: Record, record: Record,
cacheGroup: "ffxiv", cacheGroup: "ffxiv",
} }
} }
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
var articles []database.Article var articles []domain.ArticleEntity
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
return articles, err return articles, err
} }
article := database.Article{ article := domain.ArticleEntity{
Sourceid: fc.record.ID, SourceID: fc.record.ID,
Tags: tags, Tags: tags,
Title: title, Title: title,
Url: link, Url: link,
Pubdate: pubDate, PubDate: pubDate,
Videoheight: 0, Thumbnail: thumb,
Videowidth: 0, Description: description,
Thumbnail: thumb, AuthorName: authorName,
Description: description, AuthorImageUrl: authorImage,
Authorname: sql.NullString{String: authorName},
Authorimage: sql.NullString{String: authorImage},
} }
log.Printf("Collected '%v' from '%v'", article.Title, article.Url) log.Printf("Collected '%v' from '%v'", article.Title, article.Url)

16
internal/services/input/ffxiv_test.go
View File

@ -3,18 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var FFXIVRecord database.Source = database.Source{ var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 997,
Site: "ffxiv", DisplayName: "Final Fantasy XIV - NA",
Name: "Final Fantasy XIV - NA", Source: domain.SourceCollectorFfxiv,
Source: "ffxiv", Url: "https://na.finalfantasyxiv.com/lodestone/",
Url: "https://na.finalfantasyxiv.com/lodestone/", Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
} }
func TestFfxivGetParser(t *testing.T) { func TestFfxivGetParser(t *testing.T) {

93
internal/services/input/reddit.go
View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -9,7 +8,6 @@ import (
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod" "github.com/go-rod/rod"
@ -17,8 +15,8 @@ import (
) )
type RedditClient struct { type RedditClient struct {
config RedditConfig config services.Configs
record database.Source record domain.SourceEntity
} }
type RedditConfig struct { type RedditConfig struct {
@ -27,14 +25,11 @@ type RedditConfig struct {
PullNSFW string PullNSFW string
} }
func NewRedditClient(Record database.Source) *RedditClient { func NewRedditClient(record domain.SourceEntity) *RedditClient {
rc := RedditClient{ rc := RedditClient{
record: Record, record: record,
config: services.GetEnvConfig(),
} }
cc := services.NewConfig()
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
//rc.disableHttp2Client() //rc.disableHttp2Client()
@ -71,7 +66,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
// TODO Wire this to support the config options // TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url) Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name) log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
content, err := getHttpContent(Url) content, err := getHttpContent(Url)
if err != nil { if err != nil {
@ -88,24 +83,28 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
return items, nil return items, nil
} }
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article { func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
var redditArticles []database.Article var redditArticles []domain.ArticleEntity
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article database.Article var article domain.ArticleEntity
article, err := rc.convertToArticle(item.Data) article, err := rc.convertToArticle(item.Data)
if err != nil { if err != nil {
log.Printf("[Reddit] %v", err) log.Printf("[Reddit] %v", err)
continue continue
} }
redditArticles = append(redditArticles, article) redditArticles = append(redditArticles, article)
} }
return redditArticles return redditArticles
} }
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) { func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
var item database.Article var item domain.ArticleEntity
if source.Content == "" && source.Url != "" { if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source) item = rc.convertPicturePost(source)
@ -131,65 +130,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
return item, nil return item, nil
} }
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Title: source.Title, Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags), Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Pubdate: time.Now(), PubDate: time.Now(),
Video: sql.NullString{String: "null"}, IsVideo: false,
Videoheight: 0, Thumbnail: source.Thumbnail,
Videowidth: 0, Description: source.Content,
Thumbnail: source.Thumbnail, AuthorName: source.Author,
Description: source.Content, AuthorImageUrl: "",
Authorname: sql.NullString{String: source.Author},
Authorimage: sql.NullString{String: "null"},
} }
return item return item
} }
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Authorname: sql.NullString{String: source.Author}, AuthorName: source.Author,
Description: source.Content, Description: source.Content,
} }
return item return item
} }
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0, AuthorName: source.Author,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl, Description: source.Media.RedditVideo.FallBackUrl,
} }
return item return item
} }
// This post is nothing more then a redirect to another location. // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0, AuthorName: source.Author,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest, Description: source.UrlOverriddenByDest,
} }
return item return item

16
internal/services/input/reddit_test.go
View File

@ -3,18 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var RedditRecord database.Source = database.Source{ var RedditRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 999,
Name: "dadjokes", DisplayName: "dadjokes",
Source: "reddit", Source: domain.SourceCollectorReddit,
Site: "reddit", Url: "https://reddit.com/r/dadjokes",
Url: "https://reddit.com/r/dadjokes", Tags: "reddit, dadjokes",
Tags: "reddit, dadjokes",
} }
func TestGetContent(t *testing.T) { func TestGetContent(t *testing.T) {

5
internal/services/input/rss_test.go
View File

@ -8,9 +8,10 @@ import (
) )
var rssRecord = domain.SourceEntity{ var rssRecord = domain.SourceEntity{
ID: 1, ID: 1,
DisplayName: "ArsTechnica", DisplayName: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index", Source: domain.SourceCollectorRss,
Url: "https://feeds.arstechnica.com/arstechnica/index",
} }
func TestRssClientConstructor(t *testing.T) { func TestRssClientConstructor(t *testing.T) {

23
internal/services/input/twitch.go
View File

@ -1,19 +1,18 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/nicklaw5/helix/v2" "github.com/nicklaw5/helix/v2"
) )
type TwitchClient struct { type TwitchClient struct {
SourceRecord database.Source SourceRecord domain.SourceEntity
// config // config
monitorClips string monitorClips string
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
} }
// This will let you replace the bound source record to keep the same session alive. // This will let you replace the bound source record to keep the same session alive.
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
tc.SourceRecord = source tc.SourceRecord = source
} }
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
return nil return nil
} }
func (tc *TwitchClient) GetContent() ([]database.Article, error) { func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
var items []database.Article var items []domain.ArticleEntity
user, err := tc.GetUserDetails() user, err := tc.GetUserDetails()
if err != nil { if err != nil {
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
} }
for _, video := range posts { for _, video := range posts {
var article database.Article var article domain.ArticleEntity
AuthorName, err := tc.ExtractAuthor(video) AuthorName, err := tc.ExtractAuthor(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Authorname = sql.NullString{String: AuthorName} article.AuthorName = AuthorName
Authorimage, err := tc.ExtractAuthorImage(user) Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Authorimage = sql.NullString{String: Authorimage} article.AuthorImageUrl = Authorimage
article.Description, err = tc.ExtractDescription(video) article.Description, err = tc.ExtractDescription(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Pubdate, err = tc.ExtractPubDate(video) article.PubDate, err = tc.ExtractPubDate(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Sourceid = tc.SourceRecord.ID article.SourceID = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user) article.Tags, err = tc.ExtractTags(video, user)
if err != nil { if err != nil {
return items, err return items, err
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{ users, err := tc.api.GetUsers(&helix.UsersParams{
Logins: []string{tc.SourceRecord.Name}, Logins: []string{tc.SourceRecord.DisplayName},
}) })
if err != nil { if err != nil {
return blank, err return blank, err

19
internal/services/input/twitch_test.go
View File

@ -4,21 +4,20 @@ import (
"log" "log"
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var TwitchSourceRecord = database.Source{ var TwitchSourceRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "nintendo", DisplayName: "nintendo",
Source: "Twitch", Source: domain.SourceCollectorTwitch,
} }
var TwitchInvalidRecord = database.Source{ var TwitchInvalidRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "EvilNintendo", DisplayName: "EvilNintendo",
Source: "Twitch", Source: domain.SourceCollectorTwitch,
} }
func TestTwitchLogin(t *testing.T) { func TestTwitchLogin(t *testing.T) {

33
internal/services/input/youtube.go
View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@ -12,11 +11,11 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
) )
type YoutubeClient struct { type YoutubeClient struct {
record database.Source record domain.SourceEntity
// internal variables at time of collection // internal variables at time of collection
channelID string channelID string
@ -37,7 +36,7 @@ var (
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
func NewYoutubeClient(Record database.Source) YoutubeClient { func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
yc := YoutubeClient{ yc := YoutubeClient{
record: Record, record: Record,
cacheGroup: "youtube", cacheGroup: "youtube",
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
} }
// CheckSource will go and run all the commands needed to process a source. // CheckSource will go and run all the commands needed to process a source.
func (yc *YoutubeClient) GetContent() ([]database.Article, error) { func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
var items []database.Article var items []domain.ArticleEntity
docParser, err := yc.GetParser(yc.record.Url) docParser, err := yc.GetParser(yc.record.Url)
if err != nil { if err != nil {
return items, err return items, err
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
return false return false
} }
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
parser, err := yc.GetParser(item.Link) parser, err := yc.GetParser(item.Link)
if err != nil { if err != nil {
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
log.Printf("[YouTube] %v", msg) log.Printf("[YouTube] %v", msg)
} }
var article = database.Article{ var article = domain.ArticleEntity{
Sourceid: yc.record.ID, SourceID: yc.record.ID,
Tags: tags, Tags: tags,
Title: item.Title, Title: item.Title,
Url: item.Link, Url: item.Link,
Pubdate: *item.PublishedParsed, PubDate: *item.PublishedParsed,
Thumbnail: thumb, Thumbnail: thumb,
Description: item.Description, Description: item.Description,
Authorname: sql.NullString{String: item.Author.Name}, AuthorName: item.Author.Name,
Authorimage: sql.NullString{String: yc.avatarUri}, AuthorImageUrl: yc.avatarUri,
} }
return article return article
} }

14
internal/services/input/youtube_test.go
View File

@ -3,17 +3,15 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var YouTubeRecord database.Source = database.Source{ var YouTubeRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 999,
Name: "dadjokes", DisplayName: "dadjokes",
Source: "reddit", Source: domain.SourceCollectorYoutube,
Site: "reddit", Url: "https://youtube.com/gamegrumps",
Url: "https://youtube.com/gamegrumps",
} }
func TestGetPageParser(t *testing.T) { func TestGetPageParser(t *testing.T) {