From 94c2bdb312c1f4660e630de398722da05baf5a0d Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 9 May 2024 18:59:50 -0700 Subject: [PATCH] entity was moved to its own page --- internal/entity/entity.go | 140 ++++++++++++++++++++++++ internal/services/input/ffxiv.go | 12 +- internal/services/input/ffxiv_test.go | 5 +- internal/services/input/reddit.go | 31 +++--- internal/services/input/reddit_test.go | 5 +- internal/services/input/rss.go | 14 +-- internal/services/input/rss_test.go | 7 +- internal/services/input/twitch.go | 12 +- internal/services/input/twitch_test.go | 7 +- internal/services/input/youtube.go | 19 ++-- internal/services/input/youtube_test.go | 5 +- 11 files changed, 201 insertions(+), 56 deletions(-) create mode 100644 internal/entity/entity.go diff --git a/internal/entity/entity.go b/internal/entity/entity.go new file mode 100644 index 0000000..107a9d6 --- /dev/null +++ b/internal/entity/entity.go @@ -0,0 +1,140 @@ +package entity + +import ( + "time" +) + +// This links a source to a discord webhook. +// It is owned by a user so they can remove the link +type AlertDiscordEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + UserID int64 + SourceID int64 + DiscordWebHookId int64 +} + +type ArticleEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + SourceID int64 + Tags string + Title string + Url string + PubDate time.Time + IsVideo bool + Thumbnail string + Description string + AuthorName string + AuthorImageUrl string +} + +type DiscordQueueEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + ArticleId int64 + SourceId int64 +} + +type DiscordWebHookEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + UserID int64 + Url string + Server string + Channel string + Enabled bool +} + +type IconEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + FileName string + Site string +} + +type SettingEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + Key string + Value string + Options string +} + +type SourceEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + + // Who will collect from it. Used + // domain.SourceCollector... + Source string + + // Human Readable value to state what is getting collected + DisplayName string + + // Tells the parser where to look for data + Url string + + // Static tags for this defined record + Tags string + + // If the record is disabled, then it will be skipped on processing + Enabled bool +} + +//type SubscriptionEntity struct { +// ID int64 +// CreatedAt time.Time +// UpdatedAt time.Time +// DeletedAt time.Time +// UserID int64 +// SourceID int64 +// //SourceType string +// //SourceName string +// DiscordID int64 +// //DiscordName string +//} + +// This defines what sources a user wants to follow. +// These will show up for the user as a front page +type UserSourceSubscriptionEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + UserID int64 + SourceID int64 +} + +type UserEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + Username string + Hash string + Scopes string +} + +type RefreshTokenEntity struct { + ID int64 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt time.Time + Username string + Token string +} diff --git a/internal/services/input/ffxiv.go b/internal/services/input/ffxiv.go index 7072548..0cd4cba 100644 --- a/internal/services/input/ffxiv.go +++ b/internal/services/input/ffxiv.go @@ -12,7 +12,7 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/google/uuid" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" ) @@ -24,7 +24,7 @@ const ( ) type FFXIVClient struct { - record domain.SourceEntity + record entity.SourceEntity //SourceID uint //Url string //Region string @@ -32,15 +32,15 @@ type FFXIVClient struct { cacheGroup string } -func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient { +func NewFFXIVClient(Record entity.SourceEntity) FFXIVClient { return FFXIVClient{ record: Record, cacheGroup: "ffxiv", } } -func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) { - var articles []domain.ArticleEntity +func (fc *FFXIVClient) CheckSource() ([]entity.ArticleEntity, error) { + var articles []entity.ArticleEntity parser := fc.GetBrowser() defer parser.Close() @@ -96,7 +96,7 @@ func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) { return articles, err } - article := domain.ArticleEntity{ + article := entity.ArticleEntity{ SourceID: fc.record.ID, Tags: tags, Title: title, diff --git a/internal/services/input/ffxiv_test.go b/internal/services/input/ffxiv_test.go index f2fde75..4fe035e 100644 --- a/internal/services/input/ffxiv_test.go +++ b/internal/services/input/ffxiv_test.go @@ -3,11 +3,12 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ) -var FFXIVRecord domain.SourceEntity = domain.SourceEntity{ +var FFXIVRecord entity.SourceEntity = entity.SourceEntity{ ID: 9999, DisplayName: "Final Fantasy XIV - NA", Source: domain.SourceCollectorFfxiv, diff --git a/internal/services/input/reddit.go b/internal/services/input/reddit.go index 0a6aab7..bfd8844 100644 --- a/internal/services/input/reddit.go +++ b/internal/services/input/reddit.go @@ -9,6 +9,7 @@ import ( "time" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" @@ -16,7 +17,7 @@ import ( type RedditClient struct { config RedditConfig - record domain.SourceEntity + record entity.SourceEntity } type RedditConfig struct { @@ -25,7 +26,7 @@ type RedditConfig struct { PullNSFW string } -func NewRedditClient(Record domain.SourceEntity) *RedditClient { +func NewRedditClient(Record entity.SourceEntity) *RedditClient { rc := RedditClient{ record: Record, } @@ -86,10 +87,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) { return items, nil } -func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity { - var redditArticles []domain.ArticleEntity +func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []entity.ArticleEntity { + var redditArticles []entity.ArticleEntity for _, item := range items.Data.Children { - var article domain.ArticleEntity + var article entity.ArticleEntity article, err := rc.convertToArticle(item.Data) if err != nil { log.Printf("[Reddit] %v", err) @@ -102,8 +103,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []doma // ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // This data can be passed to the database. -func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) { - var item domain.ArticleEntity +func (rc *RedditClient) convertToArticle(source domain.RedditPost) (entity.ArticleEntity, error) { + var item entity.ArticleEntity if source.Content == "" && source.Url != "" { item = rc.convertPicturePost(source) @@ -129,8 +130,8 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.Artic return item, nil } -func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity { - var item = domain.ArticleEntity{ +func (rc *RedditClient) convertPicturePost(source domain.RedditPost) entity.ArticleEntity { + var item = entity.ArticleEntity{ SourceID: rc.record.ID, Title: source.Title, Tags: fmt.Sprintf("%v", rc.record.Tags), @@ -145,8 +146,8 @@ func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.Arti return item } -func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity { - var item = domain.ArticleEntity{ +func (rc *RedditClient) convertTextPost(source domain.RedditPost) entity.ArticleEntity { + var item = entity.ArticleEntity{ SourceID: rc.record.ID, Tags: "a", Title: source.Title, @@ -158,8 +159,8 @@ func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.Article return item } -func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity { - var item = domain.ArticleEntity{ +func (rc *RedditClient) convertVideoPost(source domain.RedditPost) entity.ArticleEntity { + var item = entity.ArticleEntity{ SourceID: rc.record.ID, Tags: "a", Title: source.Title, @@ -172,8 +173,8 @@ func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.Articl } // This post is nothing more then a redirect to another location. -func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity { - var item = domain.ArticleEntity{ +func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) entity.ArticleEntity { + var item = entity.ArticleEntity{ SourceID: rc.record.ID, Tags: "a", Title: source.Title, diff --git a/internal/services/input/reddit_test.go b/internal/services/input/reddit_test.go index bb7eb76..de13971 100644 --- a/internal/services/input/reddit_test.go +++ b/internal/services/input/reddit_test.go @@ -3,11 +3,12 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ) -var RedditRecord domain.SourceEntity = domain.SourceEntity{ +var RedditRecord entity.SourceEntity = entity.SourceEntity{ ID: 9999, DisplayName: "dadjokes", Source: domain.SourceCollectorRss, diff --git a/internal/services/input/rss.go b/internal/services/input/rss.go index e5dbccd..8a844d8 100644 --- a/internal/services/input/rss.go +++ b/internal/services/input/rss.go @@ -3,19 +3,19 @@ package input import ( "strings" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "github.com/mmcdole/gofeed" ) type FeedInput interface { - GetArticles() (domain.ArticleEntity, error) + GetArticles() (entity.ArticleEntity, error) } type rssClient struct { - SourceRecord domain.SourceEntity + SourceRecord entity.SourceEntity } -func NewRssClient(sourceRecord domain.SourceEntity) rssClient { +func NewRssClient(sourceRecord entity.SourceEntity) rssClient { client := rssClient{ SourceRecord: sourceRecord, } @@ -23,7 +23,7 @@ func NewRssClient(sourceRecord domain.SourceEntity) rssClient { return client } -func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) { +func (rc rssClient) GetArticles() ([]entity.ArticleEntity, error) { parser := gofeed.NewParser() feed, err := parser.ParseURL(rc.SourceRecord.Url) if err != nil { @@ -31,9 +31,9 @@ func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) { } sourceTags := strings.Split(rc.SourceRecord.Tags, ",") - var articles []domain.ArticleEntity + var articles []entity.ArticleEntity for _, post := range feed.Items { - article := domain.ArticleEntity{ + article := entity.ArticleEntity{ SourceID: rc.SourceRecord.ID, Title: post.Title, Description: post.Content, diff --git a/internal/services/input/rss_test.go b/internal/services/input/rss_test.go index b03e798..a2a0065 100644 --- a/internal/services/input/rss_test.go +++ b/internal/services/input/rss_test.go @@ -3,11 +3,12 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ) -var rssRecord = domain.SourceEntity{ +var rssRecord = entity.SourceEntity{ ID: 1, DisplayName: "ArsTechnica", Url: "https://feeds.arstechnica.com/arstechnica/index", @@ -27,7 +28,7 @@ func TestRssGetFeed(t *testing.T) { } func TestRssAgainstGita(t *testing.T) { - client := input.NewRssClient(domain.SourceEntity{ + client := input.NewRssClient(entity.SourceEntity{ ID: 2, DisplayName: "Gitea - Newsbot-api", Source: domain.SourceCollectorRss, diff --git a/internal/services/input/twitch.go b/internal/services/input/twitch.go index 317db56..652fa13 100644 --- a/internal/services/input/twitch.go +++ b/internal/services/input/twitch.go @@ -6,13 +6,13 @@ import ( "strings" "time" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "github.com/nicklaw5/helix/v2" ) type TwitchClient struct { - SourceRecord domain.SourceEntity + SourceRecord entity.SourceEntity // config monitorClips string @@ -71,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) { } // This will let you replace the bound source record to keep the same session alive. -func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) { +func (tc *TwitchClient) ReplaceSourceRecord(source entity.SourceEntity) { tc.SourceRecord = source } @@ -86,8 +86,8 @@ func (tc *TwitchClient) Login() error { return nil } -func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) { - var items []domain.ArticleEntity +func (tc *TwitchClient) GetContent() ([]entity.ArticleEntity, error) { + var items []entity.ArticleEntity user, err := tc.GetUserDetails() if err != nil { @@ -100,7 +100,7 @@ func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) { } for _, video := range posts { - var article domain.ArticleEntity + var article entity.ArticleEntity AuthorName, err := tc.ExtractAuthor(video) if err != nil { diff --git a/internal/services/input/twitch_test.go b/internal/services/input/twitch_test.go index df06ad3..0107811 100644 --- a/internal/services/input/twitch_test.go +++ b/internal/services/input/twitch_test.go @@ -4,17 +4,18 @@ import ( "log" "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ) -var TwitchSourceRecord = domain.SourceEntity{ +var TwitchSourceRecord = entity.SourceEntity{ ID: 9999, DisplayName: "nintendo", Source: domain.SourceCollectorTwitch, } -var TwitchInvalidRecord = domain.SourceEntity{ +var TwitchInvalidRecord = entity.SourceEntity{ ID: 9999, DisplayName: "EvilNintendo", Source: domain.SourceCollectorTwitch, diff --git a/internal/services/input/youtube.go b/internal/services/input/youtube.go index 1bfdfe2..1ec35c6 100644 --- a/internal/services/input/youtube.go +++ b/internal/services/input/youtube.go @@ -11,11 +11,11 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/mmcdole/gofeed" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" ) type YoutubeClient struct { - record domain.SourceEntity + record entity.SourceEntity // internal variables at time of collection channelID string @@ -25,7 +25,7 @@ type YoutubeClient struct { //debug bool // cache config - cacheGroup string + //cacheGroup string } var ( @@ -36,17 +36,16 @@ var ( const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" -func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient { +func NewYoutubeClient(Record entity.SourceEntity) YoutubeClient { yc := YoutubeClient{ - record: Record, - cacheGroup: "youtube", + record: Record, } return yc } // CheckSource will go and run all the commands needed to process a source. -func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) { - var items []domain.ArticleEntity +func (yc *YoutubeClient) GetContent() ([]entity.ArticleEntity, error) { + var items []entity.ArticleEntity docParser, err := yc.GetParser(yc.record.Url) if err != nil { return items, err @@ -246,7 +245,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool { return false } -func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity { +func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) entity.ArticleEntity { parser, err := yc.GetParser(item.Link) if err != nil { log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) @@ -264,7 +263,7 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntit log.Printf("[YouTube] %v", msg) } - var article = domain.ArticleEntity{ + var article = entity.ArticleEntity{ SourceID: yc.record.ID, Tags: tags, Title: item.Title, diff --git a/internal/services/input/youtube_test.go b/internal/services/input/youtube_test.go index 7c1a75f..0337e58 100644 --- a/internal/services/input/youtube_test.go +++ b/internal/services/input/youtube_test.go @@ -3,11 +3,12 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/entity" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ) -var YouTubeRecord = domain.SourceEntity{ +var YouTubeRecord = entity.SourceEntity{ ID: 9999, DisplayName: "dadjokes", Source: domain.SourceCollectorReddit,