inputs now use the domain package
This commit is contained in:
parent
bfa0f1023d
commit
77b5c2bfa4
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
@ -13,7 +12,7 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
)
|
||||
|
||||
@ -25,7 +24,7 @@ const (
|
||||
)
|
||||
|
||||
type FFXIVClient struct {
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
//SourceID uint
|
||||
//Url string
|
||||
//Region string
|
||||
@ -33,15 +32,15 @@ type FFXIVClient struct {
|
||||
cacheGroup string
|
||||
}
|
||||
|
||||
func NewFFXIVClient(Record database.Source) FFXIVClient {
|
||||
func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
|
||||
return FFXIVClient{
|
||||
record: Record,
|
||||
cacheGroup: "ffxiv",
|
||||
}
|
||||
}
|
||||
|
||||
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
var articles []database.Article
|
||||
func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
|
||||
var articles []domain.ArticleEntity
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
return articles, err
|
||||
}
|
||||
|
||||
article := database.Article{
|
||||
Sourceid: fc.record.ID,
|
||||
Tags: tags,
|
||||
Title: title,
|
||||
Url: link,
|
||||
Pubdate: pubDate,
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Thumbnail: thumb,
|
||||
Description: description,
|
||||
Authorname: sql.NullString{String: authorName},
|
||||
Authorimage: sql.NullString{String: authorImage},
|
||||
article := domain.ArticleEntity{
|
||||
SourceID: fc.record.ID,
|
||||
Tags: tags,
|
||||
Title: title,
|
||||
Url: link,
|
||||
PubDate: pubDate,
|
||||
Thumbnail: thumb,
|
||||
Description: description,
|
||||
AuthorName: authorName,
|
||||
AuthorImageUrl: authorImage,
|
||||
}
|
||||
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
||||
|
||||
|
@ -3,18 +3,16 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var FFXIVRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Site: "ffxiv",
|
||||
Name: "Final Fantasy XIV - NA",
|
||||
Source: "ffxiv",
|
||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||
var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "Final Fantasy XIV - NA",
|
||||
Source: domain.SourceCollectorFfxiv,
|
||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||
}
|
||||
|
||||
func TestFfxivGetParser(t *testing.T) {
|
||||
|
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -9,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/go-rod/rod"
|
||||
@ -18,7 +16,7 @@ import (
|
||||
|
||||
type RedditClient struct {
|
||||
config RedditConfig
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
}
|
||||
|
||||
type RedditConfig struct {
|
||||
@ -27,7 +25,7 @@ type RedditConfig struct {
|
||||
PullNSFW string
|
||||
}
|
||||
|
||||
func NewRedditClient(Record database.Source) *RedditClient {
|
||||
func NewRedditClient(Record domain.SourceEntity) *RedditClient {
|
||||
rc := RedditClient{
|
||||
record: Record,
|
||||
}
|
||||
@ -71,7 +69,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
// TODO Wire this to support the config options
|
||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
|
||||
|
||||
content, err := getHttpContent(Url)
|
||||
if err != nil {
|
||||
@ -88,10 +86,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
|
||||
var redditArticles []database.Article
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
|
||||
var redditArticles []domain.ArticleEntity
|
||||
for _, item := range items.Data.Children {
|
||||
var article database.Article
|
||||
var article domain.ArticleEntity
|
||||
article, err := rc.convertToArticle(item.Data)
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] %v", err)
|
||||
@ -104,8 +102,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []data
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
|
||||
var item database.Article
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
|
||||
var item domain.ArticleEntity
|
||||
|
||||
if source.Content == "" && source.Url != "" {
|
||||
item = rc.convertPicturePost(source)
|
||||
@ -131,65 +129,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Title: source.Title,
|
||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Pubdate: time.Now(),
|
||||
Video: sql.NullString{String: "null"},
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Authorimage: sql.NullString{String: "null"},
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Title: source.Title,
|
||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
PubDate: time.Now(),
|
||||
IsVideo: false,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
AuthorName: source.Author,
|
||||
AuthorImageUrl: "null",
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.Content,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.Media.RedditVideo.FallBackUrl,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// This post is nothing more then a redirect to another location.
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.UrlOverriddenByDest,
|
||||
}
|
||||
return item
|
||||
|
@ -3,18 +3,16 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var RedditRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
Url: "https://reddit.com/r/dadjokes",
|
||||
Tags: "reddit, dadjokes",
|
||||
var RedditRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "dadjokes",
|
||||
Source: domain.SourceCollectorRss,
|
||||
Url: "https://reddit.com/r/dadjokes",
|
||||
Tags: "reddit, dadjokes",
|
||||
}
|
||||
|
||||
func TestGetContent(t *testing.T) {
|
||||
|
@ -8,9 +8,10 @@ import (
|
||||
)
|
||||
|
||||
var rssRecord = domain.SourceEntity{
|
||||
ID: 1,
|
||||
ID: 1,
|
||||
DisplayName: "ArsTechnica",
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||
Source: domain.SourceCollectorRss,
|
||||
}
|
||||
|
||||
func TestRssClientConstructor(t *testing.T) {
|
||||
|
@ -1,19 +1,18 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/nicklaw5/helix/v2"
|
||||
)
|
||||
|
||||
type TwitchClient struct {
|
||||
SourceRecord database.Source
|
||||
SourceRecord domain.SourceEntity
|
||||
|
||||
// config
|
||||
monitorClips string
|
||||
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
||||
}
|
||||
|
||||
// This will let you replace the bound source record to keep the same session alive.
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
|
||||
tc.SourceRecord = source
|
||||
}
|
||||
|
||||
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||
var items []domain.ArticleEntity
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
}
|
||||
|
||||
for _, video := range posts {
|
||||
var article database.Article
|
||||
var article domain.ArticleEntity
|
||||
|
||||
AuthorName, err := tc.ExtractAuthor(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
article.Authorname = sql.NullString{String: AuthorName}
|
||||
article.AuthorName = AuthorName
|
||||
|
||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
article.Authorimage = sql.NullString{String: Authorimage}
|
||||
article.AuthorImageUrl = Authorimage
|
||||
|
||||
article.Description, err = tc.ExtractDescription(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
article.Pubdate, err = tc.ExtractPubDate(video)
|
||||
article.PubDate, err = tc.ExtractPubDate(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
article.Sourceid = tc.SourceRecord.ID
|
||||
article.SourceID = tc.SourceRecord.ID
|
||||
article.Tags, err = tc.ExtractTags(video, user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
var blank helix.User
|
||||
|
||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||
Logins: []string{tc.SourceRecord.Name},
|
||||
Logins: []string{tc.SourceRecord.DisplayName},
|
||||
})
|
||||
if err != nil {
|
||||
return blank, err
|
||||
|
@ -4,21 +4,20 @@ import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var TwitchSourceRecord = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "nintendo",
|
||||
Source: "Twitch",
|
||||
var TwitchSourceRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "nintendo",
|
||||
Source: domain.SourceCollectorTwitch,
|
||||
}
|
||||
|
||||
var TwitchInvalidRecord = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "EvilNintendo",
|
||||
Source: "Twitch",
|
||||
var TwitchInvalidRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "EvilNintendo",
|
||||
Source: domain.SourceCollectorTwitch,
|
||||
}
|
||||
|
||||
func TestTwitchLogin(t *testing.T) {
|
||||
|
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -12,11 +11,11 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
type YoutubeClient struct {
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
|
||||
// internal variables at time of collection
|
||||
channelID string
|
||||
@ -37,7 +36,7 @@ var (
|
||||
|
||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||
|
||||
func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
|
||||
yc := YoutubeClient{
|
||||
record: Record,
|
||||
cacheGroup: "youtube",
|
||||
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
}
|
||||
|
||||
// CheckSource will go and run all the commands needed to process a source.
|
||||
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||
var items []domain.ArticleEntity
|
||||
docParser, err := yc.GetParser(yc.record.Url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
|
||||
parser, err := yc.GetParser(item.Link)
|
||||
if err != nil {
|
||||
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
log.Printf("[YouTube] %v", msg)
|
||||
}
|
||||
|
||||
var article = database.Article{
|
||||
Sourceid: yc.record.ID,
|
||||
Tags: tags,
|
||||
Title: item.Title,
|
||||
Url: item.Link,
|
||||
Pubdate: *item.PublishedParsed,
|
||||
Thumbnail: thumb,
|
||||
Description: item.Description,
|
||||
Authorname: sql.NullString{String: item.Author.Name},
|
||||
Authorimage: sql.NullString{String: yc.avatarUri},
|
||||
var article = domain.ArticleEntity{
|
||||
SourceID: yc.record.ID,
|
||||
Tags: tags,
|
||||
Title: item.Title,
|
||||
Url: item.Link,
|
||||
PubDate: *item.PublishedParsed,
|
||||
Thumbnail: thumb,
|
||||
Description: item.Description,
|
||||
AuthorName: item.Author.Name,
|
||||
AuthorImageUrl: yc.avatarUri,
|
||||
}
|
||||
return article
|
||||
}
|
||||
|
@ -3,17 +3,15 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var YouTubeRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
Url: "https://youtube.com/gamegrumps",
|
||||
var YouTubeRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "dadjokes",
|
||||
Source: domain.SourceCollectorReddit,
|
||||
Url: "https://youtube.com/gamegrumps",
|
||||
}
|
||||
|
||||
func TestGetPageParser(t *testing.T) {
|
||||
|
Loading…
Reference in New Issue
Block a user