From 2b959e140c5f9082b50bc82404d8c0919b09c9d1 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 7 Apr 2022 14:53:40 -0700 Subject: [PATCH] Features/reddit and dto (#2) * updated db calls to new endpoints. Not all are finished yet * updated models to use the new dto objects to hide orm values * updated reddit to build articles based on what type of post it is * getting reddit ready to post the articles to the db * Now able to post to the db under the new DTO object * moved the reddit model to articles model to the struct and out of main --- database/articles.go | 83 +++++++++++++++++++++++++++++++++++----- database/common.go | 18 ++++++--- database/sources.go | 7 ++-- domain/model/database.go | 32 ++++++---------- domain/model/reddit.go | 3 ++ main.go | 16 ++++---- services/reddit.go | 82 +++++++++++++++++++++++++++++++++------ 7 files changed, 184 insertions(+), 57 deletions(-) diff --git a/database/articles.go b/database/articles.go index bd1754a..5f125be 100644 --- a/database/articles.go +++ b/database/articles.go @@ -1,7 +1,11 @@ package database import ( + "bytes" + "encoding/json" "errors" + "fmt" + "net/http" "github.com/jtom38/newsbot/collector/domain/model" ) @@ -11,24 +15,85 @@ type ArticlesClient struct { rootUri string } -func (ac *ArticlesClient) List() []model.Articles { +func (ac *ArticlesClient) List() ([]model.Articles, error) { var items []model.Articles - return items + url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri) + resp, err := getContent(url) + if err != nil { + return items, err + } + + err = json.Unmarshal(resp, &items) + if err != nil { + return []model.Articles{}, err + } + + return items, nil } -func (ac *ArticlesClient) Find() []model.Articles { - var items []model.Articles - return items +func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) { + var items model.Articles + url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID) + resp, err := getContent(url) + if err != nil { + return items, err + } + + err = json.Unmarshal(resp, &items) + if err != nil { + return items, err + } + + return items, nil } -func (ac *ArticlesClient) FindByUrl(url string) model.Articles { - return model.Articles{} +func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) { + var item model.Articles + get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url) + resp, err := getContent(get) + if err != nil { + return item, err + } + + err = json.Unmarshal(resp, &item) + if err != nil { + return item, err + } + + return item, nil } func (ac *ArticlesClient) Delete(id int32) error { return errors.New("not implemented") } -func (ac *ArticlesClient) Add() error { - return errors.New("not implemented") +func (ac *ArticlesClient) Add(item model.Articles) error { + //return errors.New("not implemented") + url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri) + + bItem, err := json.Marshal(item) + if err != nil { + return err + } + + client := &http.Client{} + req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + defer resp.Body.Close() + if err != nil { + return err + } + if resp.StatusCode != 200 { + return errors.New("failed to post to the DB") + } + + return nil + //body, err := ioutil.ReadAll(resp.Body) + //if err != nil { return err } + } diff --git a/database/common.go b/database/common.go index eaaf8b6..6194be1 100644 --- a/database/common.go +++ b/database/common.go @@ -1,8 +1,9 @@ package database import ( - "log" + "errors" "io/ioutil" + "log" "net/http" "github.com/jtom38/newsbot/collector/services" @@ -27,25 +28,30 @@ func NewDatabaseClient() DatabaseClient { return client } -func getContent(url string) []byte { +func getContent(url string) ([]byte, error) { client := &http.Client{} + var blank []byte req, err := http.NewRequest("GET", url, nil) - if err != nil { log.Fatalln(err) } + if err != nil { return blank, err } // set the user agent header to avoid kick backs.. as much req.Header.Set("User-Agent", getUserAgent()) log.Printf("Requesting content from %v\n", url) resp, err := client.Do(req) - if err != nil { log.Fatalln(err) } + if err != nil { return blank, err } + if resp.StatusCode == 404 { + err = errors.New("404 not found") + return blank, err + } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) - if err != nil { log.Fatalln(err) } + if err != nil { return blank, err } //log.Println(string(body)) - return body + return body, nil } func httpDelete(url string) error { diff --git a/database/sources.go b/database/sources.go index bffad50..b1cdfb8 100644 --- a/database/sources.go +++ b/database/sources.go @@ -14,10 +14,11 @@ type SourcesClient struct { func (sb *SourcesClient) List() ([]model.Sources, error) { var items []model.Sources - url := fmt.Sprintf("%v/v1/sources", sb.rootUri) - resp := getContent(url) + url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri) + resp, err := getContent(url) + if err != nil { return items, err } - err := json.Unmarshal(resp, &items) + err = json.Unmarshal(resp, &items) if err != nil { return []model.Sources{}, err } return items, nil diff --git a/domain/model/database.go b/domain/model/database.go index 21d4a11..ebe1c3a 100644 --- a/domain/model/database.go +++ b/domain/model/database.go @@ -6,19 +6,15 @@ import ( // Articles represents the model for an Article type Articles struct { - ID int64 `json:"ID"` - CreatedAt time.Time `json:"CreatedAt"` - UpdatedAt time.Time `json:"UpdatedAt"` - DeletedAt time.Time `json:"DeletedAt"` - - SourceId int32 `json:"sourceId"` + ID uint `json:"ID"` + SourceID uint `json:"sourceId"` Tags string `json:"tags"` Title string `json:"title"` Url string `json:"url"` PubDate time.Time `json:"pubdate"` Video string `json:"video"` - VideoHeight int16 `json:"videoHeight"` - VideoWidth int16 `json:"videoWidth"` + VideoHeight uint16 `json:"videoHeight"` + VideoWidth uint16 `json:"videoWidth"` Thumbnail string `json:"thumbnail"` Description string `json:"description"` AuthorName string `json:"authorName"` @@ -26,7 +22,7 @@ type Articles struct { } type DiscordQueue struct { - ID int64 `json:"ID"` + ID uint `json:"ID"` CreatedAt time.Time `json:"CreatedAt"` UpdatedAt time.Time `json:"UpdatedAt"` DeletedAt time.Time `json:"DeletedAt"` @@ -34,7 +30,7 @@ type DiscordQueue struct { } type DiscordWebHooks struct { - ID int32 `json:"ID"` + ID uint `json:"ID"` CreatedAt time.Time `json:"CreatedAt"` UpdatedAt time.Time `json:"UpdatedAt"` DeletedAt time.Time `json:"DeletedAt"` @@ -48,7 +44,7 @@ type DiscordWebHooks struct { } type Icons struct { - ID int32 `json:"ID"` + ID uint `json:"ID"` CreatedAt time.Time `json:"CreatedAt"` UpdatedAt time.Time `json:"UpdatedAt"` DeletedAt time.Time `json:"DeletedAt"` @@ -58,7 +54,7 @@ type Icons struct { } type Settings struct { - ID int16 `json:"ID"` + ID uint `json:"ID"` CreatedAt time.Time `json:"CreatedAt"` UpdatedAt time.Time `json:"UpdatedAt"` DeletedAt time.Time `json:"DeletedAt"` @@ -69,11 +65,7 @@ type Settings struct { } type Sources struct { - ID int32 `json:"ID"` - CreatedAt time.Time `json:"CreatedAt"` - UpdatedAt time.Time `json:"UpdatedAt"` - DeletedAt time.Time `json:"DeletedAt"` - + ID uint `json:"ID"` Site string `json:"site"` Name string `json:"name"` Source string `json:"source"` @@ -85,14 +77,14 @@ type Sources struct { } type SourceLinks struct { - ID int32 `json:"ID"` + ID uint `json:"ID"` CreatedAt time.Time `json:"CreatedAt"` UpdatedAt time.Time `json:"UpdatedAt"` DeletedAt time.Time `json:"DeletedAt"` - SourceID string `json:"sourceId"` + SourceID uint `json:"sourceId"` SourceType string `json:"sourceType"` SourceName string `json:"sourceName"` - DiscordID string `json:"discordId"` + DiscordID uint `json:"discordId"` DiscordName string `json:"discordName"` } diff --git a/domain/model/reddit.go b/domain/model/reddit.go index b82ca57..92e7ac2 100644 --- a/domain/model/reddit.go +++ b/domain/model/reddit.go @@ -29,6 +29,9 @@ type RedditPost struct { IsVideo bool `json:"is_video"` Media RedditPostMedia `json:"media"` Url string `json:"url"` + UrlOverriddenByDest string `json:"url_overridden_by_dest"` + + Thumbnail string `json:"thumbnail"` } // RedditPostMedia defines if the post contains a video that is hosted on Reddit. diff --git a/main.go b/main.go index 1a352bb..6a027bf 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,6 @@ func main() { func CheckReddit() { dc := database.NewDatabaseClient() - dc.Articles.List() sources, err := dc.Sources.FindBySource("reddit") if err != nil { log.Println(err) } @@ -47,12 +46,13 @@ func CheckReddit() { raw, err := rc.GetContent() if err != nil { log.Println(err) } - var redditArticles []model.Articles - for _, item := range raw.Data.Children { - var article model.Articles - article, err = rc.ConvertToArticle(item.Data) - redditArticles = append(redditArticles, article) + redditArticles := rc.ConvertToArticles(raw) + + for _, item := range redditArticles { + _, err = dc.Articles.FindByUrl(item.Url) + if err != nil { + err = dc.Articles.Add(item) + if err != nil { log.Println("Failed to post article.")} + } } - dc.Articles.Add() - } \ No newline at end of file diff --git a/services/reddit.go b/services/reddit.go index be62aa8..6eeb8a2 100644 --- a/services/reddit.go +++ b/services/reddit.go @@ -2,9 +2,10 @@ package services import ( "encoding/json" + "errors" "fmt" "log" - "errors" + "time" "github.com/jtom38/newsbot/collector/domain/model" ) @@ -12,7 +13,7 @@ import ( type RedditClient struct { subreddit string url string - sourceId int32 + sourceId uint } var ( @@ -28,7 +29,7 @@ func init() { PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW) } -func NewReddit(subreddit string, sourceID int32) RedditClient { +func NewReddit(subreddit string, sourceID uint) RedditClient { rc := RedditClient{ subreddit: subreddit, url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit), @@ -51,15 +52,38 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) { return items, nil } +func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles { + var redditArticles []model.Articles + for _, item := range items.Data.Children { + var article model.Articles + article, err := rc.convertToArticle(item.Data) + if err != nil { log.Println(err); continue } + redditArticles = append(redditArticles, article) + } + return redditArticles +} + // ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // This data can be passed to the database. -func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) { +func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) { var item model.Articles - + if source.Content == "" && source.Url != ""{ item = rc.convertPicturePost(source) } + + if source.Media.RedditVideo.FallBackUrl != "" { + item = rc.convertVideoPost(source) + } + + if source.Content != "" { + item = rc.convertTextPost(source) + } + + if source.UrlOverriddenByDest != "" { + item = rc.convertRedirectPost(source) + } if item.Description == "" { var err = errors.New("reddit post failed to parse correctly") @@ -71,9 +95,28 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles { var item = model.Articles{ - SourceId: rc.sourceId, - Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink), + SourceID: rc.sourceId, + Tags: "a", Title: source.Title, + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + PubDate: time.Now(), + Video: "null", + VideoHeight: 0, + VideoWidth: 0, + Thumbnail: source.Thumbnail, + Description: source.Content, + AuthorName: source.Author, + AuthorImage: "null", + } + return item +} + +func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles { + var item = model.Articles{ + SourceID: rc.sourceId, + Tags: "a", + Title: source.Title, + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), AuthorName: source.Author, Description: source.Content, @@ -81,10 +124,27 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Article return item } -func (rc RedditClient) isTextPost(source model.RedditPost) { - +func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles { + var item = model.Articles{ + SourceID: rc.sourceId, + Tags: "a", + Title: source.Title, + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + AuthorName: source.Author, + Description: source.Media.RedditVideo.FallBackUrl, + } + return item } -func (rc RedditClient) isVideoPost(source model.RedditPost) { - +// This post is nothing more then a redirect to another location. +func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles { + var item = model.Articles{ + SourceID: rc.sourceId, + Tags: "a", + Title: source.Title, + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + AuthorName: source.Author, + Description: source.UrlOverriddenByDest, + } + return item } \ No newline at end of file