Features/reddit and dto (#2)

* updated db calls to new endpoints.  Not all are finished yet

* updated models to use the new dto objects to hide orm values

* updated reddit to build articles based on what type of post it is

* getting reddit ready to post the articles to the db

* Now able to post to the db under the new DTO object

* moved the reddit model to articles model to the struct and out of main
This commit is contained in:
James Tombleson 2022-04-07 14:53:40 -07:00 committed by GitHub
parent 281fcb2d8a
commit 2b959e140c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 184 additions and 57 deletions

View File

@ -1,7 +1,11 @@
package database package database
import ( import (
"bytes"
"encoding/json"
"errors" "errors"
"fmt"
"net/http"
"github.com/jtom38/newsbot/collector/domain/model" "github.com/jtom38/newsbot/collector/domain/model"
) )
@ -11,24 +15,85 @@ type ArticlesClient struct {
rootUri string rootUri string
} }
func (ac *ArticlesClient) List() []model.Articles { func (ac *ArticlesClient) List() ([]model.Articles, error) {
var items []model.Articles var items []model.Articles
return items url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
resp, err := getContent(url)
if err != nil {
return items, err
} }
func (ac *ArticlesClient) Find() []model.Articles { err = json.Unmarshal(resp, &items)
var items []model.Articles if err != nil {
return items return []model.Articles{}, err
} }
func (ac *ArticlesClient) FindByUrl(url string) model.Articles { return items, nil
return model.Articles{} }
func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
var items model.Articles
url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return items, err
}
return items, nil
}
func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
var item model.Articles
get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
resp, err := getContent(get)
if err != nil {
return item, err
}
err = json.Unmarshal(resp, &item)
if err != nil {
return item, err
}
return item, nil
} }
func (ac *ArticlesClient) Delete(id int32) error { func (ac *ArticlesClient) Delete(id int32) error {
return errors.New("not implemented") return errors.New("not implemented")
} }
func (ac *ArticlesClient) Add() error { func (ac *ArticlesClient) Add(item model.Articles) error {
return errors.New("not implemented") //return errors.New("not implemented")
url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
bItem, err := json.Marshal(item)
if err != nil {
return err
}
client := &http.Client{}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
defer resp.Body.Close()
if err != nil {
return err
}
if resp.StatusCode != 200 {
return errors.New("failed to post to the DB")
}
return nil
//body, err := ioutil.ReadAll(resp.Body)
//if err != nil { return err }
} }

View File

@ -1,8 +1,9 @@
package database package database
import ( import (
"log" "errors"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"github.com/jtom38/newsbot/collector/services" "github.com/jtom38/newsbot/collector/services"
@ -27,25 +28,30 @@ func NewDatabaseClient() DatabaseClient {
return client return client
} }
func getContent(url string) []byte { func getContent(url string) ([]byte, error) {
client := &http.Client{} client := &http.Client{}
var blank []byte
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { log.Fatalln(err) } if err != nil { return blank, err }
// set the user agent header to avoid kick backs.. as much // set the user agent header to avoid kick backs.. as much
req.Header.Set("User-Agent", getUserAgent()) req.Header.Set("User-Agent", getUserAgent())
log.Printf("Requesting content from %v\n", url) log.Printf("Requesting content from %v\n", url)
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { log.Fatalln(err) } if err != nil { return blank, err }
if resp.StatusCode == 404 {
err = errors.New("404 not found")
return blank, err
}
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { log.Fatalln(err) } if err != nil { return blank, err }
//log.Println(string(body)) //log.Println(string(body))
return body return body, nil
} }
func httpDelete(url string) error { func httpDelete(url string) error {

View File

@ -14,10 +14,11 @@ type SourcesClient struct {
func (sb *SourcesClient) List() ([]model.Sources, error) { func (sb *SourcesClient) List() ([]model.Sources, error) {
var items []model.Sources var items []model.Sources
url := fmt.Sprintf("%v/v1/sources", sb.rootUri) url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
resp := getContent(url) resp, err := getContent(url)
if err != nil { return items, err }
err := json.Unmarshal(resp, &items) err = json.Unmarshal(resp, &items)
if err != nil { return []model.Sources{}, err } if err != nil { return []model.Sources{}, err }
return items, nil return items, nil

View File

@ -6,19 +6,15 @@ import (
// Articles represents the model for an Article // Articles represents the model for an Article
type Articles struct { type Articles struct {
ID int64 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"` SourceID uint `json:"sourceId"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
SourceId int32 `json:"sourceId"`
Tags string `json:"tags"` Tags string `json:"tags"`
Title string `json:"title"` Title string `json:"title"`
Url string `json:"url"` Url string `json:"url"`
PubDate time.Time `json:"pubdate"` PubDate time.Time `json:"pubdate"`
Video string `json:"video"` Video string `json:"video"`
VideoHeight int16 `json:"videoHeight"` VideoHeight uint16 `json:"videoHeight"`
VideoWidth int16 `json:"videoWidth"` VideoWidth uint16 `json:"videoWidth"`
Thumbnail string `json:"thumbnail"` Thumbnail string `json:"thumbnail"`
Description string `json:"description"` Description string `json:"description"`
AuthorName string `json:"authorName"` AuthorName string `json:"authorName"`
@ -26,7 +22,7 @@ type Articles struct {
} }
type DiscordQueue struct { type DiscordQueue struct {
ID int64 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"` CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"` UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"` DeletedAt time.Time `json:"DeletedAt"`
@ -34,7 +30,7 @@ type DiscordQueue struct {
} }
type DiscordWebHooks struct { type DiscordWebHooks struct {
ID int32 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"` CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"` UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"` DeletedAt time.Time `json:"DeletedAt"`
@ -48,7 +44,7 @@ type DiscordWebHooks struct {
} }
type Icons struct { type Icons struct {
ID int32 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"` CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"` UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"` DeletedAt time.Time `json:"DeletedAt"`
@ -58,7 +54,7 @@ type Icons struct {
} }
type Settings struct { type Settings struct {
ID int16 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"` CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"` UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"` DeletedAt time.Time `json:"DeletedAt"`
@ -69,11 +65,7 @@ type Settings struct {
} }
type Sources struct { type Sources struct {
ID int32 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
Site string `json:"site"` Site string `json:"site"`
Name string `json:"name"` Name string `json:"name"`
Source string `json:"source"` Source string `json:"source"`
@ -85,14 +77,14 @@ type Sources struct {
} }
type SourceLinks struct { type SourceLinks struct {
ID int32 `json:"ID"` ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"` CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"` UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"` DeletedAt time.Time `json:"DeletedAt"`
SourceID string `json:"sourceId"` SourceID uint `json:"sourceId"`
SourceType string `json:"sourceType"` SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"` SourceName string `json:"sourceName"`
DiscordID string `json:"discordId"` DiscordID uint `json:"discordId"`
DiscordName string `json:"discordName"` DiscordName string `json:"discordName"`
} }

View File

@ -29,6 +29,9 @@ type RedditPost struct {
IsVideo bool `json:"is_video"` IsVideo bool `json:"is_video"`
Media RedditPostMedia `json:"media"` Media RedditPostMedia `json:"media"`
Url string `json:"url"` Url string `json:"url"`
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
Thumbnail string `json:"thumbnail"`
} }
// RedditPostMedia defines if the post contains a video that is hosted on Reddit. // RedditPostMedia defines if the post contains a video that is hosted on Reddit.

16
main.go
View File

@ -39,7 +39,6 @@ func main() {
func CheckReddit() { func CheckReddit() {
dc := database.NewDatabaseClient() dc := database.NewDatabaseClient()
dc.Articles.List()
sources, err := dc.Sources.FindBySource("reddit") sources, err := dc.Sources.FindBySource("reddit")
if err != nil { log.Println(err) } if err != nil { log.Println(err) }
@ -47,12 +46,13 @@ func CheckReddit() {
raw, err := rc.GetContent() raw, err := rc.GetContent()
if err != nil { log.Println(err) } if err != nil { log.Println(err) }
var redditArticles []model.Articles redditArticles := rc.ConvertToArticles(raw)
for _, item := range raw.Data.Children {
var article model.Articles
article, err = rc.ConvertToArticle(item.Data)
redditArticles = append(redditArticles, article)
}
dc.Articles.Add()
for _, item := range redditArticles {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil {
err = dc.Articles.Add(item)
if err != nil { log.Println("Failed to post article.")}
}
}
} }

View File

@ -2,9 +2,10 @@ package services
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"log" "log"
"errors" "time"
"github.com/jtom38/newsbot/collector/domain/model" "github.com/jtom38/newsbot/collector/domain/model"
) )
@ -12,7 +13,7 @@ import (
type RedditClient struct { type RedditClient struct {
subreddit string subreddit string
url string url string
sourceId int32 sourceId uint
} }
var ( var (
@ -28,7 +29,7 @@ func init() {
PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW) PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW)
} }
func NewReddit(subreddit string, sourceID int32) RedditClient { func NewReddit(subreddit string, sourceID uint) RedditClient {
rc := RedditClient{ rc := RedditClient{
subreddit: subreddit, subreddit: subreddit,
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit), url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
@ -51,9 +52,20 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
return items, nil return items, nil
} }
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles {
var redditArticles []model.Articles
for _, item := range items.Data.Children {
var article model.Articles
article, err := rc.convertToArticle(item.Data)
if err != nil { log.Println(err); continue }
redditArticles = append(redditArticles, article)
}
return redditArticles
}
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) { func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) {
var item model.Articles var item model.Articles
@ -61,6 +73,18 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
item = rc.convertPicturePost(source) item = rc.convertPicturePost(source)
} }
if source.Media.RedditVideo.FallBackUrl != "" {
item = rc.convertVideoPost(source)
}
if source.Content != "" {
item = rc.convertTextPost(source)
}
if source.UrlOverriddenByDest != "" {
item = rc.convertRedirectPost(source)
}
if item.Description == "" { if item.Description == "" {
var err = errors.New("reddit post failed to parse correctly") var err = errors.New("reddit post failed to parse correctly")
return item, err return item, err
@ -71,9 +95,28 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles { func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
var item = model.Articles{ var item = model.Articles{
SourceId: rc.sourceId, SourceID: rc.sourceId,
Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink), Tags: "a",
Title: source.Title, Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
PubDate: time.Now(),
Video: "null",
VideoHeight: 0,
VideoWidth: 0,
Thumbnail: source.Thumbnail,
Description: source.Content,
AuthorName: source.Author,
AuthorImage: "null",
}
return item
}
func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author, AuthorName: source.Author,
Description: source.Content, Description: source.Content,
@ -81,10 +124,27 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Article
return item return item
} }
func (rc RedditClient) isTextPost(source model.RedditPost) { func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.Media.RedditVideo.FallBackUrl,
}
return item
} }
func (rc RedditClient) isVideoPost(source model.RedditPost) { // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.UrlOverriddenByDest,
}
return item
} }