Features/reddit and dto (#2)

* updated db calls to new endpoints.  Not all are finished yet

* updated models to use the new dto objects to hide orm values

* updated reddit to build articles based on what type of post it is

* getting reddit ready to post the articles to the db

* Now able to post to the db under the new DTO object

* moved the reddit model to articles model to the struct and out of main
This commit is contained in:
James Tombleson 2022-04-07 14:53:40 -07:00 committed by GitHub
parent 281fcb2d8a
commit 2b959e140c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 184 additions and 57 deletions

View File

@ -1,7 +1,11 @@
package database
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/jtom38/newsbot/collector/domain/model"
)
@ -11,24 +15,85 @@ type ArticlesClient struct {
rootUri string
}
func (ac *ArticlesClient) List() []model.Articles {
func (ac *ArticlesClient) List() ([]model.Articles, error) {
var items []model.Articles
return items
url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return []model.Articles{}, err
}
return items, nil
}
func (ac *ArticlesClient) Find() []model.Articles {
var items []model.Articles
return items
func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
var items model.Articles
url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return items, err
}
return items, nil
}
func (ac *ArticlesClient) FindByUrl(url string) model.Articles {
return model.Articles{}
func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
var item model.Articles
get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
resp, err := getContent(get)
if err != nil {
return item, err
}
err = json.Unmarshal(resp, &item)
if err != nil {
return item, err
}
return item, nil
}
func (ac *ArticlesClient) Delete(id int32) error {
return errors.New("not implemented")
}
func (ac *ArticlesClient) Add() error {
return errors.New("not implemented")
func (ac *ArticlesClient) Add(item model.Articles) error {
//return errors.New("not implemented")
url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
bItem, err := json.Marshal(item)
if err != nil {
return err
}
client := &http.Client{}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
defer resp.Body.Close()
if err != nil {
return err
}
if resp.StatusCode != 200 {
return errors.New("failed to post to the DB")
}
return nil
//body, err := ioutil.ReadAll(resp.Body)
//if err != nil { return err }
}

View File

@ -1,8 +1,9 @@
package database
import (
"log"
"errors"
"io/ioutil"
"log"
"net/http"
"github.com/jtom38/newsbot/collector/services"
@ -27,25 +28,30 @@ func NewDatabaseClient() DatabaseClient {
return client
}
func getContent(url string) []byte {
func getContent(url string) ([]byte, error) {
client := &http.Client{}
var blank []byte
req, err := http.NewRequest("GET", url, nil)
if err != nil { log.Fatalln(err) }
if err != nil { return blank, err }
// set the user agent header to avoid kick backs.. as much
req.Header.Set("User-Agent", getUserAgent())
log.Printf("Requesting content from %v\n", url)
resp, err := client.Do(req)
if err != nil { log.Fatalln(err) }
if err != nil { return blank, err }
if resp.StatusCode == 404 {
err = errors.New("404 not found")
return blank, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil { log.Fatalln(err) }
if err != nil { return blank, err }
//log.Println(string(body))
return body
return body, nil
}
func httpDelete(url string) error {

View File

@ -14,10 +14,11 @@ type SourcesClient struct {
func (sb *SourcesClient) List() ([]model.Sources, error) {
var items []model.Sources
url := fmt.Sprintf("%v/v1/sources", sb.rootUri)
resp := getContent(url)
url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
resp, err := getContent(url)
if err != nil { return items, err }
err := json.Unmarshal(resp, &items)
err = json.Unmarshal(resp, &items)
if err != nil { return []model.Sources{}, err }
return items, nil

View File

@ -6,19 +6,15 @@ import (
// Articles represents the model for an Article
type Articles struct {
ID int64 `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
SourceId int32 `json:"sourceId"`
ID uint `json:"ID"`
SourceID uint `json:"sourceId"`
Tags string `json:"tags"`
Title string `json:"title"`
Url string `json:"url"`
PubDate time.Time `json:"pubdate"`
Video string `json:"video"`
VideoHeight int16 `json:"videoHeight"`
VideoWidth int16 `json:"videoWidth"`
VideoHeight uint16 `json:"videoHeight"`
VideoWidth uint16 `json:"videoWidth"`
Thumbnail string `json:"thumbnail"`
Description string `json:"description"`
AuthorName string `json:"authorName"`
@ -26,7 +22,7 @@ type Articles struct {
}
type DiscordQueue struct {
ID int64 `json:"ID"`
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
@ -34,7 +30,7 @@ type DiscordQueue struct {
}
type DiscordWebHooks struct {
ID int32 `json:"ID"`
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
@ -48,7 +44,7 @@ type DiscordWebHooks struct {
}
type Icons struct {
ID int32 `json:"ID"`
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
@ -58,7 +54,7 @@ type Icons struct {
}
type Settings struct {
ID int16 `json:"ID"`
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
@ -69,11 +65,7 @@ type Settings struct {
}
type Sources struct {
ID int32 `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
ID uint `json:"ID"`
Site string `json:"site"`
Name string `json:"name"`
Source string `json:"source"`
@ -85,14 +77,14 @@ type Sources struct {
}
type SourceLinks struct {
ID int32 `json:"ID"`
ID uint `json:"ID"`
CreatedAt time.Time `json:"CreatedAt"`
UpdatedAt time.Time `json:"UpdatedAt"`
DeletedAt time.Time `json:"DeletedAt"`
SourceID string `json:"sourceId"`
SourceID uint `json:"sourceId"`
SourceType string `json:"sourceType"`
SourceName string `json:"sourceName"`
DiscordID string `json:"discordId"`
DiscordID uint `json:"discordId"`
DiscordName string `json:"discordName"`
}

View File

@ -29,6 +29,9 @@ type RedditPost struct {
IsVideo bool `json:"is_video"`
Media RedditPostMedia `json:"media"`
Url string `json:"url"`
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
Thumbnail string `json:"thumbnail"`
}
// RedditPostMedia defines if the post contains a video that is hosted on Reddit.

16
main.go
View File

@ -39,7 +39,6 @@ func main() {
func CheckReddit() {
dc := database.NewDatabaseClient()
dc.Articles.List()
sources, err := dc.Sources.FindBySource("reddit")
if err != nil { log.Println(err) }
@ -47,12 +46,13 @@ func CheckReddit() {
raw, err := rc.GetContent()
if err != nil { log.Println(err) }
var redditArticles []model.Articles
for _, item := range raw.Data.Children {
var article model.Articles
article, err = rc.ConvertToArticle(item.Data)
redditArticles = append(redditArticles, article)
}
dc.Articles.Add()
redditArticles := rc.ConvertToArticles(raw)
for _, item := range redditArticles {
_, err = dc.Articles.FindByUrl(item.Url)
if err != nil {
err = dc.Articles.Add(item)
if err != nil { log.Println("Failed to post article.")}
}
}
}

View File

@ -2,9 +2,10 @@ package services
import (
"encoding/json"
"errors"
"fmt"
"log"
"errors"
"time"
"github.com/jtom38/newsbot/collector/domain/model"
)
@ -12,7 +13,7 @@ import (
type RedditClient struct {
subreddit string
url string
sourceId int32
sourceId uint
}
var (
@ -28,7 +29,7 @@ func init() {
PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW)
}
func NewReddit(subreddit string, sourceID int32) RedditClient {
func NewReddit(subreddit string, sourceID uint) RedditClient {
rc := RedditClient{
subreddit: subreddit,
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
@ -51,9 +52,20 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
return items, nil
}
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles {
var redditArticles []model.Articles
for _, item := range items.Data.Children {
var article model.Articles
article, err := rc.convertToArticle(item.Data)
if err != nil { log.Println(err); continue }
redditArticles = append(redditArticles, article)
}
return redditArticles
}
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database.
func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) {
func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) {
var item model.Articles
@ -61,6 +73,18 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
item = rc.convertPicturePost(source)
}
if source.Media.RedditVideo.FallBackUrl != "" {
item = rc.convertVideoPost(source)
}
if source.Content != "" {
item = rc.convertTextPost(source)
}
if source.UrlOverriddenByDest != "" {
item = rc.convertRedirectPost(source)
}
if item.Description == "" {
var err = errors.New("reddit post failed to parse correctly")
return item, err
@ -71,9 +95,28 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceId: rc.sourceId,
Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink),
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
PubDate: time.Now(),
Video: "null",
VideoHeight: 0,
VideoWidth: 0,
Thumbnail: source.Thumbnail,
Description: source.Content,
AuthorName: source.Author,
AuthorImage: "null",
}
return item
}
func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.Content,
@ -81,10 +124,27 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Article
return item
}
func (rc RedditClient) isTextPost(source model.RedditPost) {
func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.Media.RedditVideo.FallBackUrl,
}
return item
}
func (rc RedditClient) isVideoPost(source model.RedditPost) {
// This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles {
var item = model.Articles{
SourceID: rc.sourceId,
Tags: "a",
Title: source.Title,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
AuthorName: source.Author,
Description: source.UrlOverriddenByDest,
}
return item
}