Features/reddit and dto (#2)

* updated db calls to new endpoints.  Not all are finished yet

* updated models to use the new dto objects to hide orm values

* updated reddit to build articles based on what type of post it is

* getting reddit ready to post the articles to the db

* Now able to post to the db under the new DTO object

* moved the reddit model to articles model to the struct and out of main

database/articles.go

@@ -1,7 +1,11 @@
 package database
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"net/http"
 
 	"github.com/jtom38/newsbot/collector/domain/model"
 )
@@ -11,24 +15,85 @@ type ArticlesClient struct {
 	rootUri string
 }
 
-func (ac *ArticlesClient) List() []model.Articles {
+func (ac *ArticlesClient) List() ([]model.Articles, error) {
 	var items []model.Articles
-	return items
+	url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
+	resp, err := getContent(url)
+	if err != nil {
+		return items, err
+	}
+
+	err = json.Unmarshal(resp, &items)
+	if err != nil {
+		return []model.Articles{}, err
+	}
+
+	return items, nil
 }
 
-func (ac *ArticlesClient) Find() []model.Articles {
-	var items []model.Articles
-	return items
+func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
+	var items model.Articles
+	url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
+	resp, err := getContent(url)
+	if err != nil {
+		return items, err
+	}
+
+	err = json.Unmarshal(resp, &items)
+	if err != nil {
+		return items, err
+	}
+
+	return items, nil
 }
 
-func (ac *ArticlesClient) FindByUrl(url string) model.Articles {
-	return model.Articles{}
+func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
+	var item model.Articles
+	get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
+	resp, err := getContent(get)
+	if err != nil {
+		return item, err
+	}
+
+	err = json.Unmarshal(resp, &item)
+	if err != nil {
+		return item, err
+	}
+
+	return item, nil
 }
 
 func (ac *ArticlesClient) Delete(id int32) error {
 	return errors.New("not implemented")
 }
 
-func (ac *ArticlesClient) Add() error {
-	return errors.New("not implemented")
+func (ac *ArticlesClient) Add(item model.Articles) error {
+	//return errors.New("not implemented")
+	url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
+
+	bItem, err := json.Marshal(item)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	defer resp.Body.Close()
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		return errors.New("failed to post to the DB")
+	}
+
+	return nil
+	//body, err := ioutil.ReadAll(resp.Body)
+	//if err != nil { return err }
+
 }

database/common.go

@@ -1,8 +1,9 @@
 package database
 
 import (
-	"log"
+	"errors"
 	"io/ioutil"
+	"log"
 	"net/http"
 
 	"github.com/jtom38/newsbot/collector/services"
@@ -27,25 +28,30 @@ func NewDatabaseClient() DatabaseClient {
 	return client
 }
 
-func getContent(url string) []byte {
+func getContent(url string) ([]byte, error) {
 	client := &http.Client{}
+	var blank []byte
 	
 	req, err := http.NewRequest("GET", url, nil)
-	if err != nil { log.Fatalln(err) }
+	if err != nil { return blank, err }
 
 	// set the user agent header to avoid kick backs.. as much
 	req.Header.Set("User-Agent", getUserAgent())
 
 	log.Printf("Requesting content from %v\n", url)
 	resp, err := client.Do(req)
-	if err != nil { log.Fatalln(err) }
+	if err != nil { return blank, err }
+	if resp.StatusCode == 404 { 
+		err = errors.New("404 not found")
+		return blank, err
+	}
 	defer resp.Body.Close()
 
 	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil { log.Fatalln(err) }
+	if err != nil { return blank, err }
 
 	//log.Println(string(body))
-	return body
+	return body, nil
 }
 
 func httpDelete(url string) error {

database/sources.go

@@ -14,10 +14,11 @@ type SourcesClient struct {
 
 func (sb *SourcesClient) List() ([]model.Sources, error) {
 	var items []model.Sources
-	url := fmt.Sprintf("%v/v1/sources", sb.rootUri)
-	resp := getContent(url)
+	url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
+	resp, err := getContent(url)
+	if err != nil { return items, err }
 
-	err := json.Unmarshal(resp, &items)
+	err = json.Unmarshal(resp, &items)
 	if err != nil { return []model.Sources{}, err }
 	
 	return items, nil

domain/model/database.go

@@ -6,19 +6,15 @@ import (
 
 // Articles represents the model for an Article
 type Articles struct {
-	ID int64 `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-
-	SourceId int32 `json:"sourceId"`
+	ID uint `json:"ID"`
+	SourceID uint `json:"sourceId"`
 	Tags string	`json:"tags"`
 	Title string `json:"title"`
 	Url string `json:"url"`
 	PubDate time.Time `json:"pubdate"`
 	Video string `json:"video"`
-	VideoHeight int16 `json:"videoHeight"`
-	VideoWidth int16 `json:"videoWidth"`
+	VideoHeight uint16 `json:"videoHeight"`
+	VideoWidth uint16 `json:"videoWidth"`
 	Thumbnail string `json:"thumbnail"`
 	Description string `json:"description"`
 	AuthorName string `json:"authorName"`
@@ -26,7 +22,7 @@ type Articles struct {
 }
 
 type DiscordQueue struct {
-	ID int64 `json:"ID"`
+	ID uint `json:"ID"`
 	CreatedAt time.Time `json:"CreatedAt"`
 	UpdatedAt time.Time `json:"UpdatedAt"`
 	DeletedAt time.Time `json:"DeletedAt"`
@@ -34,7 +30,7 @@ type DiscordQueue struct {
 }
 
 type DiscordWebHooks struct {
-	ID int32 `json:"ID"`
+	ID uint `json:"ID"`
 	CreatedAt time.Time `json:"CreatedAt"`
 	UpdatedAt time.Time `json:"UpdatedAt"`
 	DeletedAt time.Time `json:"DeletedAt"`
@@ -48,7 +44,7 @@ type DiscordWebHooks struct {
 }
 
 type Icons struct {
-	ID int32 `json:"ID"`
+	ID uint `json:"ID"`
 	CreatedAt time.Time `json:"CreatedAt"`
 	UpdatedAt time.Time `json:"UpdatedAt"`
 	DeletedAt time.Time `json:"DeletedAt"`
@@ -58,7 +54,7 @@ type Icons struct {
 }
 
 type Settings struct {
-	ID int16 `json:"ID"`
+	ID uint `json:"ID"`
 	CreatedAt time.Time `json:"CreatedAt"`
 	UpdatedAt time.Time `json:"UpdatedAt"`
 	DeletedAt time.Time `json:"DeletedAt"`
@@ -69,11 +65,7 @@ type Settings struct {
 }
 
 type Sources struct {
-	ID int32 `json:"ID"`
-	CreatedAt time.Time `json:"CreatedAt"`
-	UpdatedAt time.Time `json:"UpdatedAt"`
-	DeletedAt time.Time `json:"DeletedAt"`
-
+	ID uint 		`json:"ID"`
 	Site string		`json:"site"`
 	Name string		`json:"name"`
 	Source string	`json:"source"`
@@ -85,14 +77,14 @@ type Sources struct {
 }
 
 type SourceLinks struct {
-	ID int32 `json:"ID"`
+	ID uint `json:"ID"`
 	CreatedAt time.Time `json:"CreatedAt"`
 	UpdatedAt time.Time `json:"UpdatedAt"`
 	DeletedAt time.Time `json:"DeletedAt"`
 	
-	SourceID string		`json:"sourceId"`
+	SourceID uint		`json:"sourceId"`
 	SourceType string	`json:"sourceType"`
 	SourceName string	`json:"sourceName"`
-	DiscordID string	`json:"discordId"`
+	DiscordID uint	`json:"discordId"`
 	DiscordName string	`json:"discordName"`
 }

domain/model/reddit.go

@@ -29,6 +29,9 @@ type RedditPost struct {
 	IsVideo bool `json:"is_video"`
 	Media RedditPostMedia `json:"media"`
 	Url string `json:"url"`
+	UrlOverriddenByDest string `json:"url_overridden_by_dest"`
+
+	Thumbnail string `json:"thumbnail"`
 }
 
 // RedditPostMedia defines if the post contains a video that is hosted on Reddit.

main.go

@@ -39,7 +39,6 @@ func main() {
 
 func CheckReddit() {
 	dc := database.NewDatabaseClient()
-	dc.Articles.List()
 	sources, err := dc.Sources.FindBySource("reddit")
 	if err != nil { log.Println(err) }
 
@@ -47,12 +46,13 @@ func CheckReddit() {
 	raw, err := rc.GetContent()
 	if err != nil { log.Println(err) }
 	
-	var redditArticles []model.Articles
-	for _, item := range raw.Data.Children {
-		var article model.Articles
-		article, err = rc.ConvertToArticle(item.Data)
-		redditArticles = append(redditArticles, article)
-	}
-	dc.Articles.Add()
+	redditArticles := rc.ConvertToArticles(raw)
 	
+	for _, item := range redditArticles {		
+		_, err = dc.Articles.FindByUrl(item.Url)
+		if err != nil {
+			err = dc.Articles.Add(item)
+			if err != nil { log.Println("Failed to post article.")}
+		}
+	}
 }

services/reddit.go

@@ -2,9 +2,10 @@ package services
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
-	"errors"
+	"time"
 
 	"github.com/jtom38/newsbot/collector/domain/model"
 )
@@ -12,7 +13,7 @@ import (
 type RedditClient struct {
 	subreddit string
 	url string
-	sourceId int32
+	sourceId uint
 }
 
 var (
@@ -28,7 +29,7 @@ func init() {
 	PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW)
 }
 
-func NewReddit(subreddit string, sourceID int32) RedditClient {
+func NewReddit(subreddit string, sourceID uint) RedditClient {
 	rc := RedditClient{
 		subreddit: subreddit,
 		url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
@@ -51,9 +52,20 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
 	return items, nil
 }
 
+func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles {
+	var redditArticles []model.Articles
+	for _, item := range items.Data.Children {
+		var article model.Articles
+		article, err := rc.convertToArticle(item.Data)
+		if err != nil { log.Println(err); continue }
+		redditArticles = append(redditArticles, article)
+	}
+	return redditArticles
+}
+
 // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
 // This data can be passed to the database.
-func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) {
+func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) {
 	var item model.Articles
 
 	
@@ -61,6 +73,18 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
 		item = rc.convertPicturePost(source)
 	}
 	
+	if source.Media.RedditVideo.FallBackUrl != "" {
+		item = rc.convertVideoPost(source)
+	}
+
+	if source.Content != "" {
+		item = rc.convertTextPost(source)
+	}
+
+	if source.UrlOverriddenByDest != "" {
+		item = rc.convertRedirectPost(source)
+	}
+
 	if item.Description == "" {
 		var err = errors.New("reddit post failed to parse correctly")
 		return item, err
@@ -71,9 +95,28 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
 
 func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
 	var item = model.Articles{
-		SourceId: rc.sourceId,
-		Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink),
+		SourceID: rc.sourceId,
+		Tags: "a",
 		Title: source.Title,
+		Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		PubDate: time.Now(),
+		Video: "null",
+		VideoHeight: 0,
+		VideoWidth: 0,
+		Thumbnail: source.Thumbnail,
+		Description: source.Content,
+		AuthorName: source.Author,
+		AuthorImage: "null",
+	}
+	return item
+}
+
+func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles {
+	var item = model.Articles{
+		SourceID: rc.sourceId,
+		Tags: "a",
+		Title: source.Title,
+		Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
 		AuthorName: source.Author,
 		Description: source.Content,
 		
@@ -81,10 +124,27 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Article
 	return item
 }
 
-func (rc RedditClient) isTextPost(source model.RedditPost) {
-
+func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles {
+	var item = model.Articles{
+		SourceID: rc.sourceId,
+		Tags: "a",
+		Title: source.Title,
+		Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		AuthorName: source.Author,
+		Description: source.Media.RedditVideo.FallBackUrl,
+	}
+	return item
 }
 
-func (rc RedditClient) isVideoPost(source model.RedditPost) {
-
+// This post is nothing more then a redirect to another location.
+func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles {
+	var item = model.Articles{
+		SourceID: rc.sourceId,
+		Tags: "a",
+		Title: source.Title,
+		Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
+		AuthorName: source.Author,
+		Description: source.UrlOverriddenByDest,
+	}
+	return item
 }