Features/reddit and dto (#2)
* updated db calls to new endpoints. Not all are finished yet * updated models to use the new dto objects to hide orm values * updated reddit to build articles based on what type of post it is * getting reddit ready to post the articles to the db * Now able to post to the db under the new DTO object * moved the reddit model to articles model to the struct and out of main
This commit is contained in:
parent
281fcb2d8a
commit
2b959e140c
@ -1,7 +1,11 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
@ -11,24 +15,85 @@ type ArticlesClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) List() []model.Articles {
|
||||
func (ac *ArticlesClient) List() ([]model.Articles, error) {
|
||||
var items []model.Articles
|
||||
return items
|
||||
url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
|
||||
resp, err := getContent(url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(resp, &items)
|
||||
if err != nil {
|
||||
return []model.Articles{}, err
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Find() []model.Articles {
|
||||
var items []model.Articles
|
||||
return items
|
||||
func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
|
||||
var items model.Articles
|
||||
url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
|
||||
resp, err := getContent(url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(resp, &items)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) FindByUrl(url string) model.Articles {
|
||||
return model.Articles{}
|
||||
func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
|
||||
var item model.Articles
|
||||
get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
|
||||
resp, err := getContent(get)
|
||||
if err != nil {
|
||||
return item, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(resp, &item)
|
||||
if err != nil {
|
||||
return item, err
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Delete(id int32) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Add() error {
|
||||
return errors.New("not implemented")
|
||||
func (ac *ArticlesClient) Add(item model.Articles) error {
|
||||
//return errors.New("not implemented")
|
||||
url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
|
||||
|
||||
bItem, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
defer resp.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return errors.New("failed to post to the DB")
|
||||
}
|
||||
|
||||
return nil
|
||||
//body, err := ioutil.ReadAll(resp.Body)
|
||||
//if err != nil { return err }
|
||||
|
||||
}
|
||||
|
@ -1,8 +1,9 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"log"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
@ -27,25 +28,30 @@ func NewDatabaseClient() DatabaseClient {
|
||||
return client
|
||||
}
|
||||
|
||||
func getContent(url string) []byte {
|
||||
func getContent(url string) ([]byte, error) {
|
||||
client := &http.Client{}
|
||||
var blank []byte
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
if err != nil { return blank, err }
|
||||
|
||||
// set the user agent header to avoid kick backs.. as much
|
||||
req.Header.Set("User-Agent", getUserAgent())
|
||||
|
||||
log.Printf("Requesting content from %v\n", url)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
if err != nil { return blank, err }
|
||||
if resp.StatusCode == 404 {
|
||||
err = errors.New("404 not found")
|
||||
return blank, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
if err != nil { return blank, err }
|
||||
|
||||
//log.Println(string(body))
|
||||
return body
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func httpDelete(url string) error {
|
||||
|
@ -14,10 +14,11 @@ type SourcesClient struct {
|
||||
|
||||
func (sb *SourcesClient) List() ([]model.Sources, error) {
|
||||
var items []model.Sources
|
||||
url := fmt.Sprintf("%v/v1/sources", sb.rootUri)
|
||||
resp := getContent(url)
|
||||
url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
|
||||
resp, err := getContent(url)
|
||||
if err != nil { return items, err }
|
||||
|
||||
err := json.Unmarshal(resp, &items)
|
||||
err = json.Unmarshal(resp, &items)
|
||||
if err != nil { return []model.Sources{}, err }
|
||||
|
||||
return items, nil
|
||||
|
@ -6,19 +6,15 @@ import (
|
||||
|
||||
// Articles represents the model for an Article
|
||||
type Articles struct {
|
||||
ID int64 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
SourceId int32 `json:"sourceId"`
|
||||
ID uint `json:"ID"`
|
||||
SourceID uint `json:"sourceId"`
|
||||
Tags string `json:"tags"`
|
||||
Title string `json:"title"`
|
||||
Url string `json:"url"`
|
||||
PubDate time.Time `json:"pubdate"`
|
||||
Video string `json:"video"`
|
||||
VideoHeight int16 `json:"videoHeight"`
|
||||
VideoWidth int16 `json:"videoWidth"`
|
||||
VideoHeight uint16 `json:"videoHeight"`
|
||||
VideoWidth uint16 `json:"videoWidth"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
Description string `json:"description"`
|
||||
AuthorName string `json:"authorName"`
|
||||
@ -26,7 +22,7 @@ type Articles struct {
|
||||
}
|
||||
|
||||
type DiscordQueue struct {
|
||||
ID int64 `json:"ID"`
|
||||
ID uint `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
@ -34,7 +30,7 @@ type DiscordQueue struct {
|
||||
}
|
||||
|
||||
type DiscordWebHooks struct {
|
||||
ID int32 `json:"ID"`
|
||||
ID uint `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
@ -48,7 +44,7 @@ type DiscordWebHooks struct {
|
||||
}
|
||||
|
||||
type Icons struct {
|
||||
ID int32 `json:"ID"`
|
||||
ID uint `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
@ -58,7 +54,7 @@ type Icons struct {
|
||||
}
|
||||
|
||||
type Settings struct {
|
||||
ID int16 `json:"ID"`
|
||||
ID uint `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
@ -69,11 +65,7 @@ type Settings struct {
|
||||
}
|
||||
|
||||
type Sources struct {
|
||||
ID int32 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
ID uint `json:"ID"`
|
||||
Site string `json:"site"`
|
||||
Name string `json:"name"`
|
||||
Source string `json:"source"`
|
||||
@ -85,14 +77,14 @@ type Sources struct {
|
||||
}
|
||||
|
||||
type SourceLinks struct {
|
||||
ID int32 `json:"ID"`
|
||||
ID uint `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
SourceID string `json:"sourceId"`
|
||||
SourceID uint `json:"sourceId"`
|
||||
SourceType string `json:"sourceType"`
|
||||
SourceName string `json:"sourceName"`
|
||||
DiscordID string `json:"discordId"`
|
||||
DiscordID uint `json:"discordId"`
|
||||
DiscordName string `json:"discordName"`
|
||||
}
|
||||
|
@ -29,6 +29,9 @@ type RedditPost struct {
|
||||
IsVideo bool `json:"is_video"`
|
||||
Media RedditPostMedia `json:"media"`
|
||||
Url string `json:"url"`
|
||||
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
|
||||
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
}
|
||||
|
||||
// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
|
||||
|
16
main.go
16
main.go
@ -39,7 +39,6 @@ func main() {
|
||||
|
||||
func CheckReddit() {
|
||||
dc := database.NewDatabaseClient()
|
||||
dc.Articles.List()
|
||||
sources, err := dc.Sources.FindBySource("reddit")
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
@ -47,12 +46,13 @@ func CheckReddit() {
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
var redditArticles []model.Articles
|
||||
for _, item := range raw.Data.Children {
|
||||
var article model.Articles
|
||||
article, err = rc.ConvertToArticle(item.Data)
|
||||
redditArticles = append(redditArticles, article)
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
|
||||
for _, item := range redditArticles {
|
||||
_, err = dc.Articles.FindByUrl(item.Url)
|
||||
if err != nil {
|
||||
err = dc.Articles.Add(item)
|
||||
if err != nil { log.Println("Failed to post article.")}
|
||||
}
|
||||
}
|
||||
dc.Articles.Add()
|
||||
|
||||
}
|
@ -2,9 +2,10 @@ package services
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
@ -12,7 +13,7 @@ import (
|
||||
type RedditClient struct {
|
||||
subreddit string
|
||||
url string
|
||||
sourceId int32
|
||||
sourceId uint
|
||||
}
|
||||
|
||||
var (
|
||||
@ -28,7 +29,7 @@ func init() {
|
||||
PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW)
|
||||
}
|
||||
|
||||
func NewReddit(subreddit string, sourceID int32) RedditClient {
|
||||
func NewReddit(subreddit string, sourceID uint) RedditClient {
|
||||
rc := RedditClient{
|
||||
subreddit: subreddit,
|
||||
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
|
||||
@ -51,15 +52,38 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []model.Articles {
|
||||
var redditArticles []model.Articles
|
||||
for _, item := range items.Data.Children {
|
||||
var article model.Articles
|
||||
article, err := rc.convertToArticle(item.Data)
|
||||
if err != nil { log.Println(err); continue }
|
||||
redditArticles = append(redditArticles, article)
|
||||
}
|
||||
return redditArticles
|
||||
}
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) {
|
||||
func (rc RedditClient) convertToArticle(source model.RedditPost) (model.Articles, error) {
|
||||
var item model.Articles
|
||||
|
||||
|
||||
|
||||
if source.Content == "" && source.Url != ""{
|
||||
item = rc.convertPicturePost(source)
|
||||
}
|
||||
|
||||
if source.Media.RedditVideo.FallBackUrl != "" {
|
||||
item = rc.convertVideoPost(source)
|
||||
}
|
||||
|
||||
if source.Content != "" {
|
||||
item = rc.convertTextPost(source)
|
||||
}
|
||||
|
||||
if source.UrlOverriddenByDest != "" {
|
||||
item = rc.convertRedirectPost(source)
|
||||
}
|
||||
|
||||
if item.Description == "" {
|
||||
var err = errors.New("reddit post failed to parse correctly")
|
||||
@ -71,9 +95,28 @@ func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles
|
||||
|
||||
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceId: rc.sourceId,
|
||||
Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink),
|
||||
SourceID: rc.sourceId,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
PubDate: time.Now(),
|
||||
Video: "null",
|
||||
VideoHeight: 0,
|
||||
VideoWidth: 0,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
AuthorName: source.Author,
|
||||
AuthorImage: "null",
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertTextPost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
AuthorName: source.Author,
|
||||
Description: source.Content,
|
||||
|
||||
@ -81,10 +124,27 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Article
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) isTextPost(source model.RedditPost) {
|
||||
|
||||
func (rc RedditClient) convertVideoPost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
AuthorName: source.Author,
|
||||
Description: source.Media.RedditVideo.FallBackUrl,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) isVideoPost(source model.RedditPost) {
|
||||
|
||||
// This post is nothing more then a redirect to another location.
|
||||
func (rc *RedditClient) convertRedirectPost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceID: rc.sourceId,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
AuthorName: source.Author,
|
||||
Description: source.UrlOverriddenByDest,
|
||||
}
|
||||
return item
|
||||
}
|
Loading…
Reference in New Issue
Block a user