Features/subscription/features for portal (#34)
* added a route to delete subscriptions based on the ID given * added a new route to find a record based on the name and source * added a route to query Discord Web Hooks by Server and Channel names * tested the endpoints and they seem good to test more * updated some routes for subscriptions and formatted files * removed debug file * fixing some panic calls * swag
This commit is contained in:
parent
c161658487
commit
ff4075383a
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,5 +1,6 @@
|
|||||||
.env
|
.env
|
||||||
dev.session.sql
|
dev.session.sql
|
||||||
|
__debug_bin
|
||||||
|
|
||||||
.vscode
|
.vscode
|
||||||
|
|
||||||
|
@ -70,7 +70,7 @@ const docTemplate = `{
|
|||||||
"responses": {}
|
"responses": {}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/articles/{id}": {
|
"/articles/{ID}": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
"application/json"
|
"application/json"
|
||||||
@ -569,7 +569,6 @@ const docTemplate = `{
|
|||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
@ -582,7 +581,6 @@ const docTemplate = `{
|
|||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
@ -604,7 +602,6 @@ const docTemplate = `{
|
|||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
@ -644,9 +641,6 @@ const docTemplate = `{
|
|||||||
"/subscriptions/new/discordwebhook": {
|
"/subscriptions/new/discordwebhook": {
|
||||||
"post": {
|
"post": {
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Source",
|
|
||||||
"Discord",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||||
|
@ -61,7 +61,7 @@
|
|||||||
"responses": {}
|
"responses": {}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/articles/{id}": {
|
"/articles/{ID}": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
"application/json"
|
"application/json"
|
||||||
@ -560,7 +560,6 @@
|
|||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
@ -573,7 +572,6 @@
|
|||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
@ -595,7 +593,6 @@
|
|||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
@ -635,9 +632,6 @@
|
|||||||
"/subscriptions/new/discordwebhook": {
|
"/subscriptions/new/discordwebhook": {
|
||||||
"post": {
|
"post": {
|
||||||
"tags": [
|
"tags": [
|
||||||
"Config",
|
|
||||||
"Source",
|
|
||||||
"Discord",
|
|
||||||
"Subscription"
|
"Subscription"
|
||||||
],
|
],
|
||||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||||
|
@ -12,7 +12,7 @@ paths:
|
|||||||
summary: Lists the top 50 records
|
summary: Lists the top 50 records
|
||||||
tags:
|
tags:
|
||||||
- Articles
|
- Articles
|
||||||
/articles/{id}:
|
/articles/{ID}:
|
||||||
get:
|
get:
|
||||||
parameters:
|
parameters:
|
||||||
- description: uuid
|
- description: uuid
|
||||||
@ -391,7 +391,6 @@ paths:
|
|||||||
responses: {}
|
responses: {}
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
tags:
|
tags:
|
||||||
- Config
|
|
||||||
- Subscription
|
- Subscription
|
||||||
/subscriptions/byDiscordId:
|
/subscriptions/byDiscordId:
|
||||||
get:
|
get:
|
||||||
@ -406,7 +405,6 @@ paths:
|
|||||||
responses: {}
|
responses: {}
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
tags:
|
tags:
|
||||||
- Config
|
|
||||||
- Subscription
|
- Subscription
|
||||||
/subscriptions/bySourceId:
|
/subscriptions/bySourceId:
|
||||||
get:
|
get:
|
||||||
@ -421,7 +419,6 @@ paths:
|
|||||||
responses: {}
|
responses: {}
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
tags:
|
tags:
|
||||||
- Config
|
|
||||||
- Subscription
|
- Subscription
|
||||||
/subscriptions/discord/webhook/delete:
|
/subscriptions/discord/webhook/delete:
|
||||||
delete:
|
delete:
|
||||||
@ -454,8 +451,5 @@ paths:
|
|||||||
responses: {}
|
responses: {}
|
||||||
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
||||||
tags:
|
tags:
|
||||||
- Config
|
|
||||||
- Source
|
|
||||||
- Discord
|
|
||||||
- Subscription
|
- Subscription
|
||||||
swagger: "2.0"
|
swagger: "2.0"
|
||||||
|
@ -20,4 +20,3 @@ type Sources interface {
|
|||||||
ExtractTags(page *rod.Page) (string, error)
|
ExtractTags(page *rod.Page) (string, error)
|
||||||
ExtractTitle(page *rod.Page) (string, error)
|
ExtractTitle(page *rod.Page) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -10,7 +10,7 @@ type CacheItem struct {
|
|||||||
|
|
||||||
// Group defines what it should be a reference to.
|
// Group defines what it should be a reference to.
|
||||||
// youtube, reddit, ect
|
// youtube, reddit, ect
|
||||||
Group string
|
Group string
|
||||||
Expires time.Time
|
Expires time.Time
|
||||||
IsTainted bool
|
IsTainted bool
|
||||||
}
|
}
|
||||||
|
@ -6,85 +6,85 @@ import (
|
|||||||
|
|
||||||
// Articles represents the model for an Article
|
// Articles represents the model for an Article
|
||||||
type Articles struct {
|
type Articles struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
SourceID uint `json:"sourceId"`
|
SourceID uint `json:"sourceId"`
|
||||||
Tags string `json:"tags"`
|
Tags string `json:"tags"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Url string `json:"url"`
|
Url string `json:"url"`
|
||||||
PubDate time.Time `json:"pubdate"`
|
PubDate time.Time `json:"pubdate"`
|
||||||
Video string `json:"video"`
|
Video string `json:"video"`
|
||||||
VideoHeight uint16 `json:"videoHeight"`
|
VideoHeight uint16 `json:"videoHeight"`
|
||||||
VideoWidth uint16 `json:"videoWidth"`
|
VideoWidth uint16 `json:"videoWidth"`
|
||||||
Thumbnail string `json:"thumbnail"`
|
Thumbnail string `json:"thumbnail"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
AuthorName string `json:"authorName"`
|
AuthorName string `json:"authorName"`
|
||||||
AuthorImage string `json:"authorImage"`
|
AuthorImage string `json:"authorImage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type DiscordQueue struct {
|
type DiscordQueue struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
CreatedAt time.Time `json:"CreatedAt"`
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
DeletedAt time.Time `json:"DeletedAt"`
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
ArticleId string `json:"articleId"`
|
ArticleId string `json:"articleId"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type DiscordWebHooks struct {
|
type DiscordWebHooks struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
CreatedAt time.Time `json:"CreatedAt"`
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
DeletedAt time.Time `json:"DeletedAt"`
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Key string `json:"key"`
|
Key string `json:"key"`
|
||||||
Url string `json:"url"`
|
Url string `json:"url"`
|
||||||
Server string `json:"server"`
|
Server string `json:"server"`
|
||||||
Channel string `json:"channel"`
|
Channel string `json:"channel"`
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Icons struct {
|
type Icons struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
CreatedAt time.Time `json:"CreatedAt"`
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
DeletedAt time.Time `json:"DeletedAt"`
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
FileName string `json:"fileName"`
|
FileName string `json:"fileName"`
|
||||||
Site string `json:"site"`
|
Site string `json:"site"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Settings struct {
|
type Settings struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
CreatedAt time.Time `json:"CreatedAt"`
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
DeletedAt time.Time `json:"DeletedAt"`
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
Key string `json:"key"`
|
Key string `json:"key"`
|
||||||
Value string `json:"value"`
|
Value string `json:"value"`
|
||||||
Options string `json:"options"`
|
Options string `json:"options"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Sources struct {
|
type Sources struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
Site string `json:"site"`
|
Site string `json:"site"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Value string `json:"value"`
|
Value string `json:"value"`
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
Url string `json:"url"`
|
Url string `json:"url"`
|
||||||
Tags string `json:"tags"`
|
Tags string `json:"tags"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SourceLinks struct {
|
type SourceLinks struct {
|
||||||
ID uint `json:"ID"`
|
ID uint `json:"ID"`
|
||||||
CreatedAt time.Time `json:"CreatedAt"`
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
DeletedAt time.Time `json:"DeletedAt"`
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
SourceID uint `json:"sourceId"`
|
SourceID uint `json:"sourceId"`
|
||||||
SourceType string `json:"sourceType"`
|
SourceType string `json:"sourceType"`
|
||||||
SourceName string `json:"sourceName"`
|
SourceName string `json:"sourceName"`
|
||||||
DiscordID uint `json:"discordId"`
|
DiscordID uint `json:"discordId"`
|
||||||
DiscordName string `json:"discordName"`
|
DiscordName string `json:"discordName"`
|
||||||
}
|
}
|
||||||
|
@ -2,34 +2,34 @@ package model
|
|||||||
|
|
||||||
// This is the root Json object. It does not contain data that we care about though.
|
// This is the root Json object. It does not contain data that we care about though.
|
||||||
type RedditJsonContent struct {
|
type RedditJsonContent struct {
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Data RedditJsonContentData `json:"data"`
|
Data RedditJsonContentData `json:"data"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RedditJsonContentData struct {
|
type RedditJsonContentData struct {
|
||||||
After string `json:"after"`
|
After string `json:"after"`
|
||||||
Dist int `json:"dist"`
|
Dist int `json:"dist"`
|
||||||
Modhash string `json:"modhash"`
|
Modhash string `json:"modhash"`
|
||||||
Children []RedditJsonContentChildren `json:"children"`
|
Children []RedditJsonContentChildren `json:"children"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RedditJsonContentChildren struct {
|
type RedditJsonContentChildren struct {
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Data RedditPost `json:"data"`
|
Data RedditPost `json:"data"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RedditPost contains the information that was posted by a user.
|
// RedditPost contains the information that was posted by a user.
|
||||||
type RedditPost struct {
|
type RedditPost struct {
|
||||||
Subreddit string `json:"subreddit"`
|
Subreddit string `json:"subreddit"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Content string `json:"selftext"`
|
Content string `json:"selftext"`
|
||||||
ContentHtml string `json:"selftext_html"`
|
ContentHtml string `json:"selftext_html"`
|
||||||
Author string `json:"author"`
|
Author string `json:"author"`
|
||||||
Permalink string `json:"permalink"`
|
Permalink string `json:"permalink"`
|
||||||
IsVideo bool `json:"is_video"`
|
IsVideo bool `json:"is_video"`
|
||||||
Media RedditPostMedia `json:"media"`
|
Media RedditPostMedia `json:"media"`
|
||||||
Url string `json:"url"`
|
Url string `json:"url"`
|
||||||
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
|
UrlOverriddenByDest string `json:"url_overridden_by_dest"`
|
||||||
|
|
||||||
Thumbnail string `json:"thumbnail"`
|
Thumbnail string `json:"thumbnail"`
|
||||||
}
|
}
|
||||||
@ -41,14 +41,14 @@ type RedditPostMedia struct {
|
|||||||
|
|
||||||
// RedditVideo contains information about the video in the post.
|
// RedditVideo contains information about the video in the post.
|
||||||
type RedditPostMediaRedditVideo struct {
|
type RedditPostMediaRedditVideo struct {
|
||||||
BitrateKbps int `json:"bitrate_kpbs"`
|
BitrateKbps int `json:"bitrate_kpbs"`
|
||||||
FallBackUrl string `json:"fallback_url"`
|
FallBackUrl string `json:"fallback_url"`
|
||||||
Height int `json:"height"`
|
Height int `json:"height"`
|
||||||
Width int `json:"width"`
|
Width int `json:"width"`
|
||||||
ScrubberMediaUrl string `json:"scrubber_media_url"`
|
ScrubberMediaUrl string `json:"scrubber_media_url"`
|
||||||
DashUrl string `json:"dash_url"`
|
DashUrl string `json:"dash_url"`
|
||||||
Duration int `json:"duration"`
|
Duration int `json:"duration"`
|
||||||
HlsUrl string `json:"hls_url"`
|
HlsUrl string `json:"hls_url"`
|
||||||
IsGif bool `json:"is_gif"`
|
IsGif bool `json:"is_gif"`
|
||||||
TranscodingStatus string `json:"transcoding_status"`
|
TranscodingStatus string `json:"transcoding_status"`
|
||||||
}
|
}
|
||||||
|
4
main.go
4
main.go
@ -30,7 +30,7 @@ func main() {
|
|||||||
fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", address)
|
fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", address)
|
||||||
|
|
||||||
err := http.ListenAndServe(":8081", server.Router)
|
err := http.ListenAndServe(":8081", server.Router)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -35,7 +35,7 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
|
|||||||
// @Param id path string true "uuid"
|
// @Param id path string true "uuid"
|
||||||
// @Produce application/json
|
// @Produce application/json
|
||||||
// @Tags Articles
|
// @Tags Articles
|
||||||
// @Router /articles/{id} [get]
|
// @Router /articles/{ID} [get]
|
||||||
func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
@ -12,7 +12,7 @@ import (
|
|||||||
// @Router /discord/queue [get]
|
// @Router /discord/queue [get]
|
||||||
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
|
res, err := s.Db.ListDiscordQueueItems(*s.ctx, 100)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Write([]byte(err.Error()))
|
w.Write([]byte(err.Error()))
|
||||||
@ -26,4 +26,4 @@ func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
w.Write(bres)
|
w.Write(bres)
|
||||||
}
|
}
|
||||||
|
@ -44,8 +44,7 @@ func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
|
|||||||
func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
query := r.URL.Query()
|
_id := chi.URLParam(r, "ID")
|
||||||
_id := query["id"][0]
|
|
||||||
if _id == "" {
|
if _id == "" {
|
||||||
http.Error(w, "id is missing", http.StatusBadRequest)
|
http.Error(w, "id is missing", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
@ -56,7 +55,7 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request)
|
|||||||
http.Error(w, "unable to parse id value", http.StatusBadRequest)
|
http.Error(w, "unable to parse id value", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := s.Db.GetDiscordWebHooksByID(*s.ctx, uuid)
|
res, err := s.Db.GetDiscordWebHooksByID(*s.ctx, uuid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "no record found", http.StatusBadRequest)
|
http.Error(w, "no record found", http.StatusBadRequest)
|
||||||
@ -135,7 +134,7 @@ func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) {
|
|||||||
http.Error(w, "invalid url", http.StatusBadRequest)
|
http.Error(w, "invalid url", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if _server == ""{
|
if _server == "" {
|
||||||
http.Error(w, "server is missing", http.StatusBadRequest)
|
http.Error(w, "server is missing", http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
if _channel == "" {
|
if _channel == "" {
|
||||||
|
@ -1,2 +1 @@
|
|||||||
package routes_test
|
package routes_test
|
||||||
|
|
||||||
|
@ -41,4 +41,4 @@ func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.Write(bResult)
|
w.Write(bResult)
|
||||||
}
|
}
|
||||||
|
@ -12,7 +12,7 @@ import (
|
|||||||
// GetSubscriptions
|
// GetSubscriptions
|
||||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||||
// @Produce application/json
|
// @Produce application/json
|
||||||
// @Tags Config, Subscription
|
// @Tags Subscription
|
||||||
// @Router /subscriptions [get]
|
// @Router /subscriptions [get]
|
||||||
func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
@ -36,7 +36,7 @@ func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
|
|||||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||||
// @Produce application/json
|
// @Produce application/json
|
||||||
// @Param id query string true "id"
|
// @Param id query string true "id"
|
||||||
// @Tags Config, Subscription
|
// @Tags Subscription
|
||||||
// @Router /subscriptions/byDiscordId [get]
|
// @Router /subscriptions/byDiscordId [get]
|
||||||
func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
@ -73,7 +73,7 @@ func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Requ
|
|||||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||||
// @Produce application/json
|
// @Produce application/json
|
||||||
// @Param id query string true "id"
|
// @Param id query string true "id"
|
||||||
// @Tags Config, Subscription
|
// @Tags Subscription
|
||||||
// @Router /subscriptions/bySourceId [get]
|
// @Router /subscriptions/bySourceId [get]
|
||||||
func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
@ -110,7 +110,7 @@ func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Reque
|
|||||||
// @Summary Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
// @Summary Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
||||||
// @Param discordWebHookId query string true "discordWebHookId"
|
// @Param discordWebHookId query string true "discordWebHookId"
|
||||||
// @Param sourceId query string true "sourceId"
|
// @Param sourceId query string true "sourceId"
|
||||||
// @Tags Config, Source, Discord, Subscription
|
// @Tags Subscription
|
||||||
// @Router /subscriptions/new/discordwebhook [post]
|
// @Router /subscriptions/new/discordwebhook [post]
|
||||||
func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
|
||||||
// Extract the values given
|
// Extract the values given
|
||||||
|
31
services/cache/cache.go
vendored
31
services/cache/cache.go
vendored
@ -6,24 +6,24 @@ import (
|
|||||||
"github.com/jtom38/newsbot/collector/domain/model"
|
"github.com/jtom38/newsbot/collector/domain/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CacheClient struct{
|
type CacheClient struct {
|
||||||
group string
|
group string
|
||||||
DefaultTimer time.Duration
|
DefaultTimer time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewCacheClient(group string) CacheClient {
|
func NewCacheClient(group string) CacheClient {
|
||||||
return CacheClient{
|
return CacheClient{
|
||||||
group: group,
|
group: group,
|
||||||
DefaultTimer: time.Hour,
|
DefaultTimer: time.Hour,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cc *CacheClient) Insert(key string, value string) {
|
func (cc *CacheClient) Insert(key string, value string) {
|
||||||
item := model.CacheItem{
|
item := model.CacheItem{
|
||||||
Key: key,
|
Key: key,
|
||||||
Value: value,
|
Value: value,
|
||||||
Group: cc.group,
|
Group: cc.group,
|
||||||
Expires: time.Now().Add(1 * time.Hour),
|
Expires: time.Now().Add(1 * time.Hour),
|
||||||
IsTainted: false,
|
IsTainted: false,
|
||||||
}
|
}
|
||||||
cacheStorage = append(cacheStorage, &item)
|
cacheStorage = append(cacheStorage, &item)
|
||||||
@ -31,8 +31,12 @@ func (cc *CacheClient) Insert(key string, value string) {
|
|||||||
|
|
||||||
func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
|
func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
|
||||||
for _, item := range cacheStorage {
|
for _, item := range cacheStorage {
|
||||||
if item.Group != cc.group { continue }
|
if item.Group != cc.group {
|
||||||
if item.Key != key { continue }
|
continue
|
||||||
|
}
|
||||||
|
if item.Key != key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// if it was tainted, renew the timer and remove the taint as this record was still needed
|
// if it was tainted, renew the timer and remove the taint as this record was still needed
|
||||||
if item.IsTainted {
|
if item.IsTainted {
|
||||||
@ -47,8 +51,12 @@ func (cc *CacheClient) FindByKey(key string) (*model.CacheItem, error) {
|
|||||||
|
|
||||||
func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
|
func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
|
||||||
for _, item := range cacheStorage {
|
for _, item := range cacheStorage {
|
||||||
if item.Group != cc.group { continue }
|
if item.Group != cc.group {
|
||||||
if item.Value != value { continue }
|
continue
|
||||||
|
}
|
||||||
|
if item.Value != value {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// if it was tainted, renew the timer and remove the taint as this record was still needed
|
// if it was tainted, renew the timer and remove the taint as this record was still needed
|
||||||
if item.IsTainted {
|
if item.IsTainted {
|
||||||
@ -59,4 +67,3 @@ func (cc *CacheClient) FindByValue(value string) (*model.CacheItem, error) {
|
|||||||
}
|
}
|
||||||
return &model.CacheItem{}, ErrCacheRecordMissing
|
return &model.CacheItem{}, ErrCacheRecordMissing
|
||||||
}
|
}
|
||||||
|
|
||||||
|
13
services/cache/cache_test.go
vendored
13
services/cache/cache_test.go
vendored
@ -18,14 +18,18 @@ func TestInsert(t *testing.T) {
|
|||||||
func TestFindGroupMissing(t *testing.T) {
|
func TestFindGroupMissing(t *testing.T) {
|
||||||
cache := cache.NewCacheClient("faker")
|
cache := cache.NewCacheClient("faker")
|
||||||
_, err := cache.FindByKey("UnitTesting")
|
_, err := cache.FindByKey("UnitTesting")
|
||||||
if err == nil { panic("Nothing was appended with the requested group.") }
|
if err == nil {
|
||||||
|
panic("Nothing was appended with the requested group.")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFindGroupExists(t *testing.T) {
|
func TestFindGroupExists(t *testing.T) {
|
||||||
cache := cache.NewCacheClient("Testing")
|
cache := cache.NewCacheClient("Testing")
|
||||||
cache.Insert("UnitTesting", "Something")
|
cache.Insert("UnitTesting", "Something")
|
||||||
_, err := cache.FindByKey("UnitTesting")
|
_, err := cache.FindByKey("UnitTesting")
|
||||||
if err != nil { panic("") }
|
if err != nil {
|
||||||
|
panic("")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheStorage(t *testing.T) {
|
func TestCacheStorage(t *testing.T) {
|
||||||
@ -35,6 +39,7 @@ func TestCacheStorage(t *testing.T) {
|
|||||||
|
|
||||||
cache := cache.NewCacheClient("Testing")
|
cache := cache.NewCacheClient("Testing")
|
||||||
_, err := cache.FindByKey("UnitTesting02")
|
_, err := cache.FindByKey("UnitTesting02")
|
||||||
if err != nil { panic("expected to find the value")}
|
if err != nil {
|
||||||
|
panic("expected to find the value")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
2
services/cache/common.go
vendored
2
services/cache/common.go
vendored
@ -10,4 +10,4 @@ var (
|
|||||||
cacheStorage []*model.CacheItem
|
cacheStorage []*model.CacheItem
|
||||||
|
|
||||||
ErrCacheRecordMissing = errors.New("unable to find the requested record")
|
ErrCacheRecordMissing = errors.New("unable to find the requested record")
|
||||||
)
|
)
|
||||||
|
10
services/cache/monitor.go
vendored
10
services/cache/monitor.go
vendored
@ -9,7 +9,7 @@ import (
|
|||||||
// When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
|
// When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
|
||||||
// If a record is tainted and used again, the taint will be removed and a new Expires value will be set.
|
// If a record is tainted and used again, the taint will be removed and a new Expires value will be set.
|
||||||
// If its not renewed, it will be dropped.
|
// If its not renewed, it will be dropped.
|
||||||
type CacheAgeMonitor struct {}
|
type CacheAgeMonitor struct{}
|
||||||
|
|
||||||
func NewCacheAgeMonitor() CacheAgeMonitor {
|
func NewCacheAgeMonitor() CacheAgeMonitor {
|
||||||
return CacheAgeMonitor{}
|
return CacheAgeMonitor{}
|
||||||
@ -20,10 +20,10 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
|
|||||||
now := time.Now()
|
now := time.Now()
|
||||||
for index, item := range cacheStorage {
|
for index, item := range cacheStorage {
|
||||||
if now.After(item.Expires) {
|
if now.After(item.Expires) {
|
||||||
|
|
||||||
// the timer expired, and its not tainted, taint it
|
// the timer expired, and its not tainted, taint it
|
||||||
if !item.IsTainted {
|
if !item.IsTainted {
|
||||||
item.IsTainted = true
|
item.IsTainted = true
|
||||||
item.Expires = now.Add(1 * time.Hour)
|
item.Expires = now.Add(1 * time.Hour)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -39,7 +39,9 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
|
|||||||
func (cam CacheAgeMonitor) removeEntry(index int) []*model.CacheItem {
|
func (cam CacheAgeMonitor) removeEntry(index int) []*model.CacheItem {
|
||||||
var temp []*model.CacheItem
|
var temp []*model.CacheItem
|
||||||
for i, item := range cacheStorage {
|
for i, item := range cacheStorage {
|
||||||
if i != index { temp = append(temp, item )}
|
if i != index {
|
||||||
|
temp = append(temp, item)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return temp
|
return temp
|
||||||
}
|
}
|
||||||
|
4
services/cache/monitor_test.go
vendored
4
services/cache/monitor_test.go
vendored
@ -9,5 +9,5 @@ import (
|
|||||||
func TestCacheTaintItem(t *testing.T) {
|
func TestCacheTaintItem(t *testing.T) {
|
||||||
cc := cache.NewCacheClient("Testing")
|
cc := cache.NewCacheClient("Testing")
|
||||||
cc.Insert("UnitTesting01", "test")
|
cc.Insert("UnitTesting01", "test")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -49,6 +49,17 @@ func (cc *ConfigClient) GetConfig(key string) string {
|
|||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Looks for a value in the env and will panic if it does not exist.
|
||||||
|
func (c ConfigClient) MustGetString(key string) string {
|
||||||
|
res, filled := os.LookupEnv(key)
|
||||||
|
if !filled {
|
||||||
|
msg := fmt.Sprintf("No value was found for '%v'", key)
|
||||||
|
panic(msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
|
func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
|
||||||
cc.RefreshEnv()
|
cc.RefreshEnv()
|
||||||
|
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
|
||||||
"os"
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
"github.com/jtom38/newsbot/collector/services/config"
|
"github.com/jtom38/newsbot/collector/services/config"
|
||||||
)
|
)
|
||||||
@ -15,6 +15,8 @@ func TestGetConfigExpectNull(t *testing.T) {
|
|||||||
cc := config.New()
|
cc := config.New()
|
||||||
os.Setenv(config.REDDIT_PULL_HOT, "")
|
os.Setenv(config.REDDIT_PULL_HOT, "")
|
||||||
res := cc.GetConfig(config.REDDIT_PULL_HOT)
|
res := cc.GetConfig(config.REDDIT_PULL_HOT)
|
||||||
if res != "" { panic("expected blank")}
|
if res != "" {
|
||||||
|
panic("expected blank")
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -9,18 +9,20 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var FFXIVRecord database.Source = database.Source{
|
var FFXIVRecord database.Source = database.Source{
|
||||||
ID: uuid.New(),
|
ID: uuid.New(),
|
||||||
Site: "ffxiv",
|
Site: "ffxiv",
|
||||||
Name: "Final Fantasy XIV - NA",
|
Name: "Final Fantasy XIV - NA",
|
||||||
Source: "ffxiv",
|
Source: "ffxiv",
|
||||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivGetParser(t *testing.T) {
|
func TestFfxivGetParser(t *testing.T) {
|
||||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||||
_, err := fc.GetParser()
|
_, err := fc.GetParser()
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivPullFeed(t *testing.T) {
|
func TestFfxivPullFeed(t *testing.T) {
|
||||||
@ -30,8 +32,12 @@ func TestFfxivPullFeed(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if len(links) == 0 { panic("expected links to come back but got 0") }
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if len(links) == 0 {
|
||||||
|
t.Error("expected links to come back but got 0")
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -42,14 +48,20 @@ func TestFfxivExtractThumbnail(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
thumb, err := fc.ExtractThumbnail(page)
|
thumb, err := fc.ExtractThumbnail(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if thumb == "" { panic("expected a link but got nothing.")}
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if thumb == "" {
|
||||||
|
t.Error("expected a link but got nothing.")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivExtractPubDate(t *testing.T) {
|
func TestFfxivExtractPubDate(t *testing.T) {
|
||||||
@ -59,13 +71,17 @@ func TestFfxivExtractPubDate(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
_, err = fc.ExtractPubDate(page)
|
_, err = fc.ExtractPubDate(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivExtractDescription(t *testing.T) {
|
func TestFfxivExtractDescription(t *testing.T) {
|
||||||
@ -75,13 +91,17 @@ func TestFfxivExtractDescription(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
_, err = fc.ExtractDescription(page)
|
_, err = fc.ExtractDescription(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivExtractAuthor(t *testing.T) {
|
func TestFfxivExtractAuthor(t *testing.T) {
|
||||||
@ -91,14 +111,20 @@ func TestFfxivExtractAuthor(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
author, err := fc.ExtractAuthor(page)
|
author, err := fc.ExtractAuthor(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if author == "" { panic("failed to locate the author name") }
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if author == "" {
|
||||||
|
t.Error("failed to locate the author name")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivExtractTags(t *testing.T) {
|
func TestFfxivExtractTags(t *testing.T) {
|
||||||
@ -108,14 +134,20 @@ func TestFfxivExtractTags(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
res, err := fc.ExtractTags(page)
|
res, err := fc.ExtractTags(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if res == "" {panic("failed to locate the tags")}
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if res == "" {
|
||||||
|
t.Error("failed to locate the tags")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivExtractTitle(t *testing.T) {
|
func TestFfxivExtractTitle(t *testing.T) {
|
||||||
@ -125,14 +157,20 @@ func TestFfxivExtractTitle(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
res, err := fc.ExtractTitle(page)
|
res, err := fc.ExtractTitle(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if res == "" { panic("failed to locate the tags") }
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if res == "" {
|
||||||
|
t.Error("failed to locate the tags")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFFxivExtractAuthorIamge(t *testing.T) {
|
func TestFFxivExtractAuthorIamge(t *testing.T) {
|
||||||
@ -142,18 +180,24 @@ func TestFFxivExtractAuthorIamge(t *testing.T) {
|
|||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
|
|
||||||
links, err := fc.PullFeed(parser)
|
links, err := fc.PullFeed(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
page := fc.GetPage(parser, links[0])
|
page := fc.GetPage(parser, links[0])
|
||||||
defer page.Close()
|
defer page.Close()
|
||||||
|
|
||||||
res, err := fc.ExtractAuthorImage(page)
|
res, err := fc.ExtractAuthorImage(page)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if res == "" { panic("failed to locate the tags") }
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if res == "" {
|
||||||
|
t.Error("failed to locate the tags")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivCheckSource(t *testing.T) {
|
func TestFfxivCheckSource(t *testing.T) {
|
||||||
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
fc := ffxiv.NewFFXIVClient(FFXIVRecord)
|
||||||
fc.CheckSource()
|
fc.CheckSource()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -19,20 +19,26 @@ func getHttpContent(uri string) ([]byte, error) {
|
|||||||
client := &http.Client{
|
client := &http.Client{
|
||||||
Transport: tr,
|
Transport: tr,
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequest("GET", uri, nil)
|
req, err := http.NewRequest("GET", uri, nil)
|
||||||
if err != nil { return nil, err }
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// set the user agent header to avoid kick backs.. as much
|
// set the user agent header to avoid kick backs.. as much
|
||||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
||||||
|
|
||||||
log.Printf("Requesting content from %v\n", uri)
|
log.Printf("Requesting content from %v\n", uri)
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
if err != nil { log.Fatalln(err) }
|
if err != nil {
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
body, err := ioutil.ReadAll(resp.Body)
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
if err != nil { return nil, err }
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
@ -47,7 +47,6 @@ func NewRedditClient(Record database.Source) *RedditClient {
|
|||||||
// os.Setenv("GODEBUG", "http2client=0")
|
// os.Setenv("GODEBUG", "http2client=0")
|
||||||
//}
|
//}
|
||||||
|
|
||||||
|
|
||||||
func (rc *RedditClient) GetBrowser() *rod.Browser {
|
func (rc *RedditClient) GetBrowser() *rod.Browser {
|
||||||
var browser *rod.Browser
|
var browser *rod.Browser
|
||||||
if path, exists := launcher.LookPath(); exists {
|
if path, exists := launcher.LookPath(); exists {
|
||||||
@ -62,7 +61,7 @@ func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
|||||||
return page
|
return page
|
||||||
}
|
}
|
||||||
|
|
||||||
//func (rc RedditClient)
|
//func (rc RedditClient)
|
||||||
|
|
||||||
// GetContent() reaches out to Reddit and pulls the Json data.
|
// GetContent() reaches out to Reddit and pulls the Json data.
|
||||||
// It will then convert the data to a struct and return the struct.
|
// It will then convert the data to a struct and return the struct.
|
||||||
@ -74,8 +73,6 @@ func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
|
|||||||
|
|
||||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
content, err := getHttpContent(Url)
|
content, err := getHttpContent(Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
|
@ -9,12 +9,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var RedditRecord database.Source = database.Source{
|
var RedditRecord database.Source = database.Source{
|
||||||
ID: uuid.New(),
|
ID: uuid.New(),
|
||||||
Name: "dadjokes",
|
Name: "dadjokes",
|
||||||
Source: "reddit",
|
Source: "reddit",
|
||||||
Site: "reddit",
|
Site: "reddit",
|
||||||
Url: "https://reddit.com/r/dadjokes",
|
Url: "https://reddit.com/r/dadjokes",
|
||||||
Tags: "reddit, dadjokes",
|
Tags: "reddit, dadjokes",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetContent(t *testing.T) {
|
func TestGetContent(t *testing.T) {
|
||||||
@ -30,4 +30,4 @@ func TestGetContent(t *testing.T) {
|
|||||||
t.Error("Title is missing")
|
t.Error("Title is missing")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -22,7 +22,7 @@ func NewRssClient(sourceRecord model.Sources) rssClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
|
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
|
||||||
//rc.SourceRecord = source
|
//rc.SourceRecord = source
|
||||||
//}
|
//}
|
||||||
|
|
||||||
func (rc rssClient) getCacheGroup() string {
|
func (rc rssClient) getCacheGroup() string {
|
||||||
@ -31,7 +31,9 @@ func (rc rssClient) getCacheGroup() string {
|
|||||||
|
|
||||||
func (rc rssClient) GetContent() error {
|
func (rc rssClient) GetContent() error {
|
||||||
feed, err := rc.PullFeed()
|
feed, err := rc.PullFeed()
|
||||||
if err != nil { return err }
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
|
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
|
||||||
|
|
||||||
@ -49,7 +51,9 @@ func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
|
|||||||
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
|
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
|
||||||
fp := gofeed.NewParser()
|
fp := gofeed.NewParser()
|
||||||
feed, err := fp.ParseURL(feedUri)
|
feed, err := fp.ParseURL(feedUri)
|
||||||
if err != nil { return nil, err }
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return feed, nil
|
return feed, nil
|
||||||
}
|
}
|
||||||
|
@ -7,10 +7,10 @@ import (
|
|||||||
"github.com/jtom38/newsbot/collector/services/input"
|
"github.com/jtom38/newsbot/collector/services/input"
|
||||||
)
|
)
|
||||||
|
|
||||||
var rssRecord = model.Sources {
|
var rssRecord = model.Sources{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
Name: "ArsTechnica",
|
Name: "ArsTechnica",
|
||||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRssClientConstructor(t *testing.T) {
|
func TestRssClientConstructor(t *testing.T) {
|
||||||
@ -20,7 +20,11 @@ func TestRssClientConstructor(t *testing.T) {
|
|||||||
func TestRssGetFeed(t *testing.T) {
|
func TestRssGetFeed(t *testing.T) {
|
||||||
client := input.NewRssClient(rssRecord)
|
client := input.NewRssClient(rssRecord)
|
||||||
feed, err := client.PullFeed()
|
feed, err := client.PullFeed()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
if len(feed.Items) >= 0 { t.Error("failed to collect items from the fees")}
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if len(feed.Items) >= 0 {
|
||||||
|
t.Error("failed to collect items from the fees")
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -104,31 +104,47 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
|||||||
var article database.Article
|
var article database.Article
|
||||||
|
|
||||||
AuthorName, err := tc.ExtractAuthor(video)
|
AuthorName, err := tc.ExtractAuthor(video)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
article.Authorname = sql.NullString{String: AuthorName}
|
article.Authorname = sql.NullString{String: AuthorName}
|
||||||
|
|
||||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
article.Authorimage = sql.NullString{String: Authorimage}
|
article.Authorimage = sql.NullString{String: Authorimage}
|
||||||
|
|
||||||
article.Description, err = tc.ExtractDescription(video)
|
article.Description, err = tc.ExtractDescription(video)
|
||||||
if err != nil {return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
|
|
||||||
article.Pubdate, err = tc.ExtractPubDate(video)
|
article.Pubdate, err = tc.ExtractPubDate(video)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
|
|
||||||
article.Sourceid = tc.SourceRecord.ID
|
article.Sourceid = tc.SourceRecord.ID
|
||||||
article.Tags, err = tc.ExtractTags(video, user)
|
article.Tags, err = tc.ExtractTags(video, user)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
|
|
||||||
article.Thumbnail, err = tc.ExtractThumbnail(video)
|
article.Thumbnail, err = tc.ExtractThumbnail(video)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
|
|
||||||
article.Title, err = tc.ExtractTitle(video)
|
article.Title, err = tc.ExtractTitle(video)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
|
|
||||||
article.Url, err = tc.ExtractUrl(video)
|
article.Url, err = tc.ExtractUrl(video)
|
||||||
if err != nil { return items, err }
|
if err != nil {
|
||||||
|
return items, err
|
||||||
|
}
|
||||||
|
|
||||||
items = append(items, article)
|
items = append(items, article)
|
||||||
}
|
}
|
||||||
@ -210,8 +226,12 @@ func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
|
|||||||
|
|
||||||
// Extracts the avatar of the author with some validation.
|
// Extracts the avatar of the author with some validation.
|
||||||
func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
|
func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
|
||||||
if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
|
if user.ProfileImageURL == "" {
|
||||||
if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
|
return "", ErrMissingAuthorImage
|
||||||
|
}
|
||||||
|
if !strings.Contains(user.ProfileImageURL, "-profile_image-") {
|
||||||
|
return "", ErrInvalidAuthorImage
|
||||||
|
}
|
||||||
return user.ProfileImageURL, nil
|
return user.ProfileImageURL, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -231,6 +251,8 @@ func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
|
func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
|
||||||
if post.URL == "" { return "", ErrMissingUrl }
|
if post.URL == "" {
|
||||||
|
return "", ErrMissingUrl
|
||||||
|
}
|
||||||
return post.URL, nil
|
return post.URL, nil
|
||||||
}
|
}
|
||||||
|
@ -9,13 +9,13 @@ import (
|
|||||||
"github.com/jtom38/newsbot/collector/services/input"
|
"github.com/jtom38/newsbot/collector/services/input"
|
||||||
)
|
)
|
||||||
|
|
||||||
var TwitchSourceRecord = database.Source {
|
var TwitchSourceRecord = database.Source{
|
||||||
ID: uuid.New(),
|
ID: uuid.New(),
|
||||||
Name: "nintendo",
|
Name: "nintendo",
|
||||||
Source: "Twitch",
|
Source: "Twitch",
|
||||||
}
|
}
|
||||||
|
|
||||||
var TwitchInvalidRecord = database.Source {
|
var TwitchInvalidRecord = database.Source{
|
||||||
ID: uuid.New(),
|
ID: uuid.New(),
|
||||||
Name: "EvilNintendo",
|
Name: "EvilNintendo",
|
||||||
Source: "Twitch",
|
Source: "Twitch",
|
||||||
@ -115,40 +115,62 @@ func TestTwitchReturnsVideoAuthor(t *testing.T) {
|
|||||||
|
|
||||||
func TestTwitchReturnsThumbnail(t *testing.T) {
|
func TestTwitchReturnsThumbnail(t *testing.T) {
|
||||||
tc, err := input.NewTwitchClient()
|
tc, err := input.NewTwitchClient()
|
||||||
if err != nil {t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||||
|
|
||||||
err = tc.Login()
|
err = tc.Login()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
user, err := tc.GetUserDetails()
|
user, err := tc.GetUserDetails()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
posts, err := tc.GetPosts(user)
|
posts, err := tc.GetPosts(user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
value, err := tc.ExtractThumbnail(posts[0])
|
value, err := tc.ExtractThumbnail(posts[0])
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
if value == "" { t.Error("uable to parse username") }
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
t.Error("uable to parse username")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchReturnsPubDate(t *testing.T) {
|
func TestTwitchReturnsPubDate(t *testing.T) {
|
||||||
tc, err := input.NewTwitchClient()
|
tc, err := input.NewTwitchClient()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||||
|
|
||||||
err = tc.Login()
|
err = tc.Login()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
user, err := tc.GetUserDetails()
|
user, err := tc.GetUserDetails()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
posts, err := tc.GetPosts(user)
|
posts, err := tc.GetPosts(user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
date, err := tc.ExtractPubDate(posts[0])
|
date, err := tc.ExtractPubDate(posts[0])
|
||||||
log.Println(date)
|
log.Println(date)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchReturnsDescription(t *testing.T) {
|
func TestTwitchReturnsDescription(t *testing.T) {
|
||||||
@ -181,17 +203,25 @@ func TestTwitchReturnsDescription(t *testing.T) {
|
|||||||
|
|
||||||
func TestTwitchReturnsAuthorImage(t *testing.T) {
|
func TestTwitchReturnsAuthorImage(t *testing.T) {
|
||||||
tc, err := input.NewTwitchClient()
|
tc, err := input.NewTwitchClient()
|
||||||
if err != nil {t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||||
|
|
||||||
err = tc.Login()
|
err = tc.Login()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
user, err := tc.GetUserDetails()
|
user, err := tc.GetUserDetails()
|
||||||
if err != nil {t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = tc.ExtractAuthorImage(user)
|
_, err = tc.ExtractAuthorImage(user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchReturnsTags(t *testing.T) {
|
func TestTwitchReturnsTags(t *testing.T) {
|
||||||
@ -212,10 +242,14 @@ func TestTwitchReturnsTags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
posts, err := tc.GetPosts(user)
|
posts, err := tc.GetPosts(user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = tc.ExtractTags(posts[0], user)
|
_, err = tc.ExtractTags(posts[0], user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchReturnsTitle(t *testing.T) {
|
func TestTwitchReturnsTitle(t *testing.T) {
|
||||||
@ -236,42 +270,70 @@ func TestTwitchReturnsTitle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
posts, err := tc.GetPosts(user)
|
posts, err := tc.GetPosts(user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
res, err := tc.ExtractTitle(posts[0])
|
res, err := tc.ExtractTitle(posts[0])
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
if res == "" { t.Error("expected a filled string but got nil")}
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if res == "" {
|
||||||
|
t.Error("expected a filled string but got nil")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchReturnsUrl(t *testing.T) {
|
func TestTwitchReturnsUrl(t *testing.T) {
|
||||||
tc, err := input.NewTwitchClient()
|
tc, err := input.NewTwitchClient()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||||
|
|
||||||
err = tc.Login()
|
err = tc.Login()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
user, err := tc.GetUserDetails()
|
user, err := tc.GetUserDetails()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
posts, err := tc.GetPosts(user)
|
posts, err := tc.GetPosts(user)
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
res, err := tc.ExtractUrl(posts[0])
|
res, err := tc.ExtractUrl(posts[0])
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
if res == "" { t.Error("expected a filled string but got nil")}
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if res == "" {
|
||||||
|
t.Error("expected a filled string but got nil")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchGetContent(t *testing.T) {
|
func TestTwitchGetContent(t *testing.T) {
|
||||||
tc, err := input.NewTwitchClient()
|
tc, err := input.NewTwitchClient()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
tc.ReplaceSourceRecord(TwitchSourceRecord)
|
||||||
|
|
||||||
err = tc.Login()
|
err = tc.Login()
|
||||||
if err != nil { t.Error(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
posts, err := tc.GetContent()
|
posts, err := tc.GetContent()
|
||||||
if err != nil {t.Error(err) }
|
if err != nil {
|
||||||
if len(posts) == 0 { t.Error("posts came back with 0 posts") }
|
t.Error(err)
|
||||||
if len(posts) != 20 { t.Error("expected 20 posts") }
|
}
|
||||||
}
|
if len(posts) == 0 {
|
||||||
|
t.Error("posts came back with 0 posts")
|
||||||
|
}
|
||||||
|
if len(posts) != 20 {
|
||||||
|
t.Error("expected 20 posts")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -44,13 +44,6 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
|
|||||||
record: Record,
|
record: Record,
|
||||||
cacheGroup: "youtube",
|
cacheGroup: "youtube",
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
cc := NewConfigClient()
|
|
||||||
|
|
||||||
debug, err := strconv.ParseBool(cc.GetConfig(YOUTUBE_DEBUG))
|
|
||||||
if err != nil { panic("'YOUTUBE_DEBUG' was not a bool value")}
|
|
||||||
yc.Config.Debug = debug
|
|
||||||
*/
|
|
||||||
return yc
|
return yc
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -109,14 +102,18 @@ func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
|||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (yc *YoutubeClient) GetBrowser() *rod.Browser {
|
func (yc *YoutubeClient) GetBrowser() (*rod.Browser, error) {
|
||||||
//browser := rod.New().MustConnect()
|
//browser := rod.New().MustConnect()
|
||||||
var browser *rod.Browser
|
var browser *rod.Browser
|
||||||
if path, exists := launcher.LookPath(); exists {
|
if path, exists := launcher.LookPath(); exists {
|
||||||
u := launcher.New().Bin(path).MustLaunch()
|
u, err := launcher.New().Bin(path).Launch()
|
||||||
|
if err != nil {
|
||||||
|
return browser, err
|
||||||
|
}
|
||||||
|
|
||||||
browser = rod.New().ControlURL(u).MustConnect()
|
browser = rod.New().ControlURL(u).MustConnect()
|
||||||
}
|
}
|
||||||
return browser
|
return browser, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||||
@ -163,7 +160,11 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) {
|
|||||||
var AvatarUri string
|
var AvatarUri string
|
||||||
|
|
||||||
//browser := rod.New().MustConnect()
|
//browser := rod.New().MustConnect()
|
||||||
browser := yc.GetBrowser()
|
browser, err := yc.GetBrowser()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
page := browser.MustPage(yc.record.Url)
|
page := browser.MustPage(yc.record.Url)
|
||||||
|
|
||||||
res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
|
res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
|
||||||
|
@ -9,45 +9,61 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var YouTubeRecord database.Source = database.Source{
|
var YouTubeRecord database.Source = database.Source{
|
||||||
ID: uuid.New(),
|
ID: uuid.New(),
|
||||||
Name: "dadjokes",
|
Name: "dadjokes",
|
||||||
Source: "reddit",
|
Source: "reddit",
|
||||||
Site: "reddit",
|
Site: "reddit",
|
||||||
Url: "https://youtube.com/gamegrumps",
|
Url: "https://youtube.com/gamegrumps",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetPageParser(t *testing.T) {
|
func TestGetPageParser(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
_, err := yc.GetParser(YouTubeRecord.Url)
|
_, err := yc.GetParser(YouTubeRecord.Url)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetChannelId(t *testing.T) {
|
func TestGetChannelId(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = yc.GetChannelId(parser)
|
_, err = yc.GetChannelId(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPullFeed(t *testing.T) {
|
func TestPullFeed(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = yc.GetChannelId(parser)
|
_, err = yc.GetChannelId(parser)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = yc.PullFeed()
|
_, err = yc.PullFeed()
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetAvatarUri(t *testing.T) {
|
func TestGetAvatarUri(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
res, err := yc.GetAvatarUri()
|
res, err := yc.GetAvatarUri()
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
if res == "" { panic(input.ErrMissingAuthorImage)}
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if res == "" {
|
||||||
|
t.Error(input.ErrMissingAuthorImage)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetVideoTags(t *testing.T) {
|
func TestGetVideoTags(t *testing.T) {
|
||||||
@ -56,39 +72,59 @@ func TestGetVideoTags(t *testing.T) {
|
|||||||
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
|
var videoUri = "https://www.youtube.com/watch?v=k_sQEXOBe68"
|
||||||
|
|
||||||
parser, err := yc.GetParser(videoUri)
|
parser, err := yc.GetParser(videoUri)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
tags, err := yc.GetTags(parser)
|
tags, err := yc.GetTags(parser)
|
||||||
if err == nil && tags == "" { panic("err was empty but value was missing.")}
|
if err == nil && tags == "" {
|
||||||
if err != nil { panic(err) }
|
t.Error("err was empty but value was missing.")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetChannelTags(t *testing.T) {
|
func TestGetChannelTags(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
|
|
||||||
parser, err := yc.GetParser(YouTubeRecord.Url)
|
parser, err := yc.GetParser(YouTubeRecord.Url)
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
tags, err := yc.GetTags(parser)
|
tags, err := yc.GetTags(parser)
|
||||||
if err == nil && tags == "" { panic("no err but expected value was missing.")}
|
if err == nil && tags == "" {
|
||||||
if err != nil { panic(err) }
|
t.Error("no err but expected value was missing.")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetVideoThumbnail(t *testing.T) {
|
func TestGetVideoThumbnail(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
|
parser, err := yc.GetParser("https://www.youtube.com/watch?v=k_sQEXOBe68")
|
||||||
if err != nil {panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
thumb, err := yc.GetVideoThumbnail(parser)
|
thumb, err := yc.GetVideoThumbnail(parser)
|
||||||
if err == nil && thumb == "" { panic("no err but expected result was missing")}
|
if err == nil && thumb == "" {
|
||||||
if err != nil { panic(err) }
|
t.Error("no err but expected result was missing")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckSource(t *testing.T) {
|
func TestCheckSource(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
_, err := yc.GetContent()
|
_, err := yc.GetContent()
|
||||||
if err != nil { panic(err) }
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckUriCache(t *testing.T) {
|
func TestCheckUriCache(t *testing.T) {
|
||||||
@ -97,14 +133,18 @@ func TestCheckUriCache(t *testing.T) {
|
|||||||
|
|
||||||
input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
|
input.YoutubeUriCache = append(input.YoutubeUriCache, &item)
|
||||||
res := yc.CheckUriCache(&item)
|
res := yc.CheckUriCache(&item)
|
||||||
if res == false { panic("expected a value to come back")}
|
if res == false {
|
||||||
|
t.Error("expected a value to come back")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckUriCacheFails(t *testing.T) {
|
func TestCheckUriCacheFails(t *testing.T) {
|
||||||
yc := input.NewYoutubeClient(YouTubeRecord)
|
yc := input.NewYoutubeClient(YouTubeRecord)
|
||||||
item := "demo1"
|
item := "demo1"
|
||||||
|
|
||||||
res := yc.CheckUriCache(&item)
|
|
||||||
if res == true { panic("expected no value to come back")}
|
|
||||||
|
|
||||||
}
|
res := yc.CheckUriCache(&item)
|
||||||
|
if res == true {
|
||||||
|
t.Error("expected no value to come back")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
@ -14,14 +14,14 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
article database.Article = database.Article{
|
article database.Article = database.Article{
|
||||||
ID: uuid.New(),
|
ID: uuid.New(),
|
||||||
Sourceid: uuid.New(),
|
Sourceid: uuid.New(),
|
||||||
Tags: "unit, testing",
|
Tags: "unit, testing",
|
||||||
Title: "Demo",
|
Title: "Demo",
|
||||||
Url: "https://github.com/jtom38/newsbot.collector.api",
|
Url: "https://github.com/jtom38/newsbot.collector.api",
|
||||||
//Pubdate: time.Now(),
|
//Pubdate: time.Now(),
|
||||||
Videoheight: 0,
|
Videoheight: 0,
|
||||||
Videowidth: 0,
|
Videowidth: 0,
|
||||||
Description: "Hello World",
|
Description: "Hello World",
|
||||||
}
|
}
|
||||||
blank string = ""
|
blank string = ""
|
||||||
@ -33,7 +33,7 @@ func TestDiscordMessageContainsTitle(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, i := range *msg.Embeds {
|
for _, i := range *msg.Embeds {
|
||||||
if i.Title == &blank {
|
if i.Title == &blank {
|
||||||
t.Error("title missing")
|
t.Error("title missing")
|
||||||
@ -47,7 +47,7 @@ func TestDiscordMessageContainsDescription(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, i := range *msg.Embeds {
|
for _, i := range *msg.Embeds {
|
||||||
if i.Description == &blank {
|
if i.Description == &blank {
|
||||||
t.Error("description missing")
|
t.Error("description missing")
|
||||||
@ -68,7 +68,7 @@ func TestDiscordMessageFooter(t *testing.T) {
|
|||||||
}
|
}
|
||||||
if i.Footer.IconUrl == &blank {
|
if i.Footer.IconUrl == &blank {
|
||||||
t.Error("missing footer url")
|
t.Error("missing footer url")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -81,7 +81,7 @@ func TestDiscordMessageFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, embed := range *msg.Embeds {
|
for _, embed := range *msg.Embeds {
|
||||||
for _, field := range embed.Fields {
|
for _, field := range embed.Fields {
|
||||||
var fName string
|
var fName string
|
||||||
if field.Name != nil {
|
if field.Name != nil {
|
||||||
fName = *field.Name
|
fName = *field.Name
|
||||||
} else {
|
} else {
|
||||||
@ -110,7 +110,7 @@ func TestDiscordMessagePost(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = godotenv.Load()
|
err = godotenv.Load()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
@ -131,8 +131,8 @@ func TestDiscordMessagePost(t *testing.T) {
|
|||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = d.SendPayload(msg, endpoints[0])
|
err = d.SendPayload(msg, endpoints[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3,4 +3,4 @@ package output
|
|||||||
type Output interface {
|
type Output interface {
|
||||||
GeneratePayload() error
|
GeneratePayload() error
|
||||||
SendPayload() error
|
SendPayload() error
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user