From c161658487bc19f99eb1ff3653533b273594383b Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 30 Nov 2022 21:43:53 -0800 Subject: [PATCH] New endpoints for the portal to use (#31) * added a route to delete subscriptions based on the ID given * added a new route to find a record based on the name and source * added a route to query Discord Web Hooks by Server and Channel names * tested the endpoints and they seem good to test more --- database/query.sql.go | 65 ++++++++++++++++ database/schema/query.sql | 7 ++ docs/docs.go | 98 +++++++++++++++++++++++ docs/swagger.json | 98 +++++++++++++++++++++++ docs/swagger.yaml | 69 ++++++++++++++++ routes/discordwebhooks.go | 45 ++++++++++- routes/server.go | 4 + routes/sources.go | 62 ++++++++++++--- routes/subscriptions.go | 41 +++++++--- services/cron/scheduler.go | 4 +- services/input/ffxiv.go | 156 ++++++++++++++++++++++++------------- 11 files changed, 574 insertions(+), 75 deletions(-) diff --git a/database/query.sql.go b/database/query.sql.go index 6e80ac4..1a0cd0d 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -582,6 +582,45 @@ func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Dis return i, err } +const getDiscordWebHooksByServerAndChannel = `-- name: GetDiscordWebHooksByServerAndChannel :many +SELECT id, url, server, channel, enabled FROM DiscordWebHooks +WHERE Server = $1 and Channel = $2 +` + +type GetDiscordWebHooksByServerAndChannelParams struct { + Server string + Channel string +} + +func (q *Queries) GetDiscordWebHooksByServerAndChannel(ctx context.Context, arg GetDiscordWebHooksByServerAndChannelParams) ([]Discordwebhook, error) { + rows, err := q.db.QueryContext(ctx, getDiscordWebHooksByServerAndChannel, arg.Server, arg.Channel) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Discordwebhook + for rows.Next() { + var i Discordwebhook + if err := rows.Scan( + &i.ID, + &i.Url, + &i.Server, + &i.Channel, + &i.Enabled, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getIconByID = `-- name: GetIconByID :one Select id, filename, site FROM Icons Where ID = $1 Limit 1 @@ -742,6 +781,32 @@ func (q *Queries) GetSourceByName(ctx context.Context, name string) (Source, err return i, err } +const getSourceByNameAndSource = `-- name: GetSourceByNameAndSource :one +Select id, site, name, source, type, value, enabled, url, tags from Sources WHERE name = $1 and source = $2 +` + +type GetSourceByNameAndSourceParams struct { + Name string + Source string +} + +func (q *Queries) GetSourceByNameAndSource(ctx context.Context, arg GetSourceByNameAndSourceParams) (Source, error) { + row := q.db.QueryRowContext(ctx, getSourceByNameAndSource, arg.Name, arg.Source) + var i Source + err := row.Scan( + &i.ID, + &i.Site, + &i.Name, + &i.Source, + &i.Type, + &i.Value, + &i.Enabled, + &i.Url, + &i.Tags, + ) + return i, err +} + const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1 ` diff --git a/database/schema/query.sql b/database/schema/query.sql index ca88681..bff9dbd 100644 --- a/database/schema/query.sql +++ b/database/schema/query.sql @@ -74,6 +74,10 @@ Where ID = $1 LIMIT 1; Select * From DiscordWebHooks Where Server = $1; +-- name: GetDiscordWebHooksByServerAndChannel :many +SELECT * FROM DiscordWebHooks +WHERE Server = $1 and Channel = $2; + -- name: GetDiscordWebHookByUrl :one Select * From DiscordWebHooks Where url = $1; @@ -146,6 +150,9 @@ Select * From Sources where ID = $1 Limit 1; -- name: GetSourceByName :one Select * from Sources where name = $1 Limit 1; +-- name: GetSourceByNameAndSource :one +Select * from Sources WHERE name = $1 and source = $2; + -- name: ListSources :many Select * From Sources Limit $1; diff --git a/docs/docs.go b/docs/docs.go index 148aa6a..8194870 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -126,6 +126,35 @@ const docTemplate = `{ "responses": {} } }, + "/config/sources/by/sourceAndName": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Config", + "Source" + ], + "summary": "Returns a single entity by ID", + "parameters": [ + { + "type": "string", + "description": "dadjokes", + "name": "name", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "reddit", + "name": "source", + "in": "query", + "required": true + } + ], + "responses": {} + } + }, "/config/sources/new/reddit": { "post": { "tags": [ @@ -305,6 +334,36 @@ const docTemplate = `{ "responses": {} } }, + "/discord/webhooks/by/serverAndChannel": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Config", + "Discord", + "Webhook" + ], + "summary": "Returns all the known web hooks based on the Server and Channel given.", + "parameters": [ + { + "type": "string", + "description": "Fancy Server", + "name": "server", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "memes", + "name": "channel", + "in": "query", + "required": true + } + ], + "responses": {} + } + }, "/discord/webhooks/new": { "post": { "tags": [ @@ -378,6 +437,24 @@ const docTemplate = `{ } ], "responses": {} + }, + "patch": { + "tags": [ + "Config", + "Discord", + "Webhook" + ], + "summary": "Updates a valid discord webhook ID based on the body given.", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": {} } }, "/discord/webhooks/{id}/disable": { @@ -543,6 +620,27 @@ const docTemplate = `{ "responses": {} } }, + "/subscriptions/discord/webhook/delete": { + "delete": { + "tags": [ + "Config", + "Source", + "Discord", + "Subscription" + ], + "summary": "Removes a Discord WebHook Subscription based on the Subscription ID.", + "parameters": [ + { + "type": "string", + "description": "Id", + "name": "Id", + "in": "query", + "required": true + } + ], + "responses": {} + } + }, "/subscriptions/new/discordwebhook": { "post": { "tags": [ diff --git a/docs/swagger.json b/docs/swagger.json index 43d3593..2dfcee4 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -117,6 +117,35 @@ "responses": {} } }, + "/config/sources/by/sourceAndName": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Config", + "Source" + ], + "summary": "Returns a single entity by ID", + "parameters": [ + { + "type": "string", + "description": "dadjokes", + "name": "name", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "reddit", + "name": "source", + "in": "query", + "required": true + } + ], + "responses": {} + } + }, "/config/sources/new/reddit": { "post": { "tags": [ @@ -296,6 +325,36 @@ "responses": {} } }, + "/discord/webhooks/by/serverAndChannel": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Config", + "Discord", + "Webhook" + ], + "summary": "Returns all the known web hooks based on the Server and Channel given.", + "parameters": [ + { + "type": "string", + "description": "Fancy Server", + "name": "server", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "memes", + "name": "channel", + "in": "query", + "required": true + } + ], + "responses": {} + } + }, "/discord/webhooks/new": { "post": { "tags": [ @@ -369,6 +428,24 @@ } ], "responses": {} + }, + "patch": { + "tags": [ + "Config", + "Discord", + "Webhook" + ], + "summary": "Updates a valid discord webhook ID based on the body given.", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": {} } }, "/discord/webhooks/{id}/disable": { @@ -534,6 +611,27 @@ "responses": {} } }, + "/subscriptions/discord/webhook/delete": { + "delete": { + "tags": [ + "Config", + "Source", + "Discord", + "Subscription" + ], + "summary": "Removes a Discord WebHook Subscription based on the Subscription ID.", + "parameters": [ + { + "type": "string", + "description": "Id", + "name": "Id", + "in": "query", + "required": true + } + ], + "responses": {} + } + }, "/subscriptions/new/discordwebhook": { "post": { "tags": [ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 40d9436..c71198d 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -133,6 +133,26 @@ paths: tags: - Config - Source + /config/sources/by/sourceAndName: + get: + parameters: + - description: dadjokes + in: query + name: name + required: true + type: string + - description: reddit + in: query + name: source + required: true + type: string + produces: + - application/json + responses: {} + summary: Returns a single entity by ID + tags: + - Config + - Source /config/sources/new/reddit: post: parameters: @@ -234,6 +254,19 @@ paths: - Config - Discord - Webhook + patch: + parameters: + - description: id + in: path + name: id + required: true + type: string + responses: {} + summary: Updates a valid discord webhook ID based on the body given. + tags: + - Config + - Discord + - Webhook /discord/webhooks/{id}/disable: post: parameters: @@ -262,6 +295,27 @@ paths: - Config - Discord - Webhook + /discord/webhooks/by/serverAndChannel: + get: + parameters: + - description: Fancy Server + in: query + name: server + required: true + type: string + - description: memes + in: query + name: channel + required: true + type: string + produces: + - application/json + responses: {} + summary: Returns all the known web hooks based on the Server and Channel given. + tags: + - Config + - Discord + - Webhook /discord/webhooks/new: post: parameters: @@ -369,6 +423,21 @@ paths: tags: - Config - Subscription + /subscriptions/discord/webhook/delete: + delete: + parameters: + - description: Id + in: query + name: Id + required: true + type: string + responses: {} + summary: Removes a Discord WebHook Subscription based on the Subscription ID. + tags: + - Config + - Source + - Discord + - Subscription /subscriptions/new/discordwebhook: post: parameters: diff --git a/routes/discordwebhooks.go b/routes/discordwebhooks.go index 0fd438c..adf085a 100644 --- a/routes/discordwebhooks.go +++ b/routes/discordwebhooks.go @@ -1,6 +1,7 @@ package routes import ( + "context" "encoding/json" "log" "net/http" @@ -71,6 +72,48 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) w.Write(bres) } +// GetDiscordWebHookByServerAndChannel +// @Summary Returns all the known web hooks based on the Server and Channel given. +// @Produce application/json +// @Param server query string true "Fancy Server" +// @Param channel query string true "memes" +// @Tags Config, Discord, Webhook +// @Router /discord/webhooks/by/serverAndChannel [get] +func (s *Server) GetDiscordWebHooksByServerAndChannel(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + query := r.URL.Query() + _server := query["server"][0] + if _server == "" { + http.Error(w, "ID is missing", http.StatusInternalServerError) + return + } + + _channel := query["channel"][0] + if _channel == "" { + http.Error(w, "Channel is missing", http.StatusInternalServerError) + return + } + + res, err := s.Db.GetDiscordWebHooksByServerAndChannel(context.Background(), database.GetDiscordWebHooksByServerAndChannelParams{ + Server: _server, + Channel: _channel, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + bres, err := json.Marshal(res) + if err != nil { + http.Error(w, "unable to convert to json", http.StatusInternalServerError) + panic(err) + } + + w.Write(bres) +} + + // NewDiscordWebHook // @Summary Creates a new record for a discord web hook to post data to. // @Param url query string true "url" @@ -194,7 +237,7 @@ func (s *Server) deleteDiscordWebHook(w http.ResponseWriter, r *http.Request) { // @Summary Updates a valid discord webhook ID based on the body given. // @Param id path string true "id" // @Tags Config, Discord, Webhook -// @Router /discord/webhooks/{id} [delete] +// @Router /discord/webhooks/{id} [patch] func (s *Server) UpdateDiscordWebHook(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "ID") diff --git a/routes/server.go b/routes/server.go index 6121dd6..18057f2 100644 --- a/routes/server.go +++ b/routes/server.go @@ -87,6 +87,8 @@ func (s *Server) MountRoutes() { s.Router.Post("/api/discord/webhooks/new", s.NewDiscordWebHook) s.Router.Get("/api/discord/webhooks", s.GetDiscordWebHooks) //s.Router.Get("/api/discord/webhooks/byId", s.GetDiscordWebHooksById) + s.Router.Get("/api/discord/webhooks/by/serverAndChannel", s.GetDiscordWebHooksByServerAndChannel) + s.Router.Route("/api/discord/webhooks/{ID}", func(r chi.Router) { r.Get("/", s.GetDiscordWebHooksById) r.Delete("/", s.deleteDiscordWebHook) @@ -113,10 +115,12 @@ func (s *Server) MountRoutes() { r.Post("/disable", s.disableSource) r.Post("/enable", s.enableSource) }) + s.Router.Get("/api/config/sources/by/sourceAndName", s.GetSourceBySourceAndName) /* Subscriptions */ s.Router.Get("/api/subscriptions", s.ListSubscriptions) s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId) s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId) s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription) + s.Router.Delete("/api/subscriptions/discord/webhook/delete", s.DeleteDiscordWebHookSubscription) } diff --git a/routes/sources.go b/routes/sources.go index f033409..9fa37f8 100644 --- a/routes/sources.go +++ b/routes/sources.go @@ -1,6 +1,7 @@ package routes import ( + "context" "encoding/json" "fmt" "log" @@ -111,6 +112,45 @@ func (s *Server) getSources(w http.ResponseWriter, r *http.Request) { w.Write(bResult) } +// GetSourceByNameAndSource +// @Summary Returns a single entity by ID +// @Param name query string true "dadjokes" +// @Param source query string true "reddit" +// @Produce application/json +// @Tags Config, Source +// @Router /config/sources/by/sourceAndName [get] +func (s *Server) GetSourceBySourceAndName(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + + name := query["name"][0] + if name == "" { + http.Error(w, "Parameter 'name' was missing in the query.", http.StatusInternalServerError) + return + } + + source := query["source"][0] + if source == "" { + http.Error(w, "The parameter 'source' was missing in the query.", http.StatusInternalServerError) + return + } + + item, err := s.Db.GetSourceByNameAndSource(context.Background(), database.GetSourceByNameAndSourceParams{ + Name: name, + Source: source, + }) + if err != nil { + http.Error(w, "Unable to find the requested record.", http.StatusInternalServerError) + } + + bResult, err := json.Marshal(item) + if err != nil { + log.Panicln(err) + } + + w.Header().Set("Content-Type", "application/json") + w.Write(bResult) +} + // NewRedditSource // @Summary Creates a new reddit source to monitor. // @Param name query string true "name" @@ -133,14 +173,14 @@ func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) { } /* - var tags string - if _tags == "" { - tags = fmt.Sprintf("twitch, %v", _name) - } else { - } + var tags string + if _tags == "" { + tags = fmt.Sprintf("twitch, %v", _name) + } else { + } */ tags := fmt.Sprintf("twitch, %v", _name) - + params := database.CreateSourceParams{ ID: uuid.New(), Site: "reddit", @@ -183,13 +223,13 @@ func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) { } /* - if _tags == "" { - tags = fmt.Sprintf("twitch, %v", _name) - } else { - } + if _tags == "" { + tags = fmt.Sprintf("twitch, %v", _name) + } else { + } */ tags := fmt.Sprintf("twitch, %v", _name) - + params := database.CreateSourceParams{ ID: uuid.New(), Site: "youtube", diff --git a/routes/subscriptions.go b/routes/subscriptions.go index 582749b..979c4ed 100644 --- a/routes/subscriptions.go +++ b/routes/subscriptions.go @@ -1,6 +1,7 @@ package routes import ( + "context" "encoding/json" "net/http" @@ -119,11 +120,11 @@ func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Re // Check to make we didnt get a null if discordWebHookId == "" { - http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest ) + http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest) return } if sourceId == "" { - http.Error(w, "invalid sourceID given", http.StatusBadRequest ) + http.Error(w, "invalid sourceID given", http.StatusBadRequest) return } @@ -141,8 +142,8 @@ func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Re // Check if the sub already exists item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{ - Discordwebhookid: uHook, - Sourceid: uSource, + Discordwebhookid: uHook, + Sourceid: uSource, }) if err == nil { bJson, err := json.Marshal(&item) @@ -154,15 +155,15 @@ func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Re w.Write(bJson) return } - + // Does not exist, so make it. params := database.CreateSubscriptionParams{ - ID: uuid.New(), + ID: uuid.New(), Discordwebhookid: uHook, - Sourceid: uSource, + Sourceid: uSource, } s.Db.CreateSubscription(*s.ctx, params) - + bJson, err := json.Marshal(¶ms) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -170,4 +171,26 @@ func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Re } w.Header().Set("Content-Type", "application/json") w.Write(bJson) -} \ No newline at end of file +} + +// DeleteDiscordWebHookSubscription +// @Summary Removes a Discord WebHook Subscription based on the Subscription ID. +// @Param Id query string true "Id" +// @Tags Config, Source, Discord, Subscription +// @Router /subscriptions/discord/webhook/delete [delete] +func (s *Server) DeleteDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) { + var ErrMissingSubscriptionID string = "Request was missing a 'Id' or was a invalid UUID." + query := r.URL.Query() + + uid, err := uuid.Parse(query["Id"][0]) + if err != nil { + http.Error(w, ErrMissingSubscriptionID, http.StatusBadRequest) + return + } + + err = s.Db.DeleteSubscription(context.Background(), uid) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} diff --git a/services/cron/scheduler.go b/services/cron/scheduler.go index 369909f..1136840 100644 --- a/services/cron/scheduler.go +++ b/services/cron/scheduler.go @@ -283,7 +283,7 @@ func (c *Cron) checkPosts(posts []database.Article, sourceName string) error { return nil } -func (c *Cron) postArticle(id uuid.UUID,item database.Article) error { +func (c *Cron) postArticle(id uuid.UUID, item database.Article) error { err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{ ID: id, Sourceid: item.Sourceid, @@ -304,7 +304,7 @@ func (c *Cron) postArticle(id uuid.UUID,item database.Article) error { func (c *Cron) addToDiscordQueue(Id uuid.UUID) error { err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{ - ID: uuid.New(), + ID: uuid.New(), Articleid: Id, }) if err != nil { diff --git a/services/input/ffxiv.go b/services/input/ffxiv.go index 3f2f4c2..2b04f8f 100644 --- a/services/input/ffxiv.go +++ b/services/input/ffxiv.go @@ -35,7 +35,7 @@ type FFXIVClient struct { func NewFFXIVClient(Record database.Source) FFXIVClient { return FFXIVClient{ - record: Record, + record: Record, cacheGroup: "ffxiv", } } @@ -47,50 +47,67 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { defer parser.Close() links, err := fc.PullFeed(parser) - if err != nil { return articles, err } + if err != nil { + return articles, err + } cache := cache.NewCacheClient(fc.cacheGroup) for _, link := range links { // Check cache/db if this link has been seen already, skip _, err := cache.FindByValue(link) - if err == nil { continue } - + if err == nil { + continue + } page := fc.GetPage(parser, link) title, err := fc.ExtractTitle(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } thumb, err := fc.ExtractThumbnail(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } pubDate, err := fc.ExtractPubDate(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } description, err := fc.ExtractDescription(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } authorName, err := fc.ExtractAuthor(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } authorImage, err := fc.ExtractAuthorImage(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } tags, err := fc.ExtractTags(page) - if err != nil { return articles, err } + if err != nil { + return articles, err + } article := database.Article{ - Sourceid: fc.record.ID, - Tags: tags, - Title: title, - Url: link, - Pubdate: pubDate, + Sourceid: fc.record.ID, + Tags: tags, + Title: title, + Url: link, + Pubdate: pubDate, Videoheight: 0, - Videowidth: 0, - Thumbnail: thumb, + Videowidth: 0, + Thumbnail: thumb, Description: description, - Authorname: sql.NullString{String: authorName}, + Authorname: sql.NullString{String: authorName}, Authorimage: sql.NullString{String: authorImage}, } log.Printf("Collected '%v' from '%v'", article.Title, article.Url) @@ -105,15 +122,19 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { func (fc *FFXIVClient) GetParser() (*goquery.Document, error) { html, err := http.Get(fc.record.Url) - if err != nil { return nil, err } + if err != nil { + return nil, err + } defer html.Body.Close() - + doc, err := goquery.NewDocumentFromReader(html.Body) - if err != nil { return nil, err } + if err != nil { + return nil, err + } return doc, nil } -func (fc *FFXIVClient) GetBrowser() (*rod.Browser) { +func (fc *FFXIVClient) GetBrowser() *rod.Browser { var browser *rod.Browser if path, exists := launcher.LookPath(); exists { u := launcher.New().Bin(path).MustLaunch() @@ -133,26 +154,26 @@ func (fc *FFXIVClient) PullFeed(parser *rod.Browser) ([]string, error) { // find all the li items items := res.MustElements("li") - + for _, item := range items { // in each li, find the a items a, err := item.Element("a") - if err != nil { + if err != nil { log.Println("Unable to find the a item, skipping") - continue + continue } - // find the href behind the a + // find the href behind the a url, err := a.Property("href") - if err != nil { + if err != nil { log.Println("Unable to find a href link, skipping") - continue + continue } urlString := url.String() isTopic := strings.Contains(urlString, "topics") if isTopic { - links = append(links, urlString) + links = append(links, urlString) } } @@ -166,8 +187,10 @@ func (rc *FFXIVClient) GetPage(parser *rod.Browser, url string) *rod.Page { func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) { thumbnail := page.MustElementX("/html/body/div[3]/div[2]/div[1]/article/div[1]/img").MustProperty("src").String() - if thumbnail == "" { return "", errors.New("unable to find thumbnail")} - + if thumbnail == "" { + return "", errors.New("unable to find thumbnail") + } + title := page.MustElement(".news__header > h1:nth-child(2)").MustText() log.Println(title) @@ -176,17 +199,23 @@ func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) { func (fc *FFXIVClient) ExtractPubDate(page *rod.Page) (time.Time, error) { stringDate := page.MustElement(".news__ic--topics").MustText() - if stringDate == "" { return time.Now(), errors.New("unable to locate the publish date on the post")} + if stringDate == "" { + return time.Now(), errors.New("unable to locate the publish date on the post") + } PubDate, err := time.Parse(FFXIV_TIME_FORMAT, stringDate) - if err != nil { return time.Now(), err } + if err != nil { + return time.Now(), err + } return PubDate, nil } func (fc *FFXIVClient) ExtractDescription(page *rod.Page) (string, error) { res := page.MustElement(".news__detail__wrapper").MustText() - if res == "" { return "", errors.New("unable to locate the description on the post")} + if res == "" { + return "", errors.New("unable to locate the description on the post") + } return res, nil } @@ -195,12 +224,18 @@ func (fc *FFXIVClient) ExtractAuthor(page *rod.Page) (string, error) { meta := page.MustElements("head > meta") for _, item := range meta { name, err := item.Property("name") - if err != nil { return "", err } + if err != nil { + return "", err + } - if name.String() != "author" { continue } + if name.String() != "author" { + continue + } content, err := item.Property("content") - if err != nil { return "", err } - + if err != nil { + return "", err + } + return content.String(), nil } //log.Println(meta) @@ -211,12 +246,18 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) { meta := page.MustElements("head > meta") for _, item := range meta { name, err := item.Property("name") - if err != nil { return "", err } + if err != nil { + return "", err + } - if name.String() != "keywords" { continue } + if name.String() != "keywords" { + continue + } content, err := item.Property("content") - if err != nil { return "", err } - + if err != nil { + return "", err + } + return content.String(), nil } //log.Println(meta) @@ -225,13 +266,19 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) { func (fc *FFXIVClient) ExtractTitle(page *rod.Page) (string, error) { title, err := page.MustElement("head > title").Text() - if err != nil { return "", err } + if err != nil { + return "", err + } - if !strings.Contains(title, "|") { return "", errors.New("unable to split the title, missing | in the string")} + if !strings.Contains(title, "|") { + return "", errors.New("unable to split the title, missing | in the string") + } res := strings.Split(title, "|") - if title != "" { return res[0], nil } - + if title != "" { + return res[0], nil + } + //log.Println(meta) return "", errors.New("unable to find the author on the page") } @@ -240,15 +287,20 @@ func (fc *FFXIVClient) ExtractAuthorImage(page *rod.Page) (string, error) { meta := page.MustElements("head > link") for _, item := range meta { name, err := item.Property("rel") - if err != nil { return "", err } + if err != nil { + return "", err + } - if name.String() != "apple-touch-icon-precomposed" { continue } + if name.String() != "apple-touch-icon-precomposed" { + continue + } content, err := item.Property("href") - if err != nil { return "", err } - + if err != nil { + return "", err + } + return content.String(), nil } //log.Println(meta) return "", errors.New("unable to find the author image on the page") } -