From a1324ee1c1c542e3b858dd24b854856949269958 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Tue, 12 Jul 2022 15:28:31 -0700 Subject: [PATCH] Features/output discord (#12) * basic output looks to be working * cron was updated to add to the queue and post messages * new route to make discord webhook subscriptions * updated swag tags * swag * Updated delete subscription call * removed the time value as it throws off the msg template * updated logging * updated swagger * updated new subscription route * Updated logging and remove items from the queue if they dont have a subscription * updated getArticles to return the 50 newest for the portal * added endpoint to see if an item exists already * formatting * updated listArticles * added colors and updated the image * Updated to use the pointer in twitch * added the twitch login command to cron... it works now * found a better way to disable http2 for reddit. Test worked right away too * updated the cron tasks to run collected once and hour or longer depending on the service --- database/query.sql.go | 120 ++++++++++++----- database/schema/query.sql | 15 ++- docs/docs.go | 110 ++++++++++------ docs/swagger.json | 110 ++++++++++------ docs/swagger.yaml | 102 +++++++++------ routes/articles.go | 46 ++++++- routes/discordQueue.go | 2 +- routes/discordwebhooks.go | 8 +- routes/root.go | 6 +- routes/server.go | 23 ++-- routes/settings.go | 4 +- routes/sources.go | 20 +-- routes/subscriptions.go | 74 ++++++++++- services/cron/scheduler.go | 100 ++++++++++----- services/input/httpClient.go | 15 ++- services/input/reddit.go | 40 +++--- services/input/twitch.go | 29 +++-- services/input/youtube.go | 8 +- services/output/discordwebhook.go | 170 ++++++++++++++++++------- services/output/discordwebhook_test.go | 139 ++++++++++++++------ 20 files changed, 797 insertions(+), 344 deletions(-) diff --git a/database/query.sql.go b/database/query.sql.go index e1318d2..6ca1084 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -253,16 +253,11 @@ func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error { } const deleteSubscription = `-- name: DeleteSubscription :exec -Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2 +Delete From subscriptions Where id = $1 ` -type DeleteSubscriptionParams struct { - Discordwebhookid uuid.UUID - Sourceid uuid.UUID -} - -func (q *Queries) DeleteSubscription(ctx context.Context, arg DeleteSubscriptionParams) error { - _, err := q.db.ExecContext(ctx, deleteSubscription, arg.Discordwebhookid, arg.Sourceid) +func (q *Queries) DeleteSubscription(ctx context.Context, id uuid.UUID) error { + _, err := q.db.ExecContext(ctx, deleteSubscription, id) return err } @@ -534,6 +529,23 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor return i, err } +const getDiscordWebHookByUrl = `-- name: GetDiscordWebHookByUrl :one +Select id, url, server, channel, enabled From DiscordWebHooks Where url = $1 +` + +func (q *Queries) GetDiscordWebHookByUrl(ctx context.Context, url string) (Discordwebhook, error) { + row := q.db.QueryRowContext(ctx, getDiscordWebHookByUrl, url) + var i Discordwebhook + err := row.Scan( + &i.ID, + &i.Url, + &i.Server, + &i.Channel, + &i.Enabled, + ) + return i, err +} + const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one Select id, url, server, channel, enabled from DiscordWebHooks Where ID = $1 LIMIT 1 @@ -648,6 +660,27 @@ func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, erro return i, err } +const getSourceByName = `-- name: GetSourceByName :one +Select id, site, name, source, type, value, enabled, url, tags from Sources where name = $1 Limit 1 +` + +func (q *Queries) GetSourceByName(ctx context.Context, name string) (Source, error) { + row := q.db.QueryRowContext(ctx, getSourceByName, name) + var i Source + err := row.Scan( + &i.ID, + &i.Site, + &i.Name, + &i.Source, + &i.Type, + &i.Value, + &i.Enabled, + &i.Url, + &i.Tags, + ) + return i, err +} + const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1 ` @@ -743,6 +776,47 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err return items, nil } +const listArticlesByDate = `-- name: ListArticlesByDate :many +Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage From articles ORDER BY pubdate desc Limit $1 +` + +func (q *Queries) ListArticlesByDate(ctx context.Context, limit int32) ([]Article, error) { + rows, err := q.db.QueryContext(ctx, listArticlesByDate, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Article + for rows.Next() { + var i Article + if err := rows.Scan( + &i.ID, + &i.Sourceid, + &i.Tags, + &i.Title, + &i.Url, + &i.Pubdate, + &i.Video, + &i.Videoheight, + &i.Videowidth, + &i.Thumbnail, + &i.Description, + &i.Authorname, + &i.Authorimage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many Select id, articleid from DiscordQueue LIMIT $1 ` @@ -965,8 +1039,8 @@ func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid return items, nil } -const querySubscriptions = `-- name: QuerySubscriptions :many -Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 +const querySubscriptions = `-- name: QuerySubscriptions :one +Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1 ` type QuerySubscriptionsParams struct { @@ -974,25 +1048,9 @@ type QuerySubscriptionsParams struct { Sourceid uuid.UUID } -func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) ([]Subscription, error) { - rows, err := q.db.QueryContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid) - if err != nil { - return nil, err - } - defer rows.Close() - var items []Subscription - for rows.Next() { - var i Subscription - if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil +func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) (Subscription, error) { + row := q.db.QueryRowContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid) + var i Subscription + err := row.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid) + return i, err } diff --git a/database/schema/query.sql b/database/schema/query.sql index 13ff152..f37beb5 100644 --- a/database/schema/query.sql +++ b/database/schema/query.sql @@ -10,6 +10,9 @@ Where Url = $1 LIMIT 1; -- name: ListArticles :many Select * From articles Limit $1; +-- name: ListArticlesByDate :many +Select * From articles ORDER BY pubdate desc Limit $1; + -- name: GetArticlesBySource :many select * from articles INNER join sources on articles.sourceid=Sources.ID @@ -66,6 +69,9 @@ Where ID = $1 LIMIT 1; Select * From DiscordWebHooks Where Server = $1; +-- name: GetDiscordWebHookByUrl :one +Select * From DiscordWebHooks Where url = $1; + -- name: ListDiscordWebhooks :many Select * From discordwebhooks LIMIT $1; @@ -127,6 +133,9 @@ Values -- name: GetSourceByID :one Select * From Sources where ID = $1 Limit 1; +-- name: GetSourceByName :one +Select * from Sources where name = $1 Limit 1; + -- name: ListSources :many Select * From Sources Limit $1; @@ -154,8 +163,8 @@ Select * From subscriptions Limit $1; -- name: ListSubscriptionsBySourceId :many Select * From subscriptions where sourceid = $1; --- name: QuerySubscriptions :many -Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2; +-- name: QuerySubscriptions :one +Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1; -- name: GetSubscriptionsBySourceID :many Select * From subscriptions Where sourceid = $1; @@ -164,4 +173,4 @@ Select * From subscriptions Where sourceid = $1; Select * from subscriptions Where discordwebhookid = $1; -- name: DeleteSubscription :exec -Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2; \ No newline at end of file +Delete From subscriptions Where id = $1; \ No newline at end of file diff --git a/docs/docs.go b/docs/docs.go index c332165..40c1c45 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -22,7 +22,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "articles" + "Articles" ], "summary": "Lists the top 50 records", "responses": {} @@ -34,7 +34,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "articles" + "Articles" ], "summary": "Finds the articles based on the SourceID provided. Returns the top 50.", "parameters": [ @@ -55,7 +55,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "articles" + "Articles" ], "summary": "Returns an article based on defined ID.", "parameters": [ @@ -76,8 +76,8 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Lists the top 50 records", "responses": {} @@ -86,9 +86,9 @@ const docTemplate = `{ "/config/sources/new/reddit": { "post": { "tags": [ - "config", - "source", - "reddit" + "Config", + "Source", + "Reddit" ], "summary": "Creates a new reddit source to monitor.", "parameters": [ @@ -113,9 +113,9 @@ const docTemplate = `{ "/config/sources/new/twitch": { "post": { "tags": [ - "config", - "source", - "twitch" + "Config", + "Source", + "Twitch" ], "summary": "Creates a new twitch source to monitor.", "parameters": [ @@ -147,9 +147,9 @@ const docTemplate = `{ "/config/sources/new/youtube": { "post": { "tags": [ - "config", - "source", - "youtube" + "Config", + "Source", + "YouTube" ], "summary": "Creates a new youtube source to monitor.", "parameters": [ @@ -184,8 +184,8 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Returns a single entity by ID", "parameters": [ @@ -201,8 +201,8 @@ const docTemplate = `{ }, "delete": { "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Deletes a record by ID.", "parameters": [ @@ -220,8 +220,8 @@ const docTemplate = `{ "/config/sources/{id}/disable": { "post": { "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Disables a source from processing.", "parameters": [ @@ -239,8 +239,8 @@ const docTemplate = `{ "/config/sources/{id}/enable": { "post": { "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Enables a source to continue processing.", "parameters": [ @@ -261,7 +261,7 @@ const docTemplate = `{ "application/json" ], "tags": [ - "debug", + "Debug", "Discord", "Queue" ], @@ -275,9 +275,9 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", + "Config", "Discord", - "Webhooks" + "Webhook" ], "summary": "Returns the top 100 entries from the queue to be processed.", "responses": {} @@ -289,9 +289,9 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", + "Config", "Discord", - "Webhooks" + "Webhook" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -309,9 +309,9 @@ const docTemplate = `{ "/discord/webhooks/new": { "post": { "tags": [ - "config", + "Config", "Discord", - "Webhooks" + "Webhook" ], "summary": "Creates a new record for a discord web hook to post data to.", "parameters": [ @@ -331,7 +331,7 @@ const docTemplate = `{ }, { "type": "string", - "description": "Channel name.", + "description": "Channel name", "name": "channel", "in": "query", "required": true @@ -346,7 +346,7 @@ const docTemplate = `{ "text/plain" ], "tags": [ - "debug" + "Debug" ], "summary": "Responds back with \"Hello x\" depending on param passed in.", "parameters": [ @@ -367,7 +367,7 @@ const docTemplate = `{ "text/plain" ], "tags": [ - "debug" + "Debug" ], "summary": "Responds back with \"Hello world!\"", "responses": {} @@ -379,7 +379,7 @@ const docTemplate = `{ "text/plain" ], "tags": [ - "debug" + "Debug" ], "summary": "Sends back \"pong\". Good to test with.", "responses": {} @@ -391,9 +391,9 @@ const docTemplate = `{ "application/json" ], "tags": [ - "settings" + "Settings" ], - "summary": "Returns a object based on the Key that was given/", + "summary": "Returns a object based on the Key that was given.", "parameters": [ { "type": "string", @@ -412,8 +412,8 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", - "Subscriptions" + "Config", + "Subscription" ], "summary": "Returns the top 100 entries from the queue to be processed.", "responses": {} @@ -425,8 +425,8 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", - "Subscriptions" + "Config", + "Subscription" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -447,8 +447,8 @@ const docTemplate = `{ "application/json" ], "tags": [ - "config", - "Subscriptions" + "Config", + "Subscription" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -462,6 +462,34 @@ const docTemplate = `{ ], "responses": {} } + }, + "/subscriptions/new/discordwebhook": { + "post": { + "tags": [ + "Config", + "Source", + "Discord", + "Subscription" + ], + "summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.", + "parameters": [ + { + "type": "string", + "description": "discordWebHookId", + "name": "discordWebHookId", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "sourceId", + "name": "sourceId", + "in": "query", + "required": true + } + ], + "responses": {} + } } } }` diff --git a/docs/swagger.json b/docs/swagger.json index 327f8a0..6087439 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -13,7 +13,7 @@ "application/json" ], "tags": [ - "articles" + "Articles" ], "summary": "Lists the top 50 records", "responses": {} @@ -25,7 +25,7 @@ "application/json" ], "tags": [ - "articles" + "Articles" ], "summary": "Finds the articles based on the SourceID provided. Returns the top 50.", "parameters": [ @@ -46,7 +46,7 @@ "application/json" ], "tags": [ - "articles" + "Articles" ], "summary": "Returns an article based on defined ID.", "parameters": [ @@ -67,8 +67,8 @@ "application/json" ], "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Lists the top 50 records", "responses": {} @@ -77,9 +77,9 @@ "/config/sources/new/reddit": { "post": { "tags": [ - "config", - "source", - "reddit" + "Config", + "Source", + "Reddit" ], "summary": "Creates a new reddit source to monitor.", "parameters": [ @@ -104,9 +104,9 @@ "/config/sources/new/twitch": { "post": { "tags": [ - "config", - "source", - "twitch" + "Config", + "Source", + "Twitch" ], "summary": "Creates a new twitch source to monitor.", "parameters": [ @@ -138,9 +138,9 @@ "/config/sources/new/youtube": { "post": { "tags": [ - "config", - "source", - "youtube" + "Config", + "Source", + "YouTube" ], "summary": "Creates a new youtube source to monitor.", "parameters": [ @@ -175,8 +175,8 @@ "application/json" ], "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Returns a single entity by ID", "parameters": [ @@ -192,8 +192,8 @@ }, "delete": { "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Deletes a record by ID.", "parameters": [ @@ -211,8 +211,8 @@ "/config/sources/{id}/disable": { "post": { "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Disables a source from processing.", "parameters": [ @@ -230,8 +230,8 @@ "/config/sources/{id}/enable": { "post": { "tags": [ - "config", - "source" + "Config", + "Source" ], "summary": "Enables a source to continue processing.", "parameters": [ @@ -252,7 +252,7 @@ "application/json" ], "tags": [ - "debug", + "Debug", "Discord", "Queue" ], @@ -266,9 +266,9 @@ "application/json" ], "tags": [ - "config", + "Config", "Discord", - "Webhooks" + "Webhook" ], "summary": "Returns the top 100 entries from the queue to be processed.", "responses": {} @@ -280,9 +280,9 @@ "application/json" ], "tags": [ - "config", + "Config", "Discord", - "Webhooks" + "Webhook" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -300,9 +300,9 @@ "/discord/webhooks/new": { "post": { "tags": [ - "config", + "Config", "Discord", - "Webhooks" + "Webhook" ], "summary": "Creates a new record for a discord web hook to post data to.", "parameters": [ @@ -322,7 +322,7 @@ }, { "type": "string", - "description": "Channel name.", + "description": "Channel name", "name": "channel", "in": "query", "required": true @@ -337,7 +337,7 @@ "text/plain" ], "tags": [ - "debug" + "Debug" ], "summary": "Responds back with \"Hello x\" depending on param passed in.", "parameters": [ @@ -358,7 +358,7 @@ "text/plain" ], "tags": [ - "debug" + "Debug" ], "summary": "Responds back with \"Hello world!\"", "responses": {} @@ -370,7 +370,7 @@ "text/plain" ], "tags": [ - "debug" + "Debug" ], "summary": "Sends back \"pong\". Good to test with.", "responses": {} @@ -382,9 +382,9 @@ "application/json" ], "tags": [ - "settings" + "Settings" ], - "summary": "Returns a object based on the Key that was given/", + "summary": "Returns a object based on the Key that was given.", "parameters": [ { "type": "string", @@ -403,8 +403,8 @@ "application/json" ], "tags": [ - "config", - "Subscriptions" + "Config", + "Subscription" ], "summary": "Returns the top 100 entries from the queue to be processed.", "responses": {} @@ -416,8 +416,8 @@ "application/json" ], "tags": [ - "config", - "Subscriptions" + "Config", + "Subscription" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -438,8 +438,8 @@ "application/json" ], "tags": [ - "config", - "Subscriptions" + "Config", + "Subscription" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -453,6 +453,34 @@ ], "responses": {} } + }, + "/subscriptions/new/discordwebhook": { + "post": { + "tags": [ + "Config", + "Source", + "Discord", + "Subscription" + ], + "summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.", + "parameters": [ + { + "type": "string", + "description": "discordWebHookId", + "name": "discordWebHookId", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "sourceId", + "name": "sourceId", + "in": "query", + "required": true + } + ], + "responses": {} + } } } } \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index fe7d7cf..01b599b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -11,7 +11,7 @@ paths: responses: {} summary: Lists the top 50 records tags: - - articles + - Articles /articles/{id}: get: parameters: @@ -25,7 +25,7 @@ paths: responses: {} summary: Returns an article based on defined ID. tags: - - articles + - Articles /articles/by/sourceid: get: parameters: @@ -40,7 +40,7 @@ paths: summary: Finds the articles based on the SourceID provided. Returns the top 50. tags: - - articles + - Articles /config/sources: get: produces: @@ -48,8 +48,8 @@ paths: responses: {} summary: Lists the top 50 records tags: - - config - - source + - Config + - Source /config/sources/{id}: delete: parameters: @@ -61,8 +61,8 @@ paths: responses: {} summary: Deletes a record by ID. tags: - - config - - source + - Config + - Source get: parameters: - description: uuid @@ -75,8 +75,8 @@ paths: responses: {} summary: Returns a single entity by ID tags: - - config - - source + - Config + - Source /config/sources/{id}/disable: post: parameters: @@ -88,8 +88,8 @@ paths: responses: {} summary: Disables a source from processing. tags: - - config - - source + - Config + - Source /config/sources/{id}/enable: post: parameters: @@ -101,8 +101,8 @@ paths: responses: {} summary: Enables a source to continue processing. tags: - - config - - source + - Config + - Source /config/sources/new/reddit: post: parameters: @@ -119,9 +119,9 @@ paths: responses: {} summary: Creates a new reddit source to monitor. tags: - - config - - source - - reddit + - Config + - Source + - Reddit /config/sources/new/twitch: post: parameters: @@ -143,9 +143,9 @@ paths: responses: {} summary: Creates a new twitch source to monitor. tags: - - config - - source - - twitch + - Config + - Source + - Twitch /config/sources/new/youtube: post: parameters: @@ -167,9 +167,9 @@ paths: responses: {} summary: Creates a new youtube source to monitor. tags: - - config - - source - - youtube + - Config + - Source + - YouTube /discord/queue: get: produces: @@ -177,7 +177,7 @@ paths: responses: {} summary: Returns the top 100 entries from the queue to be processed. tags: - - debug + - Debug - Discord - Queue /discord/webhooks: @@ -187,9 +187,9 @@ paths: responses: {} summary: Returns the top 100 entries from the queue to be processed. tags: - - config + - Config - Discord - - Webhooks + - Webhook /discord/webhooks/byId: get: parameters: @@ -203,9 +203,9 @@ paths: responses: {} summary: Returns the top 100 entries from the queue to be processed. tags: - - config + - Config - Discord - - Webhooks + - Webhook /discord/webhooks/new: post: parameters: @@ -219,7 +219,7 @@ paths: name: server required: true type: string - - description: Channel name. + - description: Channel name in: query name: channel required: true @@ -227,9 +227,9 @@ paths: responses: {} summary: Creates a new record for a discord web hook to post data to. tags: - - config + - Config - Discord - - Webhooks + - Webhook /hello/{who}: get: parameters: @@ -243,7 +243,7 @@ paths: responses: {} summary: Responds back with "Hello x" depending on param passed in. tags: - - debug + - Debug /helloworld: get: produces: @@ -251,7 +251,7 @@ paths: responses: {} summary: Responds back with "Hello world!" tags: - - debug + - Debug /ping: get: produces: @@ -259,7 +259,7 @@ paths: responses: {} summary: Sends back "pong". Good to test with. tags: - - debug + - Debug /settings/{key}: get: parameters: @@ -271,9 +271,9 @@ paths: produces: - application/json responses: {} - summary: Returns a object based on the Key that was given/ + summary: Returns a object based on the Key that was given. tags: - - settings + - Settings /subscriptions: get: produces: @@ -281,8 +281,8 @@ paths: responses: {} summary: Returns the top 100 entries from the queue to be processed. tags: - - config - - Subscriptions + - Config + - Subscription /subscriptions/byDiscordId: get: parameters: @@ -296,8 +296,8 @@ paths: responses: {} summary: Returns the top 100 entries from the queue to be processed. tags: - - config - - Subscriptions + - Config + - Subscription /subscriptions/bySourceId: get: parameters: @@ -311,6 +311,26 @@ paths: responses: {} summary: Returns the top 100 entries from the queue to be processed. tags: - - config - - Subscriptions + - Config + - Subscription + /subscriptions/new/discordwebhook: + post: + parameters: + - description: discordWebHookId + in: query + name: discordWebHookId + required: true + type: string + - description: sourceId + in: query + name: sourceId + required: true + type: string + responses: {} + summary: Creates a new subscription to link a post from a Source to a DiscordWebHook. + tags: + - Config + - Source + - Discord + - Subscription swagger: "2.0" diff --git a/routes/articles.go b/routes/articles.go index d2ee736..c800afa 100644 --- a/routes/articles.go +++ b/routes/articles.go @@ -11,12 +11,12 @@ import ( // ListArticles // @Summary Lists the top 50 records // @Produce application/json -// @Tags articles +// @Tags Articles // @Router /articles [get] func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - res, err := s.Db.ListArticles(*s.ctx, 50) + res, err := s.Db.ListArticlesByDate(*s.ctx, 50) if err != nil { w.Write([]byte(err.Error())) panic(err) @@ -27,7 +27,6 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) { w.Write([]byte(err.Error())) panic(err) } - w.Write(bres) } @@ -35,7 +34,7 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) { // @Summary Returns an article based on defined ID. // @Param id path string true "uuid" // @Produce application/json -// @Tags articles +// @Tags Articles // @Router /articles/{id} [get] func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -67,7 +66,7 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) { // @Summary Finds the articles based on the SourceID provided. Returns the top 50. // @Param id query string true "Source ID UUID" // @Produce application/json -// @Tags articles +// @Tags Articles // @Router /articles/by/sourceid [get] func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -95,4 +94,39 @@ func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) { } w.Write(bres) -} \ No newline at end of file +} + +// TODO add page support +// GetArticlesByTag +// @Summary Finds the articles based on the SourceID provided. Returns the top 50. +// @Param Tag query string true "Tag name" +// @Produce application/json +// @Tags Articles +// @Router /articles/by/sourceid [get] +func (s *Server) GetArticlesByTag(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + r.URL.Query() + query := r.URL.Query() + _id := query["id"][0] + + uuid, err := uuid.Parse(_id) + if err != nil { + w.Write([]byte(err.Error())) + panic(err) + } + + res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid) + if err != nil { + w.Write([]byte(err.Error())) + panic(err) + } + + bres, err := json.Marshal(res) + if err != nil { + w.Write([]byte(err.Error())) + panic(err) + } + + w.Write(bres) +} diff --git a/routes/discordQueue.go b/routes/discordQueue.go index 246e5bb..d31acb2 100644 --- a/routes/discordQueue.go +++ b/routes/discordQueue.go @@ -8,7 +8,7 @@ import ( // GetDiscordQueue // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json -// @Tags debug, Discord, Queue +// @Tags Debug, Discord, Queue // @Router /discord/queue [get] func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/routes/discordwebhooks.go b/routes/discordwebhooks.go index 81d3784..5af6e0f 100644 --- a/routes/discordwebhooks.go +++ b/routes/discordwebhooks.go @@ -13,7 +13,7 @@ import ( // GetDiscordWebHooks // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json -// @Tags config, Discord, Webhooks +// @Tags Config, Discord, Webhook // @Router /discord/webhooks [get] func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -37,7 +37,7 @@ func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) { // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json // @Param id query string true "id" -// @Tags config, Discord, Webhooks +// @Tags Config, Discord, Webhook // @Router /discord/webhooks/byId [get] func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -74,8 +74,8 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) // @Summary Creates a new record for a discord web hook to post data to. // @Param url query string true "url" // @Param server query string true "Server name" -// @Param channel query string true "Channel name." -// @Tags config, Discord, Webhooks +// @Param channel query string true "Channel name" +// @Tags Config, Discord, Webhook // @Router /discord/webhooks/new [post] func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() diff --git a/routes/root.go b/routes/root.go index 1f029ce..ae0626d 100644 --- a/routes/root.go +++ b/routes/root.go @@ -22,7 +22,7 @@ func RootRoutes() chi.Router { // HelloWorld // @Summary Responds back with "Hello world!" // @Produce plain -// @Tags debug +// @Tags Debug // @Router /helloworld [get] func helloWorld(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hello World!")) @@ -31,7 +31,7 @@ func helloWorld(w http.ResponseWriter, r *http.Request) { // Ping // @Summary Sends back "pong". Good to test with. // @Produce plain -// @Tags debug +// @Tags Debug // @Router /ping [get] func ping(w http.ResponseWriter, r *http.Request) { msg := "pong" @@ -42,7 +42,7 @@ func ping(w http.ResponseWriter, r *http.Request) { // @Summary Responds back with "Hello x" depending on param passed in. // @Param who path string true "Who" // @Produce plain -// @Tags debug +// @Tags Debug // @Router /hello/{who} [get] func helloWho(w http.ResponseWriter, r *http.Request) { msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who")) diff --git a/routes/server.go b/routes/server.go index a4d23a0..f90b46b 100644 --- a/routes/server.go +++ b/routes/server.go @@ -3,12 +3,13 @@ package routes import ( "context" "database/sql" + //"net/http" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - httpSwagger "github.com/swaggo/http-swagger" _ "github.com/lib/pq" + httpSwagger "github.com/swaggo/http-swagger" "github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/services/config" @@ -16,14 +17,14 @@ import ( type Server struct { Router *chi.Mux - Db *database.Queries - ctx *context.Context + Db *database.Queries + ctx *context.Context } var ( - ErrIdValueMissing string = "id value is missing" - ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct." - ErrNoRecordFound string = "no record was found." + ErrIdValueMissing string = "id value is missing" + ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct." + ErrNoRecordFound string = "no record was found." ErrUnableToConvertToJson string = "Unable to convert to json" ) @@ -59,13 +60,14 @@ func openDatabase(ctx context.Context) (*database.Queries, error) { func (s *Server) MountMiddleware() { s.Router.Use(middleware.Logger) s.Router.Use(middleware.Recoverer) + //s.Router.Use(middleware.Heartbeat()) } func (s *Server) MountRoutes() { s.Router.Get("/swagger/*", httpSwagger.Handler( httpSwagger.URL("http://localhost:8081/swagger/doc.json"), //The url pointing to API definition )) - + /* Root Routes */ s.Router.Get("/api/helloworld", helloWorld) s.Router.Get("/api/hello/{who}", helloWho) @@ -88,10 +90,14 @@ func (s *Server) MountRoutes() { /* Settings */ s.Router.Get("/api/settings", s.getSettings) - + /* Source Routes */ s.Router.Get("/api/config/sources", s.listSources) + + /* Reddit Source Routes */ + s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource) + s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource) s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource) s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) { @@ -105,4 +111,5 @@ func (s *Server) MountRoutes() { s.Router.Get("/api/subscriptions", s.ListSubscriptions) s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId) s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId) + s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription) } diff --git a/routes/settings.go b/routes/settings.go index 0a10936..fd32216 100644 --- a/routes/settings.go +++ b/routes/settings.go @@ -10,10 +10,10 @@ import ( ) // GetSettings -// @Summary Returns a object based on the Key that was given/ +// @Summary Returns a object based on the Key that was given. // @Param key path string true "Settings Key value" // @Produce application/json -// @Tags settings +// @Tags Settings // @Router /settings/{key} [get] func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) { //var item model.Sources diff --git a/routes/sources.go b/routes/sources.go index d0696e9..d1ea1c2 100644 --- a/routes/sources.go +++ b/routes/sources.go @@ -15,7 +15,7 @@ import ( // ListSources // @Summary Lists the top 50 records // @Produce application/json -// @Tags config, source +// @Tags Config, Source // @Router /config/sources [get] func (s *Server) listSources(w http.ResponseWriter, r *http.Request) { //TODO Add top? @@ -48,7 +48,7 @@ func (s *Server) listSources(w http.ResponseWriter, r *http.Request) { // @Summary Returns a single entity by ID // @Param id path string true "uuid" // @Produce application/json -// @Tags config, source +// @Tags Config, Source // @Router /config/sources/{id} [get] func (s *Server) getSources(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "ID") @@ -78,7 +78,7 @@ func (s *Server) getSources(w http.ResponseWriter, r *http.Request) { // @Summary Creates a new reddit source to monitor. // @Param name query string true "name" // @Param url query string true "url" -// @Tags config, source, reddit +// @Tags Config, Source, Reddit // @Router /config/sources/new/reddit [post] func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() @@ -116,12 +116,16 @@ func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) { w.Write(bJson) } +func (s *Server) getSourceByType(w http.ResponseWriter, r *http.Request) { + +} + // NewYoutubeSource // @Summary Creates a new youtube source to monitor. // @Param name query string true "name" // @Param url query string true "url" // @Param tags query string true "tags" -// @Tags config, source, youtube +// @Tags Config, Source, YouTube // @Router /config/sources/new/youtube [post] func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() @@ -164,7 +168,7 @@ func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) { // @Param name query string true "name" // @Param url query string true "url" // @Param tags query string true "tags" -// @Tags config, source, twitch +// @Tags Config, Source, Twitch // @Router /config/sources/new/twitch [post] func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() @@ -205,7 +209,7 @@ func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) { // DeleteSource // @Summary Deletes a record by ID. // @Param id path string true "id" -// @Tags config, source +// @Tags Config, Source // @Router /config/sources/{id} [delete] func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) { //var item model.Sources = model.Sources{} @@ -232,7 +236,7 @@ func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) { // DisableSource // @Summary Disables a source from processing. // @Param id path string true "id" -// @Tags config, source +// @Tags Config, Source // @Router /config/sources/{id}/disable [post] func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "ID") @@ -256,7 +260,7 @@ func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) { // EnableSource // @Summary Enables a source to continue processing. // @Param id path string true "id" -// @Tags config, source +// @Tags Config, Source // @Router /config/sources/{id}/enable [post] func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "ID") diff --git a/routes/subscriptions.go b/routes/subscriptions.go index d8b5578..582749b 100644 --- a/routes/subscriptions.go +++ b/routes/subscriptions.go @@ -5,12 +5,13 @@ import ( "net/http" "github.com/google/uuid" + "github.com/jtom38/newsbot/collector/database" ) // GetSubscriptions // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json -// @Tags config, Subscriptions +// @Tags Config, Subscription // @Router /subscriptions [get] func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -34,7 +35,7 @@ func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) { // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json // @Param id query string true "id" -// @Tags config, Subscriptions +// @Tags Config, Subscription // @Router /subscriptions/byDiscordId [get] func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -71,7 +72,7 @@ func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Requ // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json // @Param id query string true "id" -// @Tags config, Subscriptions +// @Tags Config, Subscription // @Router /subscriptions/bySourceId [get] func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -102,4 +103,71 @@ func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Reque } w.Write(bres) +} + +// NewDiscordWebHookSubscription +// @Summary Creates a new subscription to link a post from a Source to a DiscordWebHook. +// @Param discordWebHookId query string true "discordWebHookId" +// @Param sourceId query string true "sourceId" +// @Tags Config, Source, Discord, Subscription +// @Router /subscriptions/new/discordwebhook [post] +func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) { + // Extract the values given + query := r.URL.Query() + discordWebHookId := query["discordWebHookId"][0] + sourceId := query["sourceId"][0] + + // Check to make we didnt get a null + if discordWebHookId == "" { + http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest ) + return + } + if sourceId == "" { + http.Error(w, "invalid sourceID given", http.StatusBadRequest ) + return + } + + // Valide they are UUID values + uHook, err := uuid.Parse(discordWebHookId) + if err != nil { + http.Error(w, "DiscordWebHooksID was not a uuid value.", http.StatusBadRequest) + return + } + uSource, err := uuid.Parse(sourceId) + if err != nil { + http.Error(w, "SourceId was not a uuid value", http.StatusBadRequest) + return + } + + // Check if the sub already exists + item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{ + Discordwebhookid: uHook, + Sourceid: uSource, + }) + if err == nil { + bJson, err := json.Marshal(&item) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(bJson) + return + } + + // Does not exist, so make it. + params := database.CreateSubscriptionParams{ + ID: uuid.New(), + Discordwebhookid: uHook, + Sourceid: uSource, + } + s.Db.CreateSubscription(*s.ctx, params) + + bJson, err := json.Marshal(¶ms) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(bJson) } \ No newline at end of file diff --git a/services/cron/scheduler.go b/services/cron/scheduler.go index ea04d70..7f7cd36 100644 --- a/services/cron/scheduler.go +++ b/services/cron/scheduler.go @@ -3,6 +3,7 @@ package cron import ( "context" "database/sql" + "fmt" "log" "time" @@ -11,8 +12,8 @@ import ( "github.com/robfig/cron/v3" "github.com/jtom38/newsbot/collector/database" - "github.com/jtom38/newsbot/collector/services/input" "github.com/jtom38/newsbot/collector/services/config" + "github.com/jtom38/newsbot/collector/services/input" "github.com/jtom38/newsbot/collector/services/output" ) @@ -51,29 +52,32 @@ func New(ctx context.Context) *Cron { res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND) if res { - timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() }) - log.Print("Reddit backend was enabled") + timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) + log.Print("[Input] Reddit backend was enabled") //go c.CheckReddit() } res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND) if res { - timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() }) - log.Print("YouTube backend was enabled") + timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) + log.Print("[Input] YouTube backend was enabled") } res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND) if res { - timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() }) - log.Print("FFXIV backend was enabled") + timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) + log.Print("[Input] FFXIV backend was enabled") } res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND) if res { - timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() }) - log.Print("Twitch backend was enabled") + timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) + log.Print("[Input] Twitch backend was enabled") } - + + timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() }) + log.Print("[Output] Discord Output was enabled") + c.timer = timer return c } @@ -162,6 +166,11 @@ func (c *Cron) CheckTwitch() error { return err } + err = tc.Login() + if err != nil { + return err + } + for _, source := range sources { if !source.Enabled { continue @@ -186,19 +195,13 @@ func (c *Cron) CheckDiscordQueue() error { return err } - for _, queue := range(queueItems) { + for _, queue := range queueItems { // Get the articleByID article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid) if err != nil { return err } - // Get the SourceByID - //source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid) - //if err != nil { - // return err - //} - var endpoints []string // List Subscription by SourceID subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid) @@ -206,8 +209,18 @@ func (c *Cron) CheckDiscordQueue() error { return err } + // if no one is subscribed to it, remove it from the index. + if len(subs) == 0 { + log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid) + err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID) + if err != nil { + return err + } + continue + } + // Get the webhhooks to send to - for _, sub := range(subs) { + for _, sub := range subs { webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid) if err != nil { return err @@ -218,16 +231,19 @@ func (c *Cron) CheckDiscordQueue() error { } // Create Discord Message - dwh := output.NewDiscordWebHookMessage(endpoints, article) - err = dwh.GeneratePayload() + dwh := output.NewDiscordWebHookMessage(article) + msg, err := dwh.GeneratePayload() if err != nil { return err } - - // Send Message - err = dwh.SendPayload() - if err != nil { - return err + + // Send Message(s) + for _, i := range endpoints { + err = dwh.SendPayload(msg, i) + + if err != nil { + return err + } } // Remove the item from the queue, given we sent our notification. @@ -235,29 +251,38 @@ func (c *Cron) CheckDiscordQueue() error { if err != nil { return err } + + time.Sleep(10 * time.Second) } return nil } -func (c *Cron) checkPosts(posts []database.Article, sourceName string) { +func (c *Cron) checkPosts(posts []database.Article, sourceName string) error { for _, item := range posts { _, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) if err != nil { - err = c.postArticle(item) + id := uuid.New() + + err := c.postArticle(id, item) if err != nil { - log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) - } else { - log.Printf("[%v] Posted article - %v\r", sourceName, item.Url) + return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) } + + err = c.addToDiscordQueue(id) + if err != nil { + return err + } + } } time.Sleep(30 * time.Second) + return nil } -func (c *Cron) postArticle(item database.Article) error { +func (c *Cron) postArticle(id uuid.UUID,item database.Article) error { err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{ - ID: uuid.New(), + ID: id, Sourceid: item.Sourceid, Tags: item.Tags, Title: item.Title, @@ -273,3 +298,14 @@ func (c *Cron) postArticle(item database.Article) error { }) return err } + +func (c *Cron) addToDiscordQueue(Id uuid.UUID) error { + err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{ + ID: uuid.New(), + Articleid: Id, + }) + if err != nil { + return err + } + return nil +} diff --git a/services/input/httpClient.go b/services/input/httpClient.go index 3039497..59fc058 100644 --- a/services/input/httpClient.go +++ b/services/input/httpClient.go @@ -1,15 +1,24 @@ package input import ( - "net/http" - "log" + "crypto/tls" "io/ioutil" + "log" + "net/http" ) // This will use the net/http client reach out to a site and collect the content. func getHttpContent(uri string) ([]byte, error) { + // Code to disable the http2 client for reddit. + // https://github.com/golang/go/issues/39302 + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.ForceAttemptHTTP2 = false + tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper) + tr.TLSClientConfig = &tls.Config{} - client := &http.Client{} + client := &http.Client{ + Transport: tr, + } req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } diff --git a/services/input/reddit.go b/services/input/reddit.go index 56183df..83487a8 100644 --- a/services/input/reddit.go +++ b/services/input/reddit.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log" - "os" "strings" "time" @@ -27,7 +26,7 @@ type RedditConfig struct { PullNSFW string } -func NewRedditClient(Record database.Source) RedditClient { +func NewRedditClient(Record database.Source) *RedditClient { rc := RedditClient{ record: Record, } @@ -36,23 +35,24 @@ func NewRedditClient(Record database.Source) RedditClient { rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW) rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP) - rc.disableHttp2Client() + //rc.disableHttp2Client() - return rc + return &rc } // This is needed for to get modern go to talk to the endpoint. // https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/ -func (rc RedditClient) disableHttp2Client() { - os.Setenv("GODEBUG", "http2client=0") -} +//func (rc *RedditClient) disableHttp2Client() { +// os.Setenv("GODEBUG", "http2client=0") +//} -func (rc RedditClient) GetBrowser() *rod.Browser { + +func (rc *RedditClient) GetBrowser() *rod.Browser { browser := rod.New().MustConnect() return browser } -func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page { +func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page { page := parser.MustPage(url) return page } @@ -61,13 +61,15 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page { // GetContent() reaches out to Reddit and pulls the Json data. // It will then convert the data to a struct and return the struct. -func (rc RedditClient) GetContent() (model.RedditJsonContent, error) { +func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) { var items model.RedditJsonContent = model.RedditJsonContent{} // TODO Wire this to support the config options Url := fmt.Sprintf("%v.json", rc.record.Url) - log.Printf("Collecting results on '%v'", rc.record.Name) + log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name) + + content, err := getHttpContent(Url) if err != nil { @@ -84,13 +86,13 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error) { return items, nil } -func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article { +func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article { var redditArticles []database.Article for _, item := range items.Data.Children { var article database.Article article, err := rc.convertToArticle(item.Data) if err != nil { - log.Println(err) + log.Printf("[Reddit] %v", err) continue } redditArticles = append(redditArticles, article) @@ -100,7 +102,7 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []databa // ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // This data can be passed to the database. -func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) { +func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) { var item database.Article if source.Content == "" && source.Url != "" { @@ -119,15 +121,15 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Artic item = rc.convertRedirectPost(source) } - if item.Description == "" { - var err = errors.New("reddit post failed to parse correctly") + if item.Description == "" && item.Title == "" { + var err = errors.New("post failed to parse correctly") return item, err } return item, nil } -func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article { +func (rc *RedditClient) convertPicturePost(source model.RedditPost) database.Article { var item = database.Article{ Sourceid: rc.record.ID, Title: source.Title, @@ -145,7 +147,7 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Arti return item } -func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article { +func (rc *RedditClient) convertTextPost(source model.RedditPost) database.Article { var item = database.Article{ Sourceid: rc.record.ID, Tags: "a", @@ -160,7 +162,7 @@ func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article return item } -func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article { +func (rc *RedditClient) convertVideoPost(source model.RedditPost) database.Article { var item = database.Article{ Sourceid: rc.record.ID, Tags: "a", diff --git a/services/input/twitch.go b/services/input/twitch.go index e251ef9..226e0e3 100644 --- a/services/input/twitch.go +++ b/services/input/twitch.go @@ -77,7 +77,7 @@ func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { } // Invokes Logon request to the API -func (tc TwitchClient) Login() error { +func (tc *TwitchClient) Login() error { token, err := tc.api.RequestAppAccessToken([]string{twitchScopes}) if err != nil { return err @@ -87,7 +87,7 @@ func (tc TwitchClient) Login() error { return nil } -func (tc TwitchClient) GetContent() ([]database.Article, error) { +func (tc *TwitchClient) GetContent() ([]database.Article, error) { var items []database.Article user, err := tc.GetUserDetails() @@ -136,7 +136,7 @@ func (tc TwitchClient) GetContent() ([]database.Article, error) { return items, nil } -func (tc TwitchClient) GetUserDetails() (helix.User, error) { +func (tc *TwitchClient) GetUserDetails() (helix.User, error) { var blank helix.User users, err := tc.api.GetUsers(&helix.UsersParams{ @@ -145,11 +145,16 @@ func (tc TwitchClient) GetUserDetails() (helix.User, error) { if err != nil { return blank, err } + + if len(users.Data.Users) == 0 { + return blank, errors.New("no results have been returned") + } + return users.Data.Users[0], nil } // This will reach out and collect the posts made by the user. -func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) { +func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) { var blank []helix.Video videos, err := tc.api.GetVideos(&helix.VideosParams{ @@ -163,14 +168,14 @@ func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) { return videos.Data.Videos, nil } -func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) { +func (tc *TwitchClient) ExtractAuthor(post helix.Video) (string, error) { if post.UserName == "" { return "", ErrMissingAuthorName } return post.UserName, nil } -func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) { +func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) { if post.ThumbnailURL == "" { return "", ErrMissingThumbnail } @@ -180,7 +185,7 @@ func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) { return thumb, nil } -func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) { +func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) { if post.PublishedAt == "" { return time.Now(), ErrMissingPublishDate } @@ -191,7 +196,7 @@ func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) { return pubDate, nil } -func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) { +func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) { // Check if the description is null but we have a title. // The poster didnt add a description but this isnt an error. if post.Description == "" && post.Title == "" { @@ -204,7 +209,7 @@ func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) { } // Extracts the avatar of the author with some validation. -func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) { +func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) { if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage } if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage } return user.ProfileImageURL, nil @@ -212,20 +217,20 @@ func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) { // Generate tags based on the video metadata. // TODO Figure out how to query what game is played -func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) { +func (tc *TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) { res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName) return res, nil } // Extracts the title from a post with some validation. -func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) { +func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) { if post.Title == "" { return "", errors.New("unable to find the title on the requested post") } return post.Title, nil } -func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) { +func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) { if post.URL == "" { return "", ErrMissingUrl } return post.URL, nil } \ No newline at end of file diff --git a/services/input/youtube.go b/services/input/youtube.go index 1f7057d..e48d14c 100644 --- a/services/input/youtube.go +++ b/services/input/youtube.go @@ -121,7 +121,7 @@ func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page { func (yc *YoutubeClient) GetParser(uri string) (*goquery.Document, error) { html, err := http.Get(uri) if err != nil { - log.Println(err) + log.Printf("[YouTube] %v", err) } defer html.Body.Close() @@ -244,19 +244,19 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { parser, err := yc.GetParser(item.Link) if err != nil { - log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link) + log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) } tags, err := yc.GetTags(parser) if err != nil { msg := fmt.Sprintf("%v. %v", err, item.Link) - log.Println(msg) + log.Printf("[YouTube] %v", msg) } thumb, err := yc.GetVideoThumbnail(parser) if err != nil { msg := fmt.Sprintf("%v. %v", err, item.Link) - log.Println(msg) + log.Printf("[YouTube] %v", msg) } var article = database.Article{ diff --git a/services/output/discordwebhook.go b/services/output/discordwebhook.go index 438a2fc..31107c6 100644 --- a/services/output/discordwebhook.go +++ b/services/output/discordwebhook.go @@ -1,80 +1,151 @@ package output import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" "strings" - "time" "github.com/jtom38/newsbot/collector/database" ) type discordField struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` - Inline bool `json:"inline,omitempty"` + Name *string `json:"name,omitempty"` + Value *string `json:"value,omitempty"` + Inline *bool `json:"inline,omitempty"` +} + +type discordFooter struct { + Value *string `json:"text,omitempty"` + IconUrl *string `json:"icon_url,omitempty"` } type discordAuthor struct { - Name string `json:"name,omitempty"` - Url string `json:"url,omitempty"` - IconUrl string `json:"icon_url,omitempty"` + Name *string `json:"name,omitempty"` + Url *string `json:"url,omitempty"` + IconUrl *string `json:"icon_url,omitempty"` } type discordImage struct { - Url string `json:"url,omitempty"` + Url *string `json:"url,omitempty"` } -type discordEmbed struct { - Title string `json:"title,omitempty"` - Description string `json:"description,omitempty"` - Url string `json:"url,omitempty"` - Color int32 `json:"color,omitempty"` - Timestamp time.Time `json:"timestamp,omitempty"` - Fields []discordField `json:"fields,omitempty"` - Author discordAuthor `json:"author,omitempty"` - Image discordImage `json:"image,omitempty"` - Thumbnail discordImage `json:"thumbnail,omitempty"` +type DiscordEmbed struct { + Title *string `json:"title,omitempty"` + Description *string `json:"description,omitempty"` + Url *string `json:"url,omitempty"` + Color *int32 `json:"color,omitempty"` + //Timestamp time.Time `json:"timestamp,omitempty"` + Fields []*discordField `json:"fields,omitempty"` + Author discordAuthor `json:"author,omitempty"` + Image discordImage `json:"image,omitempty"` + Thumbnail discordImage `json:"thumbnail,omitempty"` + Footer *discordFooter `json:"footer,omitempty"` } // Root object for Discord Webhook messages -type discordMessage struct { - Content string `json:"content,omitempty"` - Embeds []discordEmbed `json:"embeds,omitempty"` +type DiscordMessage struct { + Username *string `json:"username,omitempty"` + Content *string `json:"content,omitempty"` + Embeds *[]DiscordEmbed `json:"embeds,omitempty"` } +const ( + DefaultColor = 0 + YoutubeColor = 16711680 + TwitchColor = 0 + RedditColor = 0 + TwitterColor = 0 + FfxivColor = 0 +) + type Discord struct { Subscriptions []string - article database.Article - Message discordMessage + article database.Article + Message *DiscordMessage } -func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord { +func NewDiscordWebHookMessage(Article database.Article) Discord { return Discord{ - Subscriptions: Subscriptions, article: Article, - Message: discordMessage{ - Embeds: []discordEmbed{}, - }, } } -func (dwh Discord) GeneratePayload() error { - // Convert the message - embed := discordEmbed { - Title: dwh.article.Title, - Description: dwh.convertFromHtml(dwh.article.Description), - Url: dwh.article.Url, - Thumbnail: discordImage{ - Url: dwh.article.Thumbnail, - }, - } - var arr []discordEmbed +// Generates the link field to expose in the message +func (dwh Discord) getFields() []*discordField { + var fields []*discordField - arr = append(arr, embed) - dwh.Message.Embeds = arr - return nil + key := "Link" + linkField := discordField{ + Name: &key, + Value: &dwh.article.Url, + } + + fields = append(fields, &linkField) + + return fields } -func (dwh Discord) SendPayload() error { +// This will create the message that will be sent out. +func (dwh Discord) GeneratePayload() (*DiscordMessage, error) { + + // Create the embed + footerMessage := "Brought to you by Newsbot" + footerUrl := "" + description := dwh.convertFromHtml(dwh.article.Description) + color := dwh.getColor(dwh.article.Url) + + embed := DiscordEmbed{ + Title: &dwh.article.Title, + Description: &description, + Image: discordImage{ + Url: &dwh.article.Thumbnail, + }, + Fields: dwh.getFields(), + Footer: &discordFooter{ + Value: &footerMessage, + IconUrl: &footerUrl, + }, + Color: &color, + } + + // attach the embed to an array + var embedArray []DiscordEmbed + embedArray = append(embedArray, embed) + + // create the base message + msg := DiscordMessage{ + Embeds: &embedArray, + } + + return &msg, nil +} + +func (dwh Discord) SendPayload(Message *DiscordMessage, Url string) error { + // Convert the message to a io.reader object + buffer := new(bytes.Buffer) + json.NewEncoder(buffer).Encode(Message) + + // Send the message + resp, err := http.Post(Url, "application/json", buffer) + if err != nil { + return err + } + + // Check for 204 + if resp.StatusCode != 204 { + defer resp.Body.Close() + + errMsg, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return fmt.Errorf(string(errMsg)) + } + return nil } @@ -104,7 +175,16 @@ func (dwh Discord) convertFromHtml(body string) string { return clean } -func (dwh Discord) convertLinks(body string) string { +func (dwh *Discord) getColor(Url string) int32 { + if strings.Contains(Url, "youtube.com") { + return YoutubeColor + } + + return DefaultColor + +} + +func (dwh *Discord) convertLinks(body string) string { //items := regexp.MustCompile("") return "" -} +} diff --git a/services/output/discordwebhook_test.go b/services/output/discordwebhook_test.go index 61affa4..82ba3b4 100644 --- a/services/output/discordwebhook_test.go +++ b/services/output/discordwebhook_test.go @@ -1,11 +1,10 @@ package output_test import ( - "errors" "os" "strings" "testing" - "time" + //"time" "github.com/google/uuid" "github.com/joho/godotenv" @@ -13,61 +12,127 @@ import ( "github.com/jtom38/newsbot/collector/services/output" ) -var article database.Article = database.Article{ - ID: uuid.New(), - Sourceid: uuid.New(), - Tags: "unit, testing", - Title: "Demo", - Url: "https://github.com/jtom38/newsbot.collector.api", - Pubdate: time.Now(), - Videoheight: 0, - Videowidth: 0, - Description: "Hello World", +var ( + article database.Article = database.Article{ + ID: uuid.New(), + Sourceid: uuid.New(), + Tags: "unit, testing", + Title: "Demo", + Url: "https://github.com/jtom38/newsbot.collector.api", + //Pubdate: time.Now(), + Videoheight: 0, + Videowidth: 0, + Description: "Hello World", + } + blank string = "" +) + +func TestDiscordMessageContainsTitle(t *testing.T) { + d := output.NewDiscordWebHookMessage(article) + msg, err := d.GeneratePayload() + if err != nil { + t.Error(err) + } + + for _, i := range *msg.Embeds { + if i.Title == &blank { + t.Error("title missing") + } + } } -func getWebhook() ([]string, error){ - var endpoints []string +func TestDiscordMessageContainsDescription(t *testing.T) { + d := output.NewDiscordWebHookMessage(article) + msg, err := d.GeneratePayload() + if err != nil { + t.Error(err) + } + + for _, i := range *msg.Embeds { + if i.Description == &blank { + t.Error("description missing") + } + } +} +func TestDiscordMessageFooter(t *testing.T) { + d := output.NewDiscordWebHookMessage(article) + msg, err := d.GeneratePayload() + if err != nil { + t.Error(err) + } + for _, i := range *msg.Embeds { + blank := "" + if i.Footer.Value == &blank { + t.Error("missing footer vlue") + } + if i.Footer.IconUrl == &blank { + t.Error("missing footer url") + } + } +} + +func TestDiscordMessageFields(t *testing.T) { + header := "Link" + d := output.NewDiscordWebHookMessage(article) + msg, err := d.GeneratePayload() + if err != nil { + t.Error(err) + } + for _, embed := range *msg.Embeds { + for _, field := range embed.Fields { + var fName string + if field.Name != nil { + fName = *field.Name + } else { + t.Error("missing link field value") + } + + if fName != header { + t.Error("missing link field key") + } + + var fValue string + if field.Value != nil { + fValue = *field.Value + } + + if fValue == blank { + t.Error("missing link field value") + } + } + } +} + +// This test requires a env value to be present to work +func TestDiscordMessagePost(t *testing.T) { _, err := os.Open(".env") if err != nil { - return endpoints, err + t.Error(err) } err = godotenv.Load() if err != nil { - return endpoints, err + t.Error(err) } res := os.Getenv("TESTS_DISCORD_WEBHOOK") if res == "" { - return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing") + t.Error("TESTS_DISCORD_WEBHOOK is missing") + } + endpoints := strings.Split(res, " ") + if err != nil { + t.Error(err) } - endpoints = strings.Split(res, "") - return endpoints, nil -} -func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) { - hook, err := getWebhook() + d := output.NewDiscordWebHookMessage(article) + msg, err := d.GeneratePayload() if err != nil { t.Error(err) } - d := output.NewDiscordWebHookMessage(hook, article) - if len(d.Subscriptions) == 0 { - t.Error("no subscriptions found") - } -} -func TestDiscordMessageContainsTitle(t *testing.T) { - hook, err := getWebhook() + err = d.SendPayload(msg, endpoints[0]) if err != nil { t.Error(err) } - d := output.NewDiscordWebHookMessage(hook, article) - err = d.GeneratePayload() - if err != nil { - t.Error(err) - } - if d.Message.Embeds[0].Title == "" { - t.Error("no title was found ") - } } \ No newline at end of file