jtom38
/
newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Features/output discord (#12) 

* basic output looks to be working

* cron was updated to add to the queue and post messages

* new route to make discord webhook subscriptions

* updated swag tags

* swag

* Updated delete subscription call

* removed the time value as it throws off the msg template

* updated logging

* updated swagger

* updated new subscription route

* Updated logging and remove items from the queue if they dont have a subscription

* updated getArticles to return the 50 newest for the portal

* added endpoint to see if an item exists already

* formatting

* updated listArticles

* added colors and updated the image

* Updated to use the pointer in twitch

* added the twitch login command to cron... it works now

* found a better way to disable http2 for reddit. Test worked right away too

* updated the cron tasks to run collected once and hour or longer depending on the service
This commit is contained in:
James Tombleson 2022-07-12 15:28:31 -07:00 committed by GitHub
parent 0e0058506a
commit a1324ee1c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 797 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
database/query.sql.go
View File

@ -253,16 +253,11 @@ func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
} }
const deleteSubscription = `-- name: DeleteSubscription :exec const deleteSubscription = `-- name: DeleteSubscription :exec
Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2 Delete From subscriptions Where id = $1
` `
type DeleteSubscriptionParams struct { func (q *Queries) DeleteSubscription(ctx context.Context, id uuid.UUID) error {
Discordwebhookid uuid.UUID _, err := q.db.ExecContext(ctx, deleteSubscription, id)
Sourceid uuid.UUID
}
func (q *Queries) DeleteSubscription(ctx context.Context, arg DeleteSubscriptionParams) error {
_, err := q.db.ExecContext(ctx, deleteSubscription, arg.Discordwebhookid, arg.Sourceid)
return err return err
} }
@ -534,6 +529,23 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
return i, err return i, err
} }
const getDiscordWebHookByUrl = `-- name: GetDiscordWebHookByUrl :one
Select id, url, server, channel, enabled From DiscordWebHooks Where url = $1
`
func (q *Queries) GetDiscordWebHookByUrl(ctx context.Context, url string) (Discordwebhook, error) {
row := q.db.QueryRowContext(ctx, getDiscordWebHookByUrl, url)
var i Discordwebhook
err := row.Scan(
&i.ID,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
)
return i, err
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, url, server, channel, enabled from DiscordWebHooks Select id, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1 Where ID = $1 LIMIT 1
@ -648,6 +660,27 @@ func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, erro
return i, err return i, err
} }
const getSourceByName = `-- name: GetSourceByName :one
Select id, site, name, source, type, value, enabled, url, tags from Sources where name = $1 Limit 1
`
func (q *Queries) GetSourceByName(ctx context.Context, name string) (Source, error) {
row := q.db.QueryRowContext(ctx, getSourceByName, name)
var i Source
err := row.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
)
return i, err
}
const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many
Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1 Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1
` `
@ -743,6 +776,47 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
return items, nil return items, nil
} }
const listArticlesByDate = `-- name: ListArticlesByDate :many
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage From articles ORDER BY pubdate desc Limit $1
`
func (q *Queries) ListArticlesByDate(ctx context.Context, limit int32) ([]Article, error) {
rows, err := q.db.QueryContext(ctx, listArticlesByDate, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Article
for rows.Next() {
var i Article
if err := rows.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1 Select id, articleid from DiscordQueue LIMIT $1
` `
@ -965,8 +1039,8 @@ func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid
return items, nil return items, nil
} }
const querySubscriptions = `-- name: QuerySubscriptions :many const querySubscriptions = `-- name: QuerySubscriptions :one
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1
` `
type QuerySubscriptionsParams struct { type QuerySubscriptionsParams struct {
@ -974,25 +1048,9 @@ type QuerySubscriptionsParams struct {
Sourceid uuid.UUID Sourceid uuid.UUID
} }
func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) ([]Subscription, error) { func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) (Subscription, error) {
rows, err := q.db.QueryContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid) row := q.db.QueryRowContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid)
if err != nil { var i Subscription
return nil, err err := row.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid)
} return i, err
defer rows.Close()
var items []Subscription
for rows.Next() {
var i Subscription
if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
} }

15
database/schema/query.sql
View File

@ -10,6 +10,9 @@ Where Url = $1 LIMIT 1;
-- name: ListArticles :many -- name: ListArticles :many
Select * From articles Limit $1; Select * From articles Limit $1;
-- name: ListArticlesByDate :many
Select * From articles ORDER BY pubdate desc Limit $1;
-- name: GetArticlesBySource :many -- name: GetArticlesBySource :many
select * from articles select * from articles
INNER join sources on articles.sourceid=Sources.ID INNER join sources on articles.sourceid=Sources.ID
@ -66,6 +69,9 @@ Where ID = $1 LIMIT 1;
Select * From DiscordWebHooks Select * From DiscordWebHooks
Where Server = $1; Where Server = $1;
-- name: GetDiscordWebHookByUrl :one
Select * From DiscordWebHooks Where url = $1;
-- name: ListDiscordWebhooks :many -- name: ListDiscordWebhooks :many
Select * From discordwebhooks LIMIT $1; Select * From discordwebhooks LIMIT $1;
@ -127,6 +133,9 @@ Values
-- name: GetSourceByID :one -- name: GetSourceByID :one
Select * From Sources where ID = $1 Limit 1; Select * From Sources where ID = $1 Limit 1;
-- name: GetSourceByName :one
Select * from Sources where name = $1 Limit 1;
-- name: ListSources :many -- name: ListSources :many
Select * From Sources Limit $1; Select * From Sources Limit $1;
@ -154,8 +163,8 @@ Select * From subscriptions Limit $1;
-- name: ListSubscriptionsBySourceId :many -- name: ListSubscriptionsBySourceId :many
Select * From subscriptions where sourceid = $1; Select * From subscriptions where sourceid = $1;
-- name: QuerySubscriptions :many -- name: QuerySubscriptions :one
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2; Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1;
-- name: GetSubscriptionsBySourceID :many -- name: GetSubscriptionsBySourceID :many
Select * From subscriptions Where sourceid = $1; Select * From subscriptions Where sourceid = $1;
@ -164,4 +173,4 @@ Select * From subscriptions Where sourceid = $1;
Select * from subscriptions Where discordwebhookid = $1; Select * from subscriptions Where discordwebhookid = $1;
-- name: DeleteSubscription :exec -- name: DeleteSubscription :exec
Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2; Delete From subscriptions Where id = $1;

110
docs/docs.go
View File

@ -22,7 +22,7 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"articles" "Articles"
], ],
"summary": "Lists the top 50 records", "summary": "Lists the top 50 records",
"responses": {} "responses": {}
@ -34,7 +34,7 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"articles" "Articles"
], ],
"summary": "Finds the articles based on the SourceID provided. Returns the top 50.", "summary": "Finds the articles based on the SourceID provided. Returns the top 50.",
"parameters": [ "parameters": [
@ -55,7 +55,7 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"articles" "Articles"
], ],
"summary": "Returns an article based on defined ID.", "summary": "Returns an article based on defined ID.",
"parameters": [ "parameters": [
@ -76,8 +76,8 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Lists the top 50 records", "summary": "Lists the top 50 records",
"responses": {} "responses": {}
@ -86,9 +86,9 @@ const docTemplate = `{
"/config/sources/new/reddit": { "/config/sources/new/reddit": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source", "Source",
"reddit" "Reddit"
], ],
"summary": "Creates a new reddit source to monitor.", "summary": "Creates a new reddit source to monitor.",
"parameters": [ "parameters": [
@ -113,9 +113,9 @@ const docTemplate = `{
"/config/sources/new/twitch": { "/config/sources/new/twitch": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source", "Source",
"twitch" "Twitch"
], ],
"summary": "Creates a new twitch source to monitor.", "summary": "Creates a new twitch source to monitor.",
"parameters": [ "parameters": [
@ -147,9 +147,9 @@ const docTemplate = `{
"/config/sources/new/youtube": { "/config/sources/new/youtube": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source", "Source",
"youtube" "YouTube"
], ],
"summary": "Creates a new youtube source to monitor.", "summary": "Creates a new youtube source to monitor.",
"parameters": [ "parameters": [
@ -184,8 +184,8 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Returns a single entity by ID", "summary": "Returns a single entity by ID",
"parameters": [ "parameters": [
@ -201,8 +201,8 @@ const docTemplate = `{
}, },
"delete": { "delete": {
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Deletes a record by ID.", "summary": "Deletes a record by ID.",
"parameters": [ "parameters": [
@ -220,8 +220,8 @@ const docTemplate = `{
"/config/sources/{id}/disable": { "/config/sources/{id}/disable": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Disables a source from processing.", "summary": "Disables a source from processing.",
"parameters": [ "parameters": [
@ -239,8 +239,8 @@ const docTemplate = `{
"/config/sources/{id}/enable": { "/config/sources/{id}/enable": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Enables a source to continue processing.", "summary": "Enables a source to continue processing.",
"parameters": [ "parameters": [
@ -261,7 +261,7 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"debug", "Debug",
"Discord", "Discord",
"Queue" "Queue"
], ],
@ -275,9 +275,9 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Discord", "Discord",
"Webhooks" "Webhook"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {} "responses": {}
@ -289,9 +289,9 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Discord", "Discord",
"Webhooks" "Webhook"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -309,9 +309,9 @@ const docTemplate = `{
"/discord/webhooks/new": { "/discord/webhooks/new": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"Discord", "Discord",
"Webhooks" "Webhook"
], ],
"summary": "Creates a new record for a discord web hook to post data to.", "summary": "Creates a new record for a discord web hook to post data to.",
"parameters": [ "parameters": [
@ -331,7 +331,7 @@ const docTemplate = `{
}, },
{ {
"type": "string", "type": "string",
"description": "Channel name.", "description": "Channel name",
"name": "channel", "name": "channel",
"in": "query", "in": "query",
"required": true "required": true
@ -346,7 +346,7 @@ const docTemplate = `{
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"debug" "Debug"
], ],
"summary": "Responds back with \"Hello x\" depending on param passed in.", "summary": "Responds back with \"Hello x\" depending on param passed in.",
"parameters": [ "parameters": [
@ -367,7 +367,7 @@ const docTemplate = `{
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"debug" "Debug"
], ],
"summary": "Responds back with \"Hello world!\"", "summary": "Responds back with \"Hello world!\"",
"responses": {} "responses": {}
@ -379,7 +379,7 @@ const docTemplate = `{
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"debug" "Debug"
], ],
"summary": "Sends back \"pong\". Good to test with.", "summary": "Sends back \"pong\". Good to test with.",
"responses": {} "responses": {}
@ -391,9 +391,9 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"settings" "Settings"
], ],
"summary": "Returns a object based on the Key that was given/", "summary": "Returns a object based on the Key that was given.",
"parameters": [ "parameters": [
{ {
"type": "string", "type": "string",
@ -412,8 +412,8 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Subscriptions" "Subscription"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {} "responses": {}
@ -425,8 +425,8 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Subscriptions" "Subscription"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -447,8 +447,8 @@ const docTemplate = `{
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Subscriptions" "Subscription"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -462,6 +462,34 @@ const docTemplate = `{
], ],
"responses": {} "responses": {}
} }
},
"/subscriptions/new/discordwebhook": {
"post": {
"tags": [
"Config",
"Source",
"Discord",
"Subscription"
],
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
"parameters": [
{
"type": "string",
"description": "discordWebHookId",
"name": "discordWebHookId",
"in": "query",
"required": true
},
{
"type": "string",
"description": "sourceId",
"name": "sourceId",
"in": "query",
"required": true
}
],
"responses": {}
}
} }
} }
}` }`

110
docs/swagger.json
View File

@ -13,7 +13,7 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"articles" "Articles"
], ],
"summary": "Lists the top 50 records", "summary": "Lists the top 50 records",
"responses": {} "responses": {}
@ -25,7 +25,7 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"articles" "Articles"
], ],
"summary": "Finds the articles based on the SourceID provided. Returns the top 50.", "summary": "Finds the articles based on the SourceID provided. Returns the top 50.",
"parameters": [ "parameters": [
@ -46,7 +46,7 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"articles" "Articles"
], ],
"summary": "Returns an article based on defined ID.", "summary": "Returns an article based on defined ID.",
"parameters": [ "parameters": [
@ -67,8 +67,8 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Lists the top 50 records", "summary": "Lists the top 50 records",
"responses": {} "responses": {}
@ -77,9 +77,9 @@
"/config/sources/new/reddit": { "/config/sources/new/reddit": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source", "Source",
"reddit" "Reddit"
], ],
"summary": "Creates a new reddit source to monitor.", "summary": "Creates a new reddit source to monitor.",
"parameters": [ "parameters": [
@ -104,9 +104,9 @@
"/config/sources/new/twitch": { "/config/sources/new/twitch": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source", "Source",
"twitch" "Twitch"
], ],
"summary": "Creates a new twitch source to monitor.", "summary": "Creates a new twitch source to monitor.",
"parameters": [ "parameters": [
@ -138,9 +138,9 @@
"/config/sources/new/youtube": { "/config/sources/new/youtube": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source", "Source",
"youtube" "YouTube"
], ],
"summary": "Creates a new youtube source to monitor.", "summary": "Creates a new youtube source to monitor.",
"parameters": [ "parameters": [
@ -175,8 +175,8 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Returns a single entity by ID", "summary": "Returns a single entity by ID",
"parameters": [ "parameters": [
@ -192,8 +192,8 @@
}, },
"delete": { "delete": {
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Deletes a record by ID.", "summary": "Deletes a record by ID.",
"parameters": [ "parameters": [
@ -211,8 +211,8 @@
"/config/sources/{id}/disable": { "/config/sources/{id}/disable": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Disables a source from processing.", "summary": "Disables a source from processing.",
"parameters": [ "parameters": [
@ -230,8 +230,8 @@
"/config/sources/{id}/enable": { "/config/sources/{id}/enable": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"source" "Source"
], ],
"summary": "Enables a source to continue processing.", "summary": "Enables a source to continue processing.",
"parameters": [ "parameters": [
@ -252,7 +252,7 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"debug", "Debug",
"Discord", "Discord",
"Queue" "Queue"
], ],
@ -266,9 +266,9 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Discord", "Discord",
"Webhooks" "Webhook"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {} "responses": {}
@ -280,9 +280,9 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Discord", "Discord",
"Webhooks" "Webhook"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -300,9 +300,9 @@
"/discord/webhooks/new": { "/discord/webhooks/new": {
"post": { "post": {
"tags": [ "tags": [
"config", "Config",
"Discord", "Discord",
"Webhooks" "Webhook"
], ],
"summary": "Creates a new record for a discord web hook to post data to.", "summary": "Creates a new record for a discord web hook to post data to.",
"parameters": [ "parameters": [
@ -322,7 +322,7 @@
}, },
{ {
"type": "string", "type": "string",
"description": "Channel name.", "description": "Channel name",
"name": "channel", "name": "channel",
"in": "query", "in": "query",
"required": true "required": true
@ -337,7 +337,7 @@
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"debug" "Debug"
], ],
"summary": "Responds back with \"Hello x\" depending on param passed in.", "summary": "Responds back with \"Hello x\" depending on param passed in.",
"parameters": [ "parameters": [
@ -358,7 +358,7 @@
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"debug" "Debug"
], ],
"summary": "Responds back with \"Hello world!\"", "summary": "Responds back with \"Hello world!\"",
"responses": {} "responses": {}
@ -370,7 +370,7 @@
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"debug" "Debug"
], ],
"summary": "Sends back \"pong\". Good to test with.", "summary": "Sends back \"pong\". Good to test with.",
"responses": {} "responses": {}
@ -382,9 +382,9 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"settings" "Settings"
], ],
"summary": "Returns a object based on the Key that was given/", "summary": "Returns a object based on the Key that was given.",
"parameters": [ "parameters": [
{ {
"type": "string", "type": "string",
@ -403,8 +403,8 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Subscriptions" "Subscription"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {} "responses": {}
@ -416,8 +416,8 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Subscriptions" "Subscription"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -438,8 +438,8 @@
"application/json" "application/json"
], ],
"tags": [ "tags": [
"config", "Config",
"Subscriptions" "Subscription"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -453,6 +453,34 @@
], ],
"responses": {} "responses": {}
} }
},
"/subscriptions/new/discordwebhook": {
"post": {
"tags": [
"Config",
"Source",
"Discord",
"Subscription"
],
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
"parameters": [
{
"type": "string",
"description": "discordWebHookId",
"name": "discordWebHookId",
"in": "query",
"required": true
},
{
"type": "string",
"description": "sourceId",
"name": "sourceId",
"in": "query",
"required": true
}
],
"responses": {}
}
} }
} }
} }

102
docs/swagger.yaml
View File

@ -11,7 +11,7 @@ paths:
responses: {} responses: {}
summary: Lists the top 50 records summary: Lists the top 50 records
tags: tags:
- articles - Articles
/articles/{id}: /articles/{id}:
get: get:
parameters: parameters:
@ -25,7 +25,7 @@ paths:
responses: {} responses: {}
summary: Returns an article based on defined ID. summary: Returns an article based on defined ID.
tags: tags:
- articles - Articles
/articles/by/sourceid: /articles/by/sourceid:
get: get:
parameters: parameters:
@ -40,7 +40,7 @@ paths:
summary: Finds the articles based on the SourceID provided. Returns the top summary: Finds the articles based on the SourceID provided. Returns the top
50. 50.
tags: tags:
- articles - Articles
/config/sources: /config/sources:
get: get:
produces: produces:
@ -48,8 +48,8 @@ paths:
responses: {} responses: {}
summary: Lists the top 50 records summary: Lists the top 50 records
tags: tags:
- config - Config
- source - Source
/config/sources/{id}: /config/sources/{id}:
delete: delete:
parameters: parameters:
@ -61,8 +61,8 @@ paths:
responses: {} responses: {}
summary: Deletes a record by ID. summary: Deletes a record by ID.
tags: tags:
- config - Config
- source - Source
get: get:
parameters: parameters:
- description: uuid - description: uuid
@ -75,8 +75,8 @@ paths:
responses: {} responses: {}
summary: Returns a single entity by ID summary: Returns a single entity by ID
tags: tags:
- config - Config
- source - Source
/config/sources/{id}/disable: /config/sources/{id}/disable:
post: post:
parameters: parameters:
@ -88,8 +88,8 @@ paths:
responses: {} responses: {}
summary: Disables a source from processing. summary: Disables a source from processing.
tags: tags:
- config - Config
- source - Source
/config/sources/{id}/enable: /config/sources/{id}/enable:
post: post:
parameters: parameters:
@ -101,8 +101,8 @@ paths:
responses: {} responses: {}
summary: Enables a source to continue processing. summary: Enables a source to continue processing.
tags: tags:
- config - Config
- source - Source
/config/sources/new/reddit: /config/sources/new/reddit:
post: post:
parameters: parameters:
@ -119,9 +119,9 @@ paths:
responses: {} responses: {}
summary: Creates a new reddit source to monitor. summary: Creates a new reddit source to monitor.
tags: tags:
- config - Config
- source - Source
- reddit - Reddit
/config/sources/new/twitch: /config/sources/new/twitch:
post: post:
parameters: parameters:
@ -143,9 +143,9 @@ paths:
responses: {} responses: {}
summary: Creates a new twitch source to monitor. summary: Creates a new twitch source to monitor.
tags: tags:
- config - Config
- source - Source
- twitch - Twitch
/config/sources/new/youtube: /config/sources/new/youtube:
post: post:
parameters: parameters:
@ -167,9 +167,9 @@ paths:
responses: {} responses: {}
summary: Creates a new youtube source to monitor. summary: Creates a new youtube source to monitor.
tags: tags:
- config - Config
- source - Source
- youtube - YouTube
/discord/queue: /discord/queue:
get: get:
produces: produces:
@ -177,7 +177,7 @@ paths:
responses: {} responses: {}
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- debug - Debug
- Discord - Discord
- Queue - Queue
/discord/webhooks: /discord/webhooks:
@ -187,9 +187,9 @@ paths:
responses: {} responses: {}
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- config - Config
- Discord - Discord
- Webhooks - Webhook
/discord/webhooks/byId: /discord/webhooks/byId:
get: get:
parameters: parameters:
@ -203,9 +203,9 @@ paths:
responses: {} responses: {}
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- config - Config
- Discord - Discord
- Webhooks - Webhook
/discord/webhooks/new: /discord/webhooks/new:
post: post:
parameters: parameters:
@ -219,7 +219,7 @@ paths:
name: server name: server
required: true required: true
type: string type: string
- description: Channel name. - description: Channel name
in: query in: query
name: channel name: channel
required: true required: true
@ -227,9 +227,9 @@ paths:
responses: {} responses: {}
summary: Creates a new record for a discord web hook to post data to. summary: Creates a new record for a discord web hook to post data to.
tags: tags:
- config - Config
- Discord - Discord
- Webhooks - Webhook
/hello/{who}: /hello/{who}:
get: get:
parameters: parameters:
@ -243,7 +243,7 @@ paths:
responses: {} responses: {}
summary: Responds back with "Hello x" depending on param passed in. summary: Responds back with "Hello x" depending on param passed in.
tags: tags:
- debug - Debug
/helloworld: /helloworld:
get: get:
produces: produces:
@ -251,7 +251,7 @@ paths:
responses: {} responses: {}
summary: Responds back with "Hello world!" summary: Responds back with "Hello world!"
tags: tags:
- debug - Debug
/ping: /ping:
get: get:
produces: produces:
@ -259,7 +259,7 @@ paths:
responses: {} responses: {}
summary: Sends back "pong". Good to test with. summary: Sends back "pong". Good to test with.
tags: tags:
- debug - Debug
/settings/{key}: /settings/{key}:
get: get:
parameters: parameters:
@ -271,9 +271,9 @@ paths:
produces: produces:
- application/json - application/json
responses: {} responses: {}
summary: Returns a object based on the Key that was given/ summary: Returns a object based on the Key that was given.
tags: tags:
- settings - Settings
/subscriptions: /subscriptions:
get: get:
produces: produces:
@ -281,8 +281,8 @@ paths:
responses: {} responses: {}
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- config - Config
- Subscriptions - Subscription
/subscriptions/byDiscordId: /subscriptions/byDiscordId:
get: get:
parameters: parameters:
@ -296,8 +296,8 @@ paths:
responses: {} responses: {}
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- config - Config
- Subscriptions - Subscription
/subscriptions/bySourceId: /subscriptions/bySourceId:
get: get:
parameters: parameters:
@ -311,6 +311,26 @@ paths:
responses: {} responses: {}
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- config - Config
- Subscriptions - Subscription
/subscriptions/new/discordwebhook:
post:
parameters:
- description: discordWebHookId
in: query
name: discordWebHookId
required: true
type: string
- description: sourceId
in: query
name: sourceId
required: true
type: string
responses: {}
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
tags:
- Config
- Source
- Discord
- Subscription
swagger: "2.0" swagger: "2.0"

46
routes/articles.go
View File

@ -11,12 +11,12 @@ import (
// ListArticles // ListArticles
// @Summary Lists the top 50 records // @Summary Lists the top 50 records
// @Produce application/json // @Produce application/json
// @Tags articles // @Tags Articles
// @Router /articles [get] // @Router /articles [get]
func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) { func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
res, err := s.Db.ListArticles(*s.ctx, 50) res, err := s.Db.ListArticlesByDate(*s.ctx, 50)
if err != nil { if err != nil {
w.Write([]byte(err.Error())) w.Write([]byte(err.Error()))
panic(err) panic(err)
@ -27,7 +27,6 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(err.Error())) w.Write([]byte(err.Error()))
panic(err) panic(err)
} }
w.Write(bres) w.Write(bres)
} }
@ -35,7 +34,7 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
// @Summary Returns an article based on defined ID. // @Summary Returns an article based on defined ID.
// @Param id path string true "uuid" // @Param id path string true "uuid"
// @Produce application/json // @Produce application/json
// @Tags articles // @Tags Articles
// @Router /articles/{id} [get] // @Router /articles/{id} [get]
func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) { func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -67,7 +66,7 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
// @Summary Finds the articles based on the SourceID provided. Returns the top 50. // @Summary Finds the articles based on the SourceID provided. Returns the top 50.
// @Param id query string true "Source ID UUID" // @Param id query string true "Source ID UUID"
// @Produce application/json // @Produce application/json
// @Tags articles // @Tags Articles
// @Router /articles/by/sourceid [get] // @Router /articles/by/sourceid [get]
func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) { func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -95,4 +94,39 @@ func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
} }
w.Write(bres) w.Write(bres)
} }
// TODO add page support
// GetArticlesByTag
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
// @Param Tag query string true "Tag name"
// @Produce application/json
// @Tags Articles
// @Router /articles/by/sourceid [get]
func (s *Server) GetArticlesByTag(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
r.URL.Query()
query := r.URL.Query()
_id := query["id"][0]
uuid, err := uuid.Parse(_id)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
}
res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
}
bres, err := json.Marshal(res)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
}
w.Write(bres)
}

2
routes/discordQueue.go
View File

@ -8,7 +8,7 @@ import (
// GetDiscordQueue // GetDiscordQueue
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Tags debug, Discord, Queue // @Tags Debug, Discord, Queue
// @Router /discord/queue [get] // @Router /discord/queue [get]
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) { func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")

8
routes/discordwebhooks.go
View File

@ -13,7 +13,7 @@ import (
// GetDiscordWebHooks // GetDiscordWebHooks
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Tags config, Discord, Webhooks // @Tags Config, Discord, Webhook
// @Router /discord/webhooks [get] // @Router /discord/webhooks [get]
func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) { func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -37,7 +37,7 @@ func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Param id query string true "id" // @Param id query string true "id"
// @Tags config, Discord, Webhooks // @Tags Config, Discord, Webhook
// @Router /discord/webhooks/byId [get] // @Router /discord/webhooks/byId [get]
func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) { func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -74,8 +74,8 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request)
// @Summary Creates a new record for a discord web hook to post data to. // @Summary Creates a new record for a discord web hook to post data to.
// @Param url query string true "url" // @Param url query string true "url"
// @Param server query string true "Server name" // @Param server query string true "Server name"
// @Param channel query string true "Channel name." // @Param channel query string true "Channel name"
// @Tags config, Discord, Webhooks // @Tags Config, Discord, Webhook
// @Router /discord/webhooks/new [post] // @Router /discord/webhooks/new [post]
func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) { func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query() query := r.URL.Query()

6
routes/root.go
View File

@ -22,7 +22,7 @@ func RootRoutes() chi.Router {
// HelloWorld // HelloWorld
// @Summary Responds back with "Hello world!" // @Summary Responds back with "Hello world!"
// @Produce plain // @Produce plain
// @Tags debug // @Tags Debug
// @Router /helloworld [get] // @Router /helloworld [get]
func helloWorld(w http.ResponseWriter, r *http.Request) { func helloWorld(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello World!")) w.Write([]byte("Hello World!"))
@ -31,7 +31,7 @@ func helloWorld(w http.ResponseWriter, r *http.Request) {
// Ping // Ping
// @Summary Sends back "pong". Good to test with. // @Summary Sends back "pong". Good to test with.
// @Produce plain // @Produce plain
// @Tags debug // @Tags Debug
// @Router /ping [get] // @Router /ping [get]
func ping(w http.ResponseWriter, r *http.Request) { func ping(w http.ResponseWriter, r *http.Request) {
msg := "pong" msg := "pong"
@ -42,7 +42,7 @@ func ping(w http.ResponseWriter, r *http.Request) {
// @Summary Responds back with "Hello x" depending on param passed in. // @Summary Responds back with "Hello x" depending on param passed in.
// @Param who path string true "Who" // @Param who path string true "Who"
// @Produce plain // @Produce plain
// @Tags debug // @Tags Debug
// @Router /hello/{who} [get] // @Router /hello/{who} [get]
func helloWho(w http.ResponseWriter, r *http.Request) { func helloWho(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who")) msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who"))

23
routes/server.go
View File

@ -3,12 +3,13 @@ package routes
import ( import (
"context" "context"
"database/sql" "database/sql"
//"net/http" //"net/http"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
httpSwagger "github.com/swaggo/http-swagger"
_ "github.com/lib/pq" _ "github.com/lib/pq"
httpSwagger "github.com/swaggo/http-swagger"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/config" "github.com/jtom38/newsbot/collector/services/config"
@ -16,14 +17,14 @@ import (
type Server struct { type Server struct {
Router *chi.Mux Router *chi.Mux
Db *database.Queries Db *database.Queries
ctx *context.Context ctx *context.Context
} }
var ( var (
ErrIdValueMissing string = "id value is missing" ErrIdValueMissing string = "id value is missing"
ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct." ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct."
ErrNoRecordFound string = "no record was found." ErrNoRecordFound string = "no record was found."
ErrUnableToConvertToJson string = "Unable to convert to json" ErrUnableToConvertToJson string = "Unable to convert to json"
) )
@ -59,13 +60,14 @@ func openDatabase(ctx context.Context) (*database.Queries, error) {
func (s *Server) MountMiddleware() { func (s *Server) MountMiddleware() {
s.Router.Use(middleware.Logger) s.Router.Use(middleware.Logger)
s.Router.Use(middleware.Recoverer) s.Router.Use(middleware.Recoverer)
//s.Router.Use(middleware.Heartbeat())
} }
func (s *Server) MountRoutes() { func (s *Server) MountRoutes() {
s.Router.Get("/swagger/*", httpSwagger.Handler( s.Router.Get("/swagger/*", httpSwagger.Handler(
httpSwagger.URL("http://localhost:8081/swagger/doc.json"), //The url pointing to API definition httpSwagger.URL("http://localhost:8081/swagger/doc.json"), //The url pointing to API definition
)) ))
/* Root Routes */ /* Root Routes */
s.Router.Get("/api/helloworld", helloWorld) s.Router.Get("/api/helloworld", helloWorld)
s.Router.Get("/api/hello/{who}", helloWho) s.Router.Get("/api/hello/{who}", helloWho)
@ -88,10 +90,14 @@ func (s *Server) MountRoutes() {
/* Settings */ /* Settings */
s.Router.Get("/api/settings", s.getSettings) s.Router.Get("/api/settings", s.getSettings)
/* Source Routes */ /* Source Routes */
s.Router.Get("/api/config/sources", s.listSources) s.Router.Get("/api/config/sources", s.listSources)
/* Reddit Source Routes */
s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource) s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource)
s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource) s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource)
s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource) s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource)
s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) { s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) {
@ -105,4 +111,5 @@ func (s *Server) MountRoutes() {
s.Router.Get("/api/subscriptions", s.ListSubscriptions) s.Router.Get("/api/subscriptions", s.ListSubscriptions)
s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId) s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId)
s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId) s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId)
s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription)
} }

4
routes/settings.go
View File

@ -10,10 +10,10 @@ import (
) )
// GetSettings // GetSettings
// @Summary Returns a object based on the Key that was given/ // @Summary Returns a object based on the Key that was given.
// @Param key path string true "Settings Key value" // @Param key path string true "Settings Key value"
// @Produce application/json // @Produce application/json
// @Tags settings // @Tags Settings
// @Router /settings/{key} [get] // @Router /settings/{key} [get]
func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) { func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
//var item model.Sources //var item model.Sources

20
routes/sources.go
View File

@ -15,7 +15,7 @@ import (
// ListSources // ListSources
// @Summary Lists the top 50 records // @Summary Lists the top 50 records
// @Produce application/json // @Produce application/json
// @Tags config, source // @Tags Config, Source
// @Router /config/sources [get] // @Router /config/sources [get]
func (s *Server) listSources(w http.ResponseWriter, r *http.Request) { func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
//TODO Add top? //TODO Add top?
@ -48,7 +48,7 @@ func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
// @Summary Returns a single entity by ID // @Summary Returns a single entity by ID
// @Param id path string true "uuid" // @Param id path string true "uuid"
// @Produce application/json // @Produce application/json
// @Tags config, source // @Tags Config, Source
// @Router /config/sources/{id} [get] // @Router /config/sources/{id} [get]
func (s *Server) getSources(w http.ResponseWriter, r *http.Request) { func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID") id := chi.URLParam(r, "ID")
@ -78,7 +78,7 @@ func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
// @Summary Creates a new reddit source to monitor. // @Summary Creates a new reddit source to monitor.
// @Param name query string true "name" // @Param name query string true "name"
// @Param url query string true "url" // @Param url query string true "url"
// @Tags config, source, reddit // @Tags Config, Source, Reddit
// @Router /config/sources/new/reddit [post] // @Router /config/sources/new/reddit [post]
func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) { func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query() query := r.URL.Query()
@ -116,12 +116,16 @@ func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
w.Write(bJson) w.Write(bJson)
} }
func (s *Server) getSourceByType(w http.ResponseWriter, r *http.Request) {
}
// NewYoutubeSource // NewYoutubeSource
// @Summary Creates a new youtube source to monitor. // @Summary Creates a new youtube source to monitor.
// @Param name query string true "name" // @Param name query string true "name"
// @Param url query string true "url" // @Param url query string true "url"
// @Param tags query string true "tags" // @Param tags query string true "tags"
// @Tags config, source, youtube // @Tags Config, Source, YouTube
// @Router /config/sources/new/youtube [post] // @Router /config/sources/new/youtube [post]
func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) { func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query() query := r.URL.Query()
@ -164,7 +168,7 @@ func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
// @Param name query string true "name" // @Param name query string true "name"
// @Param url query string true "url" // @Param url query string true "url"
// @Param tags query string true "tags" // @Param tags query string true "tags"
// @Tags config, source, twitch // @Tags Config, Source, Twitch
// @Router /config/sources/new/twitch [post] // @Router /config/sources/new/twitch [post]
func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) { func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query() query := r.URL.Query()
@ -205,7 +209,7 @@ func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
// DeleteSource // DeleteSource
// @Summary Deletes a record by ID. // @Summary Deletes a record by ID.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags config, source // @Tags Config, Source
// @Router /config/sources/{id} [delete] // @Router /config/sources/{id} [delete]
func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) { func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
//var item model.Sources = model.Sources{} //var item model.Sources = model.Sources{}
@ -232,7 +236,7 @@ func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
// DisableSource // DisableSource
// @Summary Disables a source from processing. // @Summary Disables a source from processing.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags config, source // @Tags Config, Source
// @Router /config/sources/{id}/disable [post] // @Router /config/sources/{id}/disable [post]
func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) { func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID") id := chi.URLParam(r, "ID")
@ -256,7 +260,7 @@ func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
// EnableSource // EnableSource
// @Summary Enables a source to continue processing. // @Summary Enables a source to continue processing.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags config, source // @Tags Config, Source
// @Router /config/sources/{id}/enable [post] // @Router /config/sources/{id}/enable [post]
func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) { func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID") id := chi.URLParam(r, "ID")

74
routes/subscriptions.go
View File

@ -5,12 +5,13 @@ import (
"net/http" "net/http"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
) )
// GetSubscriptions // GetSubscriptions
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Tags config, Subscriptions // @Tags Config, Subscription
// @Router /subscriptions [get] // @Router /subscriptions [get]
func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) { func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -34,7 +35,7 @@ func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Param id query string true "id" // @Param id query string true "id"
// @Tags config, Subscriptions // @Tags Config, Subscription
// @Router /subscriptions/byDiscordId [get] // @Router /subscriptions/byDiscordId [get]
func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) { func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -71,7 +72,7 @@ func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Requ
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Param id query string true "id" // @Param id query string true "id"
// @Tags config, Subscriptions // @Tags Config, Subscription
// @Router /subscriptions/bySourceId [get] // @Router /subscriptions/bySourceId [get]
func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) { func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -102,4 +103,71 @@ func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Reque
} }
w.Write(bres) w.Write(bres)
}
// NewDiscordWebHookSubscription
// @Summary Creates a new subscription to link a post from a Source to a DiscordWebHook.
// @Param discordWebHookId query string true "discordWebHookId"
// @Param sourceId query string true "sourceId"
// @Tags Config, Source, Discord, Subscription
// @Router /subscriptions/new/discordwebhook [post]
func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
// Extract the values given
query := r.URL.Query()
discordWebHookId := query["discordWebHookId"][0]
sourceId := query["sourceId"][0]
// Check to make we didnt get a null
if discordWebHookId == "" {
http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest )
return
}
if sourceId == "" {
http.Error(w, "invalid sourceID given", http.StatusBadRequest )
return
}
// Valide they are UUID values
uHook, err := uuid.Parse(discordWebHookId)
if err != nil {
http.Error(w, "DiscordWebHooksID was not a uuid value.", http.StatusBadRequest)
return
}
uSource, err := uuid.Parse(sourceId)
if err != nil {
http.Error(w, "SourceId was not a uuid value", http.StatusBadRequest)
return
}
// Check if the sub already exists
item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{
Discordwebhookid: uHook,
Sourceid: uSource,
})
if err == nil {
bJson, err := json.Marshal(&item)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(bJson)
return
}
// Does not exist, so make it.
params := database.CreateSubscriptionParams{
ID: uuid.New(),
Discordwebhookid: uHook,
Sourceid: uSource,
}
s.Db.CreateSubscription(*s.ctx, params)
bJson, err := json.Marshal(&params)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(bJson)
} }

100
services/cron/scheduler.go
View File

@ -3,6 +3,7 @@ package cron
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
"log" "log"
"time" "time"
@ -11,8 +12,8 @@ import (
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/config" "github.com/jtom38/newsbot/collector/services/config"
"github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/output" "github.com/jtom38/newsbot/collector/services/output"
) )
@ -51,29 +52,32 @@ func New(ctx context.Context) *Cron {
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND) res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
if res { if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() }) timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("Reddit backend was enabled") log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit() //go c.CheckReddit()
} }
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND) res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res { if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() }) timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("YouTube backend was enabled") log.Print("[Input] YouTube backend was enabled")
} }
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND) res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
if res { if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() }) timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("FFXIV backend was enabled") log.Print("[Input] FFXIV backend was enabled")
} }
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND) res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
if res { if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() }) timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("Twitch backend was enabled") log.Print("[Input] Twitch backend was enabled")
} }
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
log.Print("[Output] Discord Output was enabled")
c.timer = timer c.timer = timer
return c return c
} }
@ -162,6 +166,11 @@ func (c *Cron) CheckTwitch() error {
return err return err
} }
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources { for _, source := range sources {
if !source.Enabled { if !source.Enabled {
continue continue
@ -186,19 +195,13 @@ func (c *Cron) CheckDiscordQueue() error {
return err return err
} }
for _, queue := range(queueItems) { for _, queue := range queueItems {
// Get the articleByID // Get the articleByID
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid) article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
if err != nil { if err != nil {
return err return err
} }
// Get the SourceByID
//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
//if err != nil {
// return err
//}
var endpoints []string var endpoints []string
// List Subscription by SourceID // List Subscription by SourceID
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid) subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
@ -206,8 +209,18 @@ func (c *Cron) CheckDiscordQueue() error {
return err return err
} }
// if no one is subscribed to it, remove it from the index.
if len(subs) == 0 {
log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
if err != nil {
return err
}
continue
}
// Get the webhhooks to send to // Get the webhhooks to send to
for _, sub := range(subs) { for _, sub := range subs {
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid) webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
if err != nil { if err != nil {
return err return err
@ -218,16 +231,19 @@ func (c *Cron) CheckDiscordQueue() error {
} }
// Create Discord Message // Create Discord Message
dwh := output.NewDiscordWebHookMessage(endpoints, article) dwh := output.NewDiscordWebHookMessage(article)
err = dwh.GeneratePayload() msg, err := dwh.GeneratePayload()
if err != nil { if err != nil {
return err return err
} }
// Send Message // Send Message(s)
err = dwh.SendPayload() for _, i := range endpoints {
if err != nil { err = dwh.SendPayload(msg, i)
return err
if err != nil {
return err
}
} }
// Remove the item from the queue, given we sent our notification. // Remove the item from the queue, given we sent our notification.
@ -235,29 +251,38 @@ func (c *Cron) CheckDiscordQueue() error {
if err != nil { if err != nil {
return err return err
} }
time.Sleep(10 * time.Second)
} }
return nil return nil
} }
func (c *Cron) checkPosts(posts []database.Article, sourceName string) { func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
for _, item := range posts { for _, item := range posts {
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) _, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
if err != nil { if err != nil {
err = c.postArticle(item) id := uuid.New()
err := c.postArticle(id, item)
if err != nil { if err != nil {
log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
} else {
log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
} }
err = c.addToDiscordQueue(id)
if err != nil {
return err
}
} }
} }
time.Sleep(30 * time.Second) time.Sleep(30 * time.Second)
return nil
} }
func (c *Cron) postArticle(item database.Article) error { func (c *Cron) postArticle(id uuid.UUID,item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{ err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: uuid.New(), ID: id,
Sourceid: item.Sourceid, Sourceid: item.Sourceid,
Tags: item.Tags, Tags: item.Tags,
Title: item.Title, Title: item.Title,
@ -273,3 +298,14 @@ func (c *Cron) postArticle(item database.Article) error {
}) })
return err return err
} }
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
ID: uuid.New(),
Articleid: Id,
})
if err != nil {
return err
}
return nil
}

15
services/input/httpClient.go
View File

@ -1,15 +1,24 @@
package input package input
import ( import (
"net/http" "crypto/tls"
"log"
"io/ioutil" "io/ioutil"
"log"
"net/http"
) )
// This will use the net/http client reach out to a site and collect the content. // This will use the net/http client reach out to a site and collect the content.
func getHttpContent(uri string) ([]byte, error) { func getHttpContent(uri string) ([]byte, error) {
// Code to disable the http2 client for reddit.
// https://github.com/golang/go/issues/39302
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.ForceAttemptHTTP2 = false
tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
tr.TLSClientConfig = &tls.Config{}
client := &http.Client{} client := &http.Client{
Transport: tr,
}
req, err := http.NewRequest("GET", uri, nil) req, err := http.NewRequest("GET", uri, nil)
if err != nil { return nil, err } if err != nil { return nil, err }

40
services/input/reddit.go
View File

@ -6,7 +6,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"log" "log"
"os"
"strings" "strings"
"time" "time"
@ -27,7 +26,7 @@ type RedditConfig struct {
PullNSFW string PullNSFW string
} }
func NewRedditClient(Record database.Source) RedditClient { func NewRedditClient(Record database.Source) *RedditClient {
rc := RedditClient{ rc := RedditClient{
record: Record, record: Record,
} }
@ -36,23 +35,24 @@ func NewRedditClient(Record database.Source) RedditClient {
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW) rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP) rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
rc.disableHttp2Client() //rc.disableHttp2Client()
return rc return &rc
} }
// This is needed for to get modern go to talk to the endpoint. // This is needed for to get modern go to talk to the endpoint.
// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/ // https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
func (rc RedditClient) disableHttp2Client() { //func (rc *RedditClient) disableHttp2Client() {
os.Setenv("GODEBUG", "http2client=0") // os.Setenv("GODEBUG", "http2client=0")
} //}
func (rc RedditClient) GetBrowser() *rod.Browser {
func (rc *RedditClient) GetBrowser() *rod.Browser {
browser := rod.New().MustConnect() browser := rod.New().MustConnect()
return browser return browser
} }
func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page { func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
page := parser.MustPage(url) page := parser.MustPage(url)
return page return page
} }
@ -61,13 +61,15 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
// GetContent() reaches out to Reddit and pulls the Json data. // GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct. // It will then convert the data to a struct and return the struct.
func (rc RedditClient) GetContent() (model.RedditJsonContent, error) { func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
var items model.RedditJsonContent = model.RedditJsonContent{} var items model.RedditJsonContent = model.RedditJsonContent{}
// TODO Wire this to support the config options // TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url) Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("Collecting results on '%v'", rc.record.Name) log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
content, err := getHttpContent(Url) content, err := getHttpContent(Url)
if err != nil { if err != nil {
@ -84,13 +86,13 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
return items, nil return items, nil
} }
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article { func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
var redditArticles []database.Article var redditArticles []database.Article
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article database.Article var article database.Article
article, err := rc.convertToArticle(item.Data) article, err := rc.convertToArticle(item.Data)
if err != nil { if err != nil {
log.Println(err) log.Printf("[Reddit] %v", err)
continue continue
} }
redditArticles = append(redditArticles, article) redditArticles = append(redditArticles, article)
@ -100,7 +102,7 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []databa
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) { func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
var item database.Article var item database.Article
if source.Content == "" && source.Url != "" { if source.Content == "" && source.Url != "" {
@ -119,15 +121,15 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Artic
item = rc.convertRedirectPost(source) item = rc.convertRedirectPost(source)
} }
if item.Description == "" { if item.Description == "" && item.Title == "" {
var err = errors.New("reddit post failed to parse correctly") var err = errors.New("post failed to parse correctly")
return item, err return item, err
} }
return item, nil return item, nil
} }
func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article { func (rc *RedditClient) convertPicturePost(source model.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Title: source.Title, Title: source.Title,
@ -145,7 +147,7 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Arti
return item return item
} }
func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article { func (rc *RedditClient) convertTextPost(source model.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
@ -160,7 +162,7 @@ func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article
return item return item
} }
func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article { func (rc *RedditClient) convertVideoPost(source model.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",

29
services/input/twitch.go
View File

@ -77,7 +77,7 @@ func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
} }
// Invokes Logon request to the API // Invokes Logon request to the API
func (tc TwitchClient) Login() error { func (tc *TwitchClient) Login() error {
token, err := tc.api.RequestAppAccessToken([]string{twitchScopes}) token, err := tc.api.RequestAppAccessToken([]string{twitchScopes})
if err != nil { if err != nil {
return err return err
@ -87,7 +87,7 @@ func (tc TwitchClient) Login() error {
return nil return nil
} }
func (tc TwitchClient) GetContent() ([]database.Article, error) { func (tc *TwitchClient) GetContent() ([]database.Article, error) {
var items []database.Article var items []database.Article
user, err := tc.GetUserDetails() user, err := tc.GetUserDetails()
@ -136,7 +136,7 @@ func (tc TwitchClient) GetContent() ([]database.Article, error) {
return items, nil return items, nil
} }
func (tc TwitchClient) GetUserDetails() (helix.User, error) { func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{ users, err := tc.api.GetUsers(&helix.UsersParams{
@ -145,11 +145,16 @@ func (tc TwitchClient) GetUserDetails() (helix.User, error) {
if err != nil { if err != nil {
return blank, err return blank, err
} }
if len(users.Data.Users) == 0 {
return blank, errors.New("no results have been returned")
}
return users.Data.Users[0], nil return users.Data.Users[0], nil
} }
// This will reach out and collect the posts made by the user. // This will reach out and collect the posts made by the user.
func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) { func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
var blank []helix.Video var blank []helix.Video
videos, err := tc.api.GetVideos(&helix.VideosParams{ videos, err := tc.api.GetVideos(&helix.VideosParams{
@ -163,14 +168,14 @@ func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
return videos.Data.Videos, nil return videos.Data.Videos, nil
} }
func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) { func (tc *TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
if post.UserName == "" { if post.UserName == "" {
return "", ErrMissingAuthorName return "", ErrMissingAuthorName
} }
return post.UserName, nil return post.UserName, nil
} }
func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) { func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
if post.ThumbnailURL == "" { if post.ThumbnailURL == "" {
return "", ErrMissingThumbnail return "", ErrMissingThumbnail
} }
@ -180,7 +185,7 @@ func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
return thumb, nil return thumb, nil
} }
func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) { func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
if post.PublishedAt == "" { if post.PublishedAt == "" {
return time.Now(), ErrMissingPublishDate return time.Now(), ErrMissingPublishDate
} }
@ -191,7 +196,7 @@ func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
return pubDate, nil return pubDate, nil
} }
func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) { func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
// Check if the description is null but we have a title. // Check if the description is null but we have a title.
// The poster didnt add a description but this isnt an error. // The poster didnt add a description but this isnt an error.
if post.Description == "" && post.Title == "" { if post.Description == "" && post.Title == "" {
@ -204,7 +209,7 @@ func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
} }
// Extracts the avatar of the author with some validation. // Extracts the avatar of the author with some validation.
func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) { func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage } if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage } if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
return user.ProfileImageURL, nil return user.ProfileImageURL, nil
@ -212,20 +217,20 @@ func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
// Generate tags based on the video metadata. // Generate tags based on the video metadata.
// TODO Figure out how to query what game is played // TODO Figure out how to query what game is played
func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) { func (tc *TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName) res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName)
return res, nil return res, nil
} }
// Extracts the title from a post with some validation. // Extracts the title from a post with some validation.
func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) { func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
if post.Title == "" { if post.Title == "" {
return "", errors.New("unable to find the title on the requested post") return "", errors.New("unable to find the title on the requested post")
} }
return post.Title, nil return post.Title, nil
} }
func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) { func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
if post.URL == "" { return "", ErrMissingUrl } if post.URL == "" { return "", ErrMissingUrl }
return post.URL, nil return post.URL, nil
} }

8
services/input/youtube.go
View File

@ -121,7 +121,7 @@ func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page {
func (yc *YoutubeClient) GetParser(uri string) (*goquery.Document, error) { func (yc *YoutubeClient) GetParser(uri string) (*goquery.Document, error) {
html, err := http.Get(uri) html, err := http.Get(uri)
if err != nil { if err != nil {
log.Println(err) log.Printf("[YouTube] %v", err)
} }
defer html.Body.Close() defer html.Body.Close()
@ -244,19 +244,19 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
parser, err := yc.GetParser(item.Link) parser, err := yc.GetParser(item.Link)
if err != nil { if err != nil {
log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link) log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
} }
tags, err := yc.GetTags(parser) tags, err := yc.GetTags(parser)
if err != nil { if err != nil {
msg := fmt.Sprintf("%v. %v", err, item.Link) msg := fmt.Sprintf("%v. %v", err, item.Link)
log.Println(msg) log.Printf("[YouTube] %v", msg)
} }
thumb, err := yc.GetVideoThumbnail(parser) thumb, err := yc.GetVideoThumbnail(parser)
if err != nil { if err != nil {
msg := fmt.Sprintf("%v. %v", err, item.Link) msg := fmt.Sprintf("%v. %v", err, item.Link)
log.Println(msg) log.Printf("[YouTube] %v", msg)
} }
var article = database.Article{ var article = database.Article{

170
services/output/discordwebhook.go
View File

@ -1,80 +1,151 @@
package output package output
import ( import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings" "strings"
"time"
"github.com/jtom38/newsbot/collector/database" "github.com/jtom38/newsbot/collector/database"
) )
type discordField struct { type discordField struct {
Name string `json:"name,omitempty"` Name *string `json:"name,omitempty"`
Value string `json:"value,omitempty"` Value *string `json:"value,omitempty"`
Inline bool `json:"inline,omitempty"` Inline *bool `json:"inline,omitempty"`
}
type discordFooter struct {
Value *string `json:"text,omitempty"`
IconUrl *string `json:"icon_url,omitempty"`
} }
type discordAuthor struct { type discordAuthor struct {
Name string `json:"name,omitempty"` Name *string `json:"name,omitempty"`
Url string `json:"url,omitempty"` Url *string `json:"url,omitempty"`
IconUrl string `json:"icon_url,omitempty"` IconUrl *string `json:"icon_url,omitempty"`
} }
type discordImage struct { type discordImage struct {
Url string `json:"url,omitempty"` Url *string `json:"url,omitempty"`
} }
type discordEmbed struct { type DiscordEmbed struct {
Title string `json:"title,omitempty"` Title *string `json:"title,omitempty"`
Description string `json:"description,omitempty"` Description *string `json:"description,omitempty"`
Url string `json:"url,omitempty"` Url *string `json:"url,omitempty"`
Color int32 `json:"color,omitempty"` Color *int32 `json:"color,omitempty"`
Timestamp time.Time `json:"timestamp,omitempty"` //Timestamp time.Time `json:"timestamp,omitempty"`
Fields []discordField `json:"fields,omitempty"` Fields []*discordField `json:"fields,omitempty"`
Author discordAuthor `json:"author,omitempty"` Author discordAuthor `json:"author,omitempty"`
Image discordImage `json:"image,omitempty"` Image discordImage `json:"image,omitempty"`
Thumbnail discordImage `json:"thumbnail,omitempty"` Thumbnail discordImage `json:"thumbnail,omitempty"`
Footer *discordFooter `json:"footer,omitempty"`
} }
// Root object for Discord Webhook messages // Root object for Discord Webhook messages
type discordMessage struct { type DiscordMessage struct {
Content string `json:"content,omitempty"` Username *string `json:"username,omitempty"`
Embeds []discordEmbed `json:"embeds,omitempty"` Content *string `json:"content,omitempty"`
Embeds *[]DiscordEmbed `json:"embeds,omitempty"`
} }
const (
DefaultColor = 0
YoutubeColor = 16711680
TwitchColor = 0
RedditColor = 0
TwitterColor = 0
FfxivColor = 0
)
type Discord struct { type Discord struct {
Subscriptions []string Subscriptions []string
article database.Article article database.Article
Message discordMessage Message *DiscordMessage
} }
func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord { func NewDiscordWebHookMessage(Article database.Article) Discord {
return Discord{ return Discord{
Subscriptions: Subscriptions,
article: Article, article: Article,
Message: discordMessage{
Embeds: []discordEmbed{},
},
} }
} }
func (dwh Discord) GeneratePayload() error { // Generates the link field to expose in the message
// Convert the message func (dwh Discord) getFields() []*discordField {
embed := discordEmbed { var fields []*discordField
Title: dwh.article.Title,
Description: dwh.convertFromHtml(dwh.article.Description),
Url: dwh.article.Url,
Thumbnail: discordImage{
Url: dwh.article.Thumbnail,
},
}
var arr []discordEmbed
arr = append(arr, embed) key := "Link"
dwh.Message.Embeds = arr linkField := discordField{
return nil Name: &key,
Value: &dwh.article.Url,
}
fields = append(fields, &linkField)
return fields
} }
func (dwh Discord) SendPayload() error { // This will create the message that will be sent out.
func (dwh Discord) GeneratePayload() (*DiscordMessage, error) {
// Create the embed
footerMessage := "Brought to you by Newsbot"
footerUrl := ""
description := dwh.convertFromHtml(dwh.article.Description)
color := dwh.getColor(dwh.article.Url)
embed := DiscordEmbed{
Title: &dwh.article.Title,
Description: &description,
Image: discordImage{
Url: &dwh.article.Thumbnail,
},
Fields: dwh.getFields(),
Footer: &discordFooter{
Value: &footerMessage,
IconUrl: &footerUrl,
},
Color: &color,
}
// attach the embed to an array
var embedArray []DiscordEmbed
embedArray = append(embedArray, embed)
// create the base message
msg := DiscordMessage{
Embeds: &embedArray,
}
return &msg, nil
}
func (dwh Discord) SendPayload(Message *DiscordMessage, Url string) error {
// Convert the message to a io.reader object
buffer := new(bytes.Buffer)
json.NewEncoder(buffer).Encode(Message)
// Send the message
resp, err := http.Post(Url, "application/json", buffer)
if err != nil {
return err
}
// Check for 204
if resp.StatusCode != 204 {
defer resp.Body.Close()
errMsg, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf(string(errMsg))
}
return nil return nil
} }
@ -104,7 +175,16 @@ func (dwh Discord) convertFromHtml(body string) string {
return clean return clean
} }
func (dwh Discord) convertLinks(body string) string { func (dwh *Discord) getColor(Url string) int32 {
if strings.Contains(Url, "youtube.com") {
return YoutubeColor
}
return DefaultColor
}
func (dwh *Discord) convertLinks(body string) string {
//items := regexp.MustCompile("<a(.*?)a>") //items := regexp.MustCompile("<a(.*?)a>")
return "" return ""
} }

139
services/output/discordwebhook_test.go
View File

@ -1,11 +1,10 @@
package output_test package output_test
import ( import (
"errors"
"os" "os"
"strings" "strings"
"testing" "testing"
"time" //"time"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/joho/godotenv" "github.com/joho/godotenv"
@ -13,61 +12,127 @@ import (
"github.com/jtom38/newsbot/collector/services/output" "github.com/jtom38/newsbot/collector/services/output"
) )
var article database.Article = database.Article{ var (
ID: uuid.New(), article database.Article = database.Article{
Sourceid: uuid.New(), ID: uuid.New(),
Tags: "unit, testing", Sourceid: uuid.New(),
Title: "Demo", Tags: "unit, testing",
Url: "https://github.com/jtom38/newsbot.collector.api", Title: "Demo",
Pubdate: time.Now(), Url: "https://github.com/jtom38/newsbot.collector.api",
Videoheight: 0, //Pubdate: time.Now(),
Videowidth: 0, Videoheight: 0,
Description: "Hello World", Videowidth: 0,
Description: "Hello World",
}
blank string = ""
)
func TestDiscordMessageContainsTitle(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
if i.Title == &blank {
t.Error("title missing")
}
}
} }
func getWebhook() ([]string, error){ func TestDiscordMessageContainsDescription(t *testing.T) {
var endpoints []string d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
if i.Description == &blank {
t.Error("description missing")
}
}
}
func TestDiscordMessageFooter(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
blank := ""
if i.Footer.Value == &blank {
t.Error("missing footer vlue")
}
if i.Footer.IconUrl == &blank {
t.Error("missing footer url")
}
}
}
func TestDiscordMessageFields(t *testing.T) {
header := "Link"
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, embed := range *msg.Embeds {
for _, field := range embed.Fields {
var fName string
if field.Name != nil {
fName = *field.Name
} else {
t.Error("missing link field value")
}
if fName != header {
t.Error("missing link field key")
}
var fValue string
if field.Value != nil {
fValue = *field.Value
}
if fValue == blank {
t.Error("missing link field value")
}
}
}
}
// This test requires a env value to be present to work
func TestDiscordMessagePost(t *testing.T) {
_, err := os.Open(".env") _, err := os.Open(".env")
if err != nil { if err != nil {
return endpoints, err t.Error(err)
} }
err = godotenv.Load() err = godotenv.Load()
if err != nil { if err != nil {
return endpoints, err t.Error(err)
} }
res := os.Getenv("TESTS_DISCORD_WEBHOOK") res := os.Getenv("TESTS_DISCORD_WEBHOOK")
if res == "" { if res == "" {
return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing") t.Error("TESTS_DISCORD_WEBHOOK is missing")
}
endpoints := strings.Split(res, " ")
if err != nil {
t.Error(err)
} }
endpoints = strings.Split(res, "")
return endpoints, nil
}
func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) { d := output.NewDiscordWebHookMessage(article)
hook, err := getWebhook() msg, err := d.GeneratePayload()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
d := output.NewDiscordWebHookMessage(hook, article)
if len(d.Subscriptions) == 0 {
t.Error("no subscriptions found")
}
}
func TestDiscordMessageContainsTitle(t *testing.T) { err = d.SendPayload(msg, endpoints[0])
hook, err := getWebhook()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
d := output.NewDiscordWebHookMessage(hook, article)
err = d.GeneratePayload()
if err != nil {
t.Error(err)
}
if d.Message.Embeds[0].Title == "" {
t.Error("no title was found ")
}
} }