jtom38
/
newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Features/output discord (#12) 

* basic output looks to be working

* cron was updated to add to the queue and post messages

* new route to make discord webhook subscriptions

* updated swag tags

* swag

* Updated delete subscription call

* removed the time value as it throws off the msg template

* updated logging

* updated swagger

* updated new subscription route

* Updated logging and remove items from the queue if they dont have a subscription

* updated getArticles to return the 50 newest for the portal

* added endpoint to see if an item exists already

* formatting

* updated listArticles

* added colors and updated the image

* Updated to use the pointer in twitch

* added the twitch login command to cron... it works now

* found a better way to disable http2 for reddit. Test worked right away too

* updated the cron tasks to run collected once and hour or longer depending on the service
This commit is contained in:
James Tombleson 2022-07-12 15:28:31 -07:00 committed by GitHub
parent 0e0058506a
commit a1324ee1c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 797 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
database/query.sql.go
View File

@ -253,16 +253,11 @@ func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
}
const deleteSubscription = `-- name: DeleteSubscription :exec
Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2
Delete From subscriptions Where id = $1
`
type DeleteSubscriptionParams struct {
Discordwebhookid uuid.UUID
Sourceid uuid.UUID
}
func (q *Queries) DeleteSubscription(ctx context.Context, arg DeleteSubscriptionParams) error {
_, err := q.db.ExecContext(ctx, deleteSubscription, arg.Discordwebhookid, arg.Sourceid)
func (q *Queries) DeleteSubscription(ctx context.Context, id uuid.UUID) error {
_, err := q.db.ExecContext(ctx, deleteSubscription, id)
return err
}
@ -534,6 +529,23 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
return i, err
}
const getDiscordWebHookByUrl = `-- name: GetDiscordWebHookByUrl :one
Select id, url, server, channel, enabled From DiscordWebHooks Where url = $1
`
func (q *Queries) GetDiscordWebHookByUrl(ctx context.Context, url string) (Discordwebhook, error) {
row := q.db.QueryRowContext(ctx, getDiscordWebHookByUrl, url)
var i Discordwebhook
err := row.Scan(
&i.ID,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
)
return i, err
}
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
Select id, url, server, channel, enabled from DiscordWebHooks
Where ID = $1 LIMIT 1
@ -648,6 +660,27 @@ func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, erro
return i, err
}
const getSourceByName = `-- name: GetSourceByName :one
Select id, site, name, source, type, value, enabled, url, tags from Sources where name = $1 Limit 1
`
func (q *Queries) GetSourceByName(ctx context.Context, name string) (Source, error) {
row := q.db.QueryRowContext(ctx, getSourceByName, name)
var i Source
err := row.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
)
return i, err
}
const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many
Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1
`
@ -743,6 +776,47 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
return items, nil
}
const listArticlesByDate = `-- name: ListArticlesByDate :many
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage From articles ORDER BY pubdate desc Limit $1
`
func (q *Queries) ListArticlesByDate(ctx context.Context, limit int32) ([]Article, error) {
rows, err := q.db.QueryContext(ctx, listArticlesByDate, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Article
for rows.Next() {
var i Article
if err := rows.Scan(
&i.ID,
&i.Sourceid,
&i.Tags,
&i.Title,
&i.Url,
&i.Pubdate,
&i.Video,
&i.Videoheight,
&i.Videowidth,
&i.Thumbnail,
&i.Description,
&i.Authorname,
&i.Authorimage,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
Select id, articleid from DiscordQueue LIMIT $1
`
@ -965,8 +1039,8 @@ func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid
return items, nil
}
const querySubscriptions = `-- name: QuerySubscriptions :many
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2
const querySubscriptions = `-- name: QuerySubscriptions :one
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1
`
type QuerySubscriptionsParams struct {
@ -974,25 +1048,9 @@ type QuerySubscriptionsParams struct {
Sourceid uuid.UUID
}
func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) ([]Subscription, error) {
rows, err := q.db.QueryContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Subscription
for rows.Next() {
var i Subscription
if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) (Subscription, error) {
row := q.db.QueryRowContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid)
var i Subscription
err := row.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid)
return i, err
}

15
database/schema/query.sql
View File

@ -10,6 +10,9 @@ Where Url = $1 LIMIT 1;
-- name: ListArticles :many
Select * From articles Limit $1;
-- name: ListArticlesByDate :many
Select * From articles ORDER BY pubdate desc Limit $1;
-- name: GetArticlesBySource :many
select * from articles
INNER join sources on articles.sourceid=Sources.ID
@ -66,6 +69,9 @@ Where ID = $1 LIMIT 1;
Select * From DiscordWebHooks
Where Server = $1;
-- name: GetDiscordWebHookByUrl :one
Select * From DiscordWebHooks Where url = $1;
-- name: ListDiscordWebhooks :many
Select * From discordwebhooks LIMIT $1;
@ -127,6 +133,9 @@ Values
-- name: GetSourceByID :one
Select * From Sources where ID = $1 Limit 1;
-- name: GetSourceByName :one
Select * from Sources where name = $1 Limit 1;
-- name: ListSources :many
Select * From Sources Limit $1;
@ -154,8 +163,8 @@ Select * From subscriptions Limit $1;
-- name: ListSubscriptionsBySourceId :many
Select * From subscriptions where sourceid = $1;
-- name: QuerySubscriptions :many
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2;
-- name: QuerySubscriptions :one
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1;
-- name: GetSubscriptionsBySourceID :many
Select * From subscriptions Where sourceid = $1;
@ -164,4 +173,4 @@ Select * From subscriptions Where sourceid = $1;
Select * from subscriptions Where discordwebhookid = $1;
-- name: DeleteSubscription :exec
Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2;
Delete From subscriptions Where id = $1;

110
docs/docs.go
View File

@ -22,7 +22,7 @@ const docTemplate = `{
"application/json"
],
"tags": [
"articles"
"Articles"
],
"summary": "Lists the top 50 records",
"responses": {}
@ -34,7 +34,7 @@ const docTemplate = `{
"application/json"
],
"tags": [
"articles"
"Articles"
],
"summary": "Finds the articles based on the SourceID provided. Returns the top 50.",
"parameters": [
@ -55,7 +55,7 @@ const docTemplate = `{
"application/json"
],
"tags": [
"articles"
"Articles"
],
"summary": "Returns an article based on defined ID.",
"parameters": [
@ -76,8 +76,8 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Lists the top 50 records",
"responses": {}
@ -86,9 +86,9 @@ const docTemplate = `{
"/config/sources/new/reddit": {
"post": {
"tags": [
"config",
"source",
"reddit"
"Config",
"Source",
"Reddit"
],
"summary": "Creates a new reddit source to monitor.",
"parameters": [
@ -113,9 +113,9 @@ const docTemplate = `{
"/config/sources/new/twitch": {
"post": {
"tags": [
"config",
"source",
"twitch"
"Config",
"Source",
"Twitch"
],
"summary": "Creates a new twitch source to monitor.",
"parameters": [
@ -147,9 +147,9 @@ const docTemplate = `{
"/config/sources/new/youtube": {
"post": {
"tags": [
"config",
"source",
"youtube"
"Config",
"Source",
"YouTube"
],
"summary": "Creates a new youtube source to monitor.",
"parameters": [
@ -184,8 +184,8 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Returns a single entity by ID",
"parameters": [
@ -201,8 +201,8 @@ const docTemplate = `{
},
"delete": {
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Deletes a record by ID.",
"parameters": [
@ -220,8 +220,8 @@ const docTemplate = `{
"/config/sources/{id}/disable": {
"post": {
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Disables a source from processing.",
"parameters": [
@ -239,8 +239,8 @@ const docTemplate = `{
"/config/sources/{id}/enable": {
"post": {
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Enables a source to continue processing.",
"parameters": [
@ -261,7 +261,7 @@ const docTemplate = `{
"application/json"
],
"tags": [
"debug",
"Debug",
"Discord",
"Queue"
],
@ -275,9 +275,9 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"Config",
"Discord",
"Webhooks"
"Webhook"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {}
@ -289,9 +289,9 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"Config",
"Discord",
"Webhooks"
"Webhook"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
@ -309,9 +309,9 @@ const docTemplate = `{
"/discord/webhooks/new": {
"post": {
"tags": [
"config",
"Config",
"Discord",
"Webhooks"
"Webhook"
],
"summary": "Creates a new record for a discord web hook to post data to.",
"parameters": [
@ -331,7 +331,7 @@ const docTemplate = `{
},
{
"type": "string",
"description": "Channel name.",
"description": "Channel name",
"name": "channel",
"in": "query",
"required": true
@ -346,7 +346,7 @@ const docTemplate = `{
"text/plain"
],
"tags": [
"debug"
"Debug"
],
"summary": "Responds back with \"Hello x\" depending on param passed in.",
"parameters": [
@ -367,7 +367,7 @@ const docTemplate = `{
"text/plain"
],
"tags": [
"debug"
"Debug"
],
"summary": "Responds back with \"Hello world!\"",
"responses": {}
@ -379,7 +379,7 @@ const docTemplate = `{
"text/plain"
],
"tags": [
"debug"
"Debug"
],
"summary": "Sends back \"pong\". Good to test with.",
"responses": {}
@ -391,9 +391,9 @@ const docTemplate = `{
"application/json"
],
"tags": [
"settings"
"Settings"
],
"summary": "Returns a object based on the Key that was given/",
"summary": "Returns a object based on the Key that was given.",
"parameters": [
{
"type": "string",
@ -412,8 +412,8 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"Subscriptions"
"Config",
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {}
@ -425,8 +425,8 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"Subscriptions"
"Config",
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
@ -447,8 +447,8 @@ const docTemplate = `{
"application/json"
],
"tags": [
"config",
"Subscriptions"
"Config",
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
@ -462,6 +462,34 @@ const docTemplate = `{
],
"responses": {}
}
},
"/subscriptions/new/discordwebhook": {
"post": {
"tags": [
"Config",
"Source",
"Discord",
"Subscription"
],
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
"parameters": [
{
"type": "string",
"description": "discordWebHookId",
"name": "discordWebHookId",
"in": "query",
"required": true
},
{
"type": "string",
"description": "sourceId",
"name": "sourceId",
"in": "query",
"required": true
}
],
"responses": {}
}
}
}
}`

110
docs/swagger.json
View File

@ -13,7 +13,7 @@
"application/json"
],
"tags": [
"articles"
"Articles"
],
"summary": "Lists the top 50 records",
"responses": {}
@ -25,7 +25,7 @@
"application/json"
],
"tags": [
"articles"
"Articles"
],
"summary": "Finds the articles based on the SourceID provided. Returns the top 50.",
"parameters": [
@ -46,7 +46,7 @@
"application/json"
],
"tags": [
"articles"
"Articles"
],
"summary": "Returns an article based on defined ID.",
"parameters": [
@ -67,8 +67,8 @@
"application/json"
],
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Lists the top 50 records",
"responses": {}
@ -77,9 +77,9 @@
"/config/sources/new/reddit": {
"post": {
"tags": [
"config",
"source",
"reddit"
"Config",
"Source",
"Reddit"
],
"summary": "Creates a new reddit source to monitor.",
"parameters": [
@ -104,9 +104,9 @@
"/config/sources/new/twitch": {
"post": {
"tags": [
"config",
"source",
"twitch"
"Config",
"Source",
"Twitch"
],
"summary": "Creates a new twitch source to monitor.",
"parameters": [
@ -138,9 +138,9 @@
"/config/sources/new/youtube": {
"post": {
"tags": [
"config",
"source",
"youtube"
"Config",
"Source",
"YouTube"
],
"summary": "Creates a new youtube source to monitor.",
"parameters": [
@ -175,8 +175,8 @@
"application/json"
],
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Returns a single entity by ID",
"parameters": [
@ -192,8 +192,8 @@
},
"delete": {
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Deletes a record by ID.",
"parameters": [
@ -211,8 +211,8 @@
"/config/sources/{id}/disable": {
"post": {
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Disables a source from processing.",
"parameters": [
@ -230,8 +230,8 @@
"/config/sources/{id}/enable": {
"post": {
"tags": [
"config",
"source"
"Config",
"Source"
],
"summary": "Enables a source to continue processing.",
"parameters": [
@ -252,7 +252,7 @@
"application/json"
],
"tags": [
"debug",
"Debug",
"Discord",
"Queue"
],
@ -266,9 +266,9 @@
"application/json"
],
"tags": [
"config",
"Config",
"Discord",
"Webhooks"
"Webhook"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {}
@ -280,9 +280,9 @@
"application/json"
],
"tags": [
"config",
"Config",
"Discord",
"Webhooks"
"Webhook"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
@ -300,9 +300,9 @@
"/discord/webhooks/new": {
"post": {
"tags": [
"config",
"Config",
"Discord",
"Webhooks"
"Webhook"
],
"summary": "Creates a new record for a discord web hook to post data to.",
"parameters": [
@ -322,7 +322,7 @@
},
{
"type": "string",
"description": "Channel name.",
"description": "Channel name",
"name": "channel",
"in": "query",
"required": true
@ -337,7 +337,7 @@
"text/plain"
],
"tags": [
"debug"
"Debug"
],
"summary": "Responds back with \"Hello x\" depending on param passed in.",
"parameters": [
@ -358,7 +358,7 @@
"text/plain"
],
"tags": [
"debug"
"Debug"
],
"summary": "Responds back with \"Hello world!\"",
"responses": {}
@ -370,7 +370,7 @@
"text/plain"
],
"tags": [
"debug"
"Debug"
],
"summary": "Sends back \"pong\". Good to test with.",
"responses": {}
@ -382,9 +382,9 @@
"application/json"
],
"tags": [
"settings"
"Settings"
],
"summary": "Returns a object based on the Key that was given/",
"summary": "Returns a object based on the Key that was given.",
"parameters": [
{
"type": "string",
@ -403,8 +403,8 @@
"application/json"
],
"tags": [
"config",
"Subscriptions"
"Config",
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {}
@ -416,8 +416,8 @@
"application/json"
],
"tags": [
"config",
"Subscriptions"
"Config",
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
@ -438,8 +438,8 @@
"application/json"
],
"tags": [
"config",
"Subscriptions"
"Config",
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
@ -453,6 +453,34 @@
],
"responses": {}
}
},
"/subscriptions/new/discordwebhook": {
"post": {
"tags": [
"Config",
"Source",
"Discord",
"Subscription"
],
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
"parameters": [
{
"type": "string",
"description": "discordWebHookId",
"name": "discordWebHookId",
"in": "query",
"required": true
},
{
"type": "string",
"description": "sourceId",
"name": "sourceId",
"in": "query",
"required": true
}
],
"responses": {}
}
}
}
}

102
docs/swagger.yaml
View File

@ -11,7 +11,7 @@ paths:
responses: {}
summary: Lists the top 50 records
tags:
- articles
- Articles
/articles/{id}:
get:
parameters:
@ -25,7 +25,7 @@ paths:
responses: {}
summary: Returns an article based on defined ID.
tags:
- articles
- Articles
/articles/by/sourceid:
get:
parameters:
@ -40,7 +40,7 @@ paths:
summary: Finds the articles based on the SourceID provided. Returns the top
50.
tags:
- articles
- Articles
/config/sources:
get:
produces:
@ -48,8 +48,8 @@ paths:
responses: {}
summary: Lists the top 50 records
tags:
- config
- source
- Config
- Source
/config/sources/{id}:
delete:
parameters:
@ -61,8 +61,8 @@ paths:
responses: {}
summary: Deletes a record by ID.
tags:
- config
- source
- Config
- Source
get:
parameters:
- description: uuid
@ -75,8 +75,8 @@ paths:
responses: {}
summary: Returns a single entity by ID
tags:
- config
- source
- Config
- Source
/config/sources/{id}/disable:
post:
parameters:
@ -88,8 +88,8 @@ paths:
responses: {}
summary: Disables a source from processing.
tags:
- config
- source
- Config
- Source
/config/sources/{id}/enable:
post:
parameters:
@ -101,8 +101,8 @@ paths:
responses: {}
summary: Enables a source to continue processing.
tags:
- config
- source
- Config
- Source
/config/sources/new/reddit:
post:
parameters:
@ -119,9 +119,9 @@ paths:
responses: {}
summary: Creates a new reddit source to monitor.
tags:
- config
- source
- reddit
- Config
- Source
- Reddit
/config/sources/new/twitch:
post:
parameters:
@ -143,9 +143,9 @@ paths:
responses: {}
summary: Creates a new twitch source to monitor.
tags:
- config
- source
- twitch
- Config
- Source
- Twitch
/config/sources/new/youtube:
post:
parameters:
@ -167,9 +167,9 @@ paths:
responses: {}
summary: Creates a new youtube source to monitor.
tags:
- config
- source
- youtube
- Config
- Source
- YouTube
/discord/queue:
get:
produces:
@ -177,7 +177,7 @@ paths:
responses: {}
summary: Returns the top 100 entries from the queue to be processed.
tags:
- debug
- Debug
- Discord
- Queue
/discord/webhooks:
@ -187,9 +187,9 @@ paths:
responses: {}
summary: Returns the top 100 entries from the queue to be processed.
tags:
- config
- Config
- Discord
- Webhooks
- Webhook
/discord/webhooks/byId:
get:
parameters:
@ -203,9 +203,9 @@ paths:
responses: {}
summary: Returns the top 100 entries from the queue to be processed.
tags:
- config
- Config
- Discord
- Webhooks
- Webhook
/discord/webhooks/new:
post:
parameters:
@ -219,7 +219,7 @@ paths:
name: server
required: true
type: string
- description: Channel name.
- description: Channel name
in: query
name: channel
required: true
@ -227,9 +227,9 @@ paths:
responses: {}
summary: Creates a new record for a discord web hook to post data to.
tags:
- config
- Config
- Discord
- Webhooks
- Webhook
/hello/{who}:
get:
parameters:
@ -243,7 +243,7 @@ paths:
responses: {}
summary: Responds back with "Hello x" depending on param passed in.
tags:
- debug
- Debug
/helloworld:
get:
produces:
@ -251,7 +251,7 @@ paths:
responses: {}
summary: Responds back with "Hello world!"
tags:
- debug
- Debug
/ping:
get:
produces:
@ -259,7 +259,7 @@ paths:
responses: {}
summary: Sends back "pong". Good to test with.
tags:
- debug
- Debug
/settings/{key}:
get:
parameters:
@ -271,9 +271,9 @@ paths:
produces:
- application/json
responses: {}
summary: Returns a object based on the Key that was given/
summary: Returns a object based on the Key that was given.
tags:
- settings
- Settings
/subscriptions:
get:
produces:
@ -281,8 +281,8 @@ paths:
responses: {}
summary: Returns the top 100 entries from the queue to be processed.
tags:
- config
- Subscriptions
- Config
- Subscription
/subscriptions/byDiscordId:
get:
parameters:
@ -296,8 +296,8 @@ paths:
responses: {}
summary: Returns the top 100 entries from the queue to be processed.
tags:
- config
- Subscriptions
- Config
- Subscription
/subscriptions/bySourceId:
get:
parameters:
@ -311,6 +311,26 @@ paths:
responses: {}
summary: Returns the top 100 entries from the queue to be processed.
tags:
- config
- Subscriptions
- Config
- Subscription
/subscriptions/new/discordwebhook:
post:
parameters:
- description: discordWebHookId
in: query
name: discordWebHookId
required: true
type: string
- description: sourceId
in: query
name: sourceId
required: true
type: string
responses: {}
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
tags:
- Config
- Source
- Discord
- Subscription
swagger: "2.0"

46
routes/articles.go
View File

@ -11,12 +11,12 @@ import (
// ListArticles
// @Summary Lists the top 50 records
// @Produce application/json
// @Tags articles
// @Tags Articles
// @Router /articles [get]
func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
res, err := s.Db.ListArticles(*s.ctx, 50)
res, err := s.Db.ListArticlesByDate(*s.ctx, 50)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
@ -27,7 +27,6 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(err.Error()))
panic(err)
}
w.Write(bres)
}
@ -35,7 +34,7 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
// @Summary Returns an article based on defined ID.
// @Param id path string true "uuid"
// @Produce application/json
// @Tags articles
// @Tags Articles
// @Router /articles/{id} [get]
func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -67,7 +66,7 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
// @Param id query string true "Source ID UUID"
// @Produce application/json
// @Tags articles
// @Tags Articles
// @Router /articles/by/sourceid [get]
func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -95,4 +94,39 @@ func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
}
w.Write(bres)
}
}
// TODO add page support
// GetArticlesByTag
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
// @Param Tag query string true "Tag name"
// @Produce application/json
// @Tags Articles
// @Router /articles/by/sourceid [get]
func (s *Server) GetArticlesByTag(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
r.URL.Query()
query := r.URL.Query()
_id := query["id"][0]
uuid, err := uuid.Parse(_id)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
}
res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
}
bres, err := json.Marshal(res)
if err != nil {
w.Write([]byte(err.Error()))
panic(err)
}
w.Write(bres)
}

2
routes/discordQueue.go
View File

@ -8,7 +8,7 @@ import (
// GetDiscordQueue
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Tags debug, Discord, Queue
// @Tags Debug, Discord, Queue
// @Router /discord/queue [get]
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")

8
routes/discordwebhooks.go
View File

@ -13,7 +13,7 @@ import (
// GetDiscordWebHooks
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Tags config, Discord, Webhooks
// @Tags Config, Discord, Webhook
// @Router /discord/webhooks [get]
func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -37,7 +37,7 @@ func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Param id query string true "id"
// @Tags config, Discord, Webhooks
// @Tags Config, Discord, Webhook
// @Router /discord/webhooks/byId [get]
func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -74,8 +74,8 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request)
// @Summary Creates a new record for a discord web hook to post data to.
// @Param url query string true "url"
// @Param server query string true "Server name"
// @Param channel query string true "Channel name."
// @Tags config, Discord, Webhooks
// @Param channel query string true "Channel name"
// @Tags Config, Discord, Webhook
// @Router /discord/webhooks/new [post]
func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()

6
routes/root.go
View File

@ -22,7 +22,7 @@ func RootRoutes() chi.Router {
// HelloWorld
// @Summary Responds back with "Hello world!"
// @Produce plain
// @Tags debug
// @Tags Debug
// @Router /helloworld [get]
func helloWorld(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello World!"))
@ -31,7 +31,7 @@ func helloWorld(w http.ResponseWriter, r *http.Request) {
// Ping
// @Summary Sends back "pong". Good to test with.
// @Produce plain
// @Tags debug
// @Tags Debug
// @Router /ping [get]
func ping(w http.ResponseWriter, r *http.Request) {
msg := "pong"
@ -42,7 +42,7 @@ func ping(w http.ResponseWriter, r *http.Request) {
// @Summary Responds back with "Hello x" depending on param passed in.
// @Param who path string true "Who"
// @Produce plain
// @Tags debug
// @Tags Debug
// @Router /hello/{who} [get]
func helloWho(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who"))

23
routes/server.go
View File

@ -3,12 +3,13 @@ package routes
import (
"context"
"database/sql"
//"net/http"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
httpSwagger "github.com/swaggo/http-swagger"
_ "github.com/lib/pq"
httpSwagger "github.com/swaggo/http-swagger"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/config"
@ -16,14 +17,14 @@ import (
type Server struct {
Router *chi.Mux
Db *database.Queries
ctx *context.Context
Db *database.Queries
ctx *context.Context
}
var (
ErrIdValueMissing string = "id value is missing"
ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct."
ErrNoRecordFound string = "no record was found."
ErrIdValueMissing string = "id value is missing"
ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct."
ErrNoRecordFound string = "no record was found."
ErrUnableToConvertToJson string = "Unable to convert to json"
)
@ -59,13 +60,14 @@ func openDatabase(ctx context.Context) (*database.Queries, error) {
func (s *Server) MountMiddleware() {
s.Router.Use(middleware.Logger)
s.Router.Use(middleware.Recoverer)
//s.Router.Use(middleware.Heartbeat())
}
func (s *Server) MountRoutes() {
s.Router.Get("/swagger/*", httpSwagger.Handler(
httpSwagger.URL("http://localhost:8081/swagger/doc.json"), //The url pointing to API definition
))
/* Root Routes */
s.Router.Get("/api/helloworld", helloWorld)
s.Router.Get("/api/hello/{who}", helloWho)
@ -88,10 +90,14 @@ func (s *Server) MountRoutes() {
/* Settings */
s.Router.Get("/api/settings", s.getSettings)
/* Source Routes */
s.Router.Get("/api/config/sources", s.listSources)
/* Reddit Source Routes */
s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource)
s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource)
s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource)
s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) {
@ -105,4 +111,5 @@ func (s *Server) MountRoutes() {
s.Router.Get("/api/subscriptions", s.ListSubscriptions)
s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId)
s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId)
s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription)
}

4
routes/settings.go
View File

@ -10,10 +10,10 @@ import (
)
// GetSettings
// @Summary Returns a object based on the Key that was given/
// @Summary Returns a object based on the Key that was given.
// @Param key path string true "Settings Key value"
// @Produce application/json
// @Tags settings
// @Tags Settings
// @Router /settings/{key} [get]
func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
//var item model.Sources

20
routes/sources.go
View File

@ -15,7 +15,7 @@ import (
// ListSources
// @Summary Lists the top 50 records
// @Produce application/json
// @Tags config, source
// @Tags Config, Source
// @Router /config/sources [get]
func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
//TODO Add top?
@ -48,7 +48,7 @@ func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
// @Summary Returns a single entity by ID
// @Param id path string true "uuid"
// @Produce application/json
// @Tags config, source
// @Tags Config, Source
// @Router /config/sources/{id} [get]
func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID")
@ -78,7 +78,7 @@ func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
// @Summary Creates a new reddit source to monitor.
// @Param name query string true "name"
// @Param url query string true "url"
// @Tags config, source, reddit
// @Tags Config, Source, Reddit
// @Router /config/sources/new/reddit [post]
func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
@ -116,12 +116,16 @@ func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
w.Write(bJson)
}
func (s *Server) getSourceByType(w http.ResponseWriter, r *http.Request) {
}
// NewYoutubeSource
// @Summary Creates a new youtube source to monitor.
// @Param name query string true "name"
// @Param url query string true "url"
// @Param tags query string true "tags"
// @Tags config, source, youtube
// @Tags Config, Source, YouTube
// @Router /config/sources/new/youtube [post]
func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
@ -164,7 +168,7 @@ func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
// @Param name query string true "name"
// @Param url query string true "url"
// @Param tags query string true "tags"
// @Tags config, source, twitch
// @Tags Config, Source, Twitch
// @Router /config/sources/new/twitch [post]
func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
@ -205,7 +209,7 @@ func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
// DeleteSource
// @Summary Deletes a record by ID.
// @Param id path string true "id"
// @Tags config, source
// @Tags Config, Source
// @Router /config/sources/{id} [delete]
func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
//var item model.Sources = model.Sources{}
@ -232,7 +236,7 @@ func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
// DisableSource
// @Summary Disables a source from processing.
// @Param id path string true "id"
// @Tags config, source
// @Tags Config, Source
// @Router /config/sources/{id}/disable [post]
func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID")
@ -256,7 +260,7 @@ func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
// EnableSource
// @Summary Enables a source to continue processing.
// @Param id path string true "id"
// @Tags config, source
// @Tags Config, Source
// @Router /config/sources/{id}/enable [post]
func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID")

74
routes/subscriptions.go
View File

@ -5,12 +5,13 @@ import (
"net/http"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
)
// GetSubscriptions
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Tags config, Subscriptions
// @Tags Config, Subscription
// @Router /subscriptions [get]
func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -34,7 +35,7 @@ func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Param id query string true "id"
// @Tags config, Subscriptions
// @Tags Config, Subscription
// @Router /subscriptions/byDiscordId [get]
func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -71,7 +72,7 @@ func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Requ
// @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json
// @Param id query string true "id"
// @Tags config, Subscriptions
// @Tags Config, Subscription
// @Router /subscriptions/bySourceId [get]
func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -102,4 +103,71 @@ func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Reque
}
w.Write(bres)
}
// NewDiscordWebHookSubscription
// @Summary Creates a new subscription to link a post from a Source to a DiscordWebHook.
// @Param discordWebHookId query string true "discordWebHookId"
// @Param sourceId query string true "sourceId"
// @Tags Config, Source, Discord, Subscription
// @Router /subscriptions/new/discordwebhook [post]
func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
// Extract the values given
query := r.URL.Query()
discordWebHookId := query["discordWebHookId"][0]
sourceId := query["sourceId"][0]
// Check to make we didnt get a null
if discordWebHookId == "" {
http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest )
return
}
if sourceId == "" {
http.Error(w, "invalid sourceID given", http.StatusBadRequest )
return
}
// Valide they are UUID values
uHook, err := uuid.Parse(discordWebHookId)
if err != nil {
http.Error(w, "DiscordWebHooksID was not a uuid value.", http.StatusBadRequest)
return
}
uSource, err := uuid.Parse(sourceId)
if err != nil {
http.Error(w, "SourceId was not a uuid value", http.StatusBadRequest)
return
}
// Check if the sub already exists
item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{
Discordwebhookid: uHook,
Sourceid: uSource,
})
if err == nil {
bJson, err := json.Marshal(&item)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(bJson)
return
}
// Does not exist, so make it.
params := database.CreateSubscriptionParams{
ID: uuid.New(),
Discordwebhookid: uHook,
Sourceid: uSource,
}
s.Db.CreateSubscription(*s.ctx, params)
bJson, err := json.Marshal(&params)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(bJson)
}

100
services/cron/scheduler.go
View File

@ -3,6 +3,7 @@ package cron
import (
"context"
"database/sql"
"fmt"
"log"
"time"
@ -11,8 +12,8 @@ import (
"github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/output"
)
@ -51,29 +52,32 @@ func New(ctx context.Context) *Cron {
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() })
log.Print("Reddit backend was enabled")
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit()
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() })
log.Print("YouTube backend was enabled")
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled")
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() })
log.Print("FFXIV backend was enabled")
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled")
}
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() })
log.Print("Twitch backend was enabled")
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled")
}
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
log.Print("[Output] Discord Output was enabled")
c.timer = timer
return c
}
@ -162,6 +166,11 @@ func (c *Cron) CheckTwitch() error {
return err
}
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
@ -186,19 +195,13 @@ func (c *Cron) CheckDiscordQueue() error {
return err
}
for _, queue := range(queueItems) {
for _, queue := range queueItems {
// Get the articleByID
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
if err != nil {
return err
}
// Get the SourceByID
//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
//if err != nil {
// return err
//}
var endpoints []string
// List Subscription by SourceID
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
@ -206,8 +209,18 @@ func (c *Cron) CheckDiscordQueue() error {
return err
}
// if no one is subscribed to it, remove it from the index.
if len(subs) == 0 {
log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
if err != nil {
return err
}
continue
}
// Get the webhhooks to send to
for _, sub := range(subs) {
for _, sub := range subs {
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
if err != nil {
return err
@ -218,16 +231,19 @@ func (c *Cron) CheckDiscordQueue() error {
}
// Create Discord Message
dwh := output.NewDiscordWebHookMessage(endpoints, article)
err = dwh.GeneratePayload()
dwh := output.NewDiscordWebHookMessage(article)
msg, err := dwh.GeneratePayload()
if err != nil {
return err
}
// Send Message
err = dwh.SendPayload()
if err != nil {
return err
// Send Message(s)
for _, i := range endpoints {
err = dwh.SendPayload(msg, i)
if err != nil {
return err
}
}
// Remove the item from the queue, given we sent our notification.
@ -235,29 +251,38 @@ func (c *Cron) CheckDiscordQueue() error {
if err != nil {
return err
}
time.Sleep(10 * time.Second)
}
return nil
}
func (c *Cron) checkPosts(posts []database.Article, sourceName string) {
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
for _, item := range posts {
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
if err != nil {
err = c.postArticle(item)
id := uuid.New()
err := c.postArticle(id, item)
if err != nil {
log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
} else {
log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
}
err = c.addToDiscordQueue(id)
if err != nil {
return err
}
}
}
time.Sleep(30 * time.Second)
return nil
}
func (c *Cron) postArticle(item database.Article) error {
func (c *Cron) postArticle(id uuid.UUID,item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: uuid.New(),
ID: id,
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
@ -273,3 +298,14 @@ func (c *Cron) postArticle(item database.Article) error {
})
return err
}
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
ID: uuid.New(),
Articleid: Id,
})
if err != nil {
return err
}
return nil
}

15
services/input/httpClient.go
View File

@ -1,15 +1,24 @@
package input
import (
"net/http"
"log"
"crypto/tls"
"io/ioutil"
"log"
"net/http"
)
// This will use the net/http client reach out to a site and collect the content.
func getHttpContent(uri string) ([]byte, error) {
// Code to disable the http2 client for reddit.
// https://github.com/golang/go/issues/39302
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.ForceAttemptHTTP2 = false
tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
tr.TLSClientConfig = &tls.Config{}
client := &http.Client{}
client := &http.Client{
Transport: tr,
}
req, err := http.NewRequest("GET", uri, nil)
if err != nil { return nil, err }

40
services/input/reddit.go
View File

@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"log"
"os"
"strings"
"time"
@ -27,7 +26,7 @@ type RedditConfig struct {
PullNSFW string
}
func NewRedditClient(Record database.Source) RedditClient {
func NewRedditClient(Record database.Source) *RedditClient {
rc := RedditClient{
record: Record,
}
@ -36,23 +35,24 @@ func NewRedditClient(Record database.Source) RedditClient {
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
rc.disableHttp2Client()
//rc.disableHttp2Client()
return rc
return &rc
}
// This is needed for to get modern go to talk to the endpoint.
// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
func (rc RedditClient) disableHttp2Client() {
os.Setenv("GODEBUG", "http2client=0")
}
//func (rc *RedditClient) disableHttp2Client() {
// os.Setenv("GODEBUG", "http2client=0")
//}
func (rc RedditClient) GetBrowser() *rod.Browser {
func (rc *RedditClient) GetBrowser() *rod.Browser {
browser := rod.New().MustConnect()
return browser
}
func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
page := parser.MustPage(url)
return page
}
@ -61,13 +61,15 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
// GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct.
func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
var items model.RedditJsonContent = model.RedditJsonContent{}
// TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("Collecting results on '%v'", rc.record.Name)
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
content, err := getHttpContent(Url)
if err != nil {
@ -84,13 +86,13 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
return items, nil
}
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
var redditArticles []database.Article
for _, item := range items.Data.Children {
var article database.Article
article, err := rc.convertToArticle(item.Data)
if err != nil {
log.Println(err)
log.Printf("[Reddit] %v", err)
continue
}
redditArticles = append(redditArticles, article)
@ -100,7 +102,7 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []databa
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database.
func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
var item database.Article
if source.Content == "" && source.Url != "" {
@ -119,15 +121,15 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Artic
item = rc.convertRedirectPost(source)
}
if item.Description == "" {
var err = errors.New("reddit post failed to parse correctly")
if item.Description == "" && item.Title == "" {
var err = errors.New("post failed to parse correctly")
return item, err
}
return item, nil
}
func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
func (rc *RedditClient) convertPicturePost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Title: source.Title,
@ -145,7 +147,7 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Arti
return item
}
func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
func (rc *RedditClient) convertTextPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
@ -160,7 +162,7 @@ func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article
return item
}
func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
func (rc *RedditClient) convertVideoPost(source model.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",

29
services/input/twitch.go
View File

@ -77,7 +77,7 @@ func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
}
// Invokes Logon request to the API
func (tc TwitchClient) Login() error {
func (tc *TwitchClient) Login() error {
token, err := tc.api.RequestAppAccessToken([]string{twitchScopes})
if err != nil {
return err
@ -87,7 +87,7 @@ func (tc TwitchClient) Login() error {
return nil
}
func (tc TwitchClient) GetContent() ([]database.Article, error) {
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
var items []database.Article
user, err := tc.GetUserDetails()
@ -136,7 +136,7 @@ func (tc TwitchClient) GetContent() ([]database.Article, error) {
return items, nil
}
func (tc TwitchClient) GetUserDetails() (helix.User, error) {
func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{
@ -145,11 +145,16 @@ func (tc TwitchClient) GetUserDetails() (helix.User, error) {
if err != nil {
return blank, err
}
if len(users.Data.Users) == 0 {
return blank, errors.New("no results have been returned")
}
return users.Data.Users[0], nil
}
// This will reach out and collect the posts made by the user.
func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
var blank []helix.Video
videos, err := tc.api.GetVideos(&helix.VideosParams{
@ -163,14 +168,14 @@ func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
return videos.Data.Videos, nil
}
func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
func (tc *TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
if post.UserName == "" {
return "", ErrMissingAuthorName
}
return post.UserName, nil
}
func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
if post.ThumbnailURL == "" {
return "", ErrMissingThumbnail
}
@ -180,7 +185,7 @@ func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
return thumb, nil
}
func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
if post.PublishedAt == "" {
return time.Now(), ErrMissingPublishDate
}
@ -191,7 +196,7 @@ func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
return pubDate, nil
}
func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
// Check if the description is null but we have a title.
// The poster didnt add a description but this isnt an error.
if post.Description == "" && post.Title == "" {
@ -204,7 +209,7 @@ func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
}
// Extracts the avatar of the author with some validation.
func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
return user.ProfileImageURL, nil
@ -212,20 +217,20 @@ func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
// Generate tags based on the video metadata.
// TODO Figure out how to query what game is played
func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
func (tc *TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName)
return res, nil
}
// Extracts the title from a post with some validation.
func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) {
func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
if post.Title == "" {
return "", errors.New("unable to find the title on the requested post")
}
return post.Title, nil
}
func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) {
func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
if post.URL == "" { return "", ErrMissingUrl }
return post.URL, nil
}

8
services/input/youtube.go
View File

@ -121,7 +121,7 @@ func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page {
func (yc *YoutubeClient) GetParser(uri string) (*goquery.Document, error) {
html, err := http.Get(uri)
if err != nil {
log.Println(err)
log.Printf("[YouTube] %v", err)
}
defer html.Body.Close()
@ -244,19 +244,19 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
parser, err := yc.GetParser(item.Link)
if err != nil {
log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link)
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
}
tags, err := yc.GetTags(parser)
if err != nil {
msg := fmt.Sprintf("%v. %v", err, item.Link)
log.Println(msg)
log.Printf("[YouTube] %v", msg)
}
thumb, err := yc.GetVideoThumbnail(parser)
if err != nil {
msg := fmt.Sprintf("%v. %v", err, item.Link)
log.Println(msg)
log.Printf("[YouTube] %v", msg)
}
var article = database.Article{

170
services/output/discordwebhook.go
View File

@ -1,80 +1,151 @@
package output
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/jtom38/newsbot/collector/database"
)
type discordField struct {
Name string `json:"name,omitempty"`
Value string `json:"value,omitempty"`
Inline bool `json:"inline,omitempty"`
Name *string `json:"name,omitempty"`
Value *string `json:"value,omitempty"`
Inline *bool `json:"inline,omitempty"`
}
type discordFooter struct {
Value *string `json:"text,omitempty"`
IconUrl *string `json:"icon_url,omitempty"`
}
type discordAuthor struct {
Name string `json:"name,omitempty"`
Url string `json:"url,omitempty"`
IconUrl string `json:"icon_url,omitempty"`
Name *string `json:"name,omitempty"`
Url *string `json:"url,omitempty"`
IconUrl *string `json:"icon_url,omitempty"`
}
type discordImage struct {
Url string `json:"url,omitempty"`
Url *string `json:"url,omitempty"`
}
type discordEmbed struct {
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Url string `json:"url,omitempty"`
Color int32 `json:"color,omitempty"`
Timestamp time.Time `json:"timestamp,omitempty"`
Fields []discordField `json:"fields,omitempty"`
Author discordAuthor `json:"author,omitempty"`
Image discordImage `json:"image,omitempty"`
Thumbnail discordImage `json:"thumbnail,omitempty"`
type DiscordEmbed struct {
Title *string `json:"title,omitempty"`
Description *string `json:"description,omitempty"`
Url *string `json:"url,omitempty"`
Color *int32 `json:"color,omitempty"`
//Timestamp time.Time `json:"timestamp,omitempty"`
Fields []*discordField `json:"fields,omitempty"`
Author discordAuthor `json:"author,omitempty"`
Image discordImage `json:"image,omitempty"`
Thumbnail discordImage `json:"thumbnail,omitempty"`
Footer *discordFooter `json:"footer,omitempty"`
}
// Root object for Discord Webhook messages
type discordMessage struct {
Content string `json:"content,omitempty"`
Embeds []discordEmbed `json:"embeds,omitempty"`
type DiscordMessage struct {
Username *string `json:"username,omitempty"`
Content *string `json:"content,omitempty"`
Embeds *[]DiscordEmbed `json:"embeds,omitempty"`
}
const (
DefaultColor = 0
YoutubeColor = 16711680
TwitchColor = 0
RedditColor = 0
TwitterColor = 0
FfxivColor = 0
)
type Discord struct {
Subscriptions []string
article database.Article
Message discordMessage
article database.Article
Message *DiscordMessage
}
func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord {
func NewDiscordWebHookMessage(Article database.Article) Discord {
return Discord{
Subscriptions: Subscriptions,
article: Article,
Message: discordMessage{
Embeds: []discordEmbed{},
},
}
}
func (dwh Discord) GeneratePayload() error {
// Convert the message
embed := discordEmbed {
Title: dwh.article.Title,
Description: dwh.convertFromHtml(dwh.article.Description),
Url: dwh.article.Url,
Thumbnail: discordImage{
Url: dwh.article.Thumbnail,
},
}
var arr []discordEmbed
// Generates the link field to expose in the message
func (dwh Discord) getFields() []*discordField {
var fields []*discordField
arr = append(arr, embed)
dwh.Message.Embeds = arr
return nil
key := "Link"
linkField := discordField{
Name: &key,
Value: &dwh.article.Url,
}
fields = append(fields, &linkField)
return fields
}
func (dwh Discord) SendPayload() error {
// This will create the message that will be sent out.
func (dwh Discord) GeneratePayload() (*DiscordMessage, error) {
// Create the embed
footerMessage := "Brought to you by Newsbot"
footerUrl := ""
description := dwh.convertFromHtml(dwh.article.Description)
color := dwh.getColor(dwh.article.Url)
embed := DiscordEmbed{
Title: &dwh.article.Title,
Description: &description,
Image: discordImage{
Url: &dwh.article.Thumbnail,
},
Fields: dwh.getFields(),
Footer: &discordFooter{
Value: &footerMessage,
IconUrl: &footerUrl,
},
Color: &color,
}
// attach the embed to an array
var embedArray []DiscordEmbed
embedArray = append(embedArray, embed)
// create the base message
msg := DiscordMessage{
Embeds: &embedArray,
}
return &msg, nil
}
func (dwh Discord) SendPayload(Message *DiscordMessage, Url string) error {
// Convert the message to a io.reader object
buffer := new(bytes.Buffer)
json.NewEncoder(buffer).Encode(Message)
// Send the message
resp, err := http.Post(Url, "application/json", buffer)
if err != nil {
return err
}
// Check for 204
if resp.StatusCode != 204 {
defer resp.Body.Close()
errMsg, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf(string(errMsg))
}
return nil
}
@ -104,7 +175,16 @@ func (dwh Discord) convertFromHtml(body string) string {
return clean
}
func (dwh Discord) convertLinks(body string) string {
func (dwh *Discord) getColor(Url string) int32 {
if strings.Contains(Url, "youtube.com") {
return YoutubeColor
}
return DefaultColor
}
func (dwh *Discord) convertLinks(body string) string {
//items := regexp.MustCompile("<a(.*?)a>")
return ""
}
}

139
services/output/discordwebhook_test.go
View File

@ -1,11 +1,10 @@
package output_test
import (
"errors"
"os"
"strings"
"testing"
"time"
//"time"
"github.com/google/uuid"
"github.com/joho/godotenv"
@ -13,61 +12,127 @@ import (
"github.com/jtom38/newsbot/collector/services/output"
)
var article database.Article = database.Article{
ID: uuid.New(),
Sourceid: uuid.New(),
Tags: "unit, testing",
Title: "Demo",
Url: "https://github.com/jtom38/newsbot.collector.api",
Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Description: "Hello World",
var (
article database.Article = database.Article{
ID: uuid.New(),
Sourceid: uuid.New(),
Tags: "unit, testing",
Title: "Demo",
Url: "https://github.com/jtom38/newsbot.collector.api",
//Pubdate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Description: "Hello World",
}
blank string = ""
)
func TestDiscordMessageContainsTitle(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
if i.Title == &blank {
t.Error("title missing")
}
}
}
func getWebhook() ([]string, error){
var endpoints []string
func TestDiscordMessageContainsDescription(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
if i.Description == &blank {
t.Error("description missing")
}
}
}
func TestDiscordMessageFooter(t *testing.T) {
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, i := range *msg.Embeds {
blank := ""
if i.Footer.Value == &blank {
t.Error("missing footer vlue")
}
if i.Footer.IconUrl == &blank {
t.Error("missing footer url")
}
}
}
func TestDiscordMessageFields(t *testing.T) {
header := "Link"
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
for _, embed := range *msg.Embeds {
for _, field := range embed.Fields {
var fName string
if field.Name != nil {
fName = *field.Name
} else {
t.Error("missing link field value")
}
if fName != header {
t.Error("missing link field key")
}
var fValue string
if field.Value != nil {
fValue = *field.Value
}
if fValue == blank {
t.Error("missing link field value")
}
}
}
}
// This test requires a env value to be present to work
func TestDiscordMessagePost(t *testing.T) {
_, err := os.Open(".env")
if err != nil {
return endpoints, err
t.Error(err)
}
err = godotenv.Load()
if err != nil {
return endpoints, err
t.Error(err)
}
res := os.Getenv("TESTS_DISCORD_WEBHOOK")
if res == "" {
return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing")
t.Error("TESTS_DISCORD_WEBHOOK is missing")
}
endpoints := strings.Split(res, " ")
if err != nil {
t.Error(err)
}
endpoints = strings.Split(res, "")
return endpoints, nil
}
func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) {
hook, err := getWebhook()
d := output.NewDiscordWebHookMessage(article)
msg, err := d.GeneratePayload()
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(hook, article)
if len(d.Subscriptions) == 0 {
t.Error("no subscriptions found")
}
}
func TestDiscordMessageContainsTitle(t *testing.T) {
hook, err := getWebhook()
err = d.SendPayload(msg, endpoints[0])
if err != nil {
t.Error(err)
}
d := output.NewDiscordWebHookMessage(hook, article)
err = d.GeneratePayload()
if err != nil {
t.Error(err)
}
if d.Message.Embeds[0].Title == "" {
t.Error("no title was found ")
}
}