Features/output discord (#12)
* basic output looks to be working * cron was updated to add to the queue and post messages * new route to make discord webhook subscriptions * updated swag tags * swag * Updated delete subscription call * removed the time value as it throws off the msg template * updated logging * updated swagger * updated new subscription route * Updated logging and remove items from the queue if they dont have a subscription * updated getArticles to return the 50 newest for the portal * added endpoint to see if an item exists already * formatting * updated listArticles * added colors and updated the image * Updated to use the pointer in twitch * added the twitch login command to cron... it works now * found a better way to disable http2 for reddit. Test worked right away too * updated the cron tasks to run collected once and hour or longer depending on the service
This commit is contained in:
GitHub
parent
0e0058506a
commit
a1324ee1c1
@ -253,16 +253,11 @@ func (q *Queries) DeleteSource(ctx context.Context, id uuid.UUID) error {
|
||||
}
|
||||
|
||||
const deleteSubscription = `-- name: DeleteSubscription :exec
|
||||
Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2
|
||||
Delete From subscriptions Where id = $1
|
||||
`
|
||||
|
||||
type DeleteSubscriptionParams struct {
|
||||
Discordwebhookid uuid.UUID
|
||||
Sourceid uuid.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) DeleteSubscription(ctx context.Context, arg DeleteSubscriptionParams) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteSubscription, arg.Discordwebhookid, arg.Sourceid)
|
||||
func (q *Queries) DeleteSubscription(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.ExecContext(ctx, deleteSubscription, id)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -534,6 +529,23 @@ func (q *Queries) GetDiscordQueueByID(ctx context.Context, id uuid.UUID) (Discor
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getDiscordWebHookByUrl = `-- name: GetDiscordWebHookByUrl :one
|
||||
Select id, url, server, channel, enabled From DiscordWebHooks Where url = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetDiscordWebHookByUrl(ctx context.Context, url string) (Discordwebhook, error) {
|
||||
row := q.db.QueryRowContext(ctx, getDiscordWebHookByUrl, url)
|
||||
var i Discordwebhook
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Url,
|
||||
&i.Server,
|
||||
&i.Channel,
|
||||
&i.Enabled,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getDiscordWebHooksByID = `-- name: GetDiscordWebHooksByID :one
|
||||
Select id, url, server, channel, enabled from DiscordWebHooks
|
||||
Where ID = $1 LIMIT 1
|
||||
@ -648,6 +660,27 @@ func (q *Queries) GetSourceByID(ctx context.Context, id uuid.UUID) (Source, erro
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getSourceByName = `-- name: GetSourceByName :one
|
||||
Select id, site, name, source, type, value, enabled, url, tags from Sources where name = $1 Limit 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetSourceByName(ctx context.Context, name string) (Source, error) {
|
||||
row := q.db.QueryRowContext(ctx, getSourceByName, name)
|
||||
var i Source
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Site,
|
||||
&i.Name,
|
||||
&i.Source,
|
||||
&i.Type,
|
||||
&i.Value,
|
||||
&i.Enabled,
|
||||
&i.Url,
|
||||
&i.Tags,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many
|
||||
Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1
|
||||
`
|
||||
@ -743,6 +776,47 @@ func (q *Queries) ListArticles(ctx context.Context, limit int32) ([]Article, err
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listArticlesByDate = `-- name: ListArticlesByDate :many
|
||||
Select id, sourceid, tags, title, url, pubdate, video, videoheight, videowidth, thumbnail, description, authorname, authorimage From articles ORDER BY pubdate desc Limit $1
|
||||
`
|
||||
|
||||
func (q *Queries) ListArticlesByDate(ctx context.Context, limit int32) ([]Article, error) {
|
||||
rows, err := q.db.QueryContext(ctx, listArticlesByDate, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Article
|
||||
for rows.Next() {
|
||||
var i Article
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Sourceid,
|
||||
&i.Tags,
|
||||
&i.Title,
|
||||
&i.Url,
|
||||
&i.Pubdate,
|
||||
&i.Video,
|
||||
&i.Videoheight,
|
||||
&i.Videowidth,
|
||||
&i.Thumbnail,
|
||||
&i.Description,
|
||||
&i.Authorname,
|
||||
&i.Authorimage,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listDiscordQueueItems = `-- name: ListDiscordQueueItems :many
|
||||
Select id, articleid from DiscordQueue LIMIT $1
|
||||
`
|
||||
@ -965,8 +1039,8 @@ func (q *Queries) ListSubscriptionsBySourceId(ctx context.Context, sourceid uuid
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const querySubscriptions = `-- name: QuerySubscriptions :many
|
||||
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2
|
||||
const querySubscriptions = `-- name: QuerySubscriptions :one
|
||||
Select id, discordwebhookid, sourceid From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1
|
||||
`
|
||||
|
||||
type QuerySubscriptionsParams struct {
|
||||
@ -974,25 +1048,9 @@ type QuerySubscriptionsParams struct {
|
||||
Sourceid uuid.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) ([]Subscription, error) {
|
||||
rows, err := q.db.QueryContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Subscription
|
||||
for rows.Next() {
|
||||
var i Subscription
|
||||
if err := rows.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
func (q *Queries) QuerySubscriptions(ctx context.Context, arg QuerySubscriptionsParams) (Subscription, error) {
|
||||
row := q.db.QueryRowContext(ctx, querySubscriptions, arg.Discordwebhookid, arg.Sourceid)
|
||||
var i Subscription
|
||||
err := row.Scan(&i.ID, &i.Discordwebhookid, &i.Sourceid)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@ -10,6 +10,9 @@ Where Url = $1 LIMIT 1;
|
||||
-- name: ListArticles :many
|
||||
Select * From articles Limit $1;
|
||||
|
||||
-- name: ListArticlesByDate :many
|
||||
Select * From articles ORDER BY pubdate desc Limit $1;
|
||||
|
||||
-- name: GetArticlesBySource :many
|
||||
select * from articles
|
||||
INNER join sources on articles.sourceid=Sources.ID
|
||||
@ -66,6 +69,9 @@ Where ID = $1 LIMIT 1;
|
||||
Select * From DiscordWebHooks
|
||||
Where Server = $1;
|
||||
|
||||
-- name: GetDiscordWebHookByUrl :one
|
||||
Select * From DiscordWebHooks Where url = $1;
|
||||
|
||||
-- name: ListDiscordWebhooks :many
|
||||
Select * From discordwebhooks LIMIT $1;
|
||||
|
||||
@ -127,6 +133,9 @@ Values
|
||||
-- name: GetSourceByID :one
|
||||
Select * From Sources where ID = $1 Limit 1;
|
||||
|
||||
-- name: GetSourceByName :one
|
||||
Select * from Sources where name = $1 Limit 1;
|
||||
|
||||
-- name: ListSources :many
|
||||
Select * From Sources Limit $1;
|
||||
|
||||
@ -154,8 +163,8 @@ Select * From subscriptions Limit $1;
|
||||
-- name: ListSubscriptionsBySourceId :many
|
||||
Select * From subscriptions where sourceid = $1;
|
||||
|
||||
-- name: QuerySubscriptions :many
|
||||
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2;
|
||||
-- name: QuerySubscriptions :one
|
||||
Select * From subscriptions Where discordwebhookid = $1 and sourceid = $2 Limit 1;
|
||||
|
||||
-- name: GetSubscriptionsBySourceID :many
|
||||
Select * From subscriptions Where sourceid = $1;
|
||||
@ -164,4 +173,4 @@ Select * From subscriptions Where sourceid = $1;
|
||||
Select * from subscriptions Where discordwebhookid = $1;
|
||||
|
||||
-- name: DeleteSubscription :exec
|
||||
Delete From subscriptions Where discordwebhookid = $1 and sourceid = $2;
|
||||
Delete From subscriptions Where id = $1;
|
||||
110
docs/docs.go
110
docs/docs.go
@ -22,7 +22,7 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"articles"
|
||||
"Articles"
|
||||
],
|
||||
"summary": "Lists the top 50 records",
|
||||
"responses": {}
|
||||
@ -34,7 +34,7 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"articles"
|
||||
"Articles"
|
||||
],
|
||||
"summary": "Finds the articles based on the SourceID provided. Returns the top 50.",
|
||||
"parameters": [
|
||||
@ -55,7 +55,7 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"articles"
|
||||
"Articles"
|
||||
],
|
||||
"summary": "Returns an article based on defined ID.",
|
||||
"parameters": [
|
||||
@ -76,8 +76,8 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Lists the top 50 records",
|
||||
"responses": {}
|
||||
@ -86,9 +86,9 @@ const docTemplate = `{
|
||||
"/config/sources/new/reddit": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source",
|
||||
"reddit"
|
||||
"Config",
|
||||
"Source",
|
||||
"Reddit"
|
||||
],
|
||||
"summary": "Creates a new reddit source to monitor.",
|
||||
"parameters": [
|
||||
@ -113,9 +113,9 @@ const docTemplate = `{
|
||||
"/config/sources/new/twitch": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source",
|
||||
"twitch"
|
||||
"Config",
|
||||
"Source",
|
||||
"Twitch"
|
||||
],
|
||||
"summary": "Creates a new twitch source to monitor.",
|
||||
"parameters": [
|
||||
@ -147,9 +147,9 @@ const docTemplate = `{
|
||||
"/config/sources/new/youtube": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source",
|
||||
"youtube"
|
||||
"Config",
|
||||
"Source",
|
||||
"YouTube"
|
||||
],
|
||||
"summary": "Creates a new youtube source to monitor.",
|
||||
"parameters": [
|
||||
@ -184,8 +184,8 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Returns a single entity by ID",
|
||||
"parameters": [
|
||||
@ -201,8 +201,8 @@ const docTemplate = `{
|
||||
},
|
||||
"delete": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Deletes a record by ID.",
|
||||
"parameters": [
|
||||
@ -220,8 +220,8 @@ const docTemplate = `{
|
||||
"/config/sources/{id}/disable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Disables a source from processing.",
|
||||
"parameters": [
|
||||
@ -239,8 +239,8 @@ const docTemplate = `{
|
||||
"/config/sources/{id}/enable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Enables a source to continue processing.",
|
||||
"parameters": [
|
||||
@ -261,7 +261,7 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"debug",
|
||||
"Debug",
|
||||
"Discord",
|
||||
"Queue"
|
||||
],
|
||||
@ -275,9 +275,9 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Config",
|
||||
"Discord",
|
||||
"Webhooks"
|
||||
"Webhook"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {}
|
||||
@ -289,9 +289,9 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Config",
|
||||
"Discord",
|
||||
"Webhooks"
|
||||
"Webhook"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -309,9 +309,9 @@ const docTemplate = `{
|
||||
"/discord/webhooks/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"Config",
|
||||
"Discord",
|
||||
"Webhooks"
|
||||
"Webhook"
|
||||
],
|
||||
"summary": "Creates a new record for a discord web hook to post data to.",
|
||||
"parameters": [
|
||||
@ -331,7 +331,7 @@ const docTemplate = `{
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Channel name.",
|
||||
"description": "Channel name",
|
||||
"name": "channel",
|
||||
"in": "query",
|
||||
"required": true
|
||||
@ -346,7 +346,7 @@ const docTemplate = `{
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"debug"
|
||||
"Debug"
|
||||
],
|
||||
"summary": "Responds back with \"Hello x\" depending on param passed in.",
|
||||
"parameters": [
|
||||
@ -367,7 +367,7 @@ const docTemplate = `{
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"debug"
|
||||
"Debug"
|
||||
],
|
||||
"summary": "Responds back with \"Hello world!\"",
|
||||
"responses": {}
|
||||
@ -379,7 +379,7 @@ const docTemplate = `{
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"debug"
|
||||
"Debug"
|
||||
],
|
||||
"summary": "Sends back \"pong\". Good to test with.",
|
||||
"responses": {}
|
||||
@ -391,9 +391,9 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"settings"
|
||||
"Settings"
|
||||
],
|
||||
"summary": "Returns a object based on the Key that was given/",
|
||||
"summary": "Returns a object based on the Key that was given.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
@ -412,8 +412,8 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Subscriptions"
|
||||
"Config",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {}
|
||||
@ -425,8 +425,8 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Subscriptions"
|
||||
"Config",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -447,8 +447,8 @@ const docTemplate = `{
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Subscriptions"
|
||||
"Config",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -462,6 +462,34 @@ const docTemplate = `{
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/subscriptions/new/discordwebhook": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Config",
|
||||
"Source",
|
||||
"Discord",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "discordWebHookId",
|
||||
"name": "discordWebHookId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "sourceId",
|
||||
"name": "sourceId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"articles"
|
||||
"Articles"
|
||||
],
|
||||
"summary": "Lists the top 50 records",
|
||||
"responses": {}
|
||||
@ -25,7 +25,7 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"articles"
|
||||
"Articles"
|
||||
],
|
||||
"summary": "Finds the articles based on the SourceID provided. Returns the top 50.",
|
||||
"parameters": [
|
||||
@ -46,7 +46,7 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"articles"
|
||||
"Articles"
|
||||
],
|
||||
"summary": "Returns an article based on defined ID.",
|
||||
"parameters": [
|
||||
@ -67,8 +67,8 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Lists the top 50 records",
|
||||
"responses": {}
|
||||
@ -77,9 +77,9 @@
|
||||
"/config/sources/new/reddit": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source",
|
||||
"reddit"
|
||||
"Config",
|
||||
"Source",
|
||||
"Reddit"
|
||||
],
|
||||
"summary": "Creates a new reddit source to monitor.",
|
||||
"parameters": [
|
||||
@ -104,9 +104,9 @@
|
||||
"/config/sources/new/twitch": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source",
|
||||
"twitch"
|
||||
"Config",
|
||||
"Source",
|
||||
"Twitch"
|
||||
],
|
||||
"summary": "Creates a new twitch source to monitor.",
|
||||
"parameters": [
|
||||
@ -138,9 +138,9 @@
|
||||
"/config/sources/new/youtube": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source",
|
||||
"youtube"
|
||||
"Config",
|
||||
"Source",
|
||||
"YouTube"
|
||||
],
|
||||
"summary": "Creates a new youtube source to monitor.",
|
||||
"parameters": [
|
||||
@ -175,8 +175,8 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Returns a single entity by ID",
|
||||
"parameters": [
|
||||
@ -192,8 +192,8 @@
|
||||
},
|
||||
"delete": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Deletes a record by ID.",
|
||||
"parameters": [
|
||||
@ -211,8 +211,8 @@
|
||||
"/config/sources/{id}/disable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Disables a source from processing.",
|
||||
"parameters": [
|
||||
@ -230,8 +230,8 @@
|
||||
"/config/sources/{id}/enable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"source"
|
||||
"Config",
|
||||
"Source"
|
||||
],
|
||||
"summary": "Enables a source to continue processing.",
|
||||
"parameters": [
|
||||
@ -252,7 +252,7 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"debug",
|
||||
"Debug",
|
||||
"Discord",
|
||||
"Queue"
|
||||
],
|
||||
@ -266,9 +266,9 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Config",
|
||||
"Discord",
|
||||
"Webhooks"
|
||||
"Webhook"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {}
|
||||
@ -280,9 +280,9 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Config",
|
||||
"Discord",
|
||||
"Webhooks"
|
||||
"Webhook"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -300,9 +300,9 @@
|
||||
"/discord/webhooks/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"config",
|
||||
"Config",
|
||||
"Discord",
|
||||
"Webhooks"
|
||||
"Webhook"
|
||||
],
|
||||
"summary": "Creates a new record for a discord web hook to post data to.",
|
||||
"parameters": [
|
||||
@ -322,7 +322,7 @@
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Channel name.",
|
||||
"description": "Channel name",
|
||||
"name": "channel",
|
||||
"in": "query",
|
||||
"required": true
|
||||
@ -337,7 +337,7 @@
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"debug"
|
||||
"Debug"
|
||||
],
|
||||
"summary": "Responds back with \"Hello x\" depending on param passed in.",
|
||||
"parameters": [
|
||||
@ -358,7 +358,7 @@
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"debug"
|
||||
"Debug"
|
||||
],
|
||||
"summary": "Responds back with \"Hello world!\"",
|
||||
"responses": {}
|
||||
@ -370,7 +370,7 @@
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"debug"
|
||||
"Debug"
|
||||
],
|
||||
"summary": "Sends back \"pong\". Good to test with.",
|
||||
"responses": {}
|
||||
@ -382,9 +382,9 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"settings"
|
||||
"Settings"
|
||||
],
|
||||
"summary": "Returns a object based on the Key that was given/",
|
||||
"summary": "Returns a object based on the Key that was given.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
@ -403,8 +403,8 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Subscriptions"
|
||||
"Config",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {}
|
||||
@ -416,8 +416,8 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Subscriptions"
|
||||
"Config",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -438,8 +438,8 @@
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"config",
|
||||
"Subscriptions"
|
||||
"Config",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -453,6 +453,34 @@
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/subscriptions/new/discordwebhook": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Config",
|
||||
"Source",
|
||||
"Discord",
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "discordWebHookId",
|
||||
"name": "discordWebHookId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "sourceId",
|
||||
"name": "sourceId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -11,7 +11,7 @@ paths:
|
||||
responses: {}
|
||||
summary: Lists the top 50 records
|
||||
tags:
|
||||
- articles
|
||||
- Articles
|
||||
/articles/{id}:
|
||||
get:
|
||||
parameters:
|
||||
@ -25,7 +25,7 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns an article based on defined ID.
|
||||
tags:
|
||||
- articles
|
||||
- Articles
|
||||
/articles/by/sourceid:
|
||||
get:
|
||||
parameters:
|
||||
@ -40,7 +40,7 @@ paths:
|
||||
summary: Finds the articles based on the SourceID provided. Returns the top
|
||||
50.
|
||||
tags:
|
||||
- articles
|
||||
- Articles
|
||||
/config/sources:
|
||||
get:
|
||||
produces:
|
||||
@ -48,8 +48,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Lists the top 50 records
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- Config
|
||||
- Source
|
||||
/config/sources/{id}:
|
||||
delete:
|
||||
parameters:
|
||||
@ -61,8 +61,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Deletes a record by ID.
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- Config
|
||||
- Source
|
||||
get:
|
||||
parameters:
|
||||
- description: uuid
|
||||
@ -75,8 +75,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns a single entity by ID
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- Config
|
||||
- Source
|
||||
/config/sources/{id}/disable:
|
||||
post:
|
||||
parameters:
|
||||
@ -88,8 +88,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Disables a source from processing.
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- Config
|
||||
- Source
|
||||
/config/sources/{id}/enable:
|
||||
post:
|
||||
parameters:
|
||||
@ -101,8 +101,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Enables a source to continue processing.
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- Config
|
||||
- Source
|
||||
/config/sources/new/reddit:
|
||||
post:
|
||||
parameters:
|
||||
@ -119,9 +119,9 @@ paths:
|
||||
responses: {}
|
||||
summary: Creates a new reddit source to monitor.
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- reddit
|
||||
- Config
|
||||
- Source
|
||||
- Reddit
|
||||
/config/sources/new/twitch:
|
||||
post:
|
||||
parameters:
|
||||
@ -143,9 +143,9 @@ paths:
|
||||
responses: {}
|
||||
summary: Creates a new twitch source to monitor.
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- twitch
|
||||
- Config
|
||||
- Source
|
||||
- Twitch
|
||||
/config/sources/new/youtube:
|
||||
post:
|
||||
parameters:
|
||||
@ -167,9 +167,9 @@ paths:
|
||||
responses: {}
|
||||
summary: Creates a new youtube source to monitor.
|
||||
tags:
|
||||
- config
|
||||
- source
|
||||
- youtube
|
||||
- Config
|
||||
- Source
|
||||
- YouTube
|
||||
/discord/queue:
|
||||
get:
|
||||
produces:
|
||||
@ -177,7 +177,7 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- debug
|
||||
- Debug
|
||||
- Discord
|
||||
- Queue
|
||||
/discord/webhooks:
|
||||
@ -187,9 +187,9 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- config
|
||||
- Config
|
||||
- Discord
|
||||
- Webhooks
|
||||
- Webhook
|
||||
/discord/webhooks/byId:
|
||||
get:
|
||||
parameters:
|
||||
@ -203,9 +203,9 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- config
|
||||
- Config
|
||||
- Discord
|
||||
- Webhooks
|
||||
- Webhook
|
||||
/discord/webhooks/new:
|
||||
post:
|
||||
parameters:
|
||||
@ -219,7 +219,7 @@ paths:
|
||||
name: server
|
||||
required: true
|
||||
type: string
|
||||
- description: Channel name.
|
||||
- description: Channel name
|
||||
in: query
|
||||
name: channel
|
||||
required: true
|
||||
@ -227,9 +227,9 @@ paths:
|
||||
responses: {}
|
||||
summary: Creates a new record for a discord web hook to post data to.
|
||||
tags:
|
||||
- config
|
||||
- Config
|
||||
- Discord
|
||||
- Webhooks
|
||||
- Webhook
|
||||
/hello/{who}:
|
||||
get:
|
||||
parameters:
|
||||
@ -243,7 +243,7 @@ paths:
|
||||
responses: {}
|
||||
summary: Responds back with "Hello x" depending on param passed in.
|
||||
tags:
|
||||
- debug
|
||||
- Debug
|
||||
/helloworld:
|
||||
get:
|
||||
produces:
|
||||
@ -251,7 +251,7 @@ paths:
|
||||
responses: {}
|
||||
summary: Responds back with "Hello world!"
|
||||
tags:
|
||||
- debug
|
||||
- Debug
|
||||
/ping:
|
||||
get:
|
||||
produces:
|
||||
@ -259,7 +259,7 @@ paths:
|
||||
responses: {}
|
||||
summary: Sends back "pong". Good to test with.
|
||||
tags:
|
||||
- debug
|
||||
- Debug
|
||||
/settings/{key}:
|
||||
get:
|
||||
parameters:
|
||||
@ -271,9 +271,9 @@ paths:
|
||||
produces:
|
||||
- application/json
|
||||
responses: {}
|
||||
summary: Returns a object based on the Key that was given/
|
||||
summary: Returns a object based on the Key that was given.
|
||||
tags:
|
||||
- settings
|
||||
- Settings
|
||||
/subscriptions:
|
||||
get:
|
||||
produces:
|
||||
@ -281,8 +281,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- config
|
||||
- Subscriptions
|
||||
- Config
|
||||
- Subscription
|
||||
/subscriptions/byDiscordId:
|
||||
get:
|
||||
parameters:
|
||||
@ -296,8 +296,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- config
|
||||
- Subscriptions
|
||||
- Config
|
||||
- Subscription
|
||||
/subscriptions/bySourceId:
|
||||
get:
|
||||
parameters:
|
||||
@ -311,6 +311,26 @@ paths:
|
||||
responses: {}
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- config
|
||||
- Subscriptions
|
||||
- Config
|
||||
- Subscription
|
||||
/subscriptions/new/discordwebhook:
|
||||
post:
|
||||
parameters:
|
||||
- description: discordWebHookId
|
||||
in: query
|
||||
name: discordWebHookId
|
||||
required: true
|
||||
type: string
|
||||
- description: sourceId
|
||||
in: query
|
||||
name: sourceId
|
||||
required: true
|
||||
type: string
|
||||
responses: {}
|
||||
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
||||
tags:
|
||||
- Config
|
||||
- Source
|
||||
- Discord
|
||||
- Subscription
|
||||
swagger: "2.0"
|
||||
|
||||
@ -11,12 +11,12 @@ import (
|
||||
// ListArticles
|
||||
// @Summary Lists the top 50 records
|
||||
// @Produce application/json
|
||||
// @Tags articles
|
||||
// @Tags Articles
|
||||
// @Router /articles [get]
|
||||
func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
res, err := s.Db.ListArticles(*s.ctx, 50)
|
||||
res, err := s.Db.ListArticlesByDate(*s.ctx, 50)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
@ -27,7 +27,6 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
w.Write(bres)
|
||||
}
|
||||
|
||||
@ -35,7 +34,7 @@ func (s *Server) listArticles(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Returns an article based on defined ID.
|
||||
// @Param id path string true "uuid"
|
||||
// @Produce application/json
|
||||
// @Tags articles
|
||||
// @Tags Articles
|
||||
// @Router /articles/{id} [get]
|
||||
func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -67,7 +66,7 @@ func (s *Server) getArticleById(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
|
||||
// @Param id query string true "Source ID UUID"
|
||||
// @Produce application/json
|
||||
// @Tags articles
|
||||
// @Tags Articles
|
||||
// @Router /articles/by/sourceid [get]
|
||||
func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -96,3 +95,38 @@ func (s *Server) GetArticlesBySourceId(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.Write(bres)
|
||||
}
|
||||
|
||||
// TODO add page support
|
||||
// GetArticlesByTag
|
||||
// @Summary Finds the articles based on the SourceID provided. Returns the top 50.
|
||||
// @Param Tag query string true "Tag name"
|
||||
// @Produce application/json
|
||||
// @Tags Articles
|
||||
// @Router /articles/by/sourceid [get]
|
||||
func (s *Server) GetArticlesByTag(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
r.URL.Query()
|
||||
query := r.URL.Query()
|
||||
_id := query["id"][0]
|
||||
|
||||
uuid, err := uuid.Parse(_id)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
res, err := s.Db.GetArticlesBySourceId(*s.ctx, uuid)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
bres, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
w.Write(bres)
|
||||
}
|
||||
|
||||
@ -8,7 +8,7 @@ import (
|
||||
// GetDiscordQueue
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Tags debug, Discord, Queue
|
||||
// @Tags Debug, Discord, Queue
|
||||
// @Router /discord/queue [get]
|
||||
func (s *Server) GetDiscordQueue(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
@ -13,7 +13,7 @@ import (
|
||||
// GetDiscordWebHooks
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Tags config, Discord, Webhooks
|
||||
// @Tags Config, Discord, Webhook
|
||||
// @Router /discord/webhooks [get]
|
||||
func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -37,7 +37,7 @@ func (s *Server) GetDiscordWebHooks(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Param id query string true "id"
|
||||
// @Tags config, Discord, Webhooks
|
||||
// @Tags Config, Discord, Webhook
|
||||
// @Router /discord/webhooks/byId [get]
|
||||
func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -74,8 +74,8 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request)
|
||||
// @Summary Creates a new record for a discord web hook to post data to.
|
||||
// @Param url query string true "url"
|
||||
// @Param server query string true "Server name"
|
||||
// @Param channel query string true "Channel name."
|
||||
// @Tags config, Discord, Webhooks
|
||||
// @Param channel query string true "Channel name"
|
||||
// @Tags Config, Discord, Webhook
|
||||
// @Router /discord/webhooks/new [post]
|
||||
func (s *Server) NewDiscordWebHook(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
|
||||
@ -22,7 +22,7 @@ func RootRoutes() chi.Router {
|
||||
// HelloWorld
|
||||
// @Summary Responds back with "Hello world!"
|
||||
// @Produce plain
|
||||
// @Tags debug
|
||||
// @Tags Debug
|
||||
// @Router /helloworld [get]
|
||||
func helloWorld(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("Hello World!"))
|
||||
@ -31,7 +31,7 @@ func helloWorld(w http.ResponseWriter, r *http.Request) {
|
||||
// Ping
|
||||
// @Summary Sends back "pong". Good to test with.
|
||||
// @Produce plain
|
||||
// @Tags debug
|
||||
// @Tags Debug
|
||||
// @Router /ping [get]
|
||||
func ping(w http.ResponseWriter, r *http.Request) {
|
||||
msg := "pong"
|
||||
@ -42,7 +42,7 @@ func ping(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Responds back with "Hello x" depending on param passed in.
|
||||
// @Param who path string true "Who"
|
||||
// @Produce plain
|
||||
// @Tags debug
|
||||
// @Tags Debug
|
||||
// @Router /hello/{who} [get]
|
||||
func helloWho(w http.ResponseWriter, r *http.Request) {
|
||||
msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "who"))
|
||||
|
||||
@ -3,12 +3,13 @@ package routes
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
//"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
httpSwagger "github.com/swaggo/http-swagger"
|
||||
_ "github.com/lib/pq"
|
||||
httpSwagger "github.com/swaggo/http-swagger"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
@ -16,14 +17,14 @@ import (
|
||||
|
||||
type Server struct {
|
||||
Router *chi.Mux
|
||||
Db *database.Queries
|
||||
ctx *context.Context
|
||||
Db *database.Queries
|
||||
ctx *context.Context
|
||||
}
|
||||
|
||||
var (
|
||||
ErrIdValueMissing string = "id value is missing"
|
||||
ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct."
|
||||
ErrNoRecordFound string = "no record was found."
|
||||
ErrIdValueMissing string = "id value is missing"
|
||||
ErrValueNotUuid string = "a value given was expected to be a uuid but was not correct."
|
||||
ErrNoRecordFound string = "no record was found."
|
||||
ErrUnableToConvertToJson string = "Unable to convert to json"
|
||||
)
|
||||
|
||||
@ -59,6 +60,7 @@ func openDatabase(ctx context.Context) (*database.Queries, error) {
|
||||
func (s *Server) MountMiddleware() {
|
||||
s.Router.Use(middleware.Logger)
|
||||
s.Router.Use(middleware.Recoverer)
|
||||
//s.Router.Use(middleware.Heartbeat())
|
||||
}
|
||||
|
||||
func (s *Server) MountRoutes() {
|
||||
@ -91,7 +93,11 @@ func (s *Server) MountRoutes() {
|
||||
|
||||
/* Source Routes */
|
||||
s.Router.Get("/api/config/sources", s.listSources)
|
||||
|
||||
/* Reddit Source Routes */
|
||||
|
||||
s.Router.Post("/api/config/sources/new/reddit", s.newRedditSource)
|
||||
|
||||
s.Router.Post("/api/config/sources/new/youtube", s.newYoutubeSource)
|
||||
s.Router.Post("/api/config/sources/new/twitch", s.newTwitchSource)
|
||||
s.Router.Route("/api/config/sources/{ID}", func(r chi.Router) {
|
||||
@ -105,4 +111,5 @@ func (s *Server) MountRoutes() {
|
||||
s.Router.Get("/api/subscriptions", s.ListSubscriptions)
|
||||
s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId)
|
||||
s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId)
|
||||
s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription)
|
||||
}
|
||||
|
||||
@ -10,10 +10,10 @@ import (
|
||||
)
|
||||
|
||||
// GetSettings
|
||||
// @Summary Returns a object based on the Key that was given/
|
||||
// @Summary Returns a object based on the Key that was given.
|
||||
// @Param key path string true "Settings Key value"
|
||||
// @Produce application/json
|
||||
// @Tags settings
|
||||
// @Tags Settings
|
||||
// @Router /settings/{key} [get]
|
||||
func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
|
||||
//var item model.Sources
|
||||
|
||||
@ -15,7 +15,7 @@ import (
|
||||
// ListSources
|
||||
// @Summary Lists the top 50 records
|
||||
// @Produce application/json
|
||||
// @Tags config, source
|
||||
// @Tags Config, Source
|
||||
// @Router /config/sources [get]
|
||||
func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
|
||||
//TODO Add top?
|
||||
@ -48,7 +48,7 @@ func (s *Server) listSources(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Returns a single entity by ID
|
||||
// @Param id path string true "uuid"
|
||||
// @Produce application/json
|
||||
// @Tags config, source
|
||||
// @Tags Config, Source
|
||||
// @Router /config/sources/{id} [get]
|
||||
func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "ID")
|
||||
@ -78,7 +78,7 @@ func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Creates a new reddit source to monitor.
|
||||
// @Param name query string true "name"
|
||||
// @Param url query string true "url"
|
||||
// @Tags config, source, reddit
|
||||
// @Tags Config, Source, Reddit
|
||||
// @Router /config/sources/new/reddit [post]
|
||||
func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
@ -116,12 +116,16 @@ func (s *Server) newRedditSource(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(bJson)
|
||||
}
|
||||
|
||||
func (s *Server) getSourceByType(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
}
|
||||
|
||||
// NewYoutubeSource
|
||||
// @Summary Creates a new youtube source to monitor.
|
||||
// @Param name query string true "name"
|
||||
// @Param url query string true "url"
|
||||
// @Param tags query string true "tags"
|
||||
// @Tags config, source, youtube
|
||||
// @Tags Config, Source, YouTube
|
||||
// @Router /config/sources/new/youtube [post]
|
||||
func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
@ -164,7 +168,7 @@ func (s *Server) newYoutubeSource(w http.ResponseWriter, r *http.Request) {
|
||||
// @Param name query string true "name"
|
||||
// @Param url query string true "url"
|
||||
// @Param tags query string true "tags"
|
||||
// @Tags config, source, twitch
|
||||
// @Tags Config, Source, Twitch
|
||||
// @Router /config/sources/new/twitch [post]
|
||||
func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
@ -205,7 +209,7 @@ func (s *Server) newTwitchSource(w http.ResponseWriter, r *http.Request) {
|
||||
// DeleteSource
|
||||
// @Summary Deletes a record by ID.
|
||||
// @Param id path string true "id"
|
||||
// @Tags config, source
|
||||
// @Tags Config, Source
|
||||
// @Router /config/sources/{id} [delete]
|
||||
func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
|
||||
//var item model.Sources = model.Sources{}
|
||||
@ -232,7 +236,7 @@ func (s *Server) deleteSources(w http.ResponseWriter, r *http.Request) {
|
||||
// DisableSource
|
||||
// @Summary Disables a source from processing.
|
||||
// @Param id path string true "id"
|
||||
// @Tags config, source
|
||||
// @Tags Config, Source
|
||||
// @Router /config/sources/{id}/disable [post]
|
||||
func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "ID")
|
||||
@ -256,7 +260,7 @@ func (s *Server) disableSource(w http.ResponseWriter, r *http.Request) {
|
||||
// EnableSource
|
||||
// @Summary Enables a source to continue processing.
|
||||
// @Param id path string true "id"
|
||||
// @Tags config, source
|
||||
// @Tags Config, Source
|
||||
// @Router /config/sources/{id}/enable [post]
|
||||
func (s *Server) enableSource(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "ID")
|
||||
|
||||
@ -5,12 +5,13 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
)
|
||||
|
||||
// GetSubscriptions
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Tags config, Subscriptions
|
||||
// @Tags Config, Subscription
|
||||
// @Router /subscriptions [get]
|
||||
func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -34,7 +35,7 @@ func (s *Server) ListSubscriptions(w http.ResponseWriter, r *http.Request) {
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Param id query string true "id"
|
||||
// @Tags config, Subscriptions
|
||||
// @Tags Config, Subscription
|
||||
// @Router /subscriptions/byDiscordId [get]
|
||||
func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -71,7 +72,7 @@ func (s *Server) GetSubscriptionsByDiscordId(w http.ResponseWriter, r *http.Requ
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Param id query string true "id"
|
||||
// @Tags config, Subscriptions
|
||||
// @Tags Config, Subscription
|
||||
// @Router /subscriptions/bySourceId [get]
|
||||
func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@ -103,3 +104,70 @@ func (s *Server) GetSubscriptionsBySourceId(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
w.Write(bres)
|
||||
}
|
||||
|
||||
// NewDiscordWebHookSubscription
|
||||
// @Summary Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
||||
// @Param discordWebHookId query string true "discordWebHookId"
|
||||
// @Param sourceId query string true "sourceId"
|
||||
// @Tags Config, Source, Discord, Subscription
|
||||
// @Router /subscriptions/new/discordwebhook [post]
|
||||
func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
|
||||
// Extract the values given
|
||||
query := r.URL.Query()
|
||||
discordWebHookId := query["discordWebHookId"][0]
|
||||
sourceId := query["sourceId"][0]
|
||||
|
||||
// Check to make we didnt get a null
|
||||
if discordWebHookId == "" {
|
||||
http.Error(w, "invalid discordWebHooksId given", http.StatusBadRequest )
|
||||
return
|
||||
}
|
||||
if sourceId == "" {
|
||||
http.Error(w, "invalid sourceID given", http.StatusBadRequest )
|
||||
return
|
||||
}
|
||||
|
||||
// Valide they are UUID values
|
||||
uHook, err := uuid.Parse(discordWebHookId)
|
||||
if err != nil {
|
||||
http.Error(w, "DiscordWebHooksID was not a uuid value.", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
uSource, err := uuid.Parse(sourceId)
|
||||
if err != nil {
|
||||
http.Error(w, "SourceId was not a uuid value", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if the sub already exists
|
||||
item, err := s.Db.QuerySubscriptions(*s.ctx, database.QuerySubscriptionsParams{
|
||||
Discordwebhookid: uHook,
|
||||
Sourceid: uSource,
|
||||
})
|
||||
if err == nil {
|
||||
bJson, err := json.Marshal(&item)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(bJson)
|
||||
return
|
||||
}
|
||||
|
||||
// Does not exist, so make it.
|
||||
params := database.CreateSubscriptionParams{
|
||||
ID: uuid.New(),
|
||||
Discordwebhookid: uHook,
|
||||
Sourceid: uSource,
|
||||
}
|
||||
s.Db.CreateSubscription(*s.ctx, params)
|
||||
|
||||
bJson, err := json.Marshal(¶ms)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(bJson)
|
||||
}
|
||||
@ -3,6 +3,7 @@ package cron
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
@ -11,8 +12,8 @@ import (
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
"github.com/jtom38/newsbot/collector/services/output"
|
||||
)
|
||||
|
||||
@ -51,29 +52,32 @@ func New(ctx context.Context) *Cron {
|
||||
|
||||
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckReddit() })
|
||||
log.Print("Reddit backend was enabled")
|
||||
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
||||
log.Print("[Input] Reddit backend was enabled")
|
||||
//go c.CheckReddit()
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckYoutube() })
|
||||
log.Print("YouTube backend was enabled")
|
||||
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
||||
log.Print("[Input] YouTube backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("* */1 * * *", func() { go c.CheckFfxiv() })
|
||||
log.Print("FFXIV backend was enabled")
|
||||
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
||||
log.Print("[Input] FFXIV backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("* */1 * * *", func() { go c.CheckTwitch() })
|
||||
log.Print("Twitch backend was enabled")
|
||||
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
||||
log.Print("[Input] Twitch backend was enabled")
|
||||
}
|
||||
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
||||
log.Print("[Output] Discord Output was enabled")
|
||||
|
||||
c.timer = timer
|
||||
return c
|
||||
}
|
||||
@ -162,6 +166,11 @@ func (c *Cron) CheckTwitch() error {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
@ -186,19 +195,13 @@ func (c *Cron) CheckDiscordQueue() error {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, queue := range(queueItems) {
|
||||
for _, queue := range queueItems {
|
||||
// Get the articleByID
|
||||
article, err := c.Db.GetArticleByID(*c.ctx, queue.Articleid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get the SourceByID
|
||||
//source, err := c.Db.GetSourceByID(*c.ctx, article.Sourceid)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
|
||||
var endpoints []string
|
||||
// List Subscription by SourceID
|
||||
subs, err := c.Db.ListSubscriptionsBySourceId(*c.ctx, article.Sourceid)
|
||||
@ -206,8 +209,18 @@ func (c *Cron) CheckDiscordQueue() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// if no one is subscribed to it, remove it from the index.
|
||||
if len(subs) == 0 {
|
||||
log.Printf("No subscriptions found bound to '%v' so it was removed.", article.Sourceid)
|
||||
err = c.Db.DeleteDiscordQueueItem(*c.ctx, queue.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Get the webhhooks to send to
|
||||
for _, sub := range(subs) {
|
||||
for _, sub := range subs {
|
||||
webhook, err := c.Db.GetDiscordWebHooksByID(*c.ctx, sub.Discordwebhookid)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -218,16 +231,19 @@ func (c *Cron) CheckDiscordQueue() error {
|
||||
}
|
||||
|
||||
// Create Discord Message
|
||||
dwh := output.NewDiscordWebHookMessage(endpoints, article)
|
||||
err = dwh.GeneratePayload()
|
||||
dwh := output.NewDiscordWebHookMessage(article)
|
||||
msg, err := dwh.GeneratePayload()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send Message
|
||||
err = dwh.SendPayload()
|
||||
if err != nil {
|
||||
return err
|
||||
// Send Message(s)
|
||||
for _, i := range endpoints {
|
||||
err = dwh.SendPayload(msg, i)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the item from the queue, given we sent our notification.
|
||||
@ -235,29 +251,38 @@ func (c *Cron) CheckDiscordQueue() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) checkPosts(posts []database.Article, sourceName string) {
|
||||
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
|
||||
for _, item := range posts {
|
||||
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
|
||||
if err != nil {
|
||||
err = c.postArticle(item)
|
||||
id := uuid.New()
|
||||
|
||||
err := c.postArticle(id, item)
|
||||
if err != nil {
|
||||
log.Printf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
||||
} else {
|
||||
log.Printf("[%v] Posted article - %v\r", sourceName, item.Url)
|
||||
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
||||
}
|
||||
|
||||
err = c.addToDiscordQueue(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
time.Sleep(30 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) postArticle(item database.Article) error {
|
||||
func (c *Cron) postArticle(id uuid.UUID,item database.Article) error {
|
||||
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
|
||||
ID: uuid.New(),
|
||||
ID: id,
|
||||
Sourceid: item.Sourceid,
|
||||
Tags: item.Tags,
|
||||
Title: item.Title,
|
||||
@ -273,3 +298,14 @@ func (c *Cron) postArticle(item database.Article) error {
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
|
||||
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
|
||||
ID: uuid.New(),
|
||||
Articleid: Id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1,15 +1,24 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"log"
|
||||
"crypto/tls"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// This will use the net/http client reach out to a site and collect the content.
|
||||
func getHttpContent(uri string) ([]byte, error) {
|
||||
// Code to disable the http2 client for reddit.
|
||||
// https://github.com/golang/go/issues/39302
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
tr.ForceAttemptHTTP2 = false
|
||||
tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
|
||||
tr.TLSClientConfig = &tls.Config{}
|
||||
|
||||
client := &http.Client{}
|
||||
client := &http.Client{
|
||||
Transport: tr,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil { return nil, err }
|
||||
|
||||
@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -27,7 +26,7 @@ type RedditConfig struct {
|
||||
PullNSFW string
|
||||
}
|
||||
|
||||
func NewRedditClient(Record database.Source) RedditClient {
|
||||
func NewRedditClient(Record database.Source) *RedditClient {
|
||||
rc := RedditClient{
|
||||
record: Record,
|
||||
}
|
||||
@ -36,23 +35,24 @@ func NewRedditClient(Record database.Source) RedditClient {
|
||||
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
|
||||
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
|
||||
|
||||
rc.disableHttp2Client()
|
||||
//rc.disableHttp2Client()
|
||||
|
||||
return rc
|
||||
return &rc
|
||||
}
|
||||
|
||||
// This is needed for to get modern go to talk to the endpoint.
|
||||
// https://www.reddit.com/r/redditdev/comments/t8e8hc/getting_nothing_but_429_responses_when_using_go/
|
||||
func (rc RedditClient) disableHttp2Client() {
|
||||
os.Setenv("GODEBUG", "http2client=0")
|
||||
}
|
||||
//func (rc *RedditClient) disableHttp2Client() {
|
||||
// os.Setenv("GODEBUG", "http2client=0")
|
||||
//}
|
||||
|
||||
func (rc RedditClient) GetBrowser() *rod.Browser {
|
||||
|
||||
func (rc *RedditClient) GetBrowser() *rod.Browser {
|
||||
browser := rod.New().MustConnect()
|
||||
return browser
|
||||
}
|
||||
|
||||
func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||
func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||
page := parser.MustPage(url)
|
||||
return page
|
||||
}
|
||||
@ -61,13 +61,15 @@ func (rc RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||
|
||||
// GetContent() reaches out to Reddit and pulls the Json data.
|
||||
// It will then convert the data to a struct and return the struct.
|
||||
func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
|
||||
func (rc *RedditClient) GetContent() (model.RedditJsonContent, error) {
|
||||
var items model.RedditJsonContent = model.RedditJsonContent{}
|
||||
|
||||
// TODO Wire this to support the config options
|
||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||
|
||||
log.Printf("Collecting results on '%v'", rc.record.Name)
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
||||
|
||||
|
||||
|
||||
content, err := getHttpContent(Url)
|
||||
if err != nil {
|
||||
@ -84,13 +86,13 @@ func (rc RedditClient) GetContent() (model.RedditJsonContent, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
|
||||
func (rc *RedditClient) ConvertToArticles(items model.RedditJsonContent) []database.Article {
|
||||
var redditArticles []database.Article
|
||||
for _, item := range items.Data.Children {
|
||||
var article database.Article
|
||||
article, err := rc.convertToArticle(item.Data)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
log.Printf("[Reddit] %v", err)
|
||||
continue
|
||||
}
|
||||
redditArticles = append(redditArticles, article)
|
||||
@ -100,7 +102,7 @@ func (rc RedditClient) ConvertToArticles(items model.RedditJsonContent) []databa
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
|
||||
func (rc *RedditClient) convertToArticle(source model.RedditPost) (database.Article, error) {
|
||||
var item database.Article
|
||||
|
||||
if source.Content == "" && source.Url != "" {
|
||||
@ -119,15 +121,15 @@ func (rc RedditClient) convertToArticle(source model.RedditPost) (database.Artic
|
||||
item = rc.convertRedirectPost(source)
|
||||
}
|
||||
|
||||
if item.Description == "" {
|
||||
var err = errors.New("reddit post failed to parse correctly")
|
||||
if item.Description == "" && item.Title == "" {
|
||||
var err = errors.New("post failed to parse correctly")
|
||||
return item, err
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertPicturePost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Title: source.Title,
|
||||
@ -145,7 +147,7 @@ func (rc RedditClient) convertPicturePost(source model.RedditPost) database.Arti
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertTextPost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
@ -160,7 +162,7 @@ func (rc RedditClient) convertTextPost(source model.RedditPost) database.Article
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertVideoPost(source model.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertVideoPost(source model.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
|
||||
@ -77,7 +77,7 @@ func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
||||
}
|
||||
|
||||
// Invokes Logon request to the API
|
||||
func (tc TwitchClient) Login() error {
|
||||
func (tc *TwitchClient) Login() error {
|
||||
token, err := tc.api.RequestAppAccessToken([]string{twitchScopes})
|
||||
if err != nil {
|
||||
return err
|
||||
@ -87,7 +87,7 @@ func (tc TwitchClient) Login() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) GetContent() ([]database.Article, error) {
|
||||
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
@ -136,7 +136,7 @@ func (tc TwitchClient) GetContent() ([]database.Article, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
var blank helix.User
|
||||
|
||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||
@ -145,11 +145,16 @@ func (tc TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
if err != nil {
|
||||
return blank, err
|
||||
}
|
||||
|
||||
if len(users.Data.Users) == 0 {
|
||||
return blank, errors.New("no results have been returned")
|
||||
}
|
||||
|
||||
return users.Data.Users[0], nil
|
||||
}
|
||||
|
||||
// This will reach out and collect the posts made by the user.
|
||||
func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
|
||||
func (tc *TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
|
||||
var blank []helix.Video
|
||||
|
||||
videos, err := tc.api.GetVideos(&helix.VideosParams{
|
||||
@ -163,14 +168,14 @@ func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
|
||||
return videos.Data.Videos, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
|
||||
func (tc *TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
|
||||
if post.UserName == "" {
|
||||
return "", ErrMissingAuthorName
|
||||
}
|
||||
return post.UserName, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
|
||||
func (tc *TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
|
||||
if post.ThumbnailURL == "" {
|
||||
return "", ErrMissingThumbnail
|
||||
}
|
||||
@ -180,7 +185,7 @@ func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
|
||||
return thumb, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
|
||||
func (tc *TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
|
||||
if post.PublishedAt == "" {
|
||||
return time.Now(), ErrMissingPublishDate
|
||||
}
|
||||
@ -191,7 +196,7 @@ func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
|
||||
return pubDate, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
|
||||
func (tc *TwitchClient) ExtractDescription(post helix.Video) (string, error) {
|
||||
// Check if the description is null but we have a title.
|
||||
// The poster didnt add a description but this isnt an error.
|
||||
if post.Description == "" && post.Title == "" {
|
||||
@ -204,7 +209,7 @@ func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
|
||||
}
|
||||
|
||||
// Extracts the avatar of the author with some validation.
|
||||
func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
|
||||
func (tc *TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
|
||||
if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
|
||||
if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
|
||||
return user.ProfileImageURL, nil
|
||||
@ -212,20 +217,20 @@ func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
|
||||
|
||||
// Generate tags based on the video metadata.
|
||||
// TODO Figure out how to query what game is played
|
||||
func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
|
||||
func (tc *TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
|
||||
res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Extracts the title from a post with some validation.
|
||||
func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) {
|
||||
func (tc *TwitchClient) ExtractTitle(post helix.Video) (string, error) {
|
||||
if post.Title == "" {
|
||||
return "", errors.New("unable to find the title on the requested post")
|
||||
}
|
||||
return post.Title, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) {
|
||||
func (tc *TwitchClient) ExtractUrl(post helix.Video) (string, error) {
|
||||
if post.URL == "" { return "", ErrMissingUrl }
|
||||
return post.URL, nil
|
||||
}
|
||||
@ -121,7 +121,7 @@ func (yc *YoutubeClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||
func (yc *YoutubeClient) GetParser(uri string) (*goquery.Document, error) {
|
||||
html, err := http.Get(uri)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
log.Printf("[YouTube] %v", err)
|
||||
}
|
||||
defer html.Body.Close()
|
||||
|
||||
@ -244,19 +244,19 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
parser, err := yc.GetParser(item.Link)
|
||||
if err != nil {
|
||||
log.Printf("Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||
}
|
||||
|
||||
tags, err := yc.GetTags(parser)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("%v. %v", err, item.Link)
|
||||
log.Println(msg)
|
||||
log.Printf("[YouTube] %v", msg)
|
||||
}
|
||||
|
||||
thumb, err := yc.GetVideoThumbnail(parser)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("%v. %v", err, item.Link)
|
||||
log.Println(msg)
|
||||
log.Printf("[YouTube] %v", msg)
|
||||
}
|
||||
|
||||
var article = database.Article{
|
||||
|
||||
@ -1,80 +1,151 @@
|
||||
package output
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
)
|
||||
|
||||
type discordField struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Value string `json:"value,omitempty"`
|
||||
Inline bool `json:"inline,omitempty"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
Value *string `json:"value,omitempty"`
|
||||
Inline *bool `json:"inline,omitempty"`
|
||||
}
|
||||
|
||||
type discordFooter struct {
|
||||
Value *string `json:"text,omitempty"`
|
||||
IconUrl *string `json:"icon_url,omitempty"`
|
||||
}
|
||||
|
||||
type discordAuthor struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Url string `json:"url,omitempty"`
|
||||
IconUrl string `json:"icon_url,omitempty"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
Url *string `json:"url,omitempty"`
|
||||
IconUrl *string `json:"icon_url,omitempty"`
|
||||
}
|
||||
|
||||
type discordImage struct {
|
||||
Url string `json:"url,omitempty"`
|
||||
Url *string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
type discordEmbed struct {
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Url string `json:"url,omitempty"`
|
||||
Color int32 `json:"color,omitempty"`
|
||||
Timestamp time.Time `json:"timestamp,omitempty"`
|
||||
Fields []discordField `json:"fields,omitempty"`
|
||||
Author discordAuthor `json:"author,omitempty"`
|
||||
Image discordImage `json:"image,omitempty"`
|
||||
Thumbnail discordImage `json:"thumbnail,omitempty"`
|
||||
type DiscordEmbed struct {
|
||||
Title *string `json:"title,omitempty"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
Url *string `json:"url,omitempty"`
|
||||
Color *int32 `json:"color,omitempty"`
|
||||
//Timestamp time.Time `json:"timestamp,omitempty"`
|
||||
Fields []*discordField `json:"fields,omitempty"`
|
||||
Author discordAuthor `json:"author,omitempty"`
|
||||
Image discordImage `json:"image,omitempty"`
|
||||
Thumbnail discordImage `json:"thumbnail,omitempty"`
|
||||
Footer *discordFooter `json:"footer,omitempty"`
|
||||
}
|
||||
|
||||
// Root object for Discord Webhook messages
|
||||
type discordMessage struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
Embeds []discordEmbed `json:"embeds,omitempty"`
|
||||
type DiscordMessage struct {
|
||||
Username *string `json:"username,omitempty"`
|
||||
Content *string `json:"content,omitempty"`
|
||||
Embeds *[]DiscordEmbed `json:"embeds,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultColor = 0
|
||||
YoutubeColor = 16711680
|
||||
TwitchColor = 0
|
||||
RedditColor = 0
|
||||
TwitterColor = 0
|
||||
FfxivColor = 0
|
||||
)
|
||||
|
||||
type Discord struct {
|
||||
Subscriptions []string
|
||||
article database.Article
|
||||
Message discordMessage
|
||||
article database.Article
|
||||
Message *DiscordMessage
|
||||
}
|
||||
|
||||
func NewDiscordWebHookMessage(Subscriptions []string, Article database.Article) Discord {
|
||||
func NewDiscordWebHookMessage(Article database.Article) Discord {
|
||||
return Discord{
|
||||
Subscriptions: Subscriptions,
|
||||
article: Article,
|
||||
Message: discordMessage{
|
||||
Embeds: []discordEmbed{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (dwh Discord) GeneratePayload() error {
|
||||
// Convert the message
|
||||
embed := discordEmbed {
|
||||
Title: dwh.article.Title,
|
||||
Description: dwh.convertFromHtml(dwh.article.Description),
|
||||
Url: dwh.article.Url,
|
||||
Thumbnail: discordImage{
|
||||
Url: dwh.article.Thumbnail,
|
||||
},
|
||||
}
|
||||
var arr []discordEmbed
|
||||
// Generates the link field to expose in the message
|
||||
func (dwh Discord) getFields() []*discordField {
|
||||
var fields []*discordField
|
||||
|
||||
arr = append(arr, embed)
|
||||
dwh.Message.Embeds = arr
|
||||
return nil
|
||||
key := "Link"
|
||||
linkField := discordField{
|
||||
Name: &key,
|
||||
Value: &dwh.article.Url,
|
||||
}
|
||||
|
||||
fields = append(fields, &linkField)
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
func (dwh Discord) SendPayload() error {
|
||||
// This will create the message that will be sent out.
|
||||
func (dwh Discord) GeneratePayload() (*DiscordMessage, error) {
|
||||
|
||||
// Create the embed
|
||||
footerMessage := "Brought to you by Newsbot"
|
||||
footerUrl := ""
|
||||
description := dwh.convertFromHtml(dwh.article.Description)
|
||||
color := dwh.getColor(dwh.article.Url)
|
||||
|
||||
embed := DiscordEmbed{
|
||||
Title: &dwh.article.Title,
|
||||
Description: &description,
|
||||
Image: discordImage{
|
||||
Url: &dwh.article.Thumbnail,
|
||||
},
|
||||
Fields: dwh.getFields(),
|
||||
Footer: &discordFooter{
|
||||
Value: &footerMessage,
|
||||
IconUrl: &footerUrl,
|
||||
},
|
||||
Color: &color,
|
||||
}
|
||||
|
||||
// attach the embed to an array
|
||||
var embedArray []DiscordEmbed
|
||||
embedArray = append(embedArray, embed)
|
||||
|
||||
// create the base message
|
||||
msg := DiscordMessage{
|
||||
Embeds: &embedArray,
|
||||
}
|
||||
|
||||
return &msg, nil
|
||||
}
|
||||
|
||||
func (dwh Discord) SendPayload(Message *DiscordMessage, Url string) error {
|
||||
// Convert the message to a io.reader object
|
||||
buffer := new(bytes.Buffer)
|
||||
json.NewEncoder(buffer).Encode(Message)
|
||||
|
||||
// Send the message
|
||||
resp, err := http.Post(Url, "application/json", buffer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check for 204
|
||||
if resp.StatusCode != 204 {
|
||||
defer resp.Body.Close()
|
||||
|
||||
errMsg, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return fmt.Errorf(string(errMsg))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -104,7 +175,16 @@ func (dwh Discord) convertFromHtml(body string) string {
|
||||
return clean
|
||||
}
|
||||
|
||||
func (dwh Discord) convertLinks(body string) string {
|
||||
func (dwh *Discord) getColor(Url string) int32 {
|
||||
if strings.Contains(Url, "youtube.com") {
|
||||
return YoutubeColor
|
||||
}
|
||||
|
||||
return DefaultColor
|
||||
|
||||
}
|
||||
|
||||
func (dwh *Discord) convertLinks(body string) string {
|
||||
//items := regexp.MustCompile("<a(.*?)a>")
|
||||
return ""
|
||||
}
|
||||
|
||||
@ -1,11 +1,10 @@
|
||||
package output_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
//"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/joho/godotenv"
|
||||
@ -13,61 +12,127 @@ import (
|
||||
"github.com/jtom38/newsbot/collector/services/output"
|
||||
)
|
||||
|
||||
var article database.Article = database.Article{
|
||||
ID: uuid.New(),
|
||||
Sourceid: uuid.New(),
|
||||
Tags: "unit, testing",
|
||||
Title: "Demo",
|
||||
Url: "https://github.com/jtom38/newsbot.collector.api",
|
||||
Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Description: "Hello World",
|
||||
var (
|
||||
article database.Article = database.Article{
|
||||
ID: uuid.New(),
|
||||
Sourceid: uuid.New(),
|
||||
Tags: "unit, testing",
|
||||
Title: "Demo",
|
||||
Url: "https://github.com/jtom38/newsbot.collector.api",
|
||||
//Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Description: "Hello World",
|
||||
}
|
||||
blank string = ""
|
||||
)
|
||||
|
||||
func TestDiscordMessageContainsTitle(t *testing.T) {
|
||||
d := output.NewDiscordWebHookMessage(article)
|
||||
msg, err := d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
for _, i := range *msg.Embeds {
|
||||
if i.Title == &blank {
|
||||
t.Error("title missing")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getWebhook() ([]string, error){
|
||||
var endpoints []string
|
||||
func TestDiscordMessageContainsDescription(t *testing.T) {
|
||||
d := output.NewDiscordWebHookMessage(article)
|
||||
msg, err := d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
for _, i := range *msg.Embeds {
|
||||
if i.Description == &blank {
|
||||
t.Error("description missing")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscordMessageFooter(t *testing.T) {
|
||||
d := output.NewDiscordWebHookMessage(article)
|
||||
msg, err := d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
for _, i := range *msg.Embeds {
|
||||
blank := ""
|
||||
if i.Footer.Value == &blank {
|
||||
t.Error("missing footer vlue")
|
||||
}
|
||||
if i.Footer.IconUrl == &blank {
|
||||
t.Error("missing footer url")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscordMessageFields(t *testing.T) {
|
||||
header := "Link"
|
||||
d := output.NewDiscordWebHookMessage(article)
|
||||
msg, err := d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
for _, embed := range *msg.Embeds {
|
||||
for _, field := range embed.Fields {
|
||||
var fName string
|
||||
if field.Name != nil {
|
||||
fName = *field.Name
|
||||
} else {
|
||||
t.Error("missing link field value")
|
||||
}
|
||||
|
||||
if fName != header {
|
||||
t.Error("missing link field key")
|
||||
}
|
||||
|
||||
var fValue string
|
||||
if field.Value != nil {
|
||||
fValue = *field.Value
|
||||
}
|
||||
|
||||
if fValue == blank {
|
||||
t.Error("missing link field value")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This test requires a env value to be present to work
|
||||
func TestDiscordMessagePost(t *testing.T) {
|
||||
_, err := os.Open(".env")
|
||||
if err != nil {
|
||||
return endpoints, err
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = godotenv.Load()
|
||||
if err != nil {
|
||||
return endpoints, err
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
res := os.Getenv("TESTS_DISCORD_WEBHOOK")
|
||||
if res == "" {
|
||||
return endpoints, errors.New("TESTS_DISCORD_WEBHOOK is missing")
|
||||
t.Error("TESTS_DISCORD_WEBHOOK is missing")
|
||||
}
|
||||
endpoints := strings.Split(res, " ")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
endpoints = strings.Split(res, "")
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func TestNewDiscordWebHookContainsSubscriptions(t *testing.T) {
|
||||
hook, err := getWebhook()
|
||||
d := output.NewDiscordWebHookMessage(article)
|
||||
msg, err := d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
d := output.NewDiscordWebHookMessage(hook, article)
|
||||
if len(d.Subscriptions) == 0 {
|
||||
t.Error("no subscriptions found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscordMessageContainsTitle(t *testing.T) {
|
||||
hook, err := getWebhook()
|
||||
err = d.SendPayload(msg, endpoints[0])
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
d := output.NewDiscordWebHookMessage(hook, article)
|
||||
err = d.GeneratePayload()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if d.Message.Embeds[0].Title == "" {
|
||||
t.Error("no title was found ")
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user