Merge pull request 'features/rss-job' (#6) from features/rss-job into main
Reviewed-on: #6
This commit is contained in:
commit
e57b115117
1
.gitignore
vendored
1
.gitignore
vendored
@ -11,6 +11,7 @@ server
|
||||
*.so
|
||||
*.dylib
|
||||
collector
|
||||
newsbot.db
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
6
api.http
Normal file
6
api.http
Normal file
@ -0,0 +1,6 @@
|
||||
### Select Sources fro mthe top
|
||||
GET http://localhost:8081/api/v1/sources/
|
||||
|
||||
|
||||
### Select Sources by type
|
||||
GET http://localhost:8081/api/v1/sources/by/source?source=rss
|
||||
@ -4,14 +4,12 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
_ "github.com/glebarez/go-sqlite"
|
||||
"github.com/pressly/goose/v3"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
)
|
||||
@ -42,19 +40,14 @@ func main() {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
queries := database.New(db)
|
||||
|
||||
c := cron.NewScheduler(ctx)
|
||||
c := cron.NewScheduler(ctx, db)
|
||||
c.Start()
|
||||
|
||||
server := v1.NewServer(ctx, queries, configs, db)
|
||||
server := v1.NewServer(ctx, configs, db)
|
||||
|
||||
fmt.Println("API is online and waiting for requests.")
|
||||
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
|
||||
fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress)
|
||||
|
||||
err = http.ListenAndServe(":8081", server.Router)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
server.Router.Start(":8081")
|
||||
}
|
||||
|
||||
79
docs/docs.go
79
docs/docs.go
@ -16,7 +16,7 @@ const docTemplate = `{
|
||||
"host": "{{.Host}}",
|
||||
"basePath": "{{.BasePath}}",
|
||||
"paths": {
|
||||
"/articles": {
|
||||
"/v1/articles": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -55,7 +55,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/articles/by/sourceid": {
|
||||
"/v1/articles/by/sourceid": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -101,7 +101,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/articles/{ID}": {
|
||||
"/v1/articles/{ID}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -141,7 +141,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/articles/{ID}/details": {
|
||||
"/v1/articles/{ID}/details": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -181,14 +181,13 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks": {
|
||||
"/v1/discord/webhooks": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Returns the top 100",
|
||||
"responses": {
|
||||
@ -213,14 +212,13 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/by/serverAndChannel": {
|
||||
"/v1/discord/webhooks/by/serverAndChannel": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Returns all the known web hooks based on the Server and Channel given.",
|
||||
"parameters": [
|
||||
@ -261,11 +259,10 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/new": {
|
||||
"/v1/discord/webhooks/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Creates a new record for a discord web hook to post data to.",
|
||||
"parameters": [
|
||||
@ -313,11 +310,10 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{ID}": {
|
||||
"/v1/discord/webhooks/{ID}": {
|
||||
"delete": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Deletes a record by ID.",
|
||||
"parameters": [
|
||||
@ -351,11 +347,10 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{ID}/disable": {
|
||||
"/v1/discord/webhooks/{ID}/disable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Disables a Webhook from being used.",
|
||||
"parameters": [
|
||||
@ -389,11 +384,10 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{ID}/enable": {
|
||||
"/v1/discord/webhooks/{ID}/enable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Enables a source to continue processing.",
|
||||
"parameters": [
|
||||
@ -408,14 +402,13 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{id}": {
|
||||
"/v1/discord/webhooks/{id}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -449,7 +442,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/queue/discord/webhooks": {
|
||||
"/v1/queue/discord/webhooks": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -468,7 +461,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/settings/{key}": {
|
||||
"/v1/settings/{key}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -489,7 +482,7 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources": {
|
||||
"/v1/sources": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -522,7 +515,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/by/source": {
|
||||
"/v1/sources/by/source": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -568,7 +561,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/by/sourceAndName": {
|
||||
"/v1/sources/by/sourceAndName": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -615,7 +608,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/new/reddit": {
|
||||
"/v1/sources/new/reddit": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -659,7 +652,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/new/rss": {
|
||||
"/v1/sources/new/rss": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -703,7 +696,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/new/twitch": {
|
||||
"/v1/sources/new/twitch": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -721,7 +714,7 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources/new/youtube": {
|
||||
"/v1/sources/new/youtube": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -746,7 +739,7 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources/{id}": {
|
||||
"/v1/sources/{id}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -802,7 +795,7 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources/{id}/disable": {
|
||||
"/v1/sources/{id}/disable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -839,7 +832,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/{id}/enable": {
|
||||
"/v1/sources/{id}/enable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -876,7 +869,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions": {
|
||||
"/v1/subscriptions": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -907,7 +900,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/SourceId": {
|
||||
"/v1/subscriptions/by/SourceId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -935,7 +928,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/discordId": {
|
||||
"/v1/subscriptions/by/discordId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -975,7 +968,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/details": {
|
||||
"/v1/subscriptions/details": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -994,7 +987,7 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/delete": {
|
||||
"/v1/subscriptions/discord/webhook/delete": {
|
||||
"delete": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
@ -1012,7 +1005,7 @@ const docTemplate = `{
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/new": {
|
||||
"/v1/subscriptions/discord/webhook/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
},
|
||||
"basePath": "/api",
|
||||
"paths": {
|
||||
"/articles": {
|
||||
"/v1/articles": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -46,7 +46,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/articles/by/sourceid": {
|
||||
"/v1/articles/by/sourceid": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -92,7 +92,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/articles/{ID}": {
|
||||
"/v1/articles/{ID}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -132,7 +132,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/articles/{ID}/details": {
|
||||
"/v1/articles/{ID}/details": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -172,14 +172,13 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks": {
|
||||
"/v1/discord/webhooks": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Returns the top 100",
|
||||
"responses": {
|
||||
@ -204,14 +203,13 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/by/serverAndChannel": {
|
||||
"/v1/discord/webhooks/by/serverAndChannel": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Returns all the known web hooks based on the Server and Channel given.",
|
||||
"parameters": [
|
||||
@ -252,11 +250,10 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/new": {
|
||||
"/v1/discord/webhooks/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Creates a new record for a discord web hook to post data to.",
|
||||
"parameters": [
|
||||
@ -304,11 +301,10 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{ID}": {
|
||||
"/v1/discord/webhooks/{ID}": {
|
||||
"delete": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Deletes a record by ID.",
|
||||
"parameters": [
|
||||
@ -342,11 +338,10 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{ID}/disable": {
|
||||
"/v1/discord/webhooks/{ID}/disable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Disables a Webhook from being used.",
|
||||
"parameters": [
|
||||
@ -380,11 +375,10 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{ID}/enable": {
|
||||
"/v1/discord/webhooks/{ID}/enable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Enables a source to continue processing.",
|
||||
"parameters": [
|
||||
@ -399,14 +393,13 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/discord/webhooks/{id}": {
|
||||
"/v1/discord/webhooks/{id}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Discord",
|
||||
"Webhook"
|
||||
"DiscordWebhook"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
@ -440,7 +433,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/queue/discord/webhooks": {
|
||||
"/v1/queue/discord/webhooks": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -459,7 +452,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/settings/{key}": {
|
||||
"/v1/settings/{key}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -480,7 +473,7 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources": {
|
||||
"/v1/sources": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -513,7 +506,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/by/source": {
|
||||
"/v1/sources/by/source": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -559,7 +552,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/by/sourceAndName": {
|
||||
"/v1/sources/by/sourceAndName": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -606,7 +599,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/new/reddit": {
|
||||
"/v1/sources/new/reddit": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -650,7 +643,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/new/rss": {
|
||||
"/v1/sources/new/rss": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -694,7 +687,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/new/twitch": {
|
||||
"/v1/sources/new/twitch": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -712,7 +705,7 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources/new/youtube": {
|
||||
"/v1/sources/new/youtube": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -737,7 +730,7 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources/{id}": {
|
||||
"/v1/sources/{id}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -793,7 +786,7 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/sources/{id}/disable": {
|
||||
"/v1/sources/{id}/disable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -830,7 +823,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/sources/{id}/enable": {
|
||||
"/v1/sources/{id}/enable": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Source"
|
||||
@ -867,7 +860,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions": {
|
||||
"/v1/subscriptions": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -898,7 +891,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/SourceId": {
|
||||
"/v1/subscriptions/by/SourceId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -926,7 +919,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/discordId": {
|
||||
"/v1/subscriptions/by/discordId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -966,7 +959,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/details": {
|
||||
"/v1/subscriptions/details": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
@ -985,7 +978,7 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/delete": {
|
||||
"/v1/subscriptions/discord/webhook/delete": {
|
||||
"delete": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
@ -1003,7 +996,7 @@
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/new": {
|
||||
"/v1/subscriptions/discord/webhook/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
|
||||
@ -241,7 +241,7 @@ info:
|
||||
title: NewsBot collector
|
||||
version: "0.1"
|
||||
paths:
|
||||
/articles:
|
||||
/v1/articles:
|
||||
get:
|
||||
parameters:
|
||||
- description: page number
|
||||
@ -266,7 +266,7 @@ paths:
|
||||
summary: Lists the top 25 records ordering from newest to oldest.
|
||||
tags:
|
||||
- Articles
|
||||
/articles/{ID}:
|
||||
/v1/articles/{ID}:
|
||||
get:
|
||||
parameters:
|
||||
- description: int
|
||||
@ -292,7 +292,7 @@ paths:
|
||||
summary: Returns an article based on defined ID.
|
||||
tags:
|
||||
- Articles
|
||||
/articles/{ID}/details:
|
||||
/v1/articles/{ID}/details:
|
||||
get:
|
||||
parameters:
|
||||
- description: int
|
||||
@ -318,7 +318,7 @@ paths:
|
||||
summary: Returns an article and source based on defined ID.
|
||||
tags:
|
||||
- Articles
|
||||
/articles/by/sourceid:
|
||||
/v1/articles/by/sourceid:
|
||||
get:
|
||||
parameters:
|
||||
- description: source id
|
||||
@ -349,7 +349,7 @@ paths:
|
||||
25.
|
||||
tags:
|
||||
- Articles
|
||||
/discord/webhooks:
|
||||
/v1/discord/webhooks:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
@ -368,9 +368,8 @@ paths:
|
||||
$ref: '#/definitions/domain.BaseResponse'
|
||||
summary: Returns the top 100
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/discord/webhooks/{ID}:
|
||||
- DiscordWebhook
|
||||
/v1/discord/webhooks/{ID}:
|
||||
delete:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -393,9 +392,8 @@ paths:
|
||||
$ref: '#/definitions/domain.BaseResponse'
|
||||
summary: Deletes a record by ID.
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/discord/webhooks/{ID}/disable:
|
||||
- DiscordWebhook
|
||||
/v1/discord/webhooks/{ID}/disable:
|
||||
post:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -418,9 +416,8 @@ paths:
|
||||
$ref: '#/definitions/domain.BaseResponse'
|
||||
summary: Disables a Webhook from being used.
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/discord/webhooks/{ID}/enable:
|
||||
- DiscordWebhook
|
||||
/v1/discord/webhooks/{ID}/enable:
|
||||
post:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -431,9 +428,8 @@ paths:
|
||||
responses: {}
|
||||
summary: Enables a source to continue processing.
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/discord/webhooks/{id}:
|
||||
- DiscordWebhook
|
||||
/v1/discord/webhooks/{id}:
|
||||
get:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -458,9 +454,8 @@ paths:
|
||||
$ref: '#/definitions/domain.BaseResponse'
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/discord/webhooks/by/serverAndChannel:
|
||||
- DiscordWebhook
|
||||
/v1/discord/webhooks/by/serverAndChannel:
|
||||
get:
|
||||
parameters:
|
||||
- description: Fancy Server
|
||||
@ -490,9 +485,8 @@ paths:
|
||||
$ref: '#/definitions/domain.BaseResponse'
|
||||
summary: Returns all the known web hooks based on the Server and Channel given.
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/discord/webhooks/new:
|
||||
- DiscordWebhook
|
||||
/v1/discord/webhooks/new:
|
||||
post:
|
||||
parameters:
|
||||
- description: url
|
||||
@ -525,9 +519,8 @@ paths:
|
||||
$ref: '#/definitions/domain.BaseResponse'
|
||||
summary: Creates a new record for a discord web hook to post data to.
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/queue/discord/webhooks:
|
||||
- DiscordWebhook
|
||||
/v1/queue/discord/webhooks:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
@ -539,7 +532,7 @@ paths:
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Queue
|
||||
/settings/{key}:
|
||||
/v1/settings/{key}:
|
||||
get:
|
||||
parameters:
|
||||
- description: Settings Key value
|
||||
@ -553,7 +546,7 @@ paths:
|
||||
summary: Returns a object based on the Key that was given.
|
||||
tags:
|
||||
- Settings
|
||||
/sources:
|
||||
/v1/sources:
|
||||
get:
|
||||
parameters:
|
||||
- description: page number
|
||||
@ -574,7 +567,7 @@ paths:
|
||||
summary: Lists the top 50 records
|
||||
tags:
|
||||
- Source
|
||||
/sources/{id}:
|
||||
/v1/sources/{id}:
|
||||
get:
|
||||
parameters:
|
||||
- description: uuid
|
||||
@ -611,7 +604,7 @@ paths:
|
||||
summary: Marks a source as deleted based on its ID value.
|
||||
tags:
|
||||
- Source
|
||||
/sources/{id}/disable:
|
||||
/v1/sources/{id}/disable:
|
||||
post:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -635,7 +628,7 @@ paths:
|
||||
summary: Disables a source from processing.
|
||||
tags:
|
||||
- Source
|
||||
/sources/{id}/enable:
|
||||
/v1/sources/{id}/enable:
|
||||
post:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -659,7 +652,7 @@ paths:
|
||||
summary: Enables a source to continue processing.
|
||||
tags:
|
||||
- Source
|
||||
/sources/by/source:
|
||||
/v1/sources/by/source:
|
||||
get:
|
||||
parameters:
|
||||
- description: Source Name
|
||||
@ -689,7 +682,7 @@ paths:
|
||||
summary: 'Lists the top 50 records based on the name given. Example: reddit'
|
||||
tags:
|
||||
- Source
|
||||
/sources/by/sourceAndName:
|
||||
/v1/sources/by/sourceAndName:
|
||||
get:
|
||||
parameters:
|
||||
- description: dadjokes
|
||||
@ -720,7 +713,7 @@ paths:
|
||||
summary: Returns a single entity by ID
|
||||
tags:
|
||||
- Source
|
||||
/sources/new/reddit:
|
||||
/v1/sources/new/reddit:
|
||||
post:
|
||||
parameters:
|
||||
- description: name
|
||||
@ -749,7 +742,7 @@ paths:
|
||||
summary: Creates a new reddit source to monitor.
|
||||
tags:
|
||||
- Source
|
||||
/sources/new/rss:
|
||||
/v1/sources/new/rss:
|
||||
post:
|
||||
parameters:
|
||||
- description: Site Name
|
||||
@ -778,7 +771,7 @@ paths:
|
||||
summary: Creates a new rss source to monitor.
|
||||
tags:
|
||||
- Source
|
||||
/sources/new/twitch:
|
||||
/v1/sources/new/twitch:
|
||||
post:
|
||||
parameters:
|
||||
- description: name
|
||||
@ -790,7 +783,7 @@ paths:
|
||||
summary: Creates a new twitch source to monitor.
|
||||
tags:
|
||||
- Source
|
||||
/sources/new/youtube:
|
||||
/v1/sources/new/youtube:
|
||||
post:
|
||||
parameters:
|
||||
- description: name
|
||||
@ -807,7 +800,7 @@ paths:
|
||||
summary: Creates a new youtube source to monitor.
|
||||
tags:
|
||||
- Source
|
||||
/subscriptions:
|
||||
/v1/subscriptions:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
@ -827,7 +820,7 @@ paths:
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/by/SourceId:
|
||||
/v1/subscriptions/by/SourceId:
|
||||
get:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -845,7 +838,7 @@ paths:
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/by/discordId:
|
||||
/v1/subscriptions/by/discordId:
|
||||
get:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -871,7 +864,7 @@ paths:
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/details:
|
||||
/v1/subscriptions/details:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
@ -883,7 +876,7 @@ paths:
|
||||
summary: Returns the top 50 entries with full deatils on the source and output.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/discord/webhook/delete:
|
||||
/v1/subscriptions/discord/webhook/delete:
|
||||
delete:
|
||||
parameters:
|
||||
- description: id
|
||||
@ -895,7 +888,7 @@ paths:
|
||||
summary: Removes a Discord WebHook Subscription based on the Subscription ID.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/discord/webhook/new:
|
||||
/v1/subscriptions/discord/webhook/new:
|
||||
post:
|
||||
parameters:
|
||||
- description: discordWebHookId
|
||||
|
||||
3
go.mod
3
go.mod
@ -5,7 +5,6 @@ go 1.22
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0
|
||||
github.com/glebarez/go-sqlite v1.22.0
|
||||
github.com/go-chi/chi/v5 v5.0.7
|
||||
github.com/go-rod/rod v0.107.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/huandu/go-sqlbuilder v1.27.1
|
||||
@ -23,6 +22,7 @@ require (
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
github.com/huandu/xstrings v1.3.2 // indirect
|
||||
github.com/labstack/gommon v0.4.2 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
@ -36,6 +36,7 @@ require (
|
||||
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/sync v0.7.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
modernc.org/libc v1.41.0 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.7.2 // indirect
|
||||
|
||||
6
go.sum
6
go.sum
@ -18,8 +18,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
||||
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
||||
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
||||
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
@ -33,6 +31,8 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK
|
||||
github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY=
|
||||
github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@ -163,6 +163,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
|
||||
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
|
||||
|
||||
@ -12,8 +12,6 @@ CREATE TABLE Articles (
|
||||
Url TEXT NOT NULL,
|
||||
PubDate DATETIME NOT NULL,
|
||||
IsVideo TEXT NOT NULL,
|
||||
--VideoHeight int NOT NULL,
|
||||
--VideoWidth int NOT NULL,
|
||||
ThumbnailUrl TEXT NOT NULL,
|
||||
Description TEXT NOT NULL,
|
||||
AuthorName TEXT NOT NULL,
|
||||
@ -34,6 +32,7 @@ CREATE Table DiscordWebHooks (
|
||||
CreatedAt DATETIME NOT NULL,
|
||||
UpdatedAt DATETIME NOT NULL,
|
||||
DeletedAt DATETIME NOT NULL,
|
||||
UserID INTEGER NOT NULL,
|
||||
--Name TEXT NOT NULL, -- Defines webhook purpose
|
||||
--Key TEXT,
|
||||
Url TEXT NOT NULL, -- Webhook Url
|
||||
@ -78,8 +77,9 @@ CREATE TABLE Subscriptions (
|
||||
CreatedAt DATETIME NOT NULL,
|
||||
UpdatedAt DATETIME NOT NULL,
|
||||
DeletedAt DATETIME,
|
||||
DiscordWebHookId NUMBER NOT NULL,
|
||||
SourceId NUMBER NOT NULL
|
||||
DiscordWebHookID NUMBER NOT NULL,
|
||||
SourceID NUMBER NOT NULL,
|
||||
UserID NUMBER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE Users (
|
||||
|
||||
@ -7,33 +7,33 @@ SELECT 'up SQL query';
|
||||
|
||||
-- Final Fantasy XIV Entries
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - NA', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - JP', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - EU', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - FR', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - DE', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
|
||||
|
||||
-- Reddit Entries
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'reddit', 'dadjokes', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'reddit', 'steamdeck', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
|
||||
|
||||
-- Youtube Entries
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'youtube', 'Game Grumps', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
|
||||
|
||||
-- RSS Entries
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered', 'steam deck', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
|
||||
|
||||
-- Twitch Entries
|
||||
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'twitch', 'Nintendo', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
|
||||
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
@ -41,10 +41,10 @@ INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabl
|
||||
-- +goose StatementBegin
|
||||
--SELECT 'down SQL query';
|
||||
|
||||
DELETE FROM sources where source = 'reddit' and name = 'dadjokes';
|
||||
DELETE FROM sources where source = 'reddit' and name = 'steamdeck';
|
||||
DELETE FROM sources where source = 'ffxiv';
|
||||
DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo';
|
||||
DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps';
|
||||
DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck';
|
||||
DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes';
|
||||
DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck';
|
||||
DELETE FROM sources where Source = 'ffxiv';
|
||||
DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo';
|
||||
DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps';
|
||||
DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck';
|
||||
-- +goose StatementEnd
|
||||
|
||||
@ -14,7 +14,7 @@ import (
|
||||
// @Produce application/json
|
||||
// @Param page query string false "page number"
|
||||
// @Tags Articles
|
||||
// @Router /articles [get]
|
||||
// @Router /v1/articles [get]
|
||||
// @Success 200 {object} domain.ArticleResponse
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -44,7 +44,7 @@ func (s *Handler) listArticles(c echo.Context) error {
|
||||
// @Param ID path string true "int"
|
||||
// @Produce application/json
|
||||
// @Tags Articles
|
||||
// @Router /articles/{ID} [get]
|
||||
// @Router /v1/articles/{ID} [get]
|
||||
// @Success 200 {object} domain.ArticleResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -78,7 +78,7 @@ func (s *Handler) getArticle(c echo.Context) error {
|
||||
// @Param ID path string true "int"
|
||||
// @Produce application/json
|
||||
// @Tags Articles
|
||||
// @Router /articles/{ID}/details [get]
|
||||
// @Router /v1/articles/{ID}/details [get]
|
||||
// @Success 200 {object} domain.ArticleDetailedResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -119,7 +119,7 @@ func (s *Handler) getArticleDetails(c echo.Context) error {
|
||||
// @Param page query int false "Page to query"
|
||||
// @Produce application/json
|
||||
// @Tags Articles
|
||||
// @Router /articles/by/sourceid [get]
|
||||
// @Router /v1/articles/by/sourceid [get]
|
||||
// @Success 200 {object} domain.ArticleResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
|
||||
@ -13,8 +13,8 @@ import (
|
||||
// ListDiscordWebhooks
|
||||
// @Summary Returns the top 100
|
||||
// @Produce application/json
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks [get]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks [get]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -37,8 +37,8 @@ func (s *Handler) ListDiscordWebHooks(c echo.Context) error {
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Param id path int true "id"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/{id} [get]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/{id} [get]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -69,8 +69,8 @@ func (s *Handler) GetDiscordWebHooksById(c echo.Context) error {
|
||||
// @Produce application/json
|
||||
// @Param server query string true "Fancy Server"
|
||||
// @Param channel query string true "memes"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/by/serverAndChannel [get]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/by/serverAndChannel [get]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -105,8 +105,8 @@ func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error {
|
||||
// @Param url query string true "url"
|
||||
// @Param server query string true "Server name"
|
||||
// @Param channel query string true "Channel name"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/new [post]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/new [post]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -164,8 +164,8 @@ func (s *Handler) NewDiscordWebHook(c echo.Context) error {
|
||||
// DisableDiscordWebHooks
|
||||
// @Summary Disables a Webhook from being used.
|
||||
// @Param id path int true "id"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/{ID}/disable [post]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/{ID}/disable [post]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -212,8 +212,8 @@ func (s *Handler) disableDiscordWebHook(c echo.Context) error {
|
||||
// EnableDiscordWebHook
|
||||
// @Summary Enables a source to continue processing.
|
||||
// @Param id path int true "id"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/{ID}/enable [post]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/{ID}/enable [post]
|
||||
func (s *Handler) enableDiscordWebHook(c echo.Context) error {
|
||||
id, err := strconv.Atoi(c.Param("ID"))
|
||||
if err != nil {
|
||||
@ -253,8 +253,8 @@ func (s *Handler) enableDiscordWebHook(c echo.Context) error {
|
||||
// DeleteDiscordWebHook
|
||||
// @Summary Deletes a record by ID.
|
||||
// @Param id path string true "id"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/{ID} [delete]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/{ID} [delete]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -298,8 +298,8 @@ func (s *Handler) deleteDiscordWebHook(c echo.Context) error {
|
||||
// UpdateDiscordWebHook
|
||||
// @Summary Updates a valid discord webhook ID based on the body given.
|
||||
// @Param id path string true "id"
|
||||
// @Tags Discord, Webhook
|
||||
// @Router /discord/webhooks/{id} [patch]
|
||||
// @Tags DiscordWebhook
|
||||
// @Router /v1/discord/webhooks/{id} [patch]
|
||||
// @Success 200 {object} domain.DiscordWebhookResponse "OK"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
|
||||
@ -5,19 +5,19 @@ import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/labstack/echo/v4/middleware"
|
||||
swagger "github.com/swaggo/echo-swagger"
|
||||
|
||||
_ "git.jamestombleson.com/jtom38/newsbot-api/docs"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Router *echo.Echo
|
||||
Db *database.Queries
|
||||
dto *dto.DtoClient
|
||||
//dto *dto.DtoClient
|
||||
config services.Configs
|
||||
repo services.RepositoryService
|
||||
}
|
||||
@ -39,23 +39,26 @@ var (
|
||||
ErrUnableToConvertToJson string = "Unable to convert to json"
|
||||
)
|
||||
|
||||
func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler {
|
||||
func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
|
||||
s := &Handler{
|
||||
Db: db,
|
||||
dto: dto.NewDtoClient(db),
|
||||
//Db: db,
|
||||
//dto: dto.NewDtoClient(db),
|
||||
config: configs,
|
||||
repo: services.NewRepositoryService(conn),
|
||||
}
|
||||
|
||||
router := echo.New()
|
||||
router.Pre(middleware.RemoveTrailingSlash())
|
||||
router.Pre(middleware.Logger())
|
||||
router.Pre(middleware.Recover())
|
||||
router.GET("/swagger/*", swagger.WrapHandler)
|
||||
|
||||
v1 := router.Group("/api/v1")
|
||||
articles := v1.Group("/articles")
|
||||
articles.GET("/", s.listArticles)
|
||||
articles.GET("/:id", s.getArticle)
|
||||
articles.GET("/:id/details", s.getArticleDetails)
|
||||
articles.GET("/by/source/:id", s.ListArticlesBySourceId)
|
||||
articles.GET("", s.listArticles)
|
||||
articles.GET(":id", s.getArticle)
|
||||
articles.GET(":id/details", s.getArticleDetails)
|
||||
articles.GET("by/source/:id", s.ListArticlesBySourceId)
|
||||
|
||||
//dwh := v1.Group("/discord/webhooks")
|
||||
//dwh.GET("/", s.ListDiscordWebHooks)
|
||||
@ -73,7 +76,7 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
|
||||
//settings.GET("/", s.getSettings)
|
||||
|
||||
sources := v1.Group("/sources")
|
||||
sources.GET("/", s.listSources)
|
||||
sources.GET("", s.listSources)
|
||||
sources.GET("/by/source", s.listSourcesBySource)
|
||||
sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName)
|
||||
//sources.POST("/new/reddit", s.newRedditSource)
|
||||
@ -93,6 +96,7 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
|
||||
subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
||||
subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
||||
|
||||
s.Router = router
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
@ -3,7 +3,6 @@ package v1
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
@ -17,7 +16,7 @@ type ListDiscordWebHooksQueueResults struct {
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Tags Queue
|
||||
// @Router /queue/discord/webhooks [get]
|
||||
// @Router /v1/queue/discord/webhooks [get]
|
||||
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
|
||||
func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
||||
p := ListDiscordWebHooksQueueResults{
|
||||
@ -28,13 +27,13 @@ func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
||||
}
|
||||
|
||||
// Get the raw resp from sql
|
||||
res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
||||
if err != nil {
|
||||
return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
//res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
||||
//if err != nil {
|
||||
// return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
||||
// Message: err.Error(),
|
||||
// })
|
||||
//}
|
||||
|
||||
p.Payload = res
|
||||
//p.Payload = res
|
||||
return c.JSON(http.StatusOK, p)
|
||||
}
|
||||
|
||||
@ -14,7 +14,7 @@ import (
|
||||
// @Param key path string true "Settings Key value"
|
||||
// @Produce application/json
|
||||
// @Tags Settings
|
||||
// @Router /settings/{key} [get]
|
||||
// @Router /v1/settings/{key} [get]
|
||||
func (s *Handler) getSettings(c echo.Context) error {
|
||||
id := c.Param("ID")
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ type GetSource struct {
|
||||
// @Param page query string false "page number"
|
||||
// @Produce application/json
|
||||
// @Tags Source
|
||||
// @Router /sources [get]
|
||||
// @Router /v1/sources [get]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse "Unable to reach SQL or Data problems"
|
||||
func (s *Handler) listSources(c echo.Context) error {
|
||||
@ -62,7 +62,7 @@ func (s *Handler) listSources(c echo.Context) error {
|
||||
// @Param page query string false "page number"
|
||||
// @Produce application/json
|
||||
// @Tags Source
|
||||
// @Router /sources/by/source [get]
|
||||
// @Router /v1/sources/by/source [get]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -100,7 +100,7 @@ func (s *Handler) listSourcesBySource(c echo.Context) error {
|
||||
// @Param id path int true "uuid"
|
||||
// @Produce application/json
|
||||
// @Tags Source
|
||||
// @Router /sources/{id} [get]
|
||||
// @Router /v1/sources/{id} [get]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -135,7 +135,7 @@ func (s *Handler) getSource(c echo.Context) error {
|
||||
// @Param source query string true "reddit"
|
||||
// @Produce application/json
|
||||
// @Tags Source
|
||||
// @Router /sources/by/sourceAndName [get]
|
||||
// @Router /v1/sources/by/sourceAndName [get]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -170,7 +170,7 @@ func (s *Handler) GetSourceBySourceAndName(c echo.Context) error {
|
||||
// @Param name query string true "name"
|
||||
// @Param url query string true "url"
|
||||
// @Tags Source
|
||||
// @Router /sources/new/reddit [post]
|
||||
// @Router /v1/sources/new/reddit [post]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -226,7 +226,7 @@ func (s *Handler) newRedditSource(c echo.Context) error {
|
||||
// @Param name query string true "name"
|
||||
// @Param url query string true "url"
|
||||
// @Tags Source
|
||||
// @Router /sources/new/youtube [post]
|
||||
// @Router /v1/sources/new/youtube [post]
|
||||
func (s *Handler) newYoutubeSource(c echo.Context) error {
|
||||
var param domain.NewSourceParamRequest
|
||||
err := c.Bind(¶m)
|
||||
@ -289,7 +289,7 @@ func (s *Handler) newYoutubeSource(c echo.Context) error {
|
||||
// @Summary Creates a new twitch source to monitor.
|
||||
// @Param name query string true "name"
|
||||
// @Tags Source
|
||||
// @Router /sources/new/twitch [post]
|
||||
// @Router /v1/sources/new/twitch [post]
|
||||
func (s *Handler) newTwitchSource(c echo.Context) error {
|
||||
var param domain.NewSourceParamRequest
|
||||
err := c.Bind(¶m)
|
||||
@ -337,7 +337,7 @@ func (s *Handler) newTwitchSource(c echo.Context) error {
|
||||
// @Param name query string true "Site Name"
|
||||
// @Param url query string true "RSS Url"
|
||||
// @Tags Source
|
||||
// @Router /sources/new/rss [post]
|
||||
// @Router /v1/sources/new/rss [post]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -387,7 +387,7 @@ func (s *Handler) newRssSource(c echo.Context) error {
|
||||
// @Summary Marks a source as deleted based on its ID value.
|
||||
// @Param id path string true "id"
|
||||
// @Tags Source
|
||||
// @Router /sources/{id} [POST]
|
||||
// @Router /v1/sources/{id} [POST]
|
||||
func (s *Handler) deleteSources(c echo.Context) error {
|
||||
id := c.Param("ID")
|
||||
uuid, err := uuid.Parse(id)
|
||||
@ -432,7 +432,7 @@ func (s *Handler) deleteSources(c echo.Context) error {
|
||||
// @Summary Disables a source from processing.
|
||||
// @Param id path int true "id"
|
||||
// @Tags Source
|
||||
// @Router /sources/{id}/disable [post]
|
||||
// @Router /v1/sources/{id}/disable [post]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
@ -474,7 +474,7 @@ func (s *Handler) disableSource(c echo.Context) error {
|
||||
// @Summary Enables a source to continue processing.
|
||||
// @Param id path string true "id"
|
||||
// @Tags Source
|
||||
// @Router /sources/{id}/enable [post]
|
||||
// @Router /v1/sources/{id}/enable [post]
|
||||
// @Success 200 {object} domain.SourcesResponse "ok"
|
||||
// @Failure 400 {object} domain.BaseResponse
|
||||
// @Failure 500 {object} domain.BaseResponse
|
||||
|
||||
@ -31,7 +31,7 @@ type ListSubscriptionDetails struct {
|
||||
// @Summary Returns the top 100 entries from the queue to be processed.
|
||||
// @Produce application/json
|
||||
// @Tags Subscription
|
||||
// @Router /subscriptions [get]
|
||||
// @Router /v1/subscriptions [get]
|
||||
// @Success 200 {object} ListSubscriptions "ok"
|
||||
// @Failure 400 {object} ApiError "Unable to reach SQL."
|
||||
// @Failure 500 {object} ApiError "Failed to process data from SQL."
|
||||
@ -43,12 +43,11 @@ func (s *Handler) ListSubscriptions(c echo.Context) error {
|
||||
},
|
||||
}
|
||||
|
||||
res, err := s.dto.ListSubscriptions(c.Request().Context(), 50)
|
||||
if err != nil {
|
||||
return s.WriteError(c, err, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
payload.Payload = res
|
||||
//res, err := s.dto.ListSubscriptions(c.Request().Context(), 50)
|
||||
//if err != nil {
|
||||
// return s.WriteError(c, err, http.StatusBadRequest)
|
||||
//}
|
||||
//payload.Payload = res
|
||||
return c.JSON(http.StatusOK, payload)
|
||||
}
|
||||
|
||||
@ -56,7 +55,7 @@ func (s *Handler) ListSubscriptions(c echo.Context) error {
|
||||
// @Summary Returns the top 50 entries with full deatils on the source and output.
|
||||
// @Produce application/json
|
||||
// @Tags Subscription
|
||||
// @Router /subscriptions/details [get]
|
||||
// @Router /v1/subscriptions/details [get]
|
||||
// @Success 200 {object} ListSubscriptionDetails "ok"
|
||||
func (s *Handler) ListSubscriptionDetails(c echo.Context) error {
|
||||
payload := ListSubscriptionDetails{
|
||||
@ -66,12 +65,11 @@ func (s *Handler) ListSubscriptionDetails(c echo.Context) error {
|
||||
},
|
||||
}
|
||||
|
||||
res, err := s.dto.ListSubscriptionDetails(c.Request().Context(), 50)
|
||||
if err != nil {
|
||||
return s.WriteError(c, err, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
payload.Payload = res
|
||||
//res, err := s.dto.ListSubscriptionDetails(c.Request().Context(), 50)
|
||||
//if err != nil {
|
||||
// return s.WriteError(c, err, http.StatusInternalServerError)
|
||||
//}
|
||||
//payload.Payload = res
|
||||
return c.JSON(http.StatusOK, payload)
|
||||
}
|
||||
|
||||
@ -80,7 +78,7 @@ func (s *Handler) ListSubscriptionDetails(c echo.Context) error {
|
||||
// @Produce application/json
|
||||
// @Param id query string true "id"
|
||||
// @Tags Subscription
|
||||
// @Router /subscriptions/by/discordId [get]
|
||||
// @Router /v1/subscriptions/by/discordId [get]
|
||||
// @Success 200 {object} ListSubscriptions "ok"
|
||||
// @Failure 400 {object} ApiError "Unable to reach SQL or Data problems"
|
||||
// @Failure 500 {object} ApiError "Data problems"
|
||||
@ -97,18 +95,16 @@ func (s *Handler) GetSubscriptionsByDiscordId(c echo.Context) error {
|
||||
return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest)
|
||||
}
|
||||
|
||||
uuid, err := uuid.Parse(id)
|
||||
if err != nil {
|
||||
return s.WriteError(c, errors.New(ErrValueNotUuid), http.StatusBadRequest)
|
||||
//uuid, err := uuid.Parse(id)
|
||||
//if err != nil {
|
||||
// return s.WriteError(c, errors.New(ErrValueNotUuid), http.StatusBadRequest)
|
||||
//}
|
||||
|
||||
}
|
||||
|
||||
res, err := s.dto.ListSubscriptionsByDiscordWebhookId(context.Background(), uuid)
|
||||
if err != nil {
|
||||
return s.WriteError(c, err, http.StatusNoContent)
|
||||
}
|
||||
|
||||
p.Payload = res
|
||||
//res, err := s.dto.ListSubscriptionsByDiscordWebhookId(context.Background(), uuid)
|
||||
//if err != nil {
|
||||
// return s.WriteError(c, err, http.StatusNoContent)
|
||||
//}
|
||||
//p.Payload = res
|
||||
return c.JSON(http.StatusOK, p)
|
||||
}
|
||||
|
||||
@ -117,7 +113,7 @@ func (s *Handler) GetSubscriptionsByDiscordId(c echo.Context) error {
|
||||
// @Produce application/json
|
||||
// @Param id query string true "id"
|
||||
// @Tags Subscription
|
||||
// @Router /subscriptions/by/SourceId [get]
|
||||
// @Router /v1/subscriptions/by/SourceId [get]
|
||||
// @Success 200 {object} ListSubscriptions "ok"
|
||||
func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error {
|
||||
p := ListSubscriptions{
|
||||
@ -132,17 +128,16 @@ func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error {
|
||||
return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest)
|
||||
}
|
||||
|
||||
uuid, err := uuid.Parse(_id)
|
||||
if err != nil {
|
||||
return s.WriteError(c, err, http.StatusBadRequest)
|
||||
}
|
||||
//uuid, err := uuid.Parse(_id)
|
||||
//if err != nil {
|
||||
// return s.WriteError(c, err, http.StatusBadRequest)
|
||||
//}
|
||||
|
||||
res, err := s.dto.ListSubscriptionsBySourceId(context.Background(), uuid)
|
||||
if err != nil {
|
||||
return s.WriteError(c, err, http.StatusNoContent)
|
||||
}
|
||||
|
||||
p.Payload = res
|
||||
//res, err := s.dto.ListSubscriptionsBySourceId(context.Background(), uuid)
|
||||
//if err != nil {
|
||||
// return s.WriteError(c, err, http.StatusNoContent)
|
||||
//}
|
||||
//p.Payload = res
|
||||
return c.JSON(http.StatusOK, p)
|
||||
}
|
||||
|
||||
@ -151,7 +146,7 @@ func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error {
|
||||
// @Param discordWebHookId query string true "discordWebHookId"
|
||||
// @Param sourceId query string true "sourceId"
|
||||
// @Tags Subscription
|
||||
// @Router /subscriptions/discord/webhook/new [post]
|
||||
// @Router /v1/subscriptions/discord/webhook/new [post]
|
||||
func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error {
|
||||
// Extract the values given
|
||||
discordWebHookId := c.QueryParam("discordWebHookId")
|
||||
@ -207,7 +202,7 @@ func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error {
|
||||
// @Summary Removes a Discord WebHook Subscription based on the Subscription ID.
|
||||
// @Param id query string true "id"
|
||||
// @Tags Subscription
|
||||
// @Router /subscriptions/discord/webhook/delete [delete]
|
||||
// @Router /v1/subscriptions/discord/webhook/delete [delete]
|
||||
func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error {
|
||||
var ErrMissingSubscriptionID string = "the request was missing a 'Id'"
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ type ArticlesRepo interface {
|
||||
ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]domain.ArticleEntity, error)
|
||||
ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]domain.ArticleEntity, error)
|
||||
Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error)
|
||||
CreateFromEntity(ctx context.Context, entity domain.ArticleEntity) (int64, error)
|
||||
}
|
||||
|
||||
type ArticleRepository struct {
|
||||
@ -192,6 +193,22 @@ func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, ti
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity domain.ArticleEntity) (int64, error) {
|
||||
dt := time.Now()
|
||||
queryBuilder := sqlbuilder.NewInsertBuilder()
|
||||
queryBuilder.InsertInto("articles")
|
||||
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
|
||||
queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl)
|
||||
query, args := queryBuilder.Build()
|
||||
|
||||
_, err := ar.conn.ExecContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (ur ArticleRepository) processRows(rows *sql.Rows) []domain.ArticleEntity {
|
||||
items := []domain.ArticleEntity{}
|
||||
|
||||
|
||||
@ -11,9 +11,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
ServerAddress = "SERVER_ADDRESS"
|
||||
ServerAddress = "ServerAddress"
|
||||
|
||||
Sql_Connection_String = "SQL_CONNECTION_STRING"
|
||||
//Sql_Connection_String = "SQL_CONNECTION_STRING"
|
||||
|
||||
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
|
||||
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||
|
||||
220
internal/services/cron/collectors.go
Normal file
220
internal/services/cron/collectors.go
Normal file
@ -0,0 +1,220 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
)
|
||||
|
||||
func (c *Cron) CollectRssPosts() {
|
||||
log.Println("Starting ")
|
||||
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for sourceIndex, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
rssClient := input.NewRssClient(source)
|
||||
articles, err := rssClient.GetArticles()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for _, article := range articles {
|
||||
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if rowsCreated != 1 {
|
||||
log.Println("Got back the wrong number of rows")
|
||||
}
|
||||
}
|
||||
|
||||
if sourceIndex != len(sources) {
|
||||
time.Sleep(time.Second * 30)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cron) CollectRedditPosts() {
|
||||
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
|
||||
rc := input.NewRedditClient(source)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
for _, article := range redditArticles {
|
||||
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
|
||||
if err != nil {
|
||||
log.Printf("Failed to add a new reddit article to the database: %s", err)
|
||||
}
|
||||
|
||||
if rowsAdded != 1 {
|
||||
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Print("[Reddit] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CollectYoutubePosts() {
|
||||
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
|
||||
if err != nil {
|
||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for sourceIndex, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
|
||||
yc := input.NewYoutubeClient(source)
|
||||
raw, err := yc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for _, article := range raw {
|
||||
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
|
||||
if err != nil {
|
||||
log.Printf("Failed to add a new youtube article to the database: %s", err)
|
||||
}
|
||||
|
||||
if rowsAdded != 1 {
|
||||
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
|
||||
}
|
||||
}
|
||||
|
||||
if sourceIndex != len(sources) {
|
||||
time.Sleep(time.Second * 30)
|
||||
}
|
||||
}
|
||||
log.Print("[YouTube] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CollectFfxivPosts() {
|
||||
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
|
||||
if err != nil {
|
||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for sourceIndex, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
fc := input.NewFFXIVClient(source)
|
||||
items, err := fc.CheckSource()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for _, article := range items {
|
||||
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
|
||||
if err != nil {
|
||||
log.Printf("Failed to add a new FFXIV article to the database: %s", err)
|
||||
}
|
||||
|
||||
if rowsAdded != 1 {
|
||||
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
|
||||
}
|
||||
}
|
||||
|
||||
if sourceIndex != len(sources) {
|
||||
time.Sleep(time.Second * 30)
|
||||
}
|
||||
}
|
||||
log.Printf("[FFXIV Done!]")
|
||||
}
|
||||
|
||||
func (c *Cron) CollectTwitchPosts() {
|
||||
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
|
||||
if err != nil {
|
||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for sourceIndex, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
|
||||
tc.ReplaceSourceRecord(source)
|
||||
items, err := tc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for _, article := range items {
|
||||
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
|
||||
if err != nil {
|
||||
log.Printf("Failed to add a new Twitch article to the database: %s", err)
|
||||
}
|
||||
|
||||
if rowsAdded != 1 {
|
||||
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
|
||||
}
|
||||
}
|
||||
|
||||
if sourceIndex != len(sources) {
|
||||
time.Sleep(time.Second * 30)
|
||||
}
|
||||
}
|
||||
|
||||
log.Print("[Twitch] Done!")
|
||||
}
|
||||
43
internal/services/cron/collectors_test.go
Normal file
43
internal/services/cron/collectors_test.go
Normal file
@ -0,0 +1,43 @@
|
||||
package cron_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
)
|
||||
|
||||
func TestRssPullsCorrectly(t *testing.T) {
|
||||
conn, err := setupInMemoryDb()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
t.FailNow()
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
db := services.NewRepositoryService(conn)
|
||||
rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
if rowsCreated != 1 {
|
||||
t.Error("failed to create the source record")
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
client := cron.NewScheduler(ctx, conn)
|
||||
client.CollectRssPosts()
|
||||
|
||||
articles, err := db.Articles.ListByPage(ctx, 0, 100)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
t.Log(len(articles))
|
||||
}
|
||||
@ -3,83 +3,37 @@ package cron
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
|
||||
)
|
||||
|
||||
type Cron struct {
|
||||
Db *database.Queries
|
||||
ctx *context.Context
|
||||
ctx context.Context
|
||||
timer *cron.Cron
|
||||
repo services.RepositoryService
|
||||
}
|
||||
|
||||
func openDatabase() (*database.Queries, error) {
|
||||
_env := services.NewConfig()
|
||||
connString := _env.GetConfig(services.Sql_Connection_String)
|
||||
if connString == "" {
|
||||
panic("Connection String is null!")
|
||||
}
|
||||
db, err := sql.Open("postgres", connString)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
queries := database.New(db)
|
||||
return queries, err
|
||||
}
|
||||
|
||||
func NewScheduler(ctx context.Context) *Cron {
|
||||
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
|
||||
c := &Cron{
|
||||
ctx: &ctx,
|
||||
ctx: ctx,
|
||||
repo: services.NewRepositoryService(conn),
|
||||
}
|
||||
|
||||
timer := cron.New()
|
||||
queries, err := openDatabase()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
c.Db = queries
|
||||
|
||||
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||
features := services.NewConfig()
|
||||
//features := services.GetEnvConfig()
|
||||
|
||||
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
||||
log.Print("[Input] Reddit backend was enabled")
|
||||
//go c.CheckReddit()
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
||||
log.Print("[Input] YouTube backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
||||
log.Print("[Input] FFXIV backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
||||
log.Print("[Input] Twitch backend was enabled")
|
||||
}
|
||||
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
||||
log.Print("[Output] Discord Output was enabled")
|
||||
timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
|
||||
//timer.AddFunc("10 * * * *", c.CollectRedditPosts)
|
||||
//timer.AddFunc("15 * * * *", c.CheckYoutube)
|
||||
//timer.AddFunc("20 * * * *", c.CheckFfxiv)
|
||||
//timer.AddFunc("25 * * * *", c.CheckTwitch)
|
||||
//timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
|
||||
|
||||
c.timer = timer
|
||||
return c
|
||||
@ -93,105 +47,8 @@ func (c *Cron) Stop() {
|
||||
c.timer.Stop()
|
||||
}
|
||||
|
||||
// This is the main entry point to query all the reddit services
|
||||
func (c *Cron) CheckReddit() {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[Reddit] Checking '%v'...", source.Name)
|
||||
rc := input.NewRedditClient(source)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
c.checkPosts(redditArticles, "Reddit")
|
||||
}
|
||||
log.Print("[Reddit] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckYoutube() {
|
||||
// Add call to the db to request youtube sources.
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
|
||||
if err != nil {
|
||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[YouTube] Checking '%v'...", source.Name)
|
||||
yc := input.NewYoutubeClient(source)
|
||||
raw, err := yc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(raw, "YouTube")
|
||||
}
|
||||
log.Print("[YouTube] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckFfxiv() {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
|
||||
if err != nil {
|
||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
fc := input.NewFFXIVClient(source)
|
||||
items, err := fc.CheckSource()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(items, "FFXIV")
|
||||
}
|
||||
log.Printf("[FFXIV Done!]")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckTwitch() error {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
|
||||
if err != nil {
|
||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[Twitch] Checking '%v'...", source.Name)
|
||||
tc.ReplaceSourceRecord(source)
|
||||
items, err := tc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(items, "Twitch")
|
||||
}
|
||||
|
||||
log.Print("[Twitch] Done!")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) CheckDiscordQueue() error {
|
||||
/*
|
||||
func (c *Cron) CheckDiscordQueue() {
|
||||
// Get items from the table
|
||||
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
||||
if err != nil {
|
||||
@ -260,55 +117,15 @@ func (c *Cron) CheckDiscordQueue() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
*/
|
||||
|
||||
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
|
||||
for _, item := range posts {
|
||||
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
|
||||
if err != nil {
|
||||
id := uuid.New()
|
||||
|
||||
err := c.postArticle(id, item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
||||
}
|
||||
|
||||
err = c.addToDiscordQueue(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
time.Sleep(30 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
|
||||
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
|
||||
ID: id,
|
||||
Sourceid: item.Sourceid,
|
||||
Tags: item.Tags,
|
||||
Title: item.Title,
|
||||
Url: item.Url,
|
||||
Pubdate: item.Pubdate,
|
||||
Video: item.Video,
|
||||
Videoheight: item.Videoheight,
|
||||
Videowidth: item.Videowidth,
|
||||
Thumbnail: item.Thumbnail,
|
||||
Description: item.Description,
|
||||
Authorname: item.Authorname,
|
||||
Authorimage: item.Authorimage,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
|
||||
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
|
||||
ID: uuid.New(),
|
||||
Articleid: Id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
|
||||
// err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
|
||||
// ID: uuid.New(),
|
||||
// Articleid: Id,
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return nil
|
||||
//}
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
package cron_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"database/sql"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
/*
|
||||
func TestInvokeTwitch(t *testing.T) {
|
||||
|
||||
}
|
||||
@ -15,7 +15,7 @@ func TestInvokeTwitch(t *testing.T) {
|
||||
func TestCheckReddit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.NewScheduler(ctx)
|
||||
c.CheckReddit()
|
||||
c.Col()
|
||||
}
|
||||
|
||||
func TestCheckYouTube(t *testing.T) {
|
||||
@ -32,3 +32,22 @@ func TestCheckTwitch(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
func setupInMemoryDb() (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = goose.SetDialect("sqlite3")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = goose.Up(db, "../../database/migrations")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
@ -1,140 +0,0 @@
|
||||
// The converter package lives between the database calls and the API calls.
|
||||
// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type DtoClient struct {
|
||||
db *database.Queries
|
||||
}
|
||||
|
||||
func NewDtoClient(db *database.Queries) *DtoClient {
|
||||
return &DtoClient{
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
|
||||
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, article := range a {
|
||||
res = append(res, c.convertArticle(article))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
|
||||
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
|
||||
Limit: limit,
|
||||
Offset: page * limit,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, article := range a {
|
||||
res = append(res, c.convertArticle(article))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) {
|
||||
a, err := c.db.GetArticleByID(ctx, ID)
|
||||
if err != nil {
|
||||
return models.ArticleDto{}, err
|
||||
}
|
||||
|
||||
return c.convertArticle(a), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) {
|
||||
a, err := c.db.GetArticleByID(ctx, ID)
|
||||
if err != nil {
|
||||
return models.ArticleDetailsDto{}, err
|
||||
}
|
||||
|
||||
s, err := c.db.GetSourceByID(ctx, a.Sourceid)
|
||||
if err != nil {
|
||||
return models.ArticleDetailsDto{}, err
|
||||
}
|
||||
|
||||
res := c.convertArticleDetails(a, s)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
|
||||
Sourceid: SourceID,
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, article := range a {
|
||||
res = append(res, c.convertArticle(article))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto {
|
||||
return models.ArticleDto{
|
||||
ID: i.ID,
|
||||
Source: i.Sourceid,
|
||||
Tags: c.SplitTags(i.Tags),
|
||||
Title: i.Title,
|
||||
Url: i.Url,
|
||||
Pubdate: i.Pubdate,
|
||||
Video: i.Video.String,
|
||||
Videoheight: i.Videoheight,
|
||||
Videowidth: i.Videoheight,
|
||||
Thumbnail: i.Thumbnail,
|
||||
Description: i.Description,
|
||||
Authorname: i.Authorname.String,
|
||||
Authorimage: i.Authorimage.String,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto {
|
||||
return models.ArticleDetailsDto{
|
||||
ID: i.ID,
|
||||
Source: c.ConvertToSource(s),
|
||||
Tags: c.SplitTags(i.Tags),
|
||||
Title: i.Title,
|
||||
Url: i.Url,
|
||||
Pubdate: i.Pubdate,
|
||||
Video: i.Video.String,
|
||||
Videoheight: i.Videoheight,
|
||||
Videowidth: i.Videoheight,
|
||||
Thumbnail: i.Thumbnail,
|
||||
Description: i.Description,
|
||||
Authorname: i.Authorname.String,
|
||||
Authorimage: i.Authorimage.String,
|
||||
}
|
||||
}
|
||||
|
||||
func (c DtoClient) SplitTags(t string) []string {
|
||||
return strings.Split(t, ", ")
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
|
||||
var res []models.DiscordWebHooksDto
|
||||
|
||||
items, err := c.db.ListDiscordWebhooks(ctx, total)
|
||||
if err != nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertDiscordWebhook(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) {
|
||||
var res models.DiscordWebHooksDto
|
||||
|
||||
item, err := c.db.GetDiscordWebHooksByID(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return c.ConvertDiscordWebhook(item), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) {
|
||||
var res []models.DiscordWebHooksDto
|
||||
|
||||
items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{
|
||||
Server: server,
|
||||
Channel: channel,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertDiscordWebhook(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto {
|
||||
return models.DiscordWebHooksDto{
|
||||
ID: i.ID,
|
||||
Url: i.Url,
|
||||
Server: i.Server,
|
||||
Channel: i.Channel,
|
||||
Enabled: i.Enabled,
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
|
||||
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) {
|
||||
var res []models.DiscordQueueDetailsDto
|
||||
|
||||
items, err := c.db.ListDiscordQueueItems(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
article, err := c.GetArticleDetails(ctx, item.ID)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res = append(res, models.DiscordQueueDetailsDto{
|
||||
ID: item.ID,
|
||||
Article: article,
|
||||
})
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto {
|
||||
return models.DiscordQueueDto{
|
||||
ID: i.ID,
|
||||
Articleid: i.Articleid,
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
|
||||
var res []models.SourceDto
|
||||
|
||||
items, err := c.db.ListSources(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertToSource(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) {
|
||||
var res []models.SourceDto
|
||||
|
||||
items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName))
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertToSource(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) {
|
||||
var res models.SourceDto
|
||||
|
||||
item, err := c.db.GetSourceByID(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return c.ConvertToSource(item), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) {
|
||||
var res models.SourceDto
|
||||
|
||||
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
|
||||
Name: name,
|
||||
Source: source,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return c.ConvertToSource(item), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto {
|
||||
var deleted bool
|
||||
if !i.Deleted.Valid {
|
||||
deleted = true
|
||||
}
|
||||
|
||||
return models.SourceDto{
|
||||
ID: i.ID,
|
||||
Site: i.Site,
|
||||
Name: i.Name,
|
||||
Source: i.Source,
|
||||
Type: i.Type,
|
||||
Value: i.Value.String,
|
||||
Enabled: i.Enabled,
|
||||
Url: i.Url,
|
||||
Tags: c.SplitTags(i.Tags),
|
||||
Deleted: deleted,
|
||||
}
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
|
||||
var res []models.SubscriptionDto
|
||||
|
||||
items, err := c.db.ListSubscriptions(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertSubscription(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) {
|
||||
var res []models.SubscriptionDetailsDto
|
||||
|
||||
items, err := c.ListSubscriptions(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
source, err := c.GetSourceById(ctx, item.SourceId)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res = append(res, models.SubscriptionDetailsDto{
|
||||
ID: item.ID,
|
||||
Source: source,
|
||||
DiscordWebHook: dwh,
|
||||
})
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
||||
var res []models.SubscriptionDto
|
||||
|
||||
items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertSubscription(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
||||
var res []models.SubscriptionDto
|
||||
|
||||
items, err := c.db.GetSubscriptionsBySourceID(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertSubscription(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto {
|
||||
return models.SubscriptionDto{
|
||||
ID: i.ID,
|
||||
DiscordWebhookId: i.Discordwebhookid,
|
||||
SourceId: i.Sourceid,
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
@ -13,7 +12,7 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
)
|
||||
|
||||
@ -25,7 +24,7 @@ const (
|
||||
)
|
||||
|
||||
type FFXIVClient struct {
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
//SourceID uint
|
||||
//Url string
|
||||
//Region string
|
||||
@ -33,15 +32,15 @@ type FFXIVClient struct {
|
||||
cacheGroup string
|
||||
}
|
||||
|
||||
func NewFFXIVClient(Record database.Source) FFXIVClient {
|
||||
func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
|
||||
return FFXIVClient{
|
||||
record: Record,
|
||||
cacheGroup: "ffxiv",
|
||||
}
|
||||
}
|
||||
|
||||
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
var articles []database.Article
|
||||
func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
|
||||
var articles []domain.ArticleEntity
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
return articles, err
|
||||
}
|
||||
|
||||
article := database.Article{
|
||||
Sourceid: fc.record.ID,
|
||||
article := domain.ArticleEntity{
|
||||
SourceID: fc.record.ID,
|
||||
Tags: tags,
|
||||
Title: title,
|
||||
Url: link,
|
||||
Pubdate: pubDate,
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
PubDate: pubDate,
|
||||
Thumbnail: thumb,
|
||||
Description: description,
|
||||
Authorname: sql.NullString{String: authorName},
|
||||
Authorimage: sql.NullString{String: authorImage},
|
||||
AuthorName: authorName,
|
||||
AuthorImageUrl: authorImage,
|
||||
}
|
||||
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
||||
|
||||
|
||||
@ -3,16 +3,14 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var FFXIVRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Site: "ffxiv",
|
||||
Name: "Final Fantasy XIV - NA",
|
||||
Source: "ffxiv",
|
||||
var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "Final Fantasy XIV - NA",
|
||||
Source: domain.SourceCollectorFfxiv,
|
||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||
}
|
||||
|
||||
@ -2,7 +2,7 @@ package input
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
@ -35,7 +35,7 @@ func getHttpContent(uri string) ([]byte, error) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -9,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/go-rod/rod"
|
||||
@ -18,7 +16,7 @@ import (
|
||||
|
||||
type RedditClient struct {
|
||||
config RedditConfig
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
}
|
||||
|
||||
type RedditConfig struct {
|
||||
@ -27,7 +25,7 @@ type RedditConfig struct {
|
||||
PullNSFW string
|
||||
}
|
||||
|
||||
func NewRedditClient(Record database.Source) *RedditClient {
|
||||
func NewRedditClient(Record domain.SourceEntity) *RedditClient {
|
||||
rc := RedditClient{
|
||||
record: Record,
|
||||
}
|
||||
@ -71,7 +69,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
// TODO Wire this to support the config options
|
||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
|
||||
|
||||
content, err := getHttpContent(Url)
|
||||
if err != nil {
|
||||
@ -88,10 +86,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
|
||||
var redditArticles []database.Article
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
|
||||
var redditArticles []domain.ArticleEntity
|
||||
for _, item := range items.Data.Children {
|
||||
var article database.Article
|
||||
var article domain.ArticleEntity
|
||||
article, err := rc.convertToArticle(item.Data)
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] %v", err)
|
||||
@ -104,8 +102,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []data
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
|
||||
var item database.Article
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
|
||||
var item domain.ArticleEntity
|
||||
|
||||
if source.Content == "" && source.Url != "" {
|
||||
item = rc.convertPicturePost(source)
|
||||
@ -131,65 +129,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Title: source.Title,
|
||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Pubdate: time.Now(),
|
||||
Video: sql.NullString{String: "null"},
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
PubDate: time.Now(),
|
||||
IsVideo: false,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Authorimage: sql.NullString{String: "null"},
|
||||
AuthorName: source.Author,
|
||||
AuthorImageUrl: "null",
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.Content,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.Media.RedditVideo.FallBackUrl,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// This post is nothing more then a redirect to another location.
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.UrlOverriddenByDest,
|
||||
}
|
||||
return item
|
||||
|
||||
@ -3,16 +3,14 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var RedditRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
var RedditRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "dadjokes",
|
||||
Source: domain.SourceCollectorRss,
|
||||
Url: "https://reddit.com/r/dadjokes",
|
||||
Tags: "reddit, dadjokes",
|
||||
}
|
||||
|
||||
@ -1,14 +1,16 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
type FeedInput interface {
|
||||
GetArticles() (domain.ArticleEntity, error)
|
||||
}
|
||||
|
||||
type rssClient struct {
|
||||
SourceRecord domain.SourceEntity
|
||||
}
|
||||
@ -21,39 +23,55 @@ func NewRssClient(sourceRecord domain.SourceEntity) rssClient {
|
||||
return client
|
||||
}
|
||||
|
||||
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) {
|
||||
//rc.SourceRecord = source
|
||||
//}
|
||||
|
||||
func (rc rssClient) getCacheGroup() string {
|
||||
return fmt.Sprintf("rss-%v", rc.SourceRecord.DisplayName)
|
||||
}
|
||||
|
||||
func (rc rssClient) GetContent() error {
|
||||
feed, err := rc.PullFeed()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
|
||||
|
||||
for _, item := range feed.Items {
|
||||
log.Println(item)
|
||||
|
||||
cacheClient.FindByValue(item.Link)
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
|
||||
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
|
||||
fp := gofeed.NewParser()
|
||||
feed, err := fp.ParseURL(feedUri)
|
||||
func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) {
|
||||
parser := gofeed.NewParser()
|
||||
feed, err := parser.ParseURL(rc.SourceRecord.Url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return feed, nil
|
||||
sourceTags := strings.Split(rc.SourceRecord.Tags, ",")
|
||||
var articles []domain.ArticleEntity
|
||||
for _, post := range feed.Items {
|
||||
article := domain.ArticleEntity{
|
||||
SourceID: rc.SourceRecord.ID,
|
||||
Title: post.Title,
|
||||
Description: post.Content,
|
||||
Url: post.Link,
|
||||
PubDate: *post.PublishedParsed,
|
||||
//AuthorName: post.Authors[0].Email,
|
||||
}
|
||||
|
||||
if len(post.Authors) != 0 {
|
||||
article.AuthorName = post.Authors[0].Email
|
||||
}
|
||||
|
||||
var postTags []string
|
||||
postTags = append(postTags, sourceTags...)
|
||||
postTags = append(postTags, post.Categories...)
|
||||
article.Tags = strings.Join(postTags, ",")
|
||||
|
||||
/*
|
||||
pageContent, err := getHttpContent(article.Url)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
htmlNode, err := html.Parse(bytes.NewReader(pageContent))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
htmlNode.
|
||||
|
||||
fmt.Println(htmlNode)
|
||||
*/
|
||||
|
||||
if post.Image == nil {
|
||||
article.Thumbnail = ""
|
||||
}
|
||||
|
||||
articles = append(articles, article)
|
||||
}
|
||||
|
||||
return articles, nil
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@ var rssRecord = domain.SourceEntity{
|
||||
ID: 1,
|
||||
DisplayName: "ArsTechnica",
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||
Source: domain.SourceCollectorRss,
|
||||
}
|
||||
|
||||
func TestRssClientConstructor(t *testing.T) {
|
||||
@ -19,12 +20,23 @@ func TestRssClientConstructor(t *testing.T) {
|
||||
|
||||
func TestRssGetFeed(t *testing.T) {
|
||||
client := input.NewRssClient(rssRecord)
|
||||
feed, err := client.PullFeed()
|
||||
_, err := client.GetArticles()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(feed.Items) >= 0 {
|
||||
t.Error("failed to collect items from the fees")
|
||||
}
|
||||
|
||||
func TestRssAgainstGita(t *testing.T) {
|
||||
client := input.NewRssClient(domain.SourceEntity{
|
||||
ID: 2,
|
||||
DisplayName: "Gitea - Newsbot-api",
|
||||
Source: domain.SourceCollectorRss,
|
||||
Url: "https://git.jamestombleson.com/jtom38/newsbot-api.rss",
|
||||
Tags: "rss,gitea,newsbot-api",
|
||||
})
|
||||
_, err := client.GetArticles()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,19 +1,18 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/nicklaw5/helix/v2"
|
||||
)
|
||||
|
||||
type TwitchClient struct {
|
||||
SourceRecord database.Source
|
||||
SourceRecord domain.SourceEntity
|
||||
|
||||
// config
|
||||
monitorClips string
|
||||
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
||||
}
|
||||
|
||||
// This will let you replace the bound source record to keep the same session alive.
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
|
||||
tc.SourceRecord = source
|
||||
}
|
||||
|
||||
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||
var items []domain.ArticleEntity
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
}
|
||||
|
||||
for _, video := range posts {
|
||||
var article database.Article
|
||||
var article domain.ArticleEntity
|
||||
|
||||
AuthorName, err := tc.ExtractAuthor(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
article.Authorname = sql.NullString{String: AuthorName}
|
||||
article.AuthorName = AuthorName
|
||||
|
||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
article.Authorimage = sql.NullString{String: Authorimage}
|
||||
article.AuthorImageUrl = Authorimage
|
||||
|
||||
article.Description, err = tc.ExtractDescription(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
article.Pubdate, err = tc.ExtractPubDate(video)
|
||||
article.PubDate, err = tc.ExtractPubDate(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
article.Sourceid = tc.SourceRecord.ID
|
||||
article.SourceID = tc.SourceRecord.ID
|
||||
article.Tags, err = tc.ExtractTags(video, user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
var blank helix.User
|
||||
|
||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||
Logins: []string{tc.SourceRecord.Name},
|
||||
Logins: []string{tc.SourceRecord.DisplayName},
|
||||
})
|
||||
if err != nil {
|
||||
return blank, err
|
||||
|
||||
@ -4,21 +4,20 @@ import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var TwitchSourceRecord = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "nintendo",
|
||||
Source: "Twitch",
|
||||
var TwitchSourceRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "nintendo",
|
||||
Source: domain.SourceCollectorTwitch,
|
||||
}
|
||||
|
||||
var TwitchInvalidRecord = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "EvilNintendo",
|
||||
Source: "Twitch",
|
||||
var TwitchInvalidRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "EvilNintendo",
|
||||
Source: domain.SourceCollectorTwitch,
|
||||
}
|
||||
|
||||
func TestTwitchLogin(t *testing.T) {
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -12,11 +11,11 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
type YoutubeClient struct {
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
|
||||
// internal variables at time of collection
|
||||
channelID string
|
||||
@ -37,7 +36,7 @@ var (
|
||||
|
||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||
|
||||
func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
|
||||
yc := YoutubeClient{
|
||||
record: Record,
|
||||
cacheGroup: "youtube",
|
||||
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
}
|
||||
|
||||
// CheckSource will go and run all the commands needed to process a source.
|
||||
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||
var items []domain.ArticleEntity
|
||||
docParser, err := yc.GetParser(yc.record.Url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
|
||||
parser, err := yc.GetParser(item.Link)
|
||||
if err != nil {
|
||||
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
log.Printf("[YouTube] %v", msg)
|
||||
}
|
||||
|
||||
var article = database.Article{
|
||||
Sourceid: yc.record.ID,
|
||||
var article = domain.ArticleEntity{
|
||||
SourceID: yc.record.ID,
|
||||
Tags: tags,
|
||||
Title: item.Title,
|
||||
Url: item.Link,
|
||||
Pubdate: *item.PublishedParsed,
|
||||
PubDate: *item.PublishedParsed,
|
||||
Thumbnail: thumb,
|
||||
Description: item.Description,
|
||||
Authorname: sql.NullString{String: item.Author.Name},
|
||||
Authorimage: sql.NullString{String: yc.avatarUri},
|
||||
AuthorName: item.Author.Name,
|
||||
AuthorImageUrl: yc.avatarUri,
|
||||
}
|
||||
return article
|
||||
}
|
||||
|
||||
@ -3,16 +3,14 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var YouTubeRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
var YouTubeRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "dadjokes",
|
||||
Source: domain.SourceCollectorReddit,
|
||||
Url: "https://youtube.com/gamegrumps",
|
||||
}
|
||||
|
||||
|
||||
8
makefile
8
makefile
@ -4,18 +4,18 @@ help: ## Shows this help command
|
||||
|
||||
build: ## builds the application with the current go runtime
|
||||
~/go/bin/swag f
|
||||
~/go/bin/swag i
|
||||
go build .
|
||||
~/go/bin/swag init -g cmd/server.go
|
||||
go build cmd/server.go
|
||||
|
||||
docker-build: ## Generates the docker image
|
||||
docker build -t "newsbot.collector.api" .
|
||||
docker image ls | grep newsbot.collector.api
|
||||
|
||||
migrate-dev: ## Apply sql migrations to dev db
|
||||
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
|
||||
goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up
|
||||
|
||||
migrate-dev-down: ## revert sql migrations to dev db
|
||||
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
|
||||
goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down
|
||||
|
||||
swag: ## Generates the swagger documentation with the swag tool
|
||||
~/go/bin/swag f
|
||||
|
||||
Loading…
Reference in New Issue
Block a user