jtom38/newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

features/rss-job #6

Merged
jtom38 merged 13 commits from features/rss-job into main 2024-05-02 17:41:12 -07:00
Conversation 0 Commits 13 Files Changed 41 +746 -1063
41 changed files with 746 additions and 1063 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,6 +11,7 @@ server
*.so *.so
*.dylib *.dylib
collector collector
newsbot.db
# Test binary, built with `go test -c` # Test binary, built with `go test -c`
*.test *.test

6
api.http Normal file
View File

@ -0,0 +1,6 @@
### Select Sources fro mthe top
GET http://localhost:8081/api/v1/sources/
### Select Sources by type
GET http://localhost:8081/api/v1/sources/by/source?source=rss

15
cmd/server.go
View File

@ -4,14 +4,12 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"net/http"
_ "github.com/glebarez/go-sqlite" _ "github.com/glebarez/go-sqlite"
"github.com/pressly/goose/v3" "github.com/pressly/goose/v3"
"git.jamestombleson.com/jtom38/newsbot-api/docs" "git.jamestombleson.com/jtom38/newsbot-api/docs"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
) )
@ -42,19 +40,14 @@ func main() {
panic(err) panic(err)
} }
queries := database.New(db) c := cron.NewScheduler(ctx, db)
c := cron.NewScheduler(ctx)
c.Start() c.Start()
server := v1.NewServer(ctx, queries, configs, db) server := v1.NewServer(ctx, configs, db)
fmt.Println("API is online and waiting for requests.") fmt.Println("API is online and waiting for requests.")
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress) fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress) fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress)
err = http.ListenAndServe(":8081", server.Router) server.Router.Start(":8081")
if err != nil {
panic(err)
}
} }

79
docs/docs.go
View File

@ -16,7 +16,7 @@ const docTemplate = `{
"host": "{{.Host}}", "host": "{{.Host}}",
"basePath": "{{.BasePath}}", "basePath": "{{.BasePath}}",
"paths": { "paths": {
"/articles": { "/v1/articles": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -55,7 +55,7 @@ const docTemplate = `{
} }
} }
}, },
"/articles/by/sourceid": { "/v1/articles/by/sourceid": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -101,7 +101,7 @@ const docTemplate = `{
} }
} }
}, },
"/articles/{ID}": { "/v1/articles/{ID}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -141,7 +141,7 @@ const docTemplate = `{
} }
} }
}, },
"/articles/{ID}/details": { "/v1/articles/{ID}/details": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -181,14 +181,13 @@ const docTemplate = `{
} }
} }
}, },
"/discord/webhooks": { "/v1/discord/webhooks": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Returns the top 100", "summary": "Returns the top 100",
"responses": { "responses": {
@ -213,14 +212,13 @@ const docTemplate = `{
} }
} }
}, },
"/discord/webhooks/by/serverAndChannel": { "/v1/discord/webhooks/by/serverAndChannel": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Returns all the known web hooks based on the Server and Channel given.", "summary": "Returns all the known web hooks based on the Server and Channel given.",
"parameters": [ "parameters": [
@ -261,11 +259,10 @@ const docTemplate = `{
} }
} }
}, },
"/discord/webhooks/new": { "/v1/discord/webhooks/new": {
"post": { "post": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Creates a new record for a discord web hook to post data to.", "summary": "Creates a new record for a discord web hook to post data to.",
"parameters": [ "parameters": [
@ -313,11 +310,10 @@ const docTemplate = `{
} }
} }
}, },
"/discord/webhooks/{ID}": { "/v1/discord/webhooks/{ID}": {
"delete": { "delete": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Deletes a record by ID.", "summary": "Deletes a record by ID.",
"parameters": [ "parameters": [
@ -351,11 +347,10 @@ const docTemplate = `{
} }
} }
}, },
"/discord/webhooks/{ID}/disable": { "/v1/discord/webhooks/{ID}/disable": {
"post": { "post": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Disables a Webhook from being used.", "summary": "Disables a Webhook from being used.",
"parameters": [ "parameters": [
@ -389,11 +384,10 @@ const docTemplate = `{
} }
} }
}, },
"/discord/webhooks/{ID}/enable": { "/v1/discord/webhooks/{ID}/enable": {
"post": { "post": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Enables a source to continue processing.", "summary": "Enables a source to continue processing.",
"parameters": [ "parameters": [
@ -408,14 +402,13 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/discord/webhooks/{id}": { "/v1/discord/webhooks/{id}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -449,7 +442,7 @@ const docTemplate = `{
} }
} }
}, },
"/queue/discord/webhooks": { "/v1/queue/discord/webhooks": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -468,7 +461,7 @@ const docTemplate = `{
} }
} }
}, },
"/settings/{key}": { "/v1/settings/{key}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -489,7 +482,7 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/sources": { "/v1/sources": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -522,7 +515,7 @@ const docTemplate = `{
} }
} }
}, },
"/sources/by/source": { "/v1/sources/by/source": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -568,7 +561,7 @@ const docTemplate = `{
} }
} }
}, },
"/sources/by/sourceAndName": { "/v1/sources/by/sourceAndName": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -615,7 +608,7 @@ const docTemplate = `{
} }
} }
}, },
"/sources/new/reddit": { "/v1/sources/new/reddit": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -659,7 +652,7 @@ const docTemplate = `{
} }
} }
}, },
"/sources/new/rss": { "/v1/sources/new/rss": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -703,7 +696,7 @@ const docTemplate = `{
} }
} }
}, },
"/sources/new/twitch": { "/v1/sources/new/twitch": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -721,7 +714,7 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/sources/new/youtube": { "/v1/sources/new/youtube": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -746,7 +739,7 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/sources/{id}": { "/v1/sources/{id}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -802,7 +795,7 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/sources/{id}/disable": { "/v1/sources/{id}/disable": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -839,7 +832,7 @@ const docTemplate = `{
} }
} }
}, },
"/sources/{id}/enable": { "/v1/sources/{id}/enable": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -876,7 +869,7 @@ const docTemplate = `{
} }
} }
}, },
"/subscriptions": { "/v1/subscriptions": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -907,7 +900,7 @@ const docTemplate = `{
} }
} }
}, },
"/subscriptions/by/SourceId": { "/v1/subscriptions/by/SourceId": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -935,7 +928,7 @@ const docTemplate = `{
} }
} }
}, },
"/subscriptions/by/discordId": { "/v1/subscriptions/by/discordId": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -975,7 +968,7 @@ const docTemplate = `{
} }
} }
}, },
"/subscriptions/details": { "/v1/subscriptions/details": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -994,7 +987,7 @@ const docTemplate = `{
} }
} }
}, },
"/subscriptions/discord/webhook/delete": { "/v1/subscriptions/discord/webhook/delete": {
"delete": { "delete": {
"tags": [ "tags": [
"Subscription" "Subscription"
@ -1012,7 +1005,7 @@ const docTemplate = `{
"responses": {} "responses": {}
} }
}, },
"/subscriptions/discord/webhook/new": { "/v1/subscriptions/discord/webhook/new": {
"post": { "post": {
"tags": [ "tags": [
"Subscription" "Subscription"

79
docs/swagger.json
View File

@ -7,7 +7,7 @@
}, },
"basePath": "/api", "basePath": "/api",
"paths": { "paths": {
"/articles": { "/v1/articles": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -46,7 +46,7 @@
} }
} }
}, },
"/articles/by/sourceid": { "/v1/articles/by/sourceid": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -92,7 +92,7 @@
} }
} }
}, },
"/articles/{ID}": { "/v1/articles/{ID}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -132,7 +132,7 @@
} }
} }
}, },
"/articles/{ID}/details": { "/v1/articles/{ID}/details": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -172,14 +172,13 @@
} }
} }
}, },
"/discord/webhooks": { "/v1/discord/webhooks": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Returns the top 100", "summary": "Returns the top 100",
"responses": { "responses": {
@ -204,14 +203,13 @@
} }
} }
}, },
"/discord/webhooks/by/serverAndChannel": { "/v1/discord/webhooks/by/serverAndChannel": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Returns all the known web hooks based on the Server and Channel given.", "summary": "Returns all the known web hooks based on the Server and Channel given.",
"parameters": [ "parameters": [
@ -252,11 +250,10 @@
} }
} }
}, },
"/discord/webhooks/new": { "/v1/discord/webhooks/new": {
"post": { "post": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Creates a new record for a discord web hook to post data to.", "summary": "Creates a new record for a discord web hook to post data to.",
"parameters": [ "parameters": [
@ -304,11 +301,10 @@
} }
} }
}, },
"/discord/webhooks/{ID}": { "/v1/discord/webhooks/{ID}": {
"delete": { "delete": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Deletes a record by ID.", "summary": "Deletes a record by ID.",
"parameters": [ "parameters": [
@ -342,11 +338,10 @@
} }
} }
}, },
"/discord/webhooks/{ID}/disable": { "/v1/discord/webhooks/{ID}/disable": {
"post": { "post": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Disables a Webhook from being used.", "summary": "Disables a Webhook from being used.",
"parameters": [ "parameters": [
@ -380,11 +375,10 @@
} }
} }
}, },
"/discord/webhooks/{ID}/enable": { "/v1/discord/webhooks/{ID}/enable": {
"post": { "post": {
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Enables a source to continue processing.", "summary": "Enables a source to continue processing.",
"parameters": [ "parameters": [
@ -399,14 +393,13 @@
"responses": {} "responses": {}
} }
}, },
"/discord/webhooks/{id}": { "/v1/discord/webhooks/{id}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"Discord", "DiscordWebhook"
"Webhook"
], ],
"summary": "Returns the top 100 entries from the queue to be processed.", "summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [ "parameters": [
@ -440,7 +433,7 @@
} }
} }
}, },
"/queue/discord/webhooks": { "/v1/queue/discord/webhooks": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -459,7 +452,7 @@
} }
} }
}, },
"/settings/{key}": { "/v1/settings/{key}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -480,7 +473,7 @@
"responses": {} "responses": {}
} }
}, },
"/sources": { "/v1/sources": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -513,7 +506,7 @@
} }
} }
}, },
"/sources/by/source": { "/v1/sources/by/source": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -559,7 +552,7 @@
} }
} }
}, },
"/sources/by/sourceAndName": { "/v1/sources/by/sourceAndName": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -606,7 +599,7 @@
} }
} }
}, },
"/sources/new/reddit": { "/v1/sources/new/reddit": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -650,7 +643,7 @@
} }
} }
}, },
"/sources/new/rss": { "/v1/sources/new/rss": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -694,7 +687,7 @@
} }
} }
}, },
"/sources/new/twitch": { "/v1/sources/new/twitch": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -712,7 +705,7 @@
"responses": {} "responses": {}
} }
}, },
"/sources/new/youtube": { "/v1/sources/new/youtube": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -737,7 +730,7 @@
"responses": {} "responses": {}
} }
}, },
"/sources/{id}": { "/v1/sources/{id}": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -793,7 +786,7 @@
"responses": {} "responses": {}
} }
}, },
"/sources/{id}/disable": { "/v1/sources/{id}/disable": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -830,7 +823,7 @@
} }
} }
}, },
"/sources/{id}/enable": { "/v1/sources/{id}/enable": {
"post": { "post": {
"tags": [ "tags": [
"Source" "Source"
@ -867,7 +860,7 @@
} }
} }
}, },
"/subscriptions": { "/v1/subscriptions": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -898,7 +891,7 @@
} }
} }
}, },
"/subscriptions/by/SourceId": { "/v1/subscriptions/by/SourceId": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -926,7 +919,7 @@
} }
} }
}, },
"/subscriptions/by/discordId": { "/v1/subscriptions/by/discordId": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -966,7 +959,7 @@
} }
} }
}, },
"/subscriptions/details": { "/v1/subscriptions/details": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "application/json"
@ -985,7 +978,7 @@
} }
} }
}, },
"/subscriptions/discord/webhook/delete": { "/v1/subscriptions/discord/webhook/delete": {
"delete": { "delete": {
"tags": [ "tags": [
"Subscription" "Subscription"
@ -1003,7 +996,7 @@
"responses": {} "responses": {}
} }
}, },
"/subscriptions/discord/webhook/new": { "/v1/subscriptions/discord/webhook/new": {
"post": { "post": {
"tags": [ "tags": [
"Subscription" "Subscription"

79
docs/swagger.yaml
View File

@ -241,7 +241,7 @@ info:
title: NewsBot collector title: NewsBot collector
version: "0.1" version: "0.1"
paths: paths:
/articles: /v1/articles:
get: get:
parameters: parameters:
- description: page number - description: page number
@ -266,7 +266,7 @@ paths:
summary: Lists the top 25 records ordering from newest to oldest. summary: Lists the top 25 records ordering from newest to oldest.
tags: tags:
- Articles - Articles
/articles/{ID}: /v1/articles/{ID}:
get: get:
parameters: parameters:
- description: int - description: int
@ -292,7 +292,7 @@ paths:
summary: Returns an article based on defined ID. summary: Returns an article based on defined ID.
tags: tags:
- Articles - Articles
/articles/{ID}/details: /v1/articles/{ID}/details:
get: get:
parameters: parameters:
- description: int - description: int
@ -318,7 +318,7 @@ paths:
summary: Returns an article and source based on defined ID. summary: Returns an article and source based on defined ID.
tags: tags:
- Articles - Articles
/articles/by/sourceid: /v1/articles/by/sourceid:
get: get:
parameters: parameters:
- description: source id - description: source id
@ -349,7 +349,7 @@ paths:
25. 25.
tags: tags:
- Articles - Articles
/discord/webhooks: /v1/discord/webhooks:
get: get:
produces: produces:
- application/json - application/json
@ -368,9 +368,8 @@ paths:
$ref: '#/definitions/domain.BaseResponse' $ref: '#/definitions/domain.BaseResponse'
summary: Returns the top 100 summary: Returns the top 100
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/discord/webhooks/{ID}:
/discord/webhooks/{ID}:
delete: delete:
parameters: parameters:
- description: id - description: id
@ -393,9 +392,8 @@ paths:
$ref: '#/definitions/domain.BaseResponse' $ref: '#/definitions/domain.BaseResponse'
summary: Deletes a record by ID. summary: Deletes a record by ID.
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/discord/webhooks/{ID}/disable:
/discord/webhooks/{ID}/disable:
post: post:
parameters: parameters:
- description: id - description: id
@ -418,9 +416,8 @@ paths:
$ref: '#/definitions/domain.BaseResponse' $ref: '#/definitions/domain.BaseResponse'
summary: Disables a Webhook from being used. summary: Disables a Webhook from being used.
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/discord/webhooks/{ID}/enable:
/discord/webhooks/{ID}/enable:
post: post:
parameters: parameters:
- description: id - description: id
@ -431,9 +428,8 @@ paths:
responses: {} responses: {}
summary: Enables a source to continue processing. summary: Enables a source to continue processing.
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/discord/webhooks/{id}:
/discord/webhooks/{id}:
get: get:
parameters: parameters:
- description: id - description: id
@ -458,9 +454,8 @@ paths:
$ref: '#/definitions/domain.BaseResponse' $ref: '#/definitions/domain.BaseResponse'
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/discord/webhooks/by/serverAndChannel:
/discord/webhooks/by/serverAndChannel:
get: get:
parameters: parameters:
- description: Fancy Server - description: Fancy Server
@ -490,9 +485,8 @@ paths:
$ref: '#/definitions/domain.BaseResponse' $ref: '#/definitions/domain.BaseResponse'
summary: Returns all the known web hooks based on the Server and Channel given. summary: Returns all the known web hooks based on the Server and Channel given.
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/discord/webhooks/new:
/discord/webhooks/new:
post: post:
parameters: parameters:
- description: url - description: url
@ -525,9 +519,8 @@ paths:
$ref: '#/definitions/domain.BaseResponse' $ref: '#/definitions/domain.BaseResponse'
summary: Creates a new record for a discord web hook to post data to. summary: Creates a new record for a discord web hook to post data to.
tags: tags:
- Discord - DiscordWebhook
- Webhook /v1/queue/discord/webhooks:
/queue/discord/webhooks:
get: get:
produces: produces:
- application/json - application/json
@ -539,7 +532,7 @@ paths:
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- Queue - Queue
/settings/{key}: /v1/settings/{key}:
get: get:
parameters: parameters:
- description: Settings Key value - description: Settings Key value
@ -553,7 +546,7 @@ paths:
summary: Returns a object based on the Key that was given. summary: Returns a object based on the Key that was given.
tags: tags:
- Settings - Settings
/sources: /v1/sources:
get: get:
parameters: parameters:
- description: page number - description: page number
@ -574,7 +567,7 @@ paths:
summary: Lists the top 50 records summary: Lists the top 50 records
tags: tags:
- Source - Source
/sources/{id}: /v1/sources/{id}:
get: get:
parameters: parameters:
- description: uuid - description: uuid
@ -611,7 +604,7 @@ paths:
summary: Marks a source as deleted based on its ID value. summary: Marks a source as deleted based on its ID value.
tags: tags:
- Source - Source
/sources/{id}/disable: /v1/sources/{id}/disable:
post: post:
parameters: parameters:
- description: id - description: id
@ -635,7 +628,7 @@ paths:
summary: Disables a source from processing. summary: Disables a source from processing.
tags: tags:
- Source - Source
/sources/{id}/enable: /v1/sources/{id}/enable:
post: post:
parameters: parameters:
- description: id - description: id
@ -659,7 +652,7 @@ paths:
summary: Enables a source to continue processing. summary: Enables a source to continue processing.
tags: tags:
- Source - Source
/sources/by/source: /v1/sources/by/source:
get: get:
parameters: parameters:
- description: Source Name - description: Source Name
@ -689,7 +682,7 @@ paths:
summary: 'Lists the top 50 records based on the name given. Example: reddit' summary: 'Lists the top 50 records based on the name given. Example: reddit'
tags: tags:
- Source - Source
/sources/by/sourceAndName: /v1/sources/by/sourceAndName:
get: get:
parameters: parameters:
- description: dadjokes - description: dadjokes
@ -720,7 +713,7 @@ paths:
summary: Returns a single entity by ID summary: Returns a single entity by ID
tags: tags:
- Source - Source
/sources/new/reddit: /v1/sources/new/reddit:
post: post:
parameters: parameters:
- description: name - description: name
@ -749,7 +742,7 @@ paths:
summary: Creates a new reddit source to monitor. summary: Creates a new reddit source to monitor.
tags: tags:
- Source - Source
/sources/new/rss: /v1/sources/new/rss:
post: post:
parameters: parameters:
- description: Site Name - description: Site Name
@ -778,7 +771,7 @@ paths:
summary: Creates a new rss source to monitor. summary: Creates a new rss source to monitor.
tags: tags:
- Source - Source
/sources/new/twitch: /v1/sources/new/twitch:
post: post:
parameters: parameters:
- description: name - description: name
@ -790,7 +783,7 @@ paths:
summary: Creates a new twitch source to monitor. summary: Creates a new twitch source to monitor.
tags: tags:
- Source - Source
/sources/new/youtube: /v1/sources/new/youtube:
post: post:
parameters: parameters:
- description: name - description: name
@ -807,7 +800,7 @@ paths:
summary: Creates a new youtube source to monitor. summary: Creates a new youtube source to monitor.
tags: tags:
- Source - Source
/subscriptions: /v1/subscriptions:
get: get:
produces: produces:
- application/json - application/json
@ -827,7 +820,7 @@ paths:
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- Subscription - Subscription
/subscriptions/by/SourceId: /v1/subscriptions/by/SourceId:
get: get:
parameters: parameters:
- description: id - description: id
@ -845,7 +838,7 @@ paths:
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- Subscription - Subscription
/subscriptions/by/discordId: /v1/subscriptions/by/discordId:
get: get:
parameters: parameters:
- description: id - description: id
@ -871,7 +864,7 @@ paths:
summary: Returns the top 100 entries from the queue to be processed. summary: Returns the top 100 entries from the queue to be processed.
tags: tags:
- Subscription - Subscription
/subscriptions/details: /v1/subscriptions/details:
get: get:
produces: produces:
- application/json - application/json
@ -883,7 +876,7 @@ paths:
summary: Returns the top 50 entries with full deatils on the source and output. summary: Returns the top 50 entries with full deatils on the source and output.
tags: tags:
- Subscription - Subscription
/subscriptions/discord/webhook/delete: /v1/subscriptions/discord/webhook/delete:
delete: delete:
parameters: parameters:
- description: id - description: id
@ -895,7 +888,7 @@ paths:
summary: Removes a Discord WebHook Subscription based on the Subscription ID. summary: Removes a Discord WebHook Subscription based on the Subscription ID.
tags: tags:
- Subscription - Subscription
/subscriptions/discord/webhook/new: /v1/subscriptions/discord/webhook/new:
post: post:
parameters: parameters:
- description: discordWebHookId - description: discordWebHookId

3
go.mod
View File

@ -5,7 +5,6 @@ go 1.22
require ( require (
github.com/PuerkitoBio/goquery v1.8.0 github.com/PuerkitoBio/goquery v1.8.0
github.com/glebarez/go-sqlite v1.22.0 github.com/glebarez/go-sqlite v1.22.0
github.com/go-chi/chi/v5 v5.0.7
github.com/go-rod/rod v0.107.1 github.com/go-rod/rod v0.107.1
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/huandu/go-sqlbuilder v1.27.1 github.com/huandu/go-sqlbuilder v1.27.1
@ -23,6 +22,7 @@ require (
require ( require (
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ghodss/yaml v1.0.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/huandu/xstrings v1.3.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect
github.com/labstack/gommon v0.4.2 // indirect github.com/labstack/gommon v0.4.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
@ -36,6 +36,7 @@ require (
github.com/valyala/fasttemplate v1.2.2 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sync v0.7.0 // indirect golang.org/x/sync v0.7.0 // indirect
golang.org/x/time v0.5.0 // indirect
modernc.org/libc v1.41.0 // indirect modernc.org/libc v1.41.0 // indirect
modernc.org/mathutil v1.6.0 // indirect modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.7.2 // indirect modernc.org/memory v1.7.2 // indirect

6
go.sum
View File

@ -18,8 +18,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@ -33,6 +31,8 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK
github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY= github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY=
github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg= github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@ -163,6 +163,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=

8
internal/database/migrations/20240425083756_init.sql
View File

@ -12,8 +12,6 @@ CREATE TABLE Articles (
Url TEXT NOT NULL, Url TEXT NOT NULL,
PubDate DATETIME NOT NULL, PubDate DATETIME NOT NULL,
IsVideo TEXT NOT NULL, IsVideo TEXT NOT NULL,
--VideoHeight int NOT NULL,
--VideoWidth int NOT NULL,
ThumbnailUrl TEXT NOT NULL, ThumbnailUrl TEXT NOT NULL,
Description TEXT NOT NULL, Description TEXT NOT NULL,
AuthorName TEXT NOT NULL, AuthorName TEXT NOT NULL,
@ -34,6 +32,7 @@ CREATE Table DiscordWebHooks (
CreatedAt DATETIME NOT NULL, CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL, UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME NOT NULL, DeletedAt DATETIME NOT NULL,
UserID INTEGER NOT NULL,
--Name TEXT NOT NULL, -- Defines webhook purpose --Name TEXT NOT NULL, -- Defines webhook purpose
--Key TEXT, --Key TEXT,
Url TEXT NOT NULL, -- Webhook Url Url TEXT NOT NULL, -- Webhook Url
@ -78,8 +77,9 @@ CREATE TABLE Subscriptions (
CreatedAt DATETIME NOT NULL, CreatedAt DATETIME NOT NULL,
UpdatedAt DATETIME NOT NULL, UpdatedAt DATETIME NOT NULL,
DeletedAt DATETIME, DeletedAt DATETIME,
DiscordWebHookId NUMBER NOT NULL, DiscordWebHookID NUMBER NOT NULL,
SourceId NUMBER NOT NULL SourceID NUMBER NOT NULL,
UserID NUMBER NOT NULL
); );
CREATE TABLE Users ( CREATE TABLE Users (

32
internal/database/migrations/20240425092459_seed.sql
View File

@ -7,33 +7,33 @@ SELECT 'up SQL query';
-- Final Fantasy XIV Entries -- Final Fantasy XIV Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - NA', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - JP', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - EU', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - FR', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - DE', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone');
-- Reddit Entries -- Reddit Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'reddit', 'dadjokes', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes');
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'reddit', 'steamdeck', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck');
-- Youtube Entries -- Youtube Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'youtube', 'Game Grumps', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps');
-- RSS Entries -- RSS Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered', 'steam deck', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck');
-- Twitch Entries -- Twitch Entries
INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES
("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'twitch', 'Nintendo', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo'); ("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo');
-- +goose StatementEnd -- +goose StatementEnd
@ -41,10 +41,10 @@ INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabl
-- +goose StatementBegin -- +goose StatementBegin
--SELECT 'down SQL query'; --SELECT 'down SQL query';
DELETE FROM sources where source = 'reddit' and name = 'dadjokes'; DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes';
DELETE FROM sources where source = 'reddit' and name = 'steamdeck'; DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck';
DELETE FROM sources where source = 'ffxiv'; DELETE FROM sources where Source = 'ffxiv';
DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo'; DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo';
DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps'; DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps';
DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck'; DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck';
-- +goose StatementEnd -- +goose StatementEnd

8
internal/handler/v1/articles.go
View File

@ -14,7 +14,7 @@ import (
// @Produce application/json // @Produce application/json
// @Param page query string false "page number" // @Param page query string false "page number"
// @Tags Articles // @Tags Articles
// @Router /articles [get] // @Router /v1/articles [get]
// @Success 200 {object} domain.ArticleResponse // @Success 200 {object} domain.ArticleResponse
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -44,7 +44,7 @@ func (s *Handler) listArticles(c echo.Context) error {
// @Param ID path string true "int" // @Param ID path string true "int"
// @Produce application/json // @Produce application/json
// @Tags Articles // @Tags Articles
// @Router /articles/{ID} [get] // @Router /v1/articles/{ID} [get]
// @Success 200 {object} domain.ArticleResponse "OK" // @Success 200 {object} domain.ArticleResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -78,7 +78,7 @@ func (s *Handler) getArticle(c echo.Context) error {
// @Param ID path string true "int" // @Param ID path string true "int"
// @Produce application/json // @Produce application/json
// @Tags Articles // @Tags Articles
// @Router /articles/{ID}/details [get] // @Router /v1/articles/{ID}/details [get]
// @Success 200 {object} domain.ArticleDetailedResponse "OK" // @Success 200 {object} domain.ArticleDetailedResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -119,7 +119,7 @@ func (s *Handler) getArticleDetails(c echo.Context) error {
// @Param page query int false "Page to query" // @Param page query int false "Page to query"
// @Produce application/json // @Produce application/json
// @Tags Articles // @Tags Articles
// @Router /articles/by/sourceid [get] // @Router /v1/articles/by/sourceid [get]
// @Success 200 {object} domain.ArticleResponse "OK" // @Success 200 {object} domain.ArticleResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse

32
internal/handler/v1/discordwebhooks.go
View File

@ -13,8 +13,8 @@ import (
// ListDiscordWebhooks // ListDiscordWebhooks
// @Summary Returns the top 100 // @Summary Returns the top 100
// @Produce application/json // @Produce application/json
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks [get] // @Router /v1/discord/webhooks [get]
// @Success 200 {object} domain.DiscordWebhookResponse // @Success 200 {object} domain.DiscordWebhookResponse
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -37,8 +37,8 @@ func (s *Handler) ListDiscordWebHooks(c echo.Context) error {
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Param id path int true "id" // @Param id path int true "id"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/{id} [get] // @Router /v1/discord/webhooks/{id} [get]
// @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -69,8 +69,8 @@ func (s *Handler) GetDiscordWebHooksById(c echo.Context) error {
// @Produce application/json // @Produce application/json
// @Param server query string true "Fancy Server" // @Param server query string true "Fancy Server"
// @Param channel query string true "memes" // @Param channel query string true "memes"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/by/serverAndChannel [get] // @Router /v1/discord/webhooks/by/serverAndChannel [get]
// @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -105,8 +105,8 @@ func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error {
// @Param url query string true "url" // @Param url query string true "url"
// @Param server query string true "Server name" // @Param server query string true "Server name"
// @Param channel query string true "Channel name" // @Param channel query string true "Channel name"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/new [post] // @Router /v1/discord/webhooks/new [post]
// @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -164,8 +164,8 @@ func (s *Handler) NewDiscordWebHook(c echo.Context) error {
// DisableDiscordWebHooks // DisableDiscordWebHooks
// @Summary Disables a Webhook from being used. // @Summary Disables a Webhook from being used.
// @Param id path int true "id" // @Param id path int true "id"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/{ID}/disable [post] // @Router /v1/discord/webhooks/{ID}/disable [post]
// @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -212,8 +212,8 @@ func (s *Handler) disableDiscordWebHook(c echo.Context) error {
// EnableDiscordWebHook // EnableDiscordWebHook
// @Summary Enables a source to continue processing. // @Summary Enables a source to continue processing.
// @Param id path int true "id" // @Param id path int true "id"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/{ID}/enable [post] // @Router /v1/discord/webhooks/{ID}/enable [post]
func (s *Handler) enableDiscordWebHook(c echo.Context) error { func (s *Handler) enableDiscordWebHook(c echo.Context) error {
id, err := strconv.Atoi(c.Param("ID")) id, err := strconv.Atoi(c.Param("ID"))
if err != nil { if err != nil {
@ -253,8 +253,8 @@ func (s *Handler) enableDiscordWebHook(c echo.Context) error {
// DeleteDiscordWebHook // DeleteDiscordWebHook
// @Summary Deletes a record by ID. // @Summary Deletes a record by ID.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/{ID} [delete] // @Router /v1/discord/webhooks/{ID} [delete]
// @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -298,8 +298,8 @@ func (s *Handler) deleteDiscordWebHook(c echo.Context) error {
// UpdateDiscordWebHook // UpdateDiscordWebHook
// @Summary Updates a valid discord webhook ID based on the body given. // @Summary Updates a valid discord webhook ID based on the body given.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags Discord, Webhook // @Tags DiscordWebhook
// @Router /discord/webhooks/{id} [patch] // @Router /v1/discord/webhooks/{id} [patch]
// @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Success 200 {object} domain.DiscordWebhookResponse "OK"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse

26
internal/handler/v1/handler.go
View File

@ -5,19 +5,19 @@ import (
"database/sql" "database/sql"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
_ "github.com/lib/pq" "github.com/labstack/echo/v4/middleware"
swagger "github.com/swaggo/echo-swagger" swagger "github.com/swaggo/echo-swagger"
_ "git.jamestombleson.com/jtom38/newsbot-api/docs"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
) )
type Handler struct { type Handler struct {
Router *echo.Echo Router *echo.Echo
Db *database.Queries Db *database.Queries
dto *dto.DtoClient //dto *dto.DtoClient
config services.Configs config services.Configs
repo services.RepositoryService repo services.RepositoryService
} }
@ -39,23 +39,26 @@ var (
ErrUnableToConvertToJson string = "Unable to convert to json" ErrUnableToConvertToJson string = "Unable to convert to json"
) )
func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler { func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
s := &Handler{ s := &Handler{
Db: db, //Db: db,
dto: dto.NewDtoClient(db), //dto: dto.NewDtoClient(db),
config: configs, config: configs,
repo: services.NewRepositoryService(conn), repo: services.NewRepositoryService(conn),
} }
router := echo.New() router := echo.New()
router.Pre(middleware.RemoveTrailingSlash())
router.Pre(middleware.Logger())
router.Pre(middleware.Recover())
router.GET("/swagger/*", swagger.WrapHandler) router.GET("/swagger/*", swagger.WrapHandler)
v1 := router.Group("/api/v1") v1 := router.Group("/api/v1")
articles := v1.Group("/articles") articles := v1.Group("/articles")
articles.GET("/", s.listArticles) articles.GET("", s.listArticles)
articles.GET("/:id", s.getArticle) articles.GET(":id", s.getArticle)
articles.GET("/:id/details", s.getArticleDetails) articles.GET(":id/details", s.getArticleDetails)
articles.GET("/by/source/:id", s.ListArticlesBySourceId) articles.GET("by/source/:id", s.ListArticlesBySourceId)
//dwh := v1.Group("/discord/webhooks") //dwh := v1.Group("/discord/webhooks")
//dwh.GET("/", s.ListDiscordWebHooks) //dwh.GET("/", s.ListDiscordWebHooks)
@ -73,7 +76,7 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
//settings.GET("/", s.getSettings) //settings.GET("/", s.getSettings)
sources := v1.Group("/sources") sources := v1.Group("/sources")
sources.GET("/", s.listSources) sources.GET("", s.listSources)
sources.GET("/by/source", s.listSourcesBySource) sources.GET("/by/source", s.listSourcesBySource)
sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName) sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName)
//sources.POST("/new/reddit", s.newRedditSource) //sources.POST("/new/reddit", s.newRedditSource)
@ -93,6 +96,7 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription) subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription) subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
s.Router = router
return s return s
} }

17
internal/handler/v1/queue.go
View File

@ -3,7 +3,6 @@ package v1
import ( import (
"net/http" "net/http"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
) )
@ -17,7 +16,7 @@ type ListDiscordWebHooksQueueResults struct {
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Tags Queue // @Tags Queue
// @Router /queue/discord/webhooks [get] // @Router /v1/queue/discord/webhooks [get]
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok" // @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error { func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
p := ListDiscordWebHooksQueueResults{ p := ListDiscordWebHooksQueueResults{
@ -28,13 +27,13 @@ func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
} }
// Get the raw resp from sql // Get the raw resp from sql
res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50) //res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
if err != nil { //if err != nil {
return c.JSON(http.StatusInternalServerError, domain.BaseResponse{ // return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
Message: err.Error(), // Message: err.Error(),
}) // })
} //}
p.Payload = res //p.Payload = res
return c.JSON(http.StatusOK, p) return c.JSON(http.StatusOK, p)
} }

2
internal/handler/v1/settings.go
View File

@ -14,7 +14,7 @@ import (
// @Param key path string true "Settings Key value" // @Param key path string true "Settings Key value"
// @Produce application/json // @Produce application/json
// @Tags Settings // @Tags Settings
// @Router /settings/{key} [get] // @Router /v1/settings/{key} [get]
func (s *Handler) getSettings(c echo.Context) error { func (s *Handler) getSettings(c echo.Context) error {
id := c.Param("ID") id := c.Param("ID")

22
internal/handler/v1/sources.go
View File

@ -31,7 +31,7 @@ type GetSource struct {
// @Param page query string false "page number" // @Param page query string false "page number"
// @Produce application/json // @Produce application/json
// @Tags Source // @Tags Source
// @Router /sources [get] // @Router /v1/sources [get]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse "Unable to reach SQL or Data problems" // @Failure 400 {object} domain.BaseResponse "Unable to reach SQL or Data problems"
func (s *Handler) listSources(c echo.Context) error { func (s *Handler) listSources(c echo.Context) error {
@ -62,7 +62,7 @@ func (s *Handler) listSources(c echo.Context) error {
// @Param page query string false "page number" // @Param page query string false "page number"
// @Produce application/json // @Produce application/json
// @Tags Source // @Tags Source
// @Router /sources/by/source [get] // @Router /v1/sources/by/source [get]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -100,7 +100,7 @@ func (s *Handler) listSourcesBySource(c echo.Context) error {
// @Param id path int true "uuid" // @Param id path int true "uuid"
// @Produce application/json // @Produce application/json
// @Tags Source // @Tags Source
// @Router /sources/{id} [get] // @Router /v1/sources/{id} [get]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -135,7 +135,7 @@ func (s *Handler) getSource(c echo.Context) error {
// @Param source query string true "reddit" // @Param source query string true "reddit"
// @Produce application/json // @Produce application/json
// @Tags Source // @Tags Source
// @Router /sources/by/sourceAndName [get] // @Router /v1/sources/by/sourceAndName [get]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -170,7 +170,7 @@ func (s *Handler) GetSourceBySourceAndName(c echo.Context) error {
// @Param name query string true "name" // @Param name query string true "name"
// @Param url query string true "url" // @Param url query string true "url"
// @Tags Source // @Tags Source
// @Router /sources/new/reddit [post] // @Router /v1/sources/new/reddit [post]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -226,7 +226,7 @@ func (s *Handler) newRedditSource(c echo.Context) error {
// @Param name query string true "name" // @Param name query string true "name"
// @Param url query string true "url" // @Param url query string true "url"
// @Tags Source // @Tags Source
// @Router /sources/new/youtube [post] // @Router /v1/sources/new/youtube [post]
func (s *Handler) newYoutubeSource(c echo.Context) error { func (s *Handler) newYoutubeSource(c echo.Context) error {
var param domain.NewSourceParamRequest var param domain.NewSourceParamRequest
err := c.Bind(&param) err := c.Bind(&param)
@ -289,7 +289,7 @@ func (s *Handler) newYoutubeSource(c echo.Context) error {
// @Summary Creates a new twitch source to monitor. // @Summary Creates a new twitch source to monitor.
// @Param name query string true "name" // @Param name query string true "name"
// @Tags Source // @Tags Source
// @Router /sources/new/twitch [post] // @Router /v1/sources/new/twitch [post]
func (s *Handler) newTwitchSource(c echo.Context) error { func (s *Handler) newTwitchSource(c echo.Context) error {
var param domain.NewSourceParamRequest var param domain.NewSourceParamRequest
err := c.Bind(&param) err := c.Bind(&param)
@ -337,7 +337,7 @@ func (s *Handler) newTwitchSource(c echo.Context) error {
// @Param name query string true "Site Name" // @Param name query string true "Site Name"
// @Param url query string true "RSS Url" // @Param url query string true "RSS Url"
// @Tags Source // @Tags Source
// @Router /sources/new/rss [post] // @Router /v1/sources/new/rss [post]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -387,7 +387,7 @@ func (s *Handler) newRssSource(c echo.Context) error {
// @Summary Marks a source as deleted based on its ID value. // @Summary Marks a source as deleted based on its ID value.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags Source // @Tags Source
// @Router /sources/{id} [POST] // @Router /v1/sources/{id} [POST]
func (s *Handler) deleteSources(c echo.Context) error { func (s *Handler) deleteSources(c echo.Context) error {
id := c.Param("ID") id := c.Param("ID")
uuid, err := uuid.Parse(id) uuid, err := uuid.Parse(id)
@ -432,7 +432,7 @@ func (s *Handler) deleteSources(c echo.Context) error {
// @Summary Disables a source from processing. // @Summary Disables a source from processing.
// @Param id path int true "id" // @Param id path int true "id"
// @Tags Source // @Tags Source
// @Router /sources/{id}/disable [post] // @Router /v1/sources/{id}/disable [post]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse
@ -474,7 +474,7 @@ func (s *Handler) disableSource(c echo.Context) error {
// @Summary Enables a source to continue processing. // @Summary Enables a source to continue processing.
// @Param id path string true "id" // @Param id path string true "id"
// @Tags Source // @Tags Source
// @Router /sources/{id}/enable [post] // @Router /v1/sources/{id}/enable [post]
// @Success 200 {object} domain.SourcesResponse "ok" // @Success 200 {object} domain.SourcesResponse "ok"
// @Failure 400 {object} domain.BaseResponse // @Failure 400 {object} domain.BaseResponse
// @Failure 500 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse

73
internal/handler/v1/subscriptions.go
View File

@ -31,7 +31,7 @@ type ListSubscriptionDetails struct {
// @Summary Returns the top 100 entries from the queue to be processed. // @Summary Returns the top 100 entries from the queue to be processed.
// @Produce application/json // @Produce application/json
// @Tags Subscription // @Tags Subscription
// @Router /subscriptions [get] // @Router /v1/subscriptions [get]
// @Success 200 {object} ListSubscriptions "ok" // @Success 200 {object} ListSubscriptions "ok"
// @Failure 400 {object} ApiError "Unable to reach SQL." // @Failure 400 {object} ApiError "Unable to reach SQL."
// @Failure 500 {object} ApiError "Failed to process data from SQL." // @Failure 500 {object} ApiError "Failed to process data from SQL."
@ -43,12 +43,11 @@ func (s *Handler) ListSubscriptions(c echo.Context) error {
}, },
} }
res, err := s.dto.ListSubscriptions(c.Request().Context(), 50) //res, err := s.dto.ListSubscriptions(c.Request().Context(), 50)
if err != nil { //if err != nil {
return s.WriteError(c, err, http.StatusBadRequest) // return s.WriteError(c, err, http.StatusBadRequest)
} //}
//payload.Payload = res
payload.Payload = res
return c.JSON(http.StatusOK, payload) return c.JSON(http.StatusOK, payload)
} }
@ -56,7 +55,7 @@ func (s *Handler) ListSubscriptions(c echo.Context) error {
// @Summary Returns the top 50 entries with full deatils on the source and output. // @Summary Returns the top 50 entries with full deatils on the source and output.
// @Produce application/json // @Produce application/json
// @Tags Subscription // @Tags Subscription
// @Router /subscriptions/details [get] // @Router /v1/subscriptions/details [get]
// @Success 200 {object} ListSubscriptionDetails "ok" // @Success 200 {object} ListSubscriptionDetails "ok"
func (s *Handler) ListSubscriptionDetails(c echo.Context) error { func (s *Handler) ListSubscriptionDetails(c echo.Context) error {
payload := ListSubscriptionDetails{ payload := ListSubscriptionDetails{
@ -66,12 +65,11 @@ func (s *Handler) ListSubscriptionDetails(c echo.Context) error {
}, },
} }
res, err := s.dto.ListSubscriptionDetails(c.Request().Context(), 50) //res, err := s.dto.ListSubscriptionDetails(c.Request().Context(), 50)
if err != nil { //if err != nil {
return s.WriteError(c, err, http.StatusInternalServerError) // return s.WriteError(c, err, http.StatusInternalServerError)
} //}
//payload.Payload = res
payload.Payload = res
return c.JSON(http.StatusOK, payload) return c.JSON(http.StatusOK, payload)
} }
@ -80,7 +78,7 @@ func (s *Handler) ListSubscriptionDetails(c echo.Context) error {
// @Produce application/json // @Produce application/json
// @Param id query string true "id" // @Param id query string true "id"
// @Tags Subscription // @Tags Subscription
// @Router /subscriptions/by/discordId [get] // @Router /v1/subscriptions/by/discordId [get]
// @Success 200 {object} ListSubscriptions "ok" // @Success 200 {object} ListSubscriptions "ok"
// @Failure 400 {object} ApiError "Unable to reach SQL or Data problems" // @Failure 400 {object} ApiError "Unable to reach SQL or Data problems"
// @Failure 500 {object} ApiError "Data problems" // @Failure 500 {object} ApiError "Data problems"
@ -97,18 +95,16 @@ func (s *Handler) GetSubscriptionsByDiscordId(c echo.Context) error {
return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest) return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest)
} }
uuid, err := uuid.Parse(id) //uuid, err := uuid.Parse(id)
if err != nil { //if err != nil {
return s.WriteError(c, errors.New(ErrValueNotUuid), http.StatusBadRequest) // return s.WriteError(c, errors.New(ErrValueNotUuid), http.StatusBadRequest)
//}
} //res, err := s.dto.ListSubscriptionsByDiscordWebhookId(context.Background(), uuid)
//if err != nil {
res, err := s.dto.ListSubscriptionsByDiscordWebhookId(context.Background(), uuid) // return s.WriteError(c, err, http.StatusNoContent)
if err != nil { //}
return s.WriteError(c, err, http.StatusNoContent) //p.Payload = res
}
p.Payload = res
return c.JSON(http.StatusOK, p) return c.JSON(http.StatusOK, p)
} }
@ -117,7 +113,7 @@ func (s *Handler) GetSubscriptionsByDiscordId(c echo.Context) error {
// @Produce application/json // @Produce application/json
// @Param id query string true "id" // @Param id query string true "id"
// @Tags Subscription // @Tags Subscription
// @Router /subscriptions/by/SourceId [get] // @Router /v1/subscriptions/by/SourceId [get]
// @Success 200 {object} ListSubscriptions "ok" // @Success 200 {object} ListSubscriptions "ok"
func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error { func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error {
p := ListSubscriptions{ p := ListSubscriptions{
@ -132,17 +128,16 @@ func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error {
return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest) return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest)
} }
uuid, err := uuid.Parse(_id) //uuid, err := uuid.Parse(_id)
if err != nil { //if err != nil {
return s.WriteError(c, err, http.StatusBadRequest) // return s.WriteError(c, err, http.StatusBadRequest)
} //}
res, err := s.dto.ListSubscriptionsBySourceId(context.Background(), uuid) //res, err := s.dto.ListSubscriptionsBySourceId(context.Background(), uuid)
if err != nil { //if err != nil {
return s.WriteError(c, err, http.StatusNoContent) // return s.WriteError(c, err, http.StatusNoContent)
} //}
//p.Payload = res
p.Payload = res
return c.JSON(http.StatusOK, p) return c.JSON(http.StatusOK, p)
} }
@ -151,7 +146,7 @@ func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error {
// @Param discordWebHookId query string true "discordWebHookId" // @Param discordWebHookId query string true "discordWebHookId"
// @Param sourceId query string true "sourceId" // @Param sourceId query string true "sourceId"
// @Tags Subscription // @Tags Subscription
// @Router /subscriptions/discord/webhook/new [post] // @Router /v1/subscriptions/discord/webhook/new [post]
func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error { func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error {
// Extract the values given // Extract the values given
discordWebHookId := c.QueryParam("discordWebHookId") discordWebHookId := c.QueryParam("discordWebHookId")
@ -207,7 +202,7 @@ func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error {
// @Summary Removes a Discord WebHook Subscription based on the Subscription ID. // @Summary Removes a Discord WebHook Subscription based on the Subscription ID.
// @Param id query string true "id" // @Param id query string true "id"
// @Tags Subscription // @Tags Subscription
// @Router /subscriptions/discord/webhook/delete [delete] // @Router /v1/subscriptions/discord/webhook/delete [delete]
func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error { func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error {
var ErrMissingSubscriptionID string = "the request was missing a 'Id'" var ErrMissingSubscriptionID string = "the request was missing a 'Id'"

17
internal/repository/article.go
View File

@ -24,6 +24,7 @@ type ArticlesRepo interface {
ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]domain.ArticleEntity, error) ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]domain.ArticleEntity, error)
ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]domain.ArticleEntity, error) ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]domain.ArticleEntity, error)
Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error) Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error)
CreateFromEntity(ctx context.Context, entity domain.ArticleEntity) (int64, error)
} }
type ArticleRepository struct { type ArticleRepository struct {
@ -192,6 +193,22 @@ func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, ti
return 1, nil return 1, nil
} }
func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity domain.ArticleEntity) (int64, error) {
dt := time.Now()
queryBuilder := sqlbuilder.NewInsertBuilder()
queryBuilder.InsertInto("articles")
queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl")
queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl)
query, args := queryBuilder.Build()
_, err := ar.conn.ExecContext(ctx, query, args...)
if err != nil {
return 0, err
}
return 1, nil
}
func (ur ArticleRepository) processRows(rows *sql.Rows) []domain.ArticleEntity { func (ur ArticleRepository) processRows(rows *sql.Rows) []domain.ArticleEntity {
items := []domain.ArticleEntity{} items := []domain.ArticleEntity{}

4
internal/services/config.go
View File

@ -11,9 +11,9 @@ import (
) )
const ( const (
ServerAddress = "SERVER_ADDRESS" ServerAddress = "ServerAddress"
Sql_Connection_String = "SQL_CONNECTION_STRING" //Sql_Connection_String = "SQL_CONNECTION_STRING"
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND" FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP" REDDIT_PULL_TOP = "REDDIT_PULL_TOP"

220
internal/services/cron/collectors.go Normal file
View File

@ -0,0 +1,220 @@
package cron
import (
"log"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CollectRssPosts() {
log.Println("Starting ")
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
rssClient := input.NewRssClient(source)
articles, err := rssClient.GetArticles()
if err != nil {
log.Println(err)
}
for _, article := range articles {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Println(err)
}
if rowsCreated != 1 {
log.Println("Got back the wrong number of rows")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
}
func (c *Cron) CollectRedditPosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit)
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
for _, article := range redditArticles {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new reddit article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CollectYoutubePosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube)
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range raw {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new youtube article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CollectFfxivPosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv)
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new FFXIV article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CollectTwitchPosts() {
sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch)
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
log.Println(err)
return
}
err = tc.Login()
if err != nil {
log.Println(err)
}
for sourceIndex, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
for _, article := range items {
_, err := c.repo.Articles.GetByUrl(c.ctx, article.Url)
if err == nil {
continue
}
rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article)
if err != nil {
log.Printf("Failed to add a new Twitch article to the database: %s", err)
}
if rowsAdded != 1 {
log.Printf("no error came back when data was added to the database but the expected row count is wrong")
}
}
if sourceIndex != len(sources) {
time.Sleep(time.Second * 30)
}
}
log.Print("[Twitch] Done!")
}

43
internal/services/cron/collectors_test.go Normal file
View File

@ -0,0 +1,43 @@
package cron_test
import (
"context"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
)
func TestRssPullsCorrectly(t *testing.T) {
conn, err := setupInMemoryDb()
if err != nil {
t.Error(err)
t.FailNow()
}
defer conn.Close()
ctx := context.Background()
db := services.NewRepositoryService(conn)
rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true)
if err != nil {
t.Error(err)
t.FailNow()
}
if rowsCreated != 1 {
t.Error("failed to create the source record")
t.FailNow()
}
client := cron.NewScheduler(ctx, conn)
client.CollectRssPosts()
articles, err := db.Articles.ListByPage(ctx, 0, 100)
if err != nil {
t.Error(err)
t.FailNow()
}
t.Log(len(articles))
}

233
internal/services/cron/scheduler.go
View File

@ -3,83 +3,37 @@ package cron
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
"log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
) )
type Cron struct { type Cron struct {
Db *database.Queries Db *database.Queries
ctx *context.Context ctx context.Context
timer *cron.Cron timer *cron.Cron
repo services.RepositoryService
} }
func openDatabase() (*database.Queries, error) { func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
_env := services.NewConfig()
connString := _env.GetConfig(services.Sql_Connection_String)
if connString == "" {
panic("Connection String is null!")
}
db, err := sql.Open("postgres", connString)
if err != nil {
panic(err)
}
queries := database.New(db)
return queries, err
}
func NewScheduler(ctx context.Context) *Cron {
c := &Cron{ c := &Cron{
ctx: &ctx, ctx: ctx,
repo: services.NewRepositoryService(conn),
} }
timer := cron.New() timer := cron.New()
queries, err := openDatabase()
if err != nil {
panic(err)
}
c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
features := services.NewConfig() //features := services.GetEnvConfig()
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND) timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() })
if res { //timer.AddFunc("10 * * * *", c.CollectRedditPosts)
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) //timer.AddFunc("15 * * * *", c.CheckYoutube)
log.Print("[Input] Reddit backend was enabled") //timer.AddFunc("20 * * * *", c.CheckFfxiv)
//go c.CheckReddit() //timer.AddFunc("25 * * * *", c.CheckTwitch)
} //timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue)
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled")
}
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled")
}
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled")
}
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
log.Print("[Output] Discord Output was enabled")
c.timer = timer c.timer = timer
return c return c
@ -93,105 +47,8 @@ func (c *Cron) Stop() {
c.timer.Stop() c.timer.Stop()
} }
// This is the main entry point to query all the reddit services /*
func (c *Cron) CheckReddit() { func (c *Cron) CheckDiscordQueue() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.Name)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
c.checkPosts(redditArticles, "Reddit")
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources.
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.Name)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(raw, "YouTube")
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CheckFfxiv() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "FFXIV")
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CheckTwitch() error {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
return err
}
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.Name)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "Twitch")
}
log.Print("[Twitch] Done!")
return nil
}
func (c *Cron) CheckDiscordQueue() error {
// Get items from the table // Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50) queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
if err != nil { if err != nil {
@ -260,55 +117,15 @@ func (c *Cron) CheckDiscordQueue() error {
return nil return nil
} }
*/
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error { //func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
for _, item := range posts { // err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) // ID: uuid.New(),
if err != nil { // Articleid: Id,
id := uuid.New() // })
// if err != nil {
err := c.postArticle(id, item) // return err
if err != nil { // }
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) // return nil
} //}
err = c.addToDiscordQueue(id)
if err != nil {
return err
}
}
}
time.Sleep(30 * time.Second)
return nil
}
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: id,
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
ID: uuid.New(),
Articleid: Id,
})
if err != nil {
return err
}
return nil
}

27
internal/services/cron/scheduler_test.go
View File

@ -1,12 +1,12 @@
package cron_test package cron_test
import ( import (
"context" "database/sql"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" "github.com/pressly/goose/v3"
) )
/*
func TestInvokeTwitch(t *testing.T) { func TestInvokeTwitch(t *testing.T) {
} }
@ -15,7 +15,7 @@ func TestInvokeTwitch(t *testing.T) {
func TestCheckReddit(t *testing.T) { func TestCheckReddit(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx)
c.CheckReddit() c.Col()
} }
func TestCheckYouTube(t *testing.T) { func TestCheckYouTube(t *testing.T) {
@ -32,3 +32,22 @@ func TestCheckTwitch(t *testing.T) {
t.Error(err) t.Error(err)
} }
} }
*/
func setupInMemoryDb() (*sql.DB, error) {
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
return nil, err
}
err = goose.SetDialect("sqlite3")
if err != nil {
return nil, err
}
err = goose.Up(db, "../../database/migrations")
if err != nil {
return nil, err
}
return db, nil
}

140
internal/services/dto/articles.go
View File

@ -1,140 +0,0 @@
// The converter package lives between the database calls and the API calls.
// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much
package dto
import (
"context"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
type DtoClient struct {
db *database.Queries
}
func NewDtoClient(db *database.Queries) *DtoClient {
return &DtoClient{
db: db,
}
}
func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
Limit: int32(limit),
Offset: int32(limit * page),
})
if err != nil {
return res, err
}
for _, article := range a {
res = append(res, c.convertArticle(article))
}
return res, nil
}
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
Limit: limit,
Offset: page * limit,
})
if err != nil {
return res, err
}
for _, article := range a {
res = append(res, c.convertArticle(article))
}
return res, nil
}
func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) {
a, err := c.db.GetArticleByID(ctx, ID)
if err != nil {
return models.ArticleDto{}, err
}
return c.convertArticle(a), nil
}
func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) {
a, err := c.db.GetArticleByID(ctx, ID)
if err != nil {
return models.ArticleDetailsDto{}, err
}
s, err := c.db.GetSourceByID(ctx, a.Sourceid)
if err != nil {
return models.ArticleDetailsDto{}, err
}
res := c.convertArticleDetails(a, s)
return res, nil
}
func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
Sourceid: SourceID,
Limit: int32(limit),
Offset: int32(limit * page),
})
if err != nil {
return res, err
}
for _, article := range a {
res = append(res, c.convertArticle(article))
}
return res, nil
}
func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto {
return models.ArticleDto{
ID: i.ID,
Source: i.Sourceid,
Tags: c.SplitTags(i.Tags),
Title: i.Title,
Url: i.Url,
Pubdate: i.Pubdate,
Video: i.Video.String,
Videoheight: i.Videoheight,
Videowidth: i.Videoheight,
Thumbnail: i.Thumbnail,
Description: i.Description,
Authorname: i.Authorname.String,
Authorimage: i.Authorimage.String,
}
}
func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto {
return models.ArticleDetailsDto{
ID: i.ID,
Source: c.ConvertToSource(s),
Tags: c.SplitTags(i.Tags),
Title: i.Title,
Url: i.Url,
Pubdate: i.Pubdate,
Video: i.Video.String,
Videoheight: i.Videoheight,
Videowidth: i.Videoheight,
Thumbnail: i.Thumbnail,
Description: i.Description,
Authorname: i.Authorname.String,
Authorimage: i.Authorimage.String,
}
}
func (c DtoClient) SplitTags(t string) []string {
return strings.Split(t, ", ")
}

63
internal/services/dto/discordwebhooks.go
View File

@ -1,63 +0,0 @@
package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
var res []models.DiscordWebHooksDto
items, err := c.db.ListDiscordWebhooks(ctx, total)
if err != nil {
return res, nil
}
for _, item := range items {
res = append(res, c.ConvertDiscordWebhook(item))
}
return res, nil
}
func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) {
var res models.DiscordWebHooksDto
item, err := c.db.GetDiscordWebHooksByID(ctx, id)
if err != nil {
return res, err
}
return c.ConvertDiscordWebhook(item), nil
}
func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) {
var res []models.DiscordWebHooksDto
items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{
Server: server,
Channel: channel,
})
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertDiscordWebhook(item))
}
return res, nil
}
func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto {
return models.DiscordWebHooksDto{
ID: i.ID,
Url: i.Url,
Server: i.Server,
Channel: i.Channel,
Enabled: i.Enabled,
}
}

42
internal/services/dto/queue.go
View File

@ -1,42 +0,0 @@
package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
)
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
}
func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) {
var res []models.DiscordQueueDetailsDto
items, err := c.db.ListDiscordQueueItems(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
article, err := c.GetArticleDetails(ctx, item.ID)
if err != nil {
return res, err
}
res = append(res, models.DiscordQueueDetailsDto{
ID: item.ID,
Article: article,
})
}
return res, nil
}
func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto {
return models.DiscordQueueDto{
ID: i.ID,
Articleid: i.Articleid,
}
}

85
internal/services/dto/sources.go
View File

@ -1,85 +0,0 @@
package dto
import (
"context"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
var res []models.SourceDto
items, err := c.db.ListSources(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertToSource(item))
}
return res, nil
}
func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) {
var res []models.SourceDto
items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName))
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertToSource(item))
}
return res, nil
}
func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) {
var res models.SourceDto
item, err := c.db.GetSourceByID(ctx, id)
if err != nil {
return res, err
}
return c.ConvertToSource(item), nil
}
func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) {
var res models.SourceDto
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
Name: name,
Source: source,
})
if err != nil {
return res, err
}
return c.ConvertToSource(item), nil
}
func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto {
var deleted bool
if !i.Deleted.Valid {
deleted = true
}
return models.SourceDto{
ID: i.ID,
Site: i.Site,
Name: i.Name,
Source: i.Source,
Type: i.Type,
Value: i.Value.String,
Enabled: i.Enabled,
Url: i.Url,
Tags: c.SplitTags(i.Tags),
Deleted: deleted,
}
}

91
internal/services/dto/subscriptions.go
View File

@ -1,91 +0,0 @@
package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
var res []models.SubscriptionDto
items, err := c.db.ListSubscriptions(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertSubscription(item))
}
return res, nil
}
func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) {
var res []models.SubscriptionDetailsDto
items, err := c.ListSubscriptions(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId)
if err != nil {
return res, err
}
source, err := c.GetSourceById(ctx, item.SourceId)
if err != nil {
return res, err
}
res = append(res, models.SubscriptionDetailsDto{
ID: item.ID,
Source: source,
DiscordWebHook: dwh,
})
}
return res, nil
}
func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
var res []models.SubscriptionDto
items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertSubscription(item))
}
return res, nil
}
func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
var res []models.SubscriptionDto
items, err := c.db.GetSubscriptionsBySourceID(ctx, id)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertSubscription(item))
}
return res, nil
}
func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto {
return models.SubscriptionDto{
ID: i.ID,
DiscordWebhookId: i.Discordwebhookid,
SourceId: i.Sourceid,
}
}

23
internal/services/input/ffxiv.go
View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"log" "log"
"net/http" "net/http"
@ -13,7 +12,7 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid" "github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
@ -25,7 +24,7 @@ const (
) )
type FFXIVClient struct { type FFXIVClient struct {
record database.Source record domain.SourceEntity
//SourceID uint //SourceID uint
//Url string //Url string
//Region string //Region string
@ -33,15 +32,15 @@ type FFXIVClient struct {
cacheGroup string cacheGroup string
} }
func NewFFXIVClient(Record database.Source) FFXIVClient { func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
return FFXIVClient{ return FFXIVClient{
record: Record, record: Record,
cacheGroup: "ffxiv", cacheGroup: "ffxiv",
} }
} }
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
var articles []database.Article var articles []domain.ArticleEntity
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
return articles, err return articles, err
} }
article := database.Article{ article := domain.ArticleEntity{
Sourceid: fc.record.ID, SourceID: fc.record.ID,
Tags: tags, Tags: tags,
Title: title, Title: title,
Url: link, Url: link,
Pubdate: pubDate, PubDate: pubDate,
Videoheight: 0,
Videowidth: 0,
Thumbnail: thumb, Thumbnail: thumb,
Description: description, Description: description,
Authorname: sql.NullString{String: authorName}, AuthorName: authorName,
Authorimage: sql.NullString{String: authorImage}, AuthorImageUrl: authorImage,
} }
log.Printf("Collected '%v' from '%v'", article.Title, article.Url) log.Printf("Collected '%v' from '%v'", article.Title, article.Url)

12
internal/services/input/ffxiv_test.go
View File

@ -3,16 +3,14 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var FFXIVRecord database.Source = database.Source{ var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Site: "ffxiv", DisplayName: "Final Fantasy XIV - NA",
Name: "Final Fantasy XIV - NA", Source: domain.SourceCollectorFfxiv,
Source: "ffxiv",
Url: "https://na.finalfantasyxiv.com/lodestone/", Url: "https://na.finalfantasyxiv.com/lodestone/",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone", Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
} }

4
internal/services/input/httpClient.go
View File

@ -2,7 +2,7 @@ package input
import ( import (
"crypto/tls" "crypto/tls"
"io/ioutil" "io"
"log" "log"
"net/http" "net/http"
) )
@ -35,7 +35,7 @@ func getHttpContent(uri string) ([]byte, error) {
} }
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return nil, err return nil, err
} }

70
internal/services/input/reddit.go
View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -9,7 +8,6 @@ import (
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod" "github.com/go-rod/rod"
@ -18,7 +16,7 @@ import (
type RedditClient struct { type RedditClient struct {
config RedditConfig config RedditConfig
record database.Source record domain.SourceEntity
} }
type RedditConfig struct { type RedditConfig struct {
@ -27,7 +25,7 @@ type RedditConfig struct {
PullNSFW string PullNSFW string
} }
func NewRedditClient(Record database.Source) *RedditClient { func NewRedditClient(Record domain.SourceEntity) *RedditClient {
rc := RedditClient{ rc := RedditClient{
record: Record, record: Record,
} }
@ -71,7 +69,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
// TODO Wire this to support the config options // TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url) Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name) log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
content, err := getHttpContent(Url) content, err := getHttpContent(Url)
if err != nil { if err != nil {
@ -88,10 +86,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
return items, nil return items, nil
} }
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article { func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
var redditArticles []database.Article var redditArticles []domain.ArticleEntity
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article database.Article var article domain.ArticleEntity
article, err := rc.convertToArticle(item.Data) article, err := rc.convertToArticle(item.Data)
if err != nil { if err != nil {
log.Printf("[Reddit] %v", err) log.Printf("[Reddit] %v", err)
@ -104,8 +102,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []data
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) { func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
var item database.Article var item domain.ArticleEntity
if source.Content == "" && source.Url != "" { if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source) item = rc.convertPicturePost(source)
@ -131,65 +129,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
return item, nil return item, nil
} }
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Title: source.Title, Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags), Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Pubdate: time.Now(), PubDate: time.Now(),
Video: sql.NullString{String: "null"}, IsVideo: false,
Videoheight: 0,
Videowidth: 0,
Thumbnail: source.Thumbnail, Thumbnail: source.Thumbnail,
Description: source.Content, Description: source.Content,
Authorname: sql.NullString{String: source.Author}, AuthorName: source.Author,
Authorimage: sql.NullString{String: "null"}, AuthorImageUrl: "null",
} }
return item return item
} }
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Authorname: sql.NullString{String: source.Author}, AuthorName: source.Author,
Description: source.Content, Description: source.Content,
} }
return item return item
} }
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0, AuthorName: source.Author,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl, Description: source.Media.RedditVideo.FallBackUrl,
} }
return item return item
} }
// This post is nothing more then a redirect to another location. // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0, AuthorName: source.Author,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest, Description: source.UrlOverriddenByDest,
} }
return item return item

12
internal/services/input/reddit_test.go
View File

@ -3,16 +3,14 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var RedditRecord database.Source = database.Source{ var RedditRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "dadjokes", DisplayName: "dadjokes",
Source: "reddit", Source: domain.SourceCollectorRss,
Site: "reddit",
Url: "https://reddit.com/r/dadjokes", Url: "https://reddit.com/r/dadjokes",
Tags: "reddit, dadjokes", Tags: "reddit, dadjokes",
} }

86
internal/services/input/rss.go
View File

@ -1,14 +1,16 @@
package input package input
import ( import (
"fmt" "strings"
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
) )
type FeedInput interface {
GetArticles() (domain.ArticleEntity, error)
}
type rssClient struct { type rssClient struct {
SourceRecord domain.SourceEntity SourceRecord domain.SourceEntity
} }
@ -21,39 +23,55 @@ func NewRssClient(sourceRecord domain.SourceEntity) rssClient {
return client return client
} }
//func (rc rssClient) ReplaceSourceRecord(source model.Sources) { func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) {
//rc.SourceRecord = source parser := gofeed.NewParser()
//} feed, err := parser.ParseURL(rc.SourceRecord.Url)
func (rc rssClient) getCacheGroup() string {
return fmt.Sprintf("rss-%v", rc.SourceRecord.DisplayName)
}
func (rc rssClient) GetContent() error {
feed, err := rc.PullFeed()
if err != nil {
return err
}
cacheClient := cache.NewCacheClient(rc.getCacheGroup())
for _, item := range feed.Items {
log.Println(item)
cacheClient.FindByValue(item.Link)
}
return nil
}
func (rc rssClient) PullFeed() (*gofeed.Feed, error) {
feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url)
fp := gofeed.NewParser()
feed, err := fp.ParseURL(feedUri)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return feed, nil sourceTags := strings.Split(rc.SourceRecord.Tags, ",")
var articles []domain.ArticleEntity
for _, post := range feed.Items {
article := domain.ArticleEntity{
SourceID: rc.SourceRecord.ID,
Title: post.Title,
Description: post.Content,
Url: post.Link,
PubDate: *post.PublishedParsed,
//AuthorName: post.Authors[0].Email,
}
if len(post.Authors) != 0 {
article.AuthorName = post.Authors[0].Email
}
var postTags []string
postTags = append(postTags, sourceTags...)
postTags = append(postTags, post.Categories...)
article.Tags = strings.Join(postTags, ",")
/*
pageContent, err := getHttpContent(article.Url)
if err != nil {
continue
}
htmlNode, err := html.Parse(bytes.NewReader(pageContent))
if err != nil {
continue
}
htmlNode.
fmt.Println(htmlNode)
*/
if post.Image == nil {
article.Thumbnail = ""
}
articles = append(articles, article)
}
return articles, nil
} }

18
internal/services/input/rss_test.go
View File

@ -11,6 +11,7 @@ var rssRecord = domain.SourceEntity{
ID: 1, ID: 1,
DisplayName: "ArsTechnica", DisplayName: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index", Url: "https://feeds.arstechnica.com/arstechnica/index",
Source: domain.SourceCollectorRss,
} }
func TestRssClientConstructor(t *testing.T) { func TestRssClientConstructor(t *testing.T) {
@ -19,12 +20,23 @@ func TestRssClientConstructor(t *testing.T) {
func TestRssGetFeed(t *testing.T) { func TestRssGetFeed(t *testing.T) {
client := input.NewRssClient(rssRecord) client := input.NewRssClient(rssRecord)
feed, err := client.PullFeed() _, err := client.GetArticles()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
if len(feed.Items) >= 0 { }
t.Error("failed to collect items from the fees")
func TestRssAgainstGita(t *testing.T) {
client := input.NewRssClient(domain.SourceEntity{
ID: 2,
DisplayName: "Gitea - Newsbot-api",
Source: domain.SourceCollectorRss,
Url: "https://git.jamestombleson.com/jtom38/newsbot-api.rss",
Tags: "rss,gitea,newsbot-api",
})
_, err := client.GetArticles()
if err != nil {
t.Error(err)
} }
} }

23
internal/services/input/twitch.go
View File

@ -1,19 +1,18 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/nicklaw5/helix/v2" "github.com/nicklaw5/helix/v2"
) )
type TwitchClient struct { type TwitchClient struct {
SourceRecord database.Source SourceRecord domain.SourceEntity
// config // config
monitorClips string monitorClips string
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
} }
// This will let you replace the bound source record to keep the same session alive. // This will let you replace the bound source record to keep the same session alive.
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
tc.SourceRecord = source tc.SourceRecord = source
} }
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
return nil return nil
} }
func (tc *TwitchClient) GetContent() ([]database.Article, error) { func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
var items []database.Article var items []domain.ArticleEntity
user, err := tc.GetUserDetails() user, err := tc.GetUserDetails()
if err != nil { if err != nil {
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
} }
for _, video := range posts { for _, video := range posts {
var article database.Article var article domain.ArticleEntity
AuthorName, err := tc.ExtractAuthor(video) AuthorName, err := tc.ExtractAuthor(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Authorname = sql.NullString{String: AuthorName} article.AuthorName = AuthorName
Authorimage, err := tc.ExtractAuthorImage(user) Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Authorimage = sql.NullString{String: Authorimage} article.AuthorImageUrl = Authorimage
article.Description, err = tc.ExtractDescription(video) article.Description, err = tc.ExtractDescription(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Pubdate, err = tc.ExtractPubDate(video) article.PubDate, err = tc.ExtractPubDate(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Sourceid = tc.SourceRecord.ID article.SourceID = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user) article.Tags, err = tc.ExtractTags(video, user)
if err != nil { if err != nil {
return items, err return items, err
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{ users, err := tc.api.GetUsers(&helix.UsersParams{
Logins: []string{tc.SourceRecord.Name}, Logins: []string{tc.SourceRecord.DisplayName},
}) })
if err != nil { if err != nil {
return blank, err return blank, err

19
internal/services/input/twitch_test.go
View File

@ -4,21 +4,20 @@ import (
"log" "log"
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var TwitchSourceRecord = database.Source{ var TwitchSourceRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "nintendo", DisplayName: "nintendo",
Source: "Twitch", Source: domain.SourceCollectorTwitch,
} }
var TwitchInvalidRecord = database.Source{ var TwitchInvalidRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "EvilNintendo", DisplayName: "EvilNintendo",
Source: "Twitch", Source: domain.SourceCollectorTwitch,
} }
func TestTwitchLogin(t *testing.T) { func TestTwitchLogin(t *testing.T) {

23
internal/services/input/youtube.go
View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@ -12,11 +11,11 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
) )
type YoutubeClient struct { type YoutubeClient struct {
record database.Source record domain.SourceEntity
// internal variables at time of collection // internal variables at time of collection
channelID string channelID string
@ -37,7 +36,7 @@ var (
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
func NewYoutubeClient(Record database.Source) YoutubeClient { func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
yc := YoutubeClient{ yc := YoutubeClient{
record: Record, record: Record,
cacheGroup: "youtube", cacheGroup: "youtube",
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
} }
// CheckSource will go and run all the commands needed to process a source. // CheckSource will go and run all the commands needed to process a source.
func (yc *YoutubeClient) GetContent() ([]database.Article, error) { func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
var items []database.Article var items []domain.ArticleEntity
docParser, err := yc.GetParser(yc.record.Url) docParser, err := yc.GetParser(yc.record.Url)
if err != nil { if err != nil {
return items, err return items, err
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
return false return false
} }
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
parser, err := yc.GetParser(item.Link) parser, err := yc.GetParser(item.Link)
if err != nil { if err != nil {
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
log.Printf("[YouTube] %v", msg) log.Printf("[YouTube] %v", msg)
} }
var article = database.Article{ var article = domain.ArticleEntity{
Sourceid: yc.record.ID, SourceID: yc.record.ID,
Tags: tags, Tags: tags,
Title: item.Title, Title: item.Title,
Url: item.Link, Url: item.Link,
Pubdate: *item.PublishedParsed, PubDate: *item.PublishedParsed,
Thumbnail: thumb, Thumbnail: thumb,
Description: item.Description, Description: item.Description,
Authorname: sql.NullString{String: item.Author.Name}, AuthorName: item.Author.Name,
Authorimage: sql.NullString{String: yc.avatarUri}, AuthorImageUrl: yc.avatarUri,
} }
return article return article
} }

12
internal/services/input/youtube_test.go
View File

@ -3,16 +3,14 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var YouTubeRecord database.Source = database.Source{ var YouTubeRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "dadjokes", DisplayName: "dadjokes",
Source: "reddit", Source: domain.SourceCollectorReddit,
Site: "reddit",
Url: "https://youtube.com/gamegrumps", Url: "https://youtube.com/gamegrumps",
} }

8
makefile
View File

@ -4,18 +4,18 @@ help: ## Shows this help command
build: ## builds the application with the current go runtime build: ## builds the application with the current go runtime
~/go/bin/swag f ~/go/bin/swag f
~/go/bin/swag i ~/go/bin/swag init -g cmd/server.go
go build . go build cmd/server.go
docker-build: ## Generates the docker image docker-build: ## Generates the docker image
docker build -t "newsbot.collector.api" . docker build -t "newsbot.collector.api" .
docker image ls | grep newsbot.collector.api docker image ls | grep newsbot.collector.api
migrate-dev: ## Apply sql migrations to dev db migrate-dev: ## Apply sql migrations to dev db
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up
migrate-dev-down: ## revert sql migrations to dev db migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down
swag: ## Generates the swagger documentation with the swag tool swag: ## Generates the swagger documentation with the swag tool
~/go/bin/swag f ~/go/bin/swag f