Compare commits
No commits in common. "fbed111fbb00a52c19e616efcb43c8b9893754a7" and "84d108f2dd40702c75a615bf47ddf59632be6986" have entirely different histories.
fbed111fbb
...
84d108f2dd
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,6 +2,7 @@
|
|||||||
dev.session.sql
|
dev.session.sql
|
||||||
__debug_bin
|
__debug_bin
|
||||||
server
|
server
|
||||||
|
.vscode
|
||||||
|
|
||||||
# Binaries for programs and plugins
|
# Binaries for programs and plugins
|
||||||
*.exe
|
*.exe
|
||||||
@ -10,7 +11,6 @@ server
|
|||||||
*.so
|
*.so
|
||||||
*.dylib
|
*.dylib
|
||||||
collector
|
collector
|
||||||
*.db
|
|
||||||
|
|
||||||
# Test binary, built with `go test -c`
|
# Test binary, built with `go test -c`
|
||||||
*.test
|
*.test
|
||||||
|
|||||||
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@ -9,7 +9,7 @@
|
|||||||
"type": "go",
|
"type": "go",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"mode": "auto",
|
"mode": "auto",
|
||||||
"program": "./cmd/server.go"
|
"program": "."
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@ -10,7 +10,8 @@ import (
|
|||||||
"github.com/pressly/goose/v3"
|
"github.com/pressly/goose/v3"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
||||||
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||||
)
|
)
|
||||||
@ -41,15 +42,12 @@ func main() {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
//queues := services.NewQueues(db)
|
queries := database.New(db)
|
||||||
//queues.RssIndex.Send(ctx, goqite.Message{
|
|
||||||
// Body: []byte("hello world"),
|
|
||||||
//})
|
|
||||||
|
|
||||||
c := cron.NewScheduler(ctx, db)
|
c := cron.NewScheduler(ctx)
|
||||||
c.Start()
|
c.Start()
|
||||||
|
|
||||||
server := v1.NewServer(ctx, configs, db)
|
server := v1.NewServer(ctx, queries, configs, db)
|
||||||
|
|
||||||
fmt.Println("API is online and waiting for requests.")
|
fmt.Println("API is online and waiting for requests.")
|
||||||
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
|
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
|
||||||
|
|||||||
386
docs/docs.go
386
docs/docs.go
@ -449,6 +449,25 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/queue/discord/webhooks": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Queue"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"/settings/{key}": {
|
"/settings/{key}": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
@ -856,6 +875,167 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"description": "Unable to reach SQL.",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"description": "Failed to process data from SQL.",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/by/SourceId": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "id",
|
||||||
|
"name": "id",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/by/discordId": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "id",
|
||||||
|
"name": "id",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"description": "Unable to reach SQL or Data problems",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"description": "Data problems",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/details": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 50 entries with full deatils on the source and output.",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptionDetails"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/discord/webhook/delete": {
|
||||||
|
"delete": {
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "id",
|
||||||
|
"name": "id",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/discord/webhook/new": {
|
||||||
|
"post": {
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "discordWebHookId",
|
||||||
|
"name": "discordWebHookId",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "sourceId",
|
||||||
|
"name": "sourceId",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
@ -1012,6 +1192,212 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"models.ArticleDetailsDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"authorImage": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"authorName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"pubdate": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"$ref": "#/definitions/models.SourceDto"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"thumbnail": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"video": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"videoHeight": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"videoWidth": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.DiscordQueueDetailsDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"article": {
|
||||||
|
"$ref": "#/definitions/models.ArticleDetailsDto"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.DiscordWebHooksDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"ID": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"channel": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"enabled": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"server": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.SourceDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"deleted": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"enabled": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"site": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"value": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.SubscriptionDetailsDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"discordwebhook": {
|
||||||
|
"$ref": "#/definitions/models.DiscordWebHooksDto"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"$ref": "#/definitions/models.SourceDto"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.SubscriptionDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"discordwebhookid": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sourceid": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ApiError": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ListDiscordWebHooksQueueResults": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"payload": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ListSubscriptionDetails": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"payload": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/models.SubscriptionDetailsDto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ListSubscriptions": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"payload": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/models.SubscriptionDto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
|
|||||||
@ -440,6 +440,25 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/queue/discord/webhooks": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Queue"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"/settings/{key}": {
|
"/settings/{key}": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
@ -847,6 +866,167 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"description": "Unable to reach SQL.",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"description": "Failed to process data from SQL.",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/by/SourceId": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "id",
|
||||||
|
"name": "id",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/by/discordId": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "id",
|
||||||
|
"name": "id",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"description": "Unable to reach SQL or Data problems",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"description": "Data problems",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/details": {
|
||||||
|
"get": {
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Returns the top 50 entries with full deatils on the source and output.",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "ok",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/v1.ListSubscriptionDetails"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/discord/webhook/delete": {
|
||||||
|
"delete": {
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "id",
|
||||||
|
"name": "id",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/subscriptions/discord/webhook/new": {
|
||||||
|
"post": {
|
||||||
|
"tags": [
|
||||||
|
"Subscription"
|
||||||
|
],
|
||||||
|
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "discordWebHookId",
|
||||||
|
"name": "discordWebHookId",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "sourceId",
|
||||||
|
"name": "sourceId",
|
||||||
|
"in": "query",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
@ -1003,6 +1183,212 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"models.ArticleDetailsDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"authorImage": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"authorName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"pubdate": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"$ref": "#/definitions/models.SourceDto"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"thumbnail": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"video": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"videoHeight": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"videoWidth": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.DiscordQueueDetailsDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"article": {
|
||||||
|
"$ref": "#/definitions/models.ArticleDetailsDto"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.DiscordWebHooksDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"ID": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"channel": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"enabled": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"server": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.SourceDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"deleted": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"enabled": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"site": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"value": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.SubscriptionDetailsDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"discordwebhook": {
|
||||||
|
"$ref": "#/definitions/models.DiscordWebHooksDto"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"$ref": "#/definitions/models.SourceDto"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models.SubscriptionDto": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"discordwebhookid": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sourceid": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ApiError": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ListDiscordWebHooksQueueResults": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"payload": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ListSubscriptionDetails": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"payload": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/models.SubscriptionDetailsDto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"v1.ListSubscriptions": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"payload": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/models.SubscriptionDto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -102,6 +102,140 @@ definitions:
|
|||||||
$ref: '#/definitions/domain.SourceDto'
|
$ref: '#/definitions/domain.SourceDto'
|
||||||
type: array
|
type: array
|
||||||
type: object
|
type: object
|
||||||
|
models.ArticleDetailsDto:
|
||||||
|
properties:
|
||||||
|
authorImage:
|
||||||
|
type: string
|
||||||
|
authorName:
|
||||||
|
type: string
|
||||||
|
description:
|
||||||
|
type: string
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
pubdate:
|
||||||
|
type: string
|
||||||
|
source:
|
||||||
|
$ref: '#/definitions/models.SourceDto'
|
||||||
|
tags:
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
thumbnail:
|
||||||
|
type: string
|
||||||
|
title:
|
||||||
|
type: string
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
video:
|
||||||
|
type: string
|
||||||
|
videoHeight:
|
||||||
|
type: integer
|
||||||
|
videoWidth:
|
||||||
|
type: integer
|
||||||
|
type: object
|
||||||
|
models.DiscordQueueDetailsDto:
|
||||||
|
properties:
|
||||||
|
article:
|
||||||
|
$ref: '#/definitions/models.ArticleDetailsDto'
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
models.DiscordWebHooksDto:
|
||||||
|
properties:
|
||||||
|
ID:
|
||||||
|
type: string
|
||||||
|
channel:
|
||||||
|
type: string
|
||||||
|
enabled:
|
||||||
|
type: boolean
|
||||||
|
server:
|
||||||
|
type: string
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
models.SourceDto:
|
||||||
|
properties:
|
||||||
|
deleted:
|
||||||
|
type: boolean
|
||||||
|
enabled:
|
||||||
|
type: boolean
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
site:
|
||||||
|
type: string
|
||||||
|
source:
|
||||||
|
type: string
|
||||||
|
tags:
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
type:
|
||||||
|
type: string
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
value:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
models.SubscriptionDetailsDto:
|
||||||
|
properties:
|
||||||
|
discordwebhook:
|
||||||
|
$ref: '#/definitions/models.DiscordWebHooksDto'
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
source:
|
||||||
|
$ref: '#/definitions/models.SourceDto'
|
||||||
|
type: object
|
||||||
|
models.SubscriptionDto:
|
||||||
|
properties:
|
||||||
|
discordwebhookid:
|
||||||
|
type: string
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
sourceid:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
|
v1.ApiError:
|
||||||
|
properties:
|
||||||
|
message:
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
type: integer
|
||||||
|
type: object
|
||||||
|
v1.ListDiscordWebHooksQueueResults:
|
||||||
|
properties:
|
||||||
|
message:
|
||||||
|
type: string
|
||||||
|
payload:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/models.DiscordQueueDetailsDto'
|
||||||
|
type: array
|
||||||
|
status:
|
||||||
|
type: integer
|
||||||
|
type: object
|
||||||
|
v1.ListSubscriptionDetails:
|
||||||
|
properties:
|
||||||
|
message:
|
||||||
|
type: string
|
||||||
|
payload:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/models.SubscriptionDetailsDto'
|
||||||
|
type: array
|
||||||
|
status:
|
||||||
|
type: integer
|
||||||
|
type: object
|
||||||
|
v1.ListSubscriptions:
|
||||||
|
properties:
|
||||||
|
message:
|
||||||
|
type: string
|
||||||
|
payload:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/models.SubscriptionDto'
|
||||||
|
type: array
|
||||||
|
status:
|
||||||
|
type: integer
|
||||||
|
type: object
|
||||||
info:
|
info:
|
||||||
contact: {}
|
contact: {}
|
||||||
title: NewsBot collector
|
title: NewsBot collector
|
||||||
@ -393,6 +527,18 @@ paths:
|
|||||||
tags:
|
tags:
|
||||||
- Discord
|
- Discord
|
||||||
- Webhook
|
- Webhook
|
||||||
|
/queue/discord/webhooks:
|
||||||
|
get:
|
||||||
|
produces:
|
||||||
|
- application/json
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: ok
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ListDiscordWebHooksQueueResults'
|
||||||
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
|
tags:
|
||||||
|
- Queue
|
||||||
/settings/{key}:
|
/settings/{key}:
|
||||||
get:
|
get:
|
||||||
parameters:
|
parameters:
|
||||||
@ -661,4 +807,109 @@ paths:
|
|||||||
summary: Creates a new youtube source to monitor.
|
summary: Creates a new youtube source to monitor.
|
||||||
tags:
|
tags:
|
||||||
- Source
|
- Source
|
||||||
|
/subscriptions:
|
||||||
|
get:
|
||||||
|
produces:
|
||||||
|
- application/json
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: ok
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ListSubscriptions'
|
||||||
|
"400":
|
||||||
|
description: Unable to reach SQL.
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ApiError'
|
||||||
|
"500":
|
||||||
|
description: Failed to process data from SQL.
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ApiError'
|
||||||
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
|
tags:
|
||||||
|
- Subscription
|
||||||
|
/subscriptions/by/SourceId:
|
||||||
|
get:
|
||||||
|
parameters:
|
||||||
|
- description: id
|
||||||
|
in: query
|
||||||
|
name: id
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
produces:
|
||||||
|
- application/json
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: ok
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ListSubscriptions'
|
||||||
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
|
tags:
|
||||||
|
- Subscription
|
||||||
|
/subscriptions/by/discordId:
|
||||||
|
get:
|
||||||
|
parameters:
|
||||||
|
- description: id
|
||||||
|
in: query
|
||||||
|
name: id
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
produces:
|
||||||
|
- application/json
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: ok
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ListSubscriptions'
|
||||||
|
"400":
|
||||||
|
description: Unable to reach SQL or Data problems
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ApiError'
|
||||||
|
"500":
|
||||||
|
description: Data problems
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ApiError'
|
||||||
|
summary: Returns the top 100 entries from the queue to be processed.
|
||||||
|
tags:
|
||||||
|
- Subscription
|
||||||
|
/subscriptions/details:
|
||||||
|
get:
|
||||||
|
produces:
|
||||||
|
- application/json
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: ok
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/v1.ListSubscriptionDetails'
|
||||||
|
summary: Returns the top 50 entries with full deatils on the source and output.
|
||||||
|
tags:
|
||||||
|
- Subscription
|
||||||
|
/subscriptions/discord/webhook/delete:
|
||||||
|
delete:
|
||||||
|
parameters:
|
||||||
|
- description: id
|
||||||
|
in: query
|
||||||
|
name: id
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
responses: {}
|
||||||
|
summary: Removes a Discord WebHook Subscription based on the Subscription ID.
|
||||||
|
tags:
|
||||||
|
- Subscription
|
||||||
|
/subscriptions/discord/webhook/new:
|
||||||
|
post:
|
||||||
|
parameters:
|
||||||
|
- description: discordWebHookId
|
||||||
|
in: query
|
||||||
|
name: discordWebHookId
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
- description: sourceId
|
||||||
|
in: query
|
||||||
|
name: sourceId
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
responses: {}
|
||||||
|
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
||||||
|
tags:
|
||||||
|
- Subscription
|
||||||
swagger: "2.0"
|
swagger: "2.0"
|
||||||
|
|||||||
2
go.mod
2
go.mod
@ -5,12 +5,12 @@ go 1.22
|
|||||||
require (
|
require (
|
||||||
github.com/PuerkitoBio/goquery v1.8.0
|
github.com/PuerkitoBio/goquery v1.8.0
|
||||||
github.com/glebarez/go-sqlite v1.22.0
|
github.com/glebarez/go-sqlite v1.22.0
|
||||||
|
github.com/go-chi/chi/v5 v5.0.7
|
||||||
github.com/go-rod/rod v0.107.1
|
github.com/go-rod/rod v0.107.1
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/huandu/go-sqlbuilder v1.27.1
|
github.com/huandu/go-sqlbuilder v1.27.1
|
||||||
github.com/joho/godotenv v1.4.0
|
github.com/joho/godotenv v1.4.0
|
||||||
github.com/labstack/echo/v4 v4.12.0
|
github.com/labstack/echo/v4 v4.12.0
|
||||||
github.com/maragudk/goqite v0.1.0
|
|
||||||
github.com/mmcdole/gofeed v1.1.3
|
github.com/mmcdole/gofeed v1.1.3
|
||||||
github.com/nicklaw5/helix/v2 v2.4.0
|
github.com/nicklaw5/helix/v2 v2.4.0
|
||||||
github.com/pressly/goose/v3 v3.20.0
|
github.com/pressly/goose/v3 v3.20.0
|
||||||
|
|||||||
8
go.sum
8
go.sum
@ -18,6 +18,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
|||||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||||
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
||||||
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
||||||
|
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
||||||
|
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||||
@ -69,17 +71,11 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
|
|||||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||||
github.com/maragudk/goqite v0.1.0 h1:k/GgnJU9DBMVAx2DxBgmB+UKQVzJNjO1JPqmUJFH4+8=
|
|
||||||
github.com/maragudk/goqite v0.1.0/go.mod h1:5430TCLkycUeLE314c9fifTrTbwcJqJXdU3iyEiF6hM=
|
|
||||||
github.com/maragudk/is v0.1.0 h1:obq9anZNmOYcaNbeT0LMyjIexdNeYTw/TLAPD/BnZHA=
|
|
||||||
github.com/maragudk/is v0.1.0/go.mod h1:W/r6+TpnISu+a88OLXQy5JQGCOhXQXXLD2e5b4xMn5c=
|
|
||||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
|
||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
|
||||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||||
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||||
|
|||||||
@ -1,25 +0,0 @@
|
|||||||
-- +goose Up
|
|
||||||
-- +goose StatementBegin
|
|
||||||
create table goqite (
|
|
||||||
id text primary key default ('m_' || lower(hex(randomblob(16)))),
|
|
||||||
created text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
|
||||||
updated text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
|
||||||
queue text not null,
|
|
||||||
body blob not null,
|
|
||||||
timeout text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
|
||||||
received integer not null default 0
|
|
||||||
) strict;
|
|
||||||
|
|
||||||
create trigger goqite_updated_timestamp after update on goqite begin
|
|
||||||
update goqite set updated = strftime('%Y-%m-%dT%H:%M:%fZ') where id = old.id;
|
|
||||||
end;
|
|
||||||
|
|
||||||
create index goqite_queue_created_idx on goqite (queue, created);
|
|
||||||
-- +goose StatementEnd
|
|
||||||
|
|
||||||
-- +goose Down
|
|
||||||
-- +goose StatementBegin
|
|
||||||
DROP TABLE goqite
|
|
||||||
DROP INDEX goqite_queue_created_idx
|
|
||||||
DROP TRIGGER goqite_updated_timestamp
|
|
||||||
-- +goose StatementEnd
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
const (
|
|
||||||
QueueRssCollection = "RssCollection"
|
|
||||||
)
|
|
||||||
|
|
||||||
type RssCollectionEvent struct {
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -11,12 +11,13 @@ import (
|
|||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
Router *echo.Echo
|
Router *echo.Echo
|
||||||
Db *database.Queries
|
Db *database.Queries
|
||||||
//dto *dto.DtoClient
|
dto *dto.DtoClient
|
||||||
config services.Configs
|
config services.Configs
|
||||||
repo services.RepositoryService
|
repo services.RepositoryService
|
||||||
}
|
}
|
||||||
@ -38,10 +39,10 @@ var (
|
|||||||
ErrUnableToConvertToJson string = "Unable to convert to json"
|
ErrUnableToConvertToJson string = "Unable to convert to json"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
|
func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler {
|
||||||
s := &Handler{
|
s := &Handler{
|
||||||
//Db: db,
|
Db: db,
|
||||||
//dto: dto.NewDtoClient(db),
|
dto: dto.NewDtoClient(db),
|
||||||
config: configs,
|
config: configs,
|
||||||
repo: services.NewRepositoryService(conn),
|
repo: services.NewRepositoryService(conn),
|
||||||
}
|
}
|
||||||
@ -84,13 +85,13 @@ func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Han
|
|||||||
sources.POST("/:ID/disable", s.disableSource)
|
sources.POST("/:ID/disable", s.disableSource)
|
||||||
sources.POST("/:ID/enable", s.enableSource)
|
sources.POST("/:ID/enable", s.enableSource)
|
||||||
|
|
||||||
//subs := v1.Group("/subscriptions")
|
subs := v1.Group("/subscriptions")
|
||||||
//subs.GET("/", s.ListSubscriptions)
|
subs.GET("/", s.ListSubscriptions)
|
||||||
//subs.GET("/details", s.ListSubscriptionDetails)
|
subs.GET("/details", s.ListSubscriptionDetails)
|
||||||
//subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
|
subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
|
||||||
//subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
|
subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
|
||||||
//subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
||||||
//subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
||||||
|
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,7 +1,11 @@
|
|||||||
package v1
|
package v1
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ListDiscordWebHooksQueueResults struct {
|
type ListDiscordWebHooksQueueResults struct {
|
||||||
@ -15,22 +19,22 @@ type ListDiscordWebHooksQueueResults struct {
|
|||||||
// @Tags Queue
|
// @Tags Queue
|
||||||
// @Router /queue/discord/webhooks [get]
|
// @Router /queue/discord/webhooks [get]
|
||||||
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
|
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
|
||||||
//func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
||||||
// p := ListDiscordWebHooksQueueResults{
|
p := ListDiscordWebHooksQueueResults{
|
||||||
// ApiStatusModel: ApiStatusModel{
|
ApiStatusModel: ApiStatusModel{
|
||||||
// Message: "OK",
|
Message: "OK",
|
||||||
// StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
// },
|
},
|
||||||
// }
|
}
|
||||||
//
|
|
||||||
// // Get the raw resp from sql
|
// Get the raw resp from sql
|
||||||
// res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
||||||
// if err != nil {
|
if err != nil {
|
||||||
// return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
||||||
// Message: err.Error(),
|
Message: err.Error(),
|
||||||
// })
|
})
|
||||||
// }
|
}
|
||||||
//
|
|
||||||
// p.Payload = res
|
p.Payload = res
|
||||||
// return c.JSON(http.StatusOK, p)
|
return c.JSON(http.StatusOK, p)
|
||||||
//}
|
}
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
package v1
|
package v1
|
||||||
|
|
||||||
/*
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@ -229,4 +228,3 @@ func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error {
|
|||||||
|
|
||||||
return c.JSON(http.StatusOK, nil)
|
return c.JSON(http.StatusOK, nil)
|
||||||
}
|
}
|
||||||
*/
|
|
||||||
@ -1,30 +0,0 @@
|
|||||||
package cron
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *Cron) CheckFfxiv() {
|
|
||||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 10, domain.SourceCollectorFfxiv)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
fc := input.NewFFXIVClient(source)
|
|
||||||
items, err := fc.CheckSource()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.SaveNewArticles(items, domain.SourceCollectorFfxiv)
|
|
||||||
}
|
|
||||||
log.Printf("[FFXIV Done!]")
|
|
||||||
}
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
package cron
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
|
||||||
)
|
|
||||||
|
|
||||||
// This is the main entry point to query all the reddit services
|
|
||||||
func (c *Cron) CheckReddit() {
|
|
||||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorReddit)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
|
|
||||||
rc := input.NewRedditClient(source)
|
|
||||||
raw, err := rc.GetContent()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
redditArticles := rc.ConvertToArticles(raw)
|
|
||||||
|
|
||||||
c.SaveNewArticles(redditArticles, domain.SourceCollectorReddit)
|
|
||||||
}
|
|
||||||
log.Print("[Reddit] Done!")
|
|
||||||
}
|
|
||||||
@ -1,34 +0,0 @@
|
|||||||
package cron
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *Cron) LoadRssQueue() {
|
|
||||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
|
|
||||||
var records []domain.SourceEntity
|
|
||||||
|
|
||||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
|
|
||||||
if err != nil {
|
|
||||||
return records, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return sources, nil
|
|
||||||
}
|
|
||||||
@ -3,57 +3,83 @@ package cron
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
_ "github.com/lib/pq"
|
_ "github.com/lib/pq"
|
||||||
"github.com/robfig/cron/v3"
|
"github.com/robfig/cron/v3"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Cron struct {
|
type Cron struct {
|
||||||
|
Db *database.Queries
|
||||||
ctx *context.Context
|
ctx *context.Context
|
||||||
timer *cron.Cron
|
timer *cron.Cron
|
||||||
repo services.RepositoryService
|
|
||||||
//queues services
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
|
func openDatabase() (*database.Queries, error) {
|
||||||
|
_env := services.NewConfig()
|
||||||
|
connString := _env.GetConfig(services.Sql_Connection_String)
|
||||||
|
if connString == "" {
|
||||||
|
panic("Connection String is null!")
|
||||||
|
}
|
||||||
|
db, err := sql.Open("postgres", connString)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
queries := database.New(db)
|
||||||
|
return queries, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewScheduler(ctx context.Context) *Cron {
|
||||||
c := &Cron{
|
c := &Cron{
|
||||||
ctx: &ctx,
|
ctx: &ctx,
|
||||||
repo: services.NewRepositoryService(conn),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
timer := cron.New()
|
timer := cron.New()
|
||||||
|
queries, err := openDatabase()
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
c.Db = queries
|
||||||
|
|
||||||
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||||
features := services.GetEnvConfig()
|
features := services.NewConfig()
|
||||||
|
|
||||||
if features.RedditEnabled {
|
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||||
|
if res {
|
||||||
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
||||||
log.Print("[Input] Reddit backend was enabled")
|
log.Print("[Input] Reddit backend was enabled")
|
||||||
//go c.CheckReddit()
|
//go c.CheckReddit()
|
||||||
}
|
}
|
||||||
|
|
||||||
if features.YoutubeEnabled {
|
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||||
|
if res {
|
||||||
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
||||||
log.Print("[Input] YouTube backend was enabled")
|
log.Print("[Input] YouTube backend was enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
if features.FfxivEnabled {
|
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||||
|
if res {
|
||||||
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
||||||
log.Print("[Input] FFXIV backend was enabled")
|
log.Print("[Input] FFXIV backend was enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
if features.TwitchEnabled {
|
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||||
|
if res {
|
||||||
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
||||||
log.Print("[Input] Twitch backend was enabled")
|
log.Print("[Input] Twitch backend was enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
//timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
||||||
//log.Print("[Output] Discord Output was enabled")
|
log.Print("[Output] Discord Output was enabled")
|
||||||
|
|
||||||
c.timer = timer
|
c.timer = timer
|
||||||
return c
|
return c
|
||||||
@ -67,7 +93,104 @@ func (c *Cron) Stop() {
|
|||||||
c.timer.Stop()
|
c.timer.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
/* TODO move to the sqlite queue
|
// This is the main entry point to query all the reddit services
|
||||||
|
func (c *Cron) CheckReddit() {
|
||||||
|
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("[Reddit] Checking '%v'...", source.Name)
|
||||||
|
rc := input.NewRedditClient(source)
|
||||||
|
raw, err := rc.GetContent()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
redditArticles := rc.ConvertToArticles(raw)
|
||||||
|
c.checkPosts(redditArticles, "Reddit")
|
||||||
|
}
|
||||||
|
log.Print("[Reddit] Done!")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cron) CheckYoutube() {
|
||||||
|
// Add call to the db to request youtube sources.
|
||||||
|
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("[YouTube] Checking '%v'...", source.Name)
|
||||||
|
yc := input.NewYoutubeClient(source)
|
||||||
|
raw, err := yc.GetContent()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
c.checkPosts(raw, "YouTube")
|
||||||
|
}
|
||||||
|
log.Print("[YouTube] Done!")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cron) CheckFfxiv() {
|
||||||
|
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fc := input.NewFFXIVClient(source)
|
||||||
|
items, err := fc.CheckSource()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
c.checkPosts(items, "FFXIV")
|
||||||
|
}
|
||||||
|
log.Printf("[FFXIV Done!]")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cron) CheckTwitch() error {
|
||||||
|
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tc, err := input.NewTwitchClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tc.Login()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("[Twitch] Checking '%v'...", source.Name)
|
||||||
|
tc.ReplaceSourceRecord(source)
|
||||||
|
items, err := tc.GetContent()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
c.checkPosts(items, "Twitch")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Print("[Twitch] Done!")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Cron) CheckDiscordQueue() error {
|
func (c *Cron) CheckDiscordQueue() error {
|
||||||
// Get items from the table
|
// Get items from the table
|
||||||
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
||||||
@ -137,26 +260,55 @@ func (c *Cron) CheckDiscordQueue() error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
*/
|
|
||||||
|
|
||||||
func (c Cron) SaveNewArticles(posts []domain.ArticleEntity, sourceName string) error {
|
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
|
||||||
for _, item := range posts {
|
for _, item := range posts {
|
||||||
_, err := c.repo.Articles.GetByUrl(*c.ctx, item.Url)
|
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
|
||||||
if err == nil {
|
|
||||||
// This url is already known, so skip it
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load the new article in the repository
|
|
||||||
rows, err := c.repo.Articles.Create(*c.ctx, item.SourceID, item.Tags, item.Title, item.Url, item.Thumbnail, item.Description, item.AuthorName, item.AuthorImageUrl, item.PubDate, item.IsVideo)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
id := uuid.New()
|
||||||
}
|
|
||||||
|
err := c.postArticle(id, item)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.addToDiscordQueue(id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if rows != 1 {
|
|
||||||
return errors.New("failed to create a new record for some reason")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
time.Sleep(30 * time.Second)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
|
||||||
|
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
|
||||||
|
ID: id,
|
||||||
|
Sourceid: item.Sourceid,
|
||||||
|
Tags: item.Tags,
|
||||||
|
Title: item.Title,
|
||||||
|
Url: item.Url,
|
||||||
|
Pubdate: item.Pubdate,
|
||||||
|
Video: item.Video,
|
||||||
|
Videoheight: item.Videoheight,
|
||||||
|
Videowidth: item.Videowidth,
|
||||||
|
Thumbnail: item.Thumbnail,
|
||||||
|
Description: item.Description,
|
||||||
|
Authorname: item.Authorname,
|
||||||
|
Authorimage: item.Authorimage,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
|
||||||
|
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Articleid: Id,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,70 +2,33 @@ package cron_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||||
_ "github.com/glebarez/go-sqlite"
|
|
||||||
"github.com/pressly/goose/v3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCheckReddit(t *testing.T) {
|
func TestInvokeTwitch(t *testing.T) {
|
||||||
db, err := setupInMemoryDb()
|
|
||||||
if err != nil {
|
|
||||||
t.Log(err)
|
|
||||||
t.FailNow()
|
|
||||||
}
|
|
||||||
defer db.Close()
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO add database mocks but not sure how to do that yet.
|
||||||
|
func TestCheckReddit(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c := cron.NewScheduler(ctx, db)
|
c := cron.NewScheduler(ctx)
|
||||||
c.CheckReddit()
|
c.CheckReddit()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckYouTube(t *testing.T) {
|
func TestCheckYouTube(t *testing.T) {
|
||||||
db, err := setupInMemoryDb()
|
|
||||||
if err != nil {
|
|
||||||
t.Log(err)
|
|
||||||
t.FailNow()
|
|
||||||
}
|
|
||||||
defer db.Close()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c := cron.NewScheduler(ctx, db)
|
c := cron.NewScheduler(ctx)
|
||||||
c.CheckYoutube()
|
c.CheckYoutube()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckTwitch(t *testing.T) {
|
func TestCheckTwitch(t *testing.T) {
|
||||||
db, err := setupInMemoryDb()
|
|
||||||
if err != nil {
|
|
||||||
t.Log(err)
|
|
||||||
t.FailNow()
|
|
||||||
}
|
|
||||||
defer db.Close()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c := cron.NewScheduler(ctx, db)
|
c := cron.NewScheduler(ctx)
|
||||||
err = c.CheckTwitch()
|
err := c.CheckTwitch()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupInMemoryDb() (*sql.DB, error) {
|
|
||||||
db, err := sql.Open("sqlite", ":memory:")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = goose.SetDialect("sqlite3")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = goose.Up(db, "../database/migrations")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return db, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,41 +0,0 @@
|
|||||||
package cron
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *Cron) CheckTwitch() error {
|
|
||||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorTwitch)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tc, err := input.NewTwitchClient()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tc.Login()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
|
|
||||||
tc.ReplaceSourceRecord(source)
|
|
||||||
items, err := tc.GetContent()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
c.SaveNewArticles(items, domain.SourceCollectorTwitch)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Print("[Twitch] Done!")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -1,34 +0,0 @@
|
|||||||
package cron
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *Cron) CheckYoutube() {
|
|
||||||
// Add call to the db to request youtube sources.
|
|
||||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorYoutube)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
|
|
||||||
|
|
||||||
yc := input.NewYoutubeClient(source)
|
|
||||||
raw, err := yc.GetContent()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
c.SaveNewArticles(raw, domain.SourceCollectorYoutube)
|
|
||||||
}
|
|
||||||
log.Print("[YouTube] Done!")
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
140
internal/services/dto/articles.go
Normal file
140
internal/services/dto/articles.go
Normal file
@ -0,0 +1,140 @@
|
|||||||
|
// The converter package lives between the database calls and the API calls.
|
||||||
|
// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much
|
||||||
|
package dto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DtoClient struct {
|
||||||
|
db *database.Queries
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewDtoClient(db *database.Queries) *DtoClient {
|
||||||
|
return &DtoClient{
|
||||||
|
db: db,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) {
|
||||||
|
var res []models.ArticleDto
|
||||||
|
|
||||||
|
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
|
||||||
|
Limit: int32(limit),
|
||||||
|
Offset: int32(limit * page),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, article := range a {
|
||||||
|
res = append(res, c.convertArticle(article))
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
|
||||||
|
var res []models.ArticleDto
|
||||||
|
|
||||||
|
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
|
||||||
|
Limit: limit,
|
||||||
|
Offset: page * limit,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, article := range a {
|
||||||
|
res = append(res, c.convertArticle(article))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) {
|
||||||
|
a, err := c.db.GetArticleByID(ctx, ID)
|
||||||
|
if err != nil {
|
||||||
|
return models.ArticleDto{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.convertArticle(a), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) {
|
||||||
|
a, err := c.db.GetArticleByID(ctx, ID)
|
||||||
|
if err != nil {
|
||||||
|
return models.ArticleDetailsDto{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s, err := c.db.GetSourceByID(ctx, a.Sourceid)
|
||||||
|
if err != nil {
|
||||||
|
return models.ArticleDetailsDto{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
res := c.convertArticleDetails(a, s)
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) {
|
||||||
|
var res []models.ArticleDto
|
||||||
|
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
|
||||||
|
Sourceid: SourceID,
|
||||||
|
Limit: int32(limit),
|
||||||
|
Offset: int32(limit * page),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, article := range a {
|
||||||
|
res = append(res, c.convertArticle(article))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto {
|
||||||
|
return models.ArticleDto{
|
||||||
|
ID: i.ID,
|
||||||
|
Source: i.Sourceid,
|
||||||
|
Tags: c.SplitTags(i.Tags),
|
||||||
|
Title: i.Title,
|
||||||
|
Url: i.Url,
|
||||||
|
Pubdate: i.Pubdate,
|
||||||
|
Video: i.Video.String,
|
||||||
|
Videoheight: i.Videoheight,
|
||||||
|
Videowidth: i.Videoheight,
|
||||||
|
Thumbnail: i.Thumbnail,
|
||||||
|
Description: i.Description,
|
||||||
|
Authorname: i.Authorname.String,
|
||||||
|
Authorimage: i.Authorimage.String,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto {
|
||||||
|
return models.ArticleDetailsDto{
|
||||||
|
ID: i.ID,
|
||||||
|
Source: c.ConvertToSource(s),
|
||||||
|
Tags: c.SplitTags(i.Tags),
|
||||||
|
Title: i.Title,
|
||||||
|
Url: i.Url,
|
||||||
|
Pubdate: i.Pubdate,
|
||||||
|
Video: i.Video.String,
|
||||||
|
Videoheight: i.Videoheight,
|
||||||
|
Videowidth: i.Videoheight,
|
||||||
|
Thumbnail: i.Thumbnail,
|
||||||
|
Description: i.Description,
|
||||||
|
Authorname: i.Authorname.String,
|
||||||
|
Authorimage: i.Authorimage.String,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c DtoClient) SplitTags(t string) []string {
|
||||||
|
return strings.Split(t, ", ")
|
||||||
|
}
|
||||||
63
internal/services/dto/discordwebhooks.go
Normal file
63
internal/services/dto/discordwebhooks.go
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
package dto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
|
||||||
|
var res []models.DiscordWebHooksDto
|
||||||
|
|
||||||
|
items, err := c.db.ListDiscordWebhooks(ctx, total)
|
||||||
|
if err != nil {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertDiscordWebhook(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) {
|
||||||
|
var res models.DiscordWebHooksDto
|
||||||
|
|
||||||
|
item, err := c.db.GetDiscordWebHooksByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.ConvertDiscordWebhook(item), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) {
|
||||||
|
var res []models.DiscordWebHooksDto
|
||||||
|
|
||||||
|
items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{
|
||||||
|
Server: server,
|
||||||
|
Channel: channel,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertDiscordWebhook(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto {
|
||||||
|
return models.DiscordWebHooksDto{
|
||||||
|
ID: i.ID,
|
||||||
|
Url: i.Url,
|
||||||
|
Server: i.Server,
|
||||||
|
Channel: i.Channel,
|
||||||
|
Enabled: i.Enabled,
|
||||||
|
}
|
||||||
|
}
|
||||||
42
internal/services/dto/queue.go
Normal file
42
internal/services/dto/queue.go
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
package dto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) {
|
||||||
|
var res []models.DiscordQueueDetailsDto
|
||||||
|
|
||||||
|
items, err := c.db.ListDiscordQueueItems(ctx, limit)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
article, err := c.GetArticleDetails(ctx, item.ID)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
res = append(res, models.DiscordQueueDetailsDto{
|
||||||
|
ID: item.ID,
|
||||||
|
Article: article,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto {
|
||||||
|
return models.DiscordQueueDto{
|
||||||
|
ID: i.ID,
|
||||||
|
Articleid: i.Articleid,
|
||||||
|
}
|
||||||
|
}
|
||||||
85
internal/services/dto/sources.go
Normal file
85
internal/services/dto/sources.go
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
package dto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
|
||||||
|
var res []models.SourceDto
|
||||||
|
|
||||||
|
items, err := c.db.ListSources(ctx, limit)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertToSource(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) {
|
||||||
|
var res []models.SourceDto
|
||||||
|
|
||||||
|
items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName))
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertToSource(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) {
|
||||||
|
var res models.SourceDto
|
||||||
|
|
||||||
|
item, err := c.db.GetSourceByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.ConvertToSource(item), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) {
|
||||||
|
var res models.SourceDto
|
||||||
|
|
||||||
|
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
|
||||||
|
Name: name,
|
||||||
|
Source: source,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.ConvertToSource(item), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto {
|
||||||
|
var deleted bool
|
||||||
|
if !i.Deleted.Valid {
|
||||||
|
deleted = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return models.SourceDto{
|
||||||
|
ID: i.ID,
|
||||||
|
Site: i.Site,
|
||||||
|
Name: i.Name,
|
||||||
|
Source: i.Source,
|
||||||
|
Type: i.Type,
|
||||||
|
Value: i.Value.String,
|
||||||
|
Enabled: i.Enabled,
|
||||||
|
Url: i.Url,
|
||||||
|
Tags: c.SplitTags(i.Tags),
|
||||||
|
Deleted: deleted,
|
||||||
|
}
|
||||||
|
}
|
||||||
91
internal/services/dto/subscriptions.go
Normal file
91
internal/services/dto/subscriptions.go
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
package dto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
|
||||||
|
var res []models.SubscriptionDto
|
||||||
|
|
||||||
|
items, err := c.db.ListSubscriptions(ctx, limit)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertSubscription(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) {
|
||||||
|
var res []models.SubscriptionDetailsDto
|
||||||
|
|
||||||
|
items, err := c.ListSubscriptions(ctx, limit)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
source, err := c.GetSourceById(ctx, item.SourceId)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
res = append(res, models.SubscriptionDetailsDto{
|
||||||
|
ID: item.ID,
|
||||||
|
Source: source,
|
||||||
|
DiscordWebHook: dwh,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
||||||
|
var res []models.SubscriptionDto
|
||||||
|
|
||||||
|
items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertSubscription(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
||||||
|
var res []models.SubscriptionDto
|
||||||
|
|
||||||
|
items, err := c.db.GetSubscriptionsBySourceID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
res = append(res, c.ConvertSubscription(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto {
|
||||||
|
return models.SubscriptionDto{
|
||||||
|
ID: i.ID,
|
||||||
|
DiscordWebhookId: i.Discordwebhookid,
|
||||||
|
SourceId: i.Sourceid,
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,6 +1,7 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -12,7 +13,7 @@ import (
|
|||||||
"github.com/go-rod/rod/lib/launcher"
|
"github.com/go-rod/rod/lib/launcher"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -24,7 +25,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type FFXIVClient struct {
|
type FFXIVClient struct {
|
||||||
record domain.SourceEntity
|
record database.Source
|
||||||
//SourceID uint
|
//SourceID uint
|
||||||
//Url string
|
//Url string
|
||||||
//Region string
|
//Region string
|
||||||
@ -32,15 +33,15 @@ type FFXIVClient struct {
|
|||||||
cacheGroup string
|
cacheGroup string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
|
func NewFFXIVClient(Record database.Source) FFXIVClient {
|
||||||
return FFXIVClient{
|
return FFXIVClient{
|
||||||
record: Record,
|
record: Record,
|
||||||
cacheGroup: "ffxiv",
|
cacheGroup: "ffxiv",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
|
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||||
var articles []domain.ArticleEntity
|
var articles []database.Article
|
||||||
|
|
||||||
parser := fc.GetBrowser()
|
parser := fc.GetBrowser()
|
||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
@ -96,16 +97,18 @@ func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
|
|||||||
return articles, err
|
return articles, err
|
||||||
}
|
}
|
||||||
|
|
||||||
article := domain.ArticleEntity{
|
article := database.Article{
|
||||||
SourceID: fc.record.ID,
|
Sourceid: fc.record.ID,
|
||||||
Tags: tags,
|
Tags: tags,
|
||||||
Title: title,
|
Title: title,
|
||||||
Url: link,
|
Url: link,
|
||||||
PubDate: pubDate,
|
Pubdate: pubDate,
|
||||||
Thumbnail: thumb,
|
Videoheight: 0,
|
||||||
Description: description,
|
Videowidth: 0,
|
||||||
AuthorName: authorName,
|
Thumbnail: thumb,
|
||||||
AuthorImageUrl: authorImage,
|
Description: description,
|
||||||
|
Authorname: sql.NullString{String: authorName},
|
||||||
|
Authorimage: sql.NullString{String: authorImage},
|
||||||
}
|
}
|
||||||
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
||||||
|
|
||||||
|
|||||||
@ -3,16 +3,18 @@ package input_test
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
|
var FFXIVRecord database.Source = database.Source{
|
||||||
ID: 997,
|
ID: uuid.New(),
|
||||||
DisplayName: "Final Fantasy XIV - NA",
|
Site: "ffxiv",
|
||||||
Source: domain.SourceCollectorFfxiv,
|
Name: "Final Fantasy XIV - NA",
|
||||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
Source: "ffxiv",
|
||||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||||
|
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivGetParser(t *testing.T) {
|
func TestFfxivGetParser(t *testing.T) {
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -8,6 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"github.com/go-rod/rod"
|
"github.com/go-rod/rod"
|
||||||
@ -15,8 +17,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type RedditClient struct {
|
type RedditClient struct {
|
||||||
config services.Configs
|
config RedditConfig
|
||||||
record domain.SourceEntity
|
record database.Source
|
||||||
}
|
}
|
||||||
|
|
||||||
type RedditConfig struct {
|
type RedditConfig struct {
|
||||||
@ -25,11 +27,14 @@ type RedditConfig struct {
|
|||||||
PullNSFW string
|
PullNSFW string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRedditClient(record domain.SourceEntity) *RedditClient {
|
func NewRedditClient(Record database.Source) *RedditClient {
|
||||||
rc := RedditClient{
|
rc := RedditClient{
|
||||||
record: record,
|
record: Record,
|
||||||
config: services.GetEnvConfig(),
|
|
||||||
}
|
}
|
||||||
|
cc := services.NewConfig()
|
||||||
|
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
|
||||||
|
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
|
||||||
|
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
|
||||||
|
|
||||||
//rc.disableHttp2Client()
|
//rc.disableHttp2Client()
|
||||||
|
|
||||||
@ -66,7 +71,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
|||||||
// TODO Wire this to support the config options
|
// TODO Wire this to support the config options
|
||||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||||
|
|
||||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
|
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
||||||
|
|
||||||
content, err := getHttpContent(Url)
|
content, err := getHttpContent(Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -83,28 +88,24 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
|||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
|
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
|
||||||
var redditArticles []domain.ArticleEntity
|
var redditArticles []database.Article
|
||||||
|
|
||||||
for _, item := range items.Data.Children {
|
for _, item := range items.Data.Children {
|
||||||
var article domain.ArticleEntity
|
var article database.Article
|
||||||
article, err := rc.convertToArticle(item.Data)
|
article, err := rc.convertToArticle(item.Data)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[Reddit] %v", err)
|
log.Printf("[Reddit] %v", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
redditArticles = append(redditArticles, article)
|
redditArticles = append(redditArticles, article)
|
||||||
}
|
}
|
||||||
|
|
||||||
return redditArticles
|
return redditArticles
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||||
// This data can be passed to the database.
|
// This data can be passed to the database.
|
||||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
|
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
|
||||||
var item domain.ArticleEntity
|
var item database.Article
|
||||||
|
|
||||||
if source.Content == "" && source.Url != "" {
|
if source.Content == "" && source.Url != "" {
|
||||||
item = rc.convertPicturePost(source)
|
item = rc.convertPicturePost(source)
|
||||||
@ -130,57 +131,65 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.Artic
|
|||||||
return item, nil
|
return item, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
|
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
|
||||||
var item = domain.ArticleEntity{
|
var item = database.Article{
|
||||||
SourceID: rc.record.ID,
|
Sourceid: rc.record.ID,
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
PubDate: time.Now(),
|
Pubdate: time.Now(),
|
||||||
IsVideo: false,
|
Video: sql.NullString{String: "null"},
|
||||||
Thumbnail: source.Thumbnail,
|
Videoheight: 0,
|
||||||
Description: source.Content,
|
Videowidth: 0,
|
||||||
AuthorName: source.Author,
|
Thumbnail: source.Thumbnail,
|
||||||
AuthorImageUrl: "",
|
Description: source.Content,
|
||||||
|
Authorname: sql.NullString{String: source.Author},
|
||||||
|
Authorimage: sql.NullString{String: "null"},
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
|
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
|
||||||
var item = domain.ArticleEntity{
|
var item = database.Article{
|
||||||
SourceID: rc.record.ID,
|
Sourceid: rc.record.ID,
|
||||||
Tags: "a",
|
Tags: "a",
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
PubDate: time.Now(),
|
Pubdate: time.Now(),
|
||||||
|
Videoheight: 0,
|
||||||
|
Videowidth: 0,
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
AuthorName: source.Author,
|
Authorname: sql.NullString{String: source.Author},
|
||||||
Description: source.Content,
|
Description: source.Content,
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
|
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
|
||||||
var item = domain.ArticleEntity{
|
var item = database.Article{
|
||||||
SourceID: rc.record.ID,
|
Sourceid: rc.record.ID,
|
||||||
Tags: "a",
|
Tags: "a",
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
PubDate: time.Now(),
|
Pubdate: time.Now(),
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
AuthorName: source.Author,
|
Videoheight: 0,
|
||||||
|
Videowidth: 0,
|
||||||
|
Authorname: sql.NullString{String: source.Author},
|
||||||
Description: source.Media.RedditVideo.FallBackUrl,
|
Description: source.Media.RedditVideo.FallBackUrl,
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
// This post is nothing more then a redirect to another location.
|
// This post is nothing more then a redirect to another location.
|
||||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
|
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
|
||||||
var item = domain.ArticleEntity{
|
var item = database.Article{
|
||||||
SourceID: rc.record.ID,
|
Sourceid: rc.record.ID,
|
||||||
Tags: "a",
|
Tags: "a",
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
PubDate: time.Now(),
|
Pubdate: time.Now(),
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
AuthorName: source.Author,
|
Videoheight: 0,
|
||||||
|
Videowidth: 0,
|
||||||
|
Authorname: sql.NullString{String: source.Author},
|
||||||
Description: source.UrlOverriddenByDest,
|
Description: source.UrlOverriddenByDest,
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
|
|||||||
@ -3,16 +3,18 @@ package input_test
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
var RedditRecord domain.SourceEntity = domain.SourceEntity{
|
var RedditRecord database.Source = database.Source{
|
||||||
ID: 999,
|
ID: uuid.New(),
|
||||||
DisplayName: "dadjokes",
|
Name: "dadjokes",
|
||||||
Source: domain.SourceCollectorReddit,
|
Source: "reddit",
|
||||||
Url: "https://reddit.com/r/dadjokes",
|
Site: "reddit",
|
||||||
Tags: "reddit, dadjokes",
|
Url: "https://reddit.com/r/dadjokes",
|
||||||
|
Tags: "reddit, dadjokes",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetContent(t *testing.T) {
|
func TestGetContent(t *testing.T) {
|
||||||
|
|||||||
@ -8,10 +8,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var rssRecord = domain.SourceEntity{
|
var rssRecord = domain.SourceEntity{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
DisplayName: "ArsTechnica",
|
DisplayName: "ArsTechnica",
|
||||||
Source: domain.SourceCollectorRss,
|
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRssClientConstructor(t *testing.T) {
|
func TestRssClientConstructor(t *testing.T) {
|
||||||
|
|||||||
@ -1,18 +1,19 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"github.com/nicklaw5/helix/v2"
|
"github.com/nicklaw5/helix/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TwitchClient struct {
|
type TwitchClient struct {
|
||||||
SourceRecord domain.SourceEntity
|
SourceRecord database.Source
|
||||||
|
|
||||||
// config
|
// config
|
||||||
monitorClips string
|
monitorClips string
|
||||||
@ -71,7 +72,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// This will let you replace the bound source record to keep the same session alive.
|
// This will let you replace the bound source record to keep the same session alive.
|
||||||
func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
|
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
||||||
tc.SourceRecord = source
|
tc.SourceRecord = source
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,8 +87,8 @@ func (tc *TwitchClient) Login() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
|
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||||
var items []domain.ArticleEntity
|
var items []database.Article
|
||||||
|
|
||||||
user, err := tc.GetUserDetails()
|
user, err := tc.GetUserDetails()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -100,31 +101,31 @@ func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, video := range posts {
|
for _, video := range posts {
|
||||||
var article domain.ArticleEntity
|
var article database.Article
|
||||||
|
|
||||||
AuthorName, err := tc.ExtractAuthor(video)
|
AuthorName, err := tc.ExtractAuthor(video)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
article.AuthorName = AuthorName
|
article.Authorname = sql.NullString{String: AuthorName}
|
||||||
|
|
||||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
article.AuthorImageUrl = Authorimage
|
article.Authorimage = sql.NullString{String: Authorimage}
|
||||||
|
|
||||||
article.Description, err = tc.ExtractDescription(video)
|
article.Description, err = tc.ExtractDescription(video)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
|
|
||||||
article.PubDate, err = tc.ExtractPubDate(video)
|
article.Pubdate, err = tc.ExtractPubDate(video)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
|
|
||||||
article.SourceID = tc.SourceRecord.ID
|
article.Sourceid = tc.SourceRecord.ID
|
||||||
article.Tags, err = tc.ExtractTags(video, user)
|
article.Tags, err = tc.ExtractTags(video, user)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
@ -155,7 +156,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
|
|||||||
var blank helix.User
|
var blank helix.User
|
||||||
|
|
||||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||||
Logins: []string{tc.SourceRecord.DisplayName},
|
Logins: []string{tc.SourceRecord.Name},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return blank, err
|
return blank, err
|
||||||
|
|||||||
@ -4,20 +4,21 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
var TwitchSourceRecord = domain.SourceEntity{
|
var TwitchSourceRecord = database.Source{
|
||||||
ID: 9999,
|
ID: uuid.New(),
|
||||||
DisplayName: "nintendo",
|
Name: "nintendo",
|
||||||
Source: domain.SourceCollectorTwitch,
|
Source: "Twitch",
|
||||||
}
|
}
|
||||||
|
|
||||||
var TwitchInvalidRecord = domain.SourceEntity{
|
var TwitchInvalidRecord = database.Source{
|
||||||
ID: 9999,
|
ID: uuid.New(),
|
||||||
DisplayName: "EvilNintendo",
|
Name: "EvilNintendo",
|
||||||
Source: domain.SourceCollectorTwitch,
|
Source: "Twitch",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchLogin(t *testing.T) {
|
func TestTwitchLogin(t *testing.T) {
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
@ -11,11 +12,11 @@ import (
|
|||||||
"github.com/go-rod/rod/lib/launcher"
|
"github.com/go-rod/rod/lib/launcher"
|
||||||
"github.com/mmcdole/gofeed"
|
"github.com/mmcdole/gofeed"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
type YoutubeClient struct {
|
type YoutubeClient struct {
|
||||||
record domain.SourceEntity
|
record database.Source
|
||||||
|
|
||||||
// internal variables at time of collection
|
// internal variables at time of collection
|
||||||
channelID string
|
channelID string
|
||||||
@ -36,7 +37,7 @@ var (
|
|||||||
|
|
||||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||||
|
|
||||||
func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
|
func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||||
yc := YoutubeClient{
|
yc := YoutubeClient{
|
||||||
record: Record,
|
record: Record,
|
||||||
cacheGroup: "youtube",
|
cacheGroup: "youtube",
|
||||||
@ -45,8 +46,8 @@ func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CheckSource will go and run all the commands needed to process a source.
|
// CheckSource will go and run all the commands needed to process a source.
|
||||||
func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
|
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
||||||
var items []domain.ArticleEntity
|
var items []database.Article
|
||||||
docParser, err := yc.GetParser(yc.record.Url)
|
docParser, err := yc.GetParser(yc.record.Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
@ -246,7 +247,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
|
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||||
parser, err := yc.GetParser(item.Link)
|
parser, err := yc.GetParser(item.Link)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||||
@ -264,16 +265,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntit
|
|||||||
log.Printf("[YouTube] %v", msg)
|
log.Printf("[YouTube] %v", msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
var article = domain.ArticleEntity{
|
var article = database.Article{
|
||||||
SourceID: yc.record.ID,
|
Sourceid: yc.record.ID,
|
||||||
Tags: tags,
|
Tags: tags,
|
||||||
Title: item.Title,
|
Title: item.Title,
|
||||||
Url: item.Link,
|
Url: item.Link,
|
||||||
PubDate: *item.PublishedParsed,
|
Pubdate: *item.PublishedParsed,
|
||||||
Thumbnail: thumb,
|
Thumbnail: thumb,
|
||||||
Description: item.Description,
|
Description: item.Description,
|
||||||
AuthorName: item.Author.Name,
|
Authorname: sql.NullString{String: item.Author.Name},
|
||||||
AuthorImageUrl: yc.avatarUri,
|
Authorimage: sql.NullString{String: yc.avatarUri},
|
||||||
}
|
}
|
||||||
return article
|
return article
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,15 +3,17 @@ package input_test
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
var YouTubeRecord domain.SourceEntity = domain.SourceEntity{
|
var YouTubeRecord database.Source = database.Source{
|
||||||
ID: 999,
|
ID: uuid.New(),
|
||||||
DisplayName: "dadjokes",
|
Name: "dadjokes",
|
||||||
Source: domain.SourceCollectorYoutube,
|
Source: "reddit",
|
||||||
Url: "https://youtube.com/gamegrumps",
|
Site: "reddit",
|
||||||
|
Url: "https://youtube.com/gamegrumps",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetPageParser(t *testing.T) {
|
func TestGetPageParser(t *testing.T) {
|
||||||
|
|||||||
@ -1,26 +0,0 @@
|
|||||||
package services
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
"github.com/maragudk/goqite"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
type queues struct {
|
|
||||||
repos RepositoryService
|
|
||||||
|
|
||||||
RssCollection goqite.Queue
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewQueues(conn *sql.DB) queues {
|
|
||||||
return queues{
|
|
||||||
repos: NewRepositoryService(conn),
|
|
||||||
RssCollection: *goqite.New(goqite.NewOpts{
|
|
||||||
DB: conn,
|
|
||||||
Name: domain.QueueRssCollection,
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
4
makefile
4
makefile
@ -4,8 +4,8 @@ help: ## Shows this help command
|
|||||||
|
|
||||||
build: ## builds the application with the current go runtime
|
build: ## builds the application with the current go runtime
|
||||||
~/go/bin/swag f
|
~/go/bin/swag f
|
||||||
~/go/bin/swag init -g cmd/server.go
|
~/go/bin/swag i
|
||||||
go build cmd/server.go
|
go build .
|
||||||
|
|
||||||
docker-build: ## Generates the docker image
|
docker-build: ## Generates the docker image
|
||||||
docker build -t "newsbot.collector.api" .
|
docker build -t "newsbot.collector.api" .
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user