Compare commits
6 Commits
main
...
features/h
| Author | SHA1 | Date | |
|---|---|---|---|
| fbed111fbb | |||
| 43a086cc02 | |||
| 2afad5d761 | |||
| fcf80ec5fe | |||
| ce45b509d3 | |||
| e48b64bbaa |
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,7 +2,6 @@
|
|||||||
dev.session.sql
|
dev.session.sql
|
||||||
__debug_bin
|
__debug_bin
|
||||||
server
|
server
|
||||||
.vscode
|
|
||||||
|
|
||||||
# Binaries for programs and plugins
|
# Binaries for programs and plugins
|
||||||
*.exe
|
*.exe
|
||||||
@ -11,6 +10,7 @@ server
|
|||||||
*.so
|
*.so
|
||||||
*.dylib
|
*.dylib
|
||||||
collector
|
collector
|
||||||
|
*.db
|
||||||
|
|
||||||
# Test binary, built with `go test -c`
|
# Test binary, built with `go test -c`
|
||||||
*.test
|
*.test
|
||||||
|
|||||||
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@ -9,7 +9,7 @@
|
|||||||
"type": "go",
|
"type": "go",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"mode": "auto",
|
"mode": "auto",
|
||||||
"program": "."
|
"program": "./cmd/server.go"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@ -10,8 +10,7 @@ import (
|
|||||||
"github.com/pressly/goose/v3"
|
"github.com/pressly/goose/v3"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||||
)
|
)
|
||||||
@ -42,12 +41,15 @@ func main() {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
queries := database.New(db)
|
//queues := services.NewQueues(db)
|
||||||
|
//queues.RssIndex.Send(ctx, goqite.Message{
|
||||||
|
// Body: []byte("hello world"),
|
||||||
|
//})
|
||||||
|
|
||||||
c := cron.NewScheduler(ctx)
|
c := cron.NewScheduler(ctx, db)
|
||||||
c.Start()
|
c.Start()
|
||||||
|
|
||||||
server := v1.NewServer(ctx, queries, configs, db)
|
server := v1.NewServer(ctx, configs, db)
|
||||||
|
|
||||||
fmt.Println("API is online and waiting for requests.")
|
fmt.Println("API is online and waiting for requests.")
|
||||||
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
|
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
|
||||||
|
|||||||
386
docs/docs.go
386
docs/docs.go
@ -449,25 +449,6 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/queue/discord/webhooks": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Queue"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/settings/{key}": {
|
"/settings/{key}": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
@ -875,167 +856,6 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"/subscriptions": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"400": {
|
|
||||||
"description": "Unable to reach SQL.",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"500": {
|
|
||||||
"description": "Failed to process data from SQL.",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/by/SourceId": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "id",
|
|
||||||
"name": "id",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/by/discordId": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "id",
|
|
||||||
"name": "id",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"400": {
|
|
||||||
"description": "Unable to reach SQL or Data problems",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"500": {
|
|
||||||
"description": "Data problems",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/details": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 50 entries with full deatils on the source and output.",
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptionDetails"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/discord/webhook/delete": {
|
|
||||||
"delete": {
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "id",
|
|
||||||
"name": "id",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/discord/webhook/new": {
|
|
||||||
"post": {
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "discordWebHookId",
|
|
||||||
"name": "discordWebHookId",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "sourceId",
|
|
||||||
"name": "sourceId",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
@ -1192,212 +1012,6 @@ const docTemplate = `{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"models.ArticleDetailsDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"authorImage": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"authorName": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"description": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"pubdate": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"$ref": "#/definitions/models.SourceDto"
|
|
||||||
},
|
|
||||||
"tags": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"thumbnail": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"title": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"video": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"videoHeight": {
|
|
||||||
"type": "integer"
|
|
||||||
},
|
|
||||||
"videoWidth": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.DiscordQueueDetailsDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"article": {
|
|
||||||
"$ref": "#/definitions/models.ArticleDetailsDto"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.DiscordWebHooksDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"ID": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"channel": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"enabled": {
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"server": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.SourceDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"deleted": {
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"enabled": {
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"name": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"site": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"tags": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"type": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"value": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.SubscriptionDetailsDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"discordwebhook": {
|
|
||||||
"$ref": "#/definitions/models.DiscordWebHooksDto"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"$ref": "#/definitions/models.SourceDto"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.SubscriptionDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"discordwebhookid": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"sourceid": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ApiError": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ListDiscordWebHooksQueueResults": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"payload": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ListSubscriptionDetails": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"payload": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/models.SubscriptionDetailsDto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ListSubscriptions": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"payload": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/models.SubscriptionDto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
|
|||||||
@ -440,25 +440,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/queue/discord/webhooks": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Queue"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/settings/{key}": {
|
"/settings/{key}": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
@ -866,167 +847,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"/subscriptions": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"400": {
|
|
||||||
"description": "Unable to reach SQL.",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"500": {
|
|
||||||
"description": "Failed to process data from SQL.",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/by/SourceId": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "id",
|
|
||||||
"name": "id",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/by/discordId": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "id",
|
|
||||||
"name": "id",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"400": {
|
|
||||||
"description": "Unable to reach SQL or Data problems",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"500": {
|
|
||||||
"description": "Data problems",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/details": {
|
|
||||||
"get": {
|
|
||||||
"produces": [
|
|
||||||
"application/json"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Returns the top 50 entries with full deatils on the source and output.",
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "ok",
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/definitions/v1.ListSubscriptionDetails"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/discord/webhook/delete": {
|
|
||||||
"delete": {
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "id",
|
|
||||||
"name": "id",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/subscriptions/discord/webhook/new": {
|
|
||||||
"post": {
|
|
||||||
"tags": [
|
|
||||||
"Subscription"
|
|
||||||
],
|
|
||||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "discordWebHookId",
|
|
||||||
"name": "discordWebHookId",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "string",
|
|
||||||
"description": "sourceId",
|
|
||||||
"name": "sourceId",
|
|
||||||
"in": "query",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
@ -1183,212 +1003,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"models.ArticleDetailsDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"authorImage": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"authorName": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"description": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"pubdate": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"$ref": "#/definitions/models.SourceDto"
|
|
||||||
},
|
|
||||||
"tags": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"thumbnail": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"title": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"video": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"videoHeight": {
|
|
||||||
"type": "integer"
|
|
||||||
},
|
|
||||||
"videoWidth": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.DiscordQueueDetailsDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"article": {
|
|
||||||
"$ref": "#/definitions/models.ArticleDetailsDto"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.DiscordWebHooksDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"ID": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"channel": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"enabled": {
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"server": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.SourceDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"deleted": {
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"enabled": {
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"name": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"site": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"tags": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"type": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"value": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.SubscriptionDetailsDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"discordwebhook": {
|
|
||||||
"$ref": "#/definitions/models.DiscordWebHooksDto"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"$ref": "#/definitions/models.SourceDto"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models.SubscriptionDto": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"discordwebhookid": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"id": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"sourceid": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ApiError": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ListDiscordWebHooksQueueResults": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"payload": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ListSubscriptionDetails": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"payload": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/models.SubscriptionDetailsDto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"v1.ListSubscriptions": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"message": {
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"payload": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/models.SubscriptionDto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
"type": "integer"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -102,140 +102,6 @@ definitions:
|
|||||||
$ref: '#/definitions/domain.SourceDto'
|
$ref: '#/definitions/domain.SourceDto'
|
||||||
type: array
|
type: array
|
||||||
type: object
|
type: object
|
||||||
models.ArticleDetailsDto:
|
|
||||||
properties:
|
|
||||||
authorImage:
|
|
||||||
type: string
|
|
||||||
authorName:
|
|
||||||
type: string
|
|
||||||
description:
|
|
||||||
type: string
|
|
||||||
id:
|
|
||||||
type: string
|
|
||||||
pubdate:
|
|
||||||
type: string
|
|
||||||
source:
|
|
||||||
$ref: '#/definitions/models.SourceDto'
|
|
||||||
tags:
|
|
||||||
items:
|
|
||||||
type: string
|
|
||||||
type: array
|
|
||||||
thumbnail:
|
|
||||||
type: string
|
|
||||||
title:
|
|
||||||
type: string
|
|
||||||
url:
|
|
||||||
type: string
|
|
||||||
video:
|
|
||||||
type: string
|
|
||||||
videoHeight:
|
|
||||||
type: integer
|
|
||||||
videoWidth:
|
|
||||||
type: integer
|
|
||||||
type: object
|
|
||||||
models.DiscordQueueDetailsDto:
|
|
||||||
properties:
|
|
||||||
article:
|
|
||||||
$ref: '#/definitions/models.ArticleDetailsDto'
|
|
||||||
id:
|
|
||||||
type: string
|
|
||||||
type: object
|
|
||||||
models.DiscordWebHooksDto:
|
|
||||||
properties:
|
|
||||||
ID:
|
|
||||||
type: string
|
|
||||||
channel:
|
|
||||||
type: string
|
|
||||||
enabled:
|
|
||||||
type: boolean
|
|
||||||
server:
|
|
||||||
type: string
|
|
||||||
url:
|
|
||||||
type: string
|
|
||||||
type: object
|
|
||||||
models.SourceDto:
|
|
||||||
properties:
|
|
||||||
deleted:
|
|
||||||
type: boolean
|
|
||||||
enabled:
|
|
||||||
type: boolean
|
|
||||||
id:
|
|
||||||
type: string
|
|
||||||
name:
|
|
||||||
type: string
|
|
||||||
site:
|
|
||||||
type: string
|
|
||||||
source:
|
|
||||||
type: string
|
|
||||||
tags:
|
|
||||||
items:
|
|
||||||
type: string
|
|
||||||
type: array
|
|
||||||
type:
|
|
||||||
type: string
|
|
||||||
url:
|
|
||||||
type: string
|
|
||||||
value:
|
|
||||||
type: string
|
|
||||||
type: object
|
|
||||||
models.SubscriptionDetailsDto:
|
|
||||||
properties:
|
|
||||||
discordwebhook:
|
|
||||||
$ref: '#/definitions/models.DiscordWebHooksDto'
|
|
||||||
id:
|
|
||||||
type: string
|
|
||||||
source:
|
|
||||||
$ref: '#/definitions/models.SourceDto'
|
|
||||||
type: object
|
|
||||||
models.SubscriptionDto:
|
|
||||||
properties:
|
|
||||||
discordwebhookid:
|
|
||||||
type: string
|
|
||||||
id:
|
|
||||||
type: string
|
|
||||||
sourceid:
|
|
||||||
type: string
|
|
||||||
type: object
|
|
||||||
v1.ApiError:
|
|
||||||
properties:
|
|
||||||
message:
|
|
||||||
type: string
|
|
||||||
status:
|
|
||||||
type: integer
|
|
||||||
type: object
|
|
||||||
v1.ListDiscordWebHooksQueueResults:
|
|
||||||
properties:
|
|
||||||
message:
|
|
||||||
type: string
|
|
||||||
payload:
|
|
||||||
items:
|
|
||||||
$ref: '#/definitions/models.DiscordQueueDetailsDto'
|
|
||||||
type: array
|
|
||||||
status:
|
|
||||||
type: integer
|
|
||||||
type: object
|
|
||||||
v1.ListSubscriptionDetails:
|
|
||||||
properties:
|
|
||||||
message:
|
|
||||||
type: string
|
|
||||||
payload:
|
|
||||||
items:
|
|
||||||
$ref: '#/definitions/models.SubscriptionDetailsDto'
|
|
||||||
type: array
|
|
||||||
status:
|
|
||||||
type: integer
|
|
||||||
type: object
|
|
||||||
v1.ListSubscriptions:
|
|
||||||
properties:
|
|
||||||
message:
|
|
||||||
type: string
|
|
||||||
payload:
|
|
||||||
items:
|
|
||||||
$ref: '#/definitions/models.SubscriptionDto'
|
|
||||||
type: array
|
|
||||||
status:
|
|
||||||
type: integer
|
|
||||||
type: object
|
|
||||||
info:
|
info:
|
||||||
contact: {}
|
contact: {}
|
||||||
title: NewsBot collector
|
title: NewsBot collector
|
||||||
@ -527,18 +393,6 @@ paths:
|
|||||||
tags:
|
tags:
|
||||||
- Discord
|
- Discord
|
||||||
- Webhook
|
- Webhook
|
||||||
/queue/discord/webhooks:
|
|
||||||
get:
|
|
||||||
produces:
|
|
||||||
- application/json
|
|
||||||
responses:
|
|
||||||
"200":
|
|
||||||
description: ok
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ListDiscordWebHooksQueueResults'
|
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
|
||||||
tags:
|
|
||||||
- Queue
|
|
||||||
/settings/{key}:
|
/settings/{key}:
|
||||||
get:
|
get:
|
||||||
parameters:
|
parameters:
|
||||||
@ -807,109 +661,4 @@ paths:
|
|||||||
summary: Creates a new youtube source to monitor.
|
summary: Creates a new youtube source to monitor.
|
||||||
tags:
|
tags:
|
||||||
- Source
|
- Source
|
||||||
/subscriptions:
|
|
||||||
get:
|
|
||||||
produces:
|
|
||||||
- application/json
|
|
||||||
responses:
|
|
||||||
"200":
|
|
||||||
description: ok
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ListSubscriptions'
|
|
||||||
"400":
|
|
||||||
description: Unable to reach SQL.
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ApiError'
|
|
||||||
"500":
|
|
||||||
description: Failed to process data from SQL.
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ApiError'
|
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
|
||||||
tags:
|
|
||||||
- Subscription
|
|
||||||
/subscriptions/by/SourceId:
|
|
||||||
get:
|
|
||||||
parameters:
|
|
||||||
- description: id
|
|
||||||
in: query
|
|
||||||
name: id
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
produces:
|
|
||||||
- application/json
|
|
||||||
responses:
|
|
||||||
"200":
|
|
||||||
description: ok
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ListSubscriptions'
|
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
|
||||||
tags:
|
|
||||||
- Subscription
|
|
||||||
/subscriptions/by/discordId:
|
|
||||||
get:
|
|
||||||
parameters:
|
|
||||||
- description: id
|
|
||||||
in: query
|
|
||||||
name: id
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
produces:
|
|
||||||
- application/json
|
|
||||||
responses:
|
|
||||||
"200":
|
|
||||||
description: ok
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ListSubscriptions'
|
|
||||||
"400":
|
|
||||||
description: Unable to reach SQL or Data problems
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ApiError'
|
|
||||||
"500":
|
|
||||||
description: Data problems
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ApiError'
|
|
||||||
summary: Returns the top 100 entries from the queue to be processed.
|
|
||||||
tags:
|
|
||||||
- Subscription
|
|
||||||
/subscriptions/details:
|
|
||||||
get:
|
|
||||||
produces:
|
|
||||||
- application/json
|
|
||||||
responses:
|
|
||||||
"200":
|
|
||||||
description: ok
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/v1.ListSubscriptionDetails'
|
|
||||||
summary: Returns the top 50 entries with full deatils on the source and output.
|
|
||||||
tags:
|
|
||||||
- Subscription
|
|
||||||
/subscriptions/discord/webhook/delete:
|
|
||||||
delete:
|
|
||||||
parameters:
|
|
||||||
- description: id
|
|
||||||
in: query
|
|
||||||
name: id
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
responses: {}
|
|
||||||
summary: Removes a Discord WebHook Subscription based on the Subscription ID.
|
|
||||||
tags:
|
|
||||||
- Subscription
|
|
||||||
/subscriptions/discord/webhook/new:
|
|
||||||
post:
|
|
||||||
parameters:
|
|
||||||
- description: discordWebHookId
|
|
||||||
in: query
|
|
||||||
name: discordWebHookId
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
- description: sourceId
|
|
||||||
in: query
|
|
||||||
name: sourceId
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
responses: {}
|
|
||||||
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
|
||||||
tags:
|
|
||||||
- Subscription
|
|
||||||
swagger: "2.0"
|
swagger: "2.0"
|
||||||
|
|||||||
2
go.mod
2
go.mod
@ -5,12 +5,12 @@ go 1.22
|
|||||||
require (
|
require (
|
||||||
github.com/PuerkitoBio/goquery v1.8.0
|
github.com/PuerkitoBio/goquery v1.8.0
|
||||||
github.com/glebarez/go-sqlite v1.22.0
|
github.com/glebarez/go-sqlite v1.22.0
|
||||||
github.com/go-chi/chi/v5 v5.0.7
|
|
||||||
github.com/go-rod/rod v0.107.1
|
github.com/go-rod/rod v0.107.1
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/huandu/go-sqlbuilder v1.27.1
|
github.com/huandu/go-sqlbuilder v1.27.1
|
||||||
github.com/joho/godotenv v1.4.0
|
github.com/joho/godotenv v1.4.0
|
||||||
github.com/labstack/echo/v4 v4.12.0
|
github.com/labstack/echo/v4 v4.12.0
|
||||||
|
github.com/maragudk/goqite v0.1.0
|
||||||
github.com/mmcdole/gofeed v1.1.3
|
github.com/mmcdole/gofeed v1.1.3
|
||||||
github.com/nicklaw5/helix/v2 v2.4.0
|
github.com/nicklaw5/helix/v2 v2.4.0
|
||||||
github.com/pressly/goose/v3 v3.20.0
|
github.com/pressly/goose/v3 v3.20.0
|
||||||
|
|||||||
8
go.sum
8
go.sum
@ -18,8 +18,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
|||||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||||
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
||||||
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
||||||
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
|
||||||
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
|
||||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||||
@ -71,11 +69,17 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
|
|||||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||||
|
github.com/maragudk/goqite v0.1.0 h1:k/GgnJU9DBMVAx2DxBgmB+UKQVzJNjO1JPqmUJFH4+8=
|
||||||
|
github.com/maragudk/goqite v0.1.0/go.mod h1:5430TCLkycUeLE314c9fifTrTbwcJqJXdU3iyEiF6hM=
|
||||||
|
github.com/maragudk/is v0.1.0 h1:obq9anZNmOYcaNbeT0LMyjIexdNeYTw/TLAPD/BnZHA=
|
||||||
|
github.com/maragudk/is v0.1.0/go.mod h1:W/r6+TpnISu+a88OLXQy5JQGCOhXQXXLD2e5b4xMn5c=
|
||||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||||
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||||
|
|||||||
25
internal/database/migrations/20240428201449_goqite.sql
Normal file
25
internal/database/migrations/20240428201449_goqite.sql
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- +goose StatementBegin
|
||||||
|
create table goqite (
|
||||||
|
id text primary key default ('m_' || lower(hex(randomblob(16)))),
|
||||||
|
created text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
||||||
|
updated text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
||||||
|
queue text not null,
|
||||||
|
body blob not null,
|
||||||
|
timeout text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
||||||
|
received integer not null default 0
|
||||||
|
) strict;
|
||||||
|
|
||||||
|
create trigger goqite_updated_timestamp after update on goqite begin
|
||||||
|
update goqite set updated = strftime('%Y-%m-%dT%H:%M:%fZ') where id = old.id;
|
||||||
|
end;
|
||||||
|
|
||||||
|
create index goqite_queue_created_idx on goqite (queue, created);
|
||||||
|
-- +goose StatementEnd
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
-- +goose StatementBegin
|
||||||
|
DROP TABLE goqite
|
||||||
|
DROP INDEX goqite_queue_created_idx
|
||||||
|
DROP TRIGGER goqite_updated_timestamp
|
||||||
|
-- +goose StatementEnd
|
||||||
9
internal/domain/events.go
Normal file
9
internal/domain/events.go
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
const (
|
||||||
|
QueueRssCollection = "RssCollection"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RssCollectionEvent struct {
|
||||||
|
|
||||||
|
}
|
||||||
@ -11,13 +11,12 @@ import (
|
|||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
Router *echo.Echo
|
Router *echo.Echo
|
||||||
Db *database.Queries
|
Db *database.Queries
|
||||||
dto *dto.DtoClient
|
//dto *dto.DtoClient
|
||||||
config services.Configs
|
config services.Configs
|
||||||
repo services.RepositoryService
|
repo services.RepositoryService
|
||||||
}
|
}
|
||||||
@ -39,10 +38,10 @@ var (
|
|||||||
ErrUnableToConvertToJson string = "Unable to convert to json"
|
ErrUnableToConvertToJson string = "Unable to convert to json"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler {
|
func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
|
||||||
s := &Handler{
|
s := &Handler{
|
||||||
Db: db,
|
//Db: db,
|
||||||
dto: dto.NewDtoClient(db),
|
//dto: dto.NewDtoClient(db),
|
||||||
config: configs,
|
config: configs,
|
||||||
repo: services.NewRepositoryService(conn),
|
repo: services.NewRepositoryService(conn),
|
||||||
}
|
}
|
||||||
@ -85,13 +84,13 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
|
|||||||
sources.POST("/:ID/disable", s.disableSource)
|
sources.POST("/:ID/disable", s.disableSource)
|
||||||
sources.POST("/:ID/enable", s.enableSource)
|
sources.POST("/:ID/enable", s.enableSource)
|
||||||
|
|
||||||
subs := v1.Group("/subscriptions")
|
//subs := v1.Group("/subscriptions")
|
||||||
subs.GET("/", s.ListSubscriptions)
|
//subs.GET("/", s.ListSubscriptions)
|
||||||
subs.GET("/details", s.ListSubscriptionDetails)
|
//subs.GET("/details", s.ListSubscriptionDetails)
|
||||||
subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
|
//subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
|
||||||
subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
|
//subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
|
||||||
subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
//subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
||||||
subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
//subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
||||||
|
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,11 +1,7 @@
|
|||||||
package v1
|
package v1
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||||
"github.com/labstack/echo/v4"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ListDiscordWebHooksQueueResults struct {
|
type ListDiscordWebHooksQueueResults struct {
|
||||||
@ -19,22 +15,22 @@ type ListDiscordWebHooksQueueResults struct {
|
|||||||
// @Tags Queue
|
// @Tags Queue
|
||||||
// @Router /queue/discord/webhooks [get]
|
// @Router /queue/discord/webhooks [get]
|
||||||
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
|
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
|
||||||
func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
//func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
||||||
p := ListDiscordWebHooksQueueResults{
|
// p := ListDiscordWebHooksQueueResults{
|
||||||
ApiStatusModel: ApiStatusModel{
|
// ApiStatusModel: ApiStatusModel{
|
||||||
Message: "OK",
|
// Message: "OK",
|
||||||
StatusCode: http.StatusOK,
|
// StatusCode: http.StatusOK,
|
||||||
},
|
// },
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
// Get the raw resp from sql
|
// // Get the raw resp from sql
|
||||||
res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
// res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
// return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
||||||
Message: err.Error(),
|
// Message: err.Error(),
|
||||||
})
|
// })
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
p.Payload = res
|
// p.Payload = res
|
||||||
return c.JSON(http.StatusOK, p)
|
// return c.JSON(http.StatusOK, p)
|
||||||
}
|
//}
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
package v1
|
package v1
|
||||||
|
|
||||||
|
/*
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@ -228,3 +229,4 @@ func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error {
|
|||||||
|
|
||||||
return c.JSON(http.StatusOK, nil)
|
return c.JSON(http.StatusOK, nil)
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
30
internal/services/cron/ffxivJob.go
Normal file
30
internal/services/cron/ffxivJob.go
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
package cron
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *Cron) CheckFfxiv() {
|
||||||
|
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 10, domain.SourceCollectorFfxiv)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fc := input.NewFFXIVClient(source)
|
||||||
|
items, err := fc.CheckSource()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.SaveNewArticles(items, domain.SourceCollectorFfxiv)
|
||||||
|
}
|
||||||
|
log.Printf("[FFXIV Done!]")
|
||||||
|
}
|
||||||
33
internal/services/cron/redditJob.go
Normal file
33
internal/services/cron/redditJob.go
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
package cron
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This is the main entry point to query all the reddit services
|
||||||
|
func (c *Cron) CheckReddit() {
|
||||||
|
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorReddit)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
|
||||||
|
rc := input.NewRedditClient(source)
|
||||||
|
raw, err := rc.GetContent()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
redditArticles := rc.ConvertToArticles(raw)
|
||||||
|
|
||||||
|
c.SaveNewArticles(redditArticles, domain.SourceCollectorReddit)
|
||||||
|
}
|
||||||
|
log.Print("[Reddit] Done!")
|
||||||
|
}
|
||||||
34
internal/services/cron/rss.go
Normal file
34
internal/services/cron/rss.go
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
package cron
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *Cron) LoadRssQueue() {
|
||||||
|
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
|
||||||
|
var records []domain.SourceEntity
|
||||||
|
|
||||||
|
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
|
||||||
|
if err != nil {
|
||||||
|
return records, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return sources, nil
|
||||||
|
}
|
||||||
@ -3,83 +3,57 @@ package cron
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
_ "github.com/lib/pq"
|
_ "github.com/lib/pq"
|
||||||
"github.com/robfig/cron/v3"
|
"github.com/robfig/cron/v3"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Cron struct {
|
type Cron struct {
|
||||||
Db *database.Queries
|
|
||||||
ctx *context.Context
|
ctx *context.Context
|
||||||
timer *cron.Cron
|
timer *cron.Cron
|
||||||
|
repo services.RepositoryService
|
||||||
|
//queues services
|
||||||
}
|
}
|
||||||
|
|
||||||
func openDatabase() (*database.Queries, error) {
|
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
|
||||||
_env := services.NewConfig()
|
|
||||||
connString := _env.GetConfig(services.Sql_Connection_String)
|
|
||||||
if connString == "" {
|
|
||||||
panic("Connection String is null!")
|
|
||||||
}
|
|
||||||
db, err := sql.Open("postgres", connString)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
queries := database.New(db)
|
|
||||||
return queries, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewScheduler(ctx context.Context) *Cron {
|
|
||||||
c := &Cron{
|
c := &Cron{
|
||||||
ctx: &ctx,
|
ctx: &ctx,
|
||||||
|
repo: services.NewRepositoryService(conn),
|
||||||
}
|
}
|
||||||
|
|
||||||
timer := cron.New()
|
timer := cron.New()
|
||||||
queries, err := openDatabase()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
c.Db = queries
|
|
||||||
|
|
||||||
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||||
features := services.NewConfig()
|
features := services.GetEnvConfig()
|
||||||
|
|
||||||
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
|
if features.RedditEnabled {
|
||||||
if res {
|
|
||||||
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
||||||
log.Print("[Input] Reddit backend was enabled")
|
log.Print("[Input] Reddit backend was enabled")
|
||||||
//go c.CheckReddit()
|
//go c.CheckReddit()
|
||||||
}
|
}
|
||||||
|
|
||||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
if features.YoutubeEnabled {
|
||||||
if res {
|
|
||||||
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
||||||
log.Print("[Input] YouTube backend was enabled")
|
log.Print("[Input] YouTube backend was enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
|
if features.FfxivEnabled {
|
||||||
if res {
|
|
||||||
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
||||||
log.Print("[Input] FFXIV backend was enabled")
|
log.Print("[Input] FFXIV backend was enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
|
if features.TwitchEnabled {
|
||||||
if res {
|
|
||||||
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
||||||
log.Print("[Input] Twitch backend was enabled")
|
log.Print("[Input] Twitch backend was enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
//timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
||||||
log.Print("[Output] Discord Output was enabled")
|
//log.Print("[Output] Discord Output was enabled")
|
||||||
|
|
||||||
c.timer = timer
|
c.timer = timer
|
||||||
return c
|
return c
|
||||||
@ -93,104 +67,7 @@ func (c *Cron) Stop() {
|
|||||||
c.timer.Stop()
|
c.timer.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// This is the main entry point to query all the reddit services
|
/* TODO move to the sqlite queue
|
||||||
func (c *Cron) CheckReddit() {
|
|
||||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Printf("[Reddit] Checking '%v'...", source.Name)
|
|
||||||
rc := input.NewRedditClient(source)
|
|
||||||
raw, err := rc.GetContent()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
redditArticles := rc.ConvertToArticles(raw)
|
|
||||||
c.checkPosts(redditArticles, "Reddit")
|
|
||||||
}
|
|
||||||
log.Print("[Reddit] Done!")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) CheckYoutube() {
|
|
||||||
// Add call to the db to request youtube sources.
|
|
||||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Printf("[YouTube] Checking '%v'...", source.Name)
|
|
||||||
yc := input.NewYoutubeClient(source)
|
|
||||||
raw, err := yc.GetContent()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
c.checkPosts(raw, "YouTube")
|
|
||||||
}
|
|
||||||
log.Print("[YouTube] Done!")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) CheckFfxiv() {
|
|
||||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fc := input.NewFFXIVClient(source)
|
|
||||||
items, err := fc.CheckSource()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
c.checkPosts(items, "FFXIV")
|
|
||||||
}
|
|
||||||
log.Printf("[FFXIV Done!]")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) CheckTwitch() error {
|
|
||||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tc, err := input.NewTwitchClient()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tc.Login()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, source := range sources {
|
|
||||||
if !source.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Printf("[Twitch] Checking '%v'...", source.Name)
|
|
||||||
tc.ReplaceSourceRecord(source)
|
|
||||||
items, err := tc.GetContent()
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
c.checkPosts(items, "Twitch")
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Print("[Twitch] Done!")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) CheckDiscordQueue() error {
|
func (c *Cron) CheckDiscordQueue() error {
|
||||||
// Get items from the table
|
// Get items from the table
|
||||||
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
||||||
@ -260,55 +137,26 @@ func (c *Cron) CheckDiscordQueue() error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
|
func (c Cron) SaveNewArticles(posts []domain.ArticleEntity, sourceName string) error {
|
||||||
for _, item := range posts {
|
for _, item := range posts {
|
||||||
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
|
_, err := c.repo.Articles.GetByUrl(*c.ctx, item.Url)
|
||||||
|
if err == nil {
|
||||||
|
// This url is already known, so skip it
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the new article in the repository
|
||||||
|
rows, err := c.repo.Articles.Create(*c.ctx, item.SourceID, item.Tags, item.Title, item.Url, item.Thumbnail, item.Description, item.AuthorName, item.AuthorImageUrl, item.PubDate, item.IsVideo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
id := uuid.New()
|
return err
|
||||||
|
}
|
||||||
err := c.postArticle(id, item)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = c.addToDiscordQueue(id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
|
if rows != 1 {
|
||||||
|
return errors.New("failed to create a new record for some reason")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
time.Sleep(30 * time.Second)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
|
|
||||||
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
|
|
||||||
ID: id,
|
|
||||||
Sourceid: item.Sourceid,
|
|
||||||
Tags: item.Tags,
|
|
||||||
Title: item.Title,
|
|
||||||
Url: item.Url,
|
|
||||||
Pubdate: item.Pubdate,
|
|
||||||
Video: item.Video,
|
|
||||||
Videoheight: item.Videoheight,
|
|
||||||
Videowidth: item.Videowidth,
|
|
||||||
Thumbnail: item.Thumbnail,
|
|
||||||
Description: item.Description,
|
|
||||||
Authorname: item.Authorname,
|
|
||||||
Authorimage: item.Authorimage,
|
|
||||||
})
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
|
|
||||||
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
|
|
||||||
ID: uuid.New(),
|
|
||||||
Articleid: Id,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,33 +2,70 @@ package cron_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"database/sql"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||||
|
_ "github.com/glebarez/go-sqlite"
|
||||||
|
"github.com/pressly/goose/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestInvokeTwitch(t *testing.T) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO add database mocks but not sure how to do that yet.
|
|
||||||
func TestCheckReddit(t *testing.T) {
|
func TestCheckReddit(t *testing.T) {
|
||||||
|
db, err := setupInMemoryDb()
|
||||||
|
if err != nil {
|
||||||
|
t.Log(err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c := cron.NewScheduler(ctx)
|
c := cron.NewScheduler(ctx, db)
|
||||||
c.CheckReddit()
|
c.CheckReddit()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckYouTube(t *testing.T) {
|
func TestCheckYouTube(t *testing.T) {
|
||||||
|
db, err := setupInMemoryDb()
|
||||||
|
if err != nil {
|
||||||
|
t.Log(err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c := cron.NewScheduler(ctx)
|
c := cron.NewScheduler(ctx, db)
|
||||||
c.CheckYoutube()
|
c.CheckYoutube()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckTwitch(t *testing.T) {
|
func TestCheckTwitch(t *testing.T) {
|
||||||
|
db, err := setupInMemoryDb()
|
||||||
|
if err != nil {
|
||||||
|
t.Log(err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c := cron.NewScheduler(ctx)
|
c := cron.NewScheduler(ctx, db)
|
||||||
err := c.CheckTwitch()
|
err = c.CheckTwitch()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setupInMemoryDb() (*sql.DB, error) {
|
||||||
|
db, err := sql.Open("sqlite", ":memory:")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = goose.SetDialect("sqlite3")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = goose.Up(db, "../database/migrations")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|||||||
41
internal/services/cron/twitchJob.go
Normal file
41
internal/services/cron/twitchJob.go
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
package cron
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *Cron) CheckTwitch() error {
|
||||||
|
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorTwitch)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tc, err := input.NewTwitchClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tc.Login()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
|
||||||
|
tc.ReplaceSourceRecord(source)
|
||||||
|
items, err := tc.GetContent()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
c.SaveNewArticles(items, domain.SourceCollectorTwitch)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Print("[Twitch] Done!")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
34
internal/services/cron/youtube.go
Normal file
34
internal/services/cron/youtube.go
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
package cron
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *Cron) CheckYoutube() {
|
||||||
|
// Add call to the db to request youtube sources.
|
||||||
|
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorYoutube)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
if !source.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
|
||||||
|
|
||||||
|
yc := input.NewYoutubeClient(source)
|
||||||
|
raw, err := yc.GetContent()
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
c.SaveNewArticles(raw, domain.SourceCollectorYoutube)
|
||||||
|
}
|
||||||
|
log.Print("[YouTube] Done!")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1,140 +0,0 @@
|
|||||||
// The converter package lives between the database calls and the API calls.
|
|
||||||
// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much
|
|
||||||
package dto
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DtoClient struct {
|
|
||||||
db *database.Queries
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewDtoClient(db *database.Queries) *DtoClient {
|
|
||||||
return &DtoClient{
|
|
||||||
db: db,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) {
|
|
||||||
var res []models.ArticleDto
|
|
||||||
|
|
||||||
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
|
|
||||||
Limit: int32(limit),
|
|
||||||
Offset: int32(limit * page),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, article := range a {
|
|
||||||
res = append(res, c.convertArticle(article))
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
|
|
||||||
var res []models.ArticleDto
|
|
||||||
|
|
||||||
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
|
|
||||||
Limit: limit,
|
|
||||||
Offset: page * limit,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, article := range a {
|
|
||||||
res = append(res, c.convertArticle(article))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) {
|
|
||||||
a, err := c.db.GetArticleByID(ctx, ID)
|
|
||||||
if err != nil {
|
|
||||||
return models.ArticleDto{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.convertArticle(a), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) {
|
|
||||||
a, err := c.db.GetArticleByID(ctx, ID)
|
|
||||||
if err != nil {
|
|
||||||
return models.ArticleDetailsDto{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
s, err := c.db.GetSourceByID(ctx, a.Sourceid)
|
|
||||||
if err != nil {
|
|
||||||
return models.ArticleDetailsDto{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
res := c.convertArticleDetails(a, s)
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) {
|
|
||||||
var res []models.ArticleDto
|
|
||||||
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
|
|
||||||
Sourceid: SourceID,
|
|
||||||
Limit: int32(limit),
|
|
||||||
Offset: int32(limit * page),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, article := range a {
|
|
||||||
res = append(res, c.convertArticle(article))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto {
|
|
||||||
return models.ArticleDto{
|
|
||||||
ID: i.ID,
|
|
||||||
Source: i.Sourceid,
|
|
||||||
Tags: c.SplitTags(i.Tags),
|
|
||||||
Title: i.Title,
|
|
||||||
Url: i.Url,
|
|
||||||
Pubdate: i.Pubdate,
|
|
||||||
Video: i.Video.String,
|
|
||||||
Videoheight: i.Videoheight,
|
|
||||||
Videowidth: i.Videoheight,
|
|
||||||
Thumbnail: i.Thumbnail,
|
|
||||||
Description: i.Description,
|
|
||||||
Authorname: i.Authorname.String,
|
|
||||||
Authorimage: i.Authorimage.String,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto {
|
|
||||||
return models.ArticleDetailsDto{
|
|
||||||
ID: i.ID,
|
|
||||||
Source: c.ConvertToSource(s),
|
|
||||||
Tags: c.SplitTags(i.Tags),
|
|
||||||
Title: i.Title,
|
|
||||||
Url: i.Url,
|
|
||||||
Pubdate: i.Pubdate,
|
|
||||||
Video: i.Video.String,
|
|
||||||
Videoheight: i.Videoheight,
|
|
||||||
Videowidth: i.Videoheight,
|
|
||||||
Thumbnail: i.Thumbnail,
|
|
||||||
Description: i.Description,
|
|
||||||
Authorname: i.Authorname.String,
|
|
||||||
Authorimage: i.Authorimage.String,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c DtoClient) SplitTags(t string) []string {
|
|
||||||
return strings.Split(t, ", ")
|
|
||||||
}
|
|
||||||
@ -1,63 +0,0 @@
|
|||||||
package dto
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
|
|
||||||
var res []models.DiscordWebHooksDto
|
|
||||||
|
|
||||||
items, err := c.db.ListDiscordWebhooks(ctx, total)
|
|
||||||
if err != nil {
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertDiscordWebhook(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) {
|
|
||||||
var res models.DiscordWebHooksDto
|
|
||||||
|
|
||||||
item, err := c.db.GetDiscordWebHooksByID(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.ConvertDiscordWebhook(item), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) {
|
|
||||||
var res []models.DiscordWebHooksDto
|
|
||||||
|
|
||||||
items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{
|
|
||||||
Server: server,
|
|
||||||
Channel: channel,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertDiscordWebhook(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto {
|
|
||||||
return models.DiscordWebHooksDto{
|
|
||||||
ID: i.ID,
|
|
||||||
Url: i.Url,
|
|
||||||
Server: i.Server,
|
|
||||||
Channel: i.Channel,
|
|
||||||
Enabled: i.Enabled,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
package dto
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) {
|
|
||||||
var res []models.DiscordQueueDetailsDto
|
|
||||||
|
|
||||||
items, err := c.db.ListDiscordQueueItems(ctx, limit)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
article, err := c.GetArticleDetails(ctx, item.ID)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
res = append(res, models.DiscordQueueDetailsDto{
|
|
||||||
ID: item.ID,
|
|
||||||
Article: article,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto {
|
|
||||||
return models.DiscordQueueDto{
|
|
||||||
ID: i.ID,
|
|
||||||
Articleid: i.Articleid,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,85 +0,0 @@
|
|||||||
package dto
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
|
|
||||||
var res []models.SourceDto
|
|
||||||
|
|
||||||
items, err := c.db.ListSources(ctx, limit)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertToSource(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) {
|
|
||||||
var res []models.SourceDto
|
|
||||||
|
|
||||||
items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName))
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertToSource(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) {
|
|
||||||
var res models.SourceDto
|
|
||||||
|
|
||||||
item, err := c.db.GetSourceByID(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.ConvertToSource(item), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) {
|
|
||||||
var res models.SourceDto
|
|
||||||
|
|
||||||
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
|
|
||||||
Name: name,
|
|
||||||
Source: source,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.ConvertToSource(item), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto {
|
|
||||||
var deleted bool
|
|
||||||
if !i.Deleted.Valid {
|
|
||||||
deleted = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return models.SourceDto{
|
|
||||||
ID: i.ID,
|
|
||||||
Site: i.Site,
|
|
||||||
Name: i.Name,
|
|
||||||
Source: i.Source,
|
|
||||||
Type: i.Type,
|
|
||||||
Value: i.Value.String,
|
|
||||||
Enabled: i.Enabled,
|
|
||||||
Url: i.Url,
|
|
||||||
Tags: c.SplitTags(i.Tags),
|
|
||||||
Deleted: deleted,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,91 +0,0 @@
|
|||||||
package dto
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
|
|
||||||
var res []models.SubscriptionDto
|
|
||||||
|
|
||||||
items, err := c.db.ListSubscriptions(ctx, limit)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertSubscription(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) {
|
|
||||||
var res []models.SubscriptionDetailsDto
|
|
||||||
|
|
||||||
items, err := c.ListSubscriptions(ctx, limit)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
source, err := c.GetSourceById(ctx, item.SourceId)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
res = append(res, models.SubscriptionDetailsDto{
|
|
||||||
ID: item.ID,
|
|
||||||
Source: source,
|
|
||||||
DiscordWebHook: dwh,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
|
||||||
var res []models.SubscriptionDto
|
|
||||||
|
|
||||||
items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertSubscription(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
|
||||||
var res []models.SubscriptionDto
|
|
||||||
|
|
||||||
items, err := c.db.GetSubscriptionsBySourceID(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range items {
|
|
||||||
res = append(res, c.ConvertSubscription(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto {
|
|
||||||
return models.SubscriptionDto{
|
|
||||||
ID: i.ID,
|
|
||||||
DiscordWebhookId: i.Discordwebhookid,
|
|
||||||
SourceId: i.Sourceid,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,7 +1,6 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
|
||||||
"errors"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -13,7 +12,7 @@ import (
|
|||||||
"github.com/go-rod/rod/lib/launcher"
|
"github.com/go-rod/rod/lib/launcher"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -25,7 +24,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type FFXIVClient struct {
|
type FFXIVClient struct {
|
||||||
record database.Source
|
record domain.SourceEntity
|
||||||
//SourceID uint
|
//SourceID uint
|
||||||
//Url string
|
//Url string
|
||||||
//Region string
|
//Region string
|
||||||
@ -33,15 +32,15 @@ type FFXIVClient struct {
|
|||||||
cacheGroup string
|
cacheGroup string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFFXIVClient(Record database.Source) FFXIVClient {
|
func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
|
||||||
return FFXIVClient{
|
return FFXIVClient{
|
||||||
record: Record,
|
record: Record,
|
||||||
cacheGroup: "ffxiv",
|
cacheGroup: "ffxiv",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
|
||||||
var articles []database.Article
|
var articles []domain.ArticleEntity
|
||||||
|
|
||||||
parser := fc.GetBrowser()
|
parser := fc.GetBrowser()
|
||||||
defer parser.Close()
|
defer parser.Close()
|
||||||
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
|||||||
return articles, err
|
return articles, err
|
||||||
}
|
}
|
||||||
|
|
||||||
article := database.Article{
|
article := domain.ArticleEntity{
|
||||||
Sourceid: fc.record.ID,
|
SourceID: fc.record.ID,
|
||||||
Tags: tags,
|
Tags: tags,
|
||||||
Title: title,
|
Title: title,
|
||||||
Url: link,
|
Url: link,
|
||||||
Pubdate: pubDate,
|
PubDate: pubDate,
|
||||||
Videoheight: 0,
|
Thumbnail: thumb,
|
||||||
Videowidth: 0,
|
Description: description,
|
||||||
Thumbnail: thumb,
|
AuthorName: authorName,
|
||||||
Description: description,
|
AuthorImageUrl: authorImage,
|
||||||
Authorname: sql.NullString{String: authorName},
|
|
||||||
Authorimage: sql.NullString{String: authorImage},
|
|
||||||
}
|
}
|
||||||
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
||||||
|
|
||||||
|
|||||||
@ -3,18 +3,16 @@ package input_test
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var FFXIVRecord database.Source = database.Source{
|
var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
|
||||||
ID: uuid.New(),
|
ID: 997,
|
||||||
Site: "ffxiv",
|
DisplayName: "Final Fantasy XIV - NA",
|
||||||
Name: "Final Fantasy XIV - NA",
|
Source: domain.SourceCollectorFfxiv,
|
||||||
Source: "ffxiv",
|
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFfxivGetParser(t *testing.T) {
|
func TestFfxivGetParser(t *testing.T) {
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -9,7 +8,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"github.com/go-rod/rod"
|
"github.com/go-rod/rod"
|
||||||
@ -17,8 +15,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type RedditClient struct {
|
type RedditClient struct {
|
||||||
config RedditConfig
|
config services.Configs
|
||||||
record database.Source
|
record domain.SourceEntity
|
||||||
}
|
}
|
||||||
|
|
||||||
type RedditConfig struct {
|
type RedditConfig struct {
|
||||||
@ -27,14 +25,11 @@ type RedditConfig struct {
|
|||||||
PullNSFW string
|
PullNSFW string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRedditClient(Record database.Source) *RedditClient {
|
func NewRedditClient(record domain.SourceEntity) *RedditClient {
|
||||||
rc := RedditClient{
|
rc := RedditClient{
|
||||||
record: Record,
|
record: record,
|
||||||
|
config: services.GetEnvConfig(),
|
||||||
}
|
}
|
||||||
cc := services.NewConfig()
|
|
||||||
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
|
|
||||||
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
|
|
||||||
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
|
|
||||||
|
|
||||||
//rc.disableHttp2Client()
|
//rc.disableHttp2Client()
|
||||||
|
|
||||||
@ -71,7 +66,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
|||||||
// TODO Wire this to support the config options
|
// TODO Wire this to support the config options
|
||||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||||
|
|
||||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
|
||||||
|
|
||||||
content, err := getHttpContent(Url)
|
content, err := getHttpContent(Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -88,24 +83,28 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
|||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
|
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
|
||||||
var redditArticles []database.Article
|
var redditArticles []domain.ArticleEntity
|
||||||
|
|
||||||
for _, item := range items.Data.Children {
|
for _, item := range items.Data.Children {
|
||||||
var article database.Article
|
var article domain.ArticleEntity
|
||||||
article, err := rc.convertToArticle(item.Data)
|
article, err := rc.convertToArticle(item.Data)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[Reddit] %v", err)
|
log.Printf("[Reddit] %v", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
redditArticles = append(redditArticles, article)
|
redditArticles = append(redditArticles, article)
|
||||||
}
|
}
|
||||||
|
|
||||||
return redditArticles
|
return redditArticles
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||||
// This data can be passed to the database.
|
// This data can be passed to the database.
|
||||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
|
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
|
||||||
var item database.Article
|
var item domain.ArticleEntity
|
||||||
|
|
||||||
if source.Content == "" && source.Url != "" {
|
if source.Content == "" && source.Url != "" {
|
||||||
item = rc.convertPicturePost(source)
|
item = rc.convertPicturePost(source)
|
||||||
@ -131,65 +130,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
|
|||||||
return item, nil
|
return item, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
|
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
|
||||||
var item = database.Article{
|
var item = domain.ArticleEntity{
|
||||||
Sourceid: rc.record.ID,
|
SourceID: rc.record.ID,
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
Pubdate: time.Now(),
|
PubDate: time.Now(),
|
||||||
Video: sql.NullString{String: "null"},
|
IsVideo: false,
|
||||||
Videoheight: 0,
|
Thumbnail: source.Thumbnail,
|
||||||
Videowidth: 0,
|
Description: source.Content,
|
||||||
Thumbnail: source.Thumbnail,
|
AuthorName: source.Author,
|
||||||
Description: source.Content,
|
AuthorImageUrl: "",
|
||||||
Authorname: sql.NullString{String: source.Author},
|
|
||||||
Authorimage: sql.NullString{String: "null"},
|
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
|
func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
|
||||||
var item = database.Article{
|
var item = domain.ArticleEntity{
|
||||||
Sourceid: rc.record.ID,
|
SourceID: rc.record.ID,
|
||||||
Tags: "a",
|
Tags: "a",
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
Pubdate: time.Now(),
|
PubDate: time.Now(),
|
||||||
Videoheight: 0,
|
|
||||||
Videowidth: 0,
|
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
Authorname: sql.NullString{String: source.Author},
|
AuthorName: source.Author,
|
||||||
Description: source.Content,
|
Description: source.Content,
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
|
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
|
||||||
var item = database.Article{
|
var item = domain.ArticleEntity{
|
||||||
Sourceid: rc.record.ID,
|
SourceID: rc.record.ID,
|
||||||
Tags: "a",
|
Tags: "a",
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
Pubdate: time.Now(),
|
PubDate: time.Now(),
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
Videoheight: 0,
|
AuthorName: source.Author,
|
||||||
Videowidth: 0,
|
|
||||||
Authorname: sql.NullString{String: source.Author},
|
|
||||||
Description: source.Media.RedditVideo.FallBackUrl,
|
Description: source.Media.RedditVideo.FallBackUrl,
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
// This post is nothing more then a redirect to another location.
|
// This post is nothing more then a redirect to another location.
|
||||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
|
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
|
||||||
var item = database.Article{
|
var item = domain.ArticleEntity{
|
||||||
Sourceid: rc.record.ID,
|
SourceID: rc.record.ID,
|
||||||
Tags: "a",
|
Tags: "a",
|
||||||
Title: source.Title,
|
Title: source.Title,
|
||||||
Pubdate: time.Now(),
|
PubDate: time.Now(),
|
||||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||||
Videoheight: 0,
|
AuthorName: source.Author,
|
||||||
Videowidth: 0,
|
|
||||||
Authorname: sql.NullString{String: source.Author},
|
|
||||||
Description: source.UrlOverriddenByDest,
|
Description: source.UrlOverriddenByDest,
|
||||||
}
|
}
|
||||||
return item
|
return item
|
||||||
|
|||||||
@ -3,18 +3,16 @@ package input_test
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var RedditRecord database.Source = database.Source{
|
var RedditRecord domain.SourceEntity = domain.SourceEntity{
|
||||||
ID: uuid.New(),
|
ID: 999,
|
||||||
Name: "dadjokes",
|
DisplayName: "dadjokes",
|
||||||
Source: "reddit",
|
Source: domain.SourceCollectorReddit,
|
||||||
Site: "reddit",
|
Url: "https://reddit.com/r/dadjokes",
|
||||||
Url: "https://reddit.com/r/dadjokes",
|
Tags: "reddit, dadjokes",
|
||||||
Tags: "reddit, dadjokes",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetContent(t *testing.T) {
|
func TestGetContent(t *testing.T) {
|
||||||
|
|||||||
@ -8,9 +8,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var rssRecord = domain.SourceEntity{
|
var rssRecord = domain.SourceEntity{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
DisplayName: "ArsTechnica",
|
DisplayName: "ArsTechnica",
|
||||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
Source: domain.SourceCollectorRss,
|
||||||
|
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRssClientConstructor(t *testing.T) {
|
func TestRssClientConstructor(t *testing.T) {
|
||||||
|
|||||||
@ -1,19 +1,18 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||||
"github.com/nicklaw5/helix/v2"
|
"github.com/nicklaw5/helix/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TwitchClient struct {
|
type TwitchClient struct {
|
||||||
SourceRecord database.Source
|
SourceRecord domain.SourceEntity
|
||||||
|
|
||||||
// config
|
// config
|
||||||
monitorClips string
|
monitorClips string
|
||||||
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// This will let you replace the bound source record to keep the same session alive.
|
// This will let you replace the bound source record to keep the same session alive.
|
||||||
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
|
||||||
tc.SourceRecord = source
|
tc.SourceRecord = source
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||||
var items []database.Article
|
var items []domain.ArticleEntity
|
||||||
|
|
||||||
user, err := tc.GetUserDetails()
|
user, err := tc.GetUserDetails()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, video := range posts {
|
for _, video := range posts {
|
||||||
var article database.Article
|
var article domain.ArticleEntity
|
||||||
|
|
||||||
AuthorName, err := tc.ExtractAuthor(video)
|
AuthorName, err := tc.ExtractAuthor(video)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
article.Authorname = sql.NullString{String: AuthorName}
|
article.AuthorName = AuthorName
|
||||||
|
|
||||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
article.Authorimage = sql.NullString{String: Authorimage}
|
article.AuthorImageUrl = Authorimage
|
||||||
|
|
||||||
article.Description, err = tc.ExtractDescription(video)
|
article.Description, err = tc.ExtractDescription(video)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
|
|
||||||
article.Pubdate, err = tc.ExtractPubDate(video)
|
article.PubDate, err = tc.ExtractPubDate(video)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
}
|
}
|
||||||
|
|
||||||
article.Sourceid = tc.SourceRecord.ID
|
article.SourceID = tc.SourceRecord.ID
|
||||||
article.Tags, err = tc.ExtractTags(video, user)
|
article.Tags, err = tc.ExtractTags(video, user)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
|
|||||||
var blank helix.User
|
var blank helix.User
|
||||||
|
|
||||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||||
Logins: []string{tc.SourceRecord.Name},
|
Logins: []string{tc.SourceRecord.DisplayName},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return blank, err
|
return blank, err
|
||||||
|
|||||||
@ -4,21 +4,20 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var TwitchSourceRecord = database.Source{
|
var TwitchSourceRecord = domain.SourceEntity{
|
||||||
ID: uuid.New(),
|
ID: 9999,
|
||||||
Name: "nintendo",
|
DisplayName: "nintendo",
|
||||||
Source: "Twitch",
|
Source: domain.SourceCollectorTwitch,
|
||||||
}
|
}
|
||||||
|
|
||||||
var TwitchInvalidRecord = database.Source{
|
var TwitchInvalidRecord = domain.SourceEntity{
|
||||||
ID: uuid.New(),
|
ID: 9999,
|
||||||
Name: "EvilNintendo",
|
DisplayName: "EvilNintendo",
|
||||||
Source: "Twitch",
|
Source: domain.SourceCollectorTwitch,
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTwitchLogin(t *testing.T) {
|
func TestTwitchLogin(t *testing.T) {
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package input
|
package input
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
@ -12,11 +11,11 @@ import (
|
|||||||
"github.com/go-rod/rod/lib/launcher"
|
"github.com/go-rod/rod/lib/launcher"
|
||||||
"github.com/mmcdole/gofeed"
|
"github.com/mmcdole/gofeed"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
type YoutubeClient struct {
|
type YoutubeClient struct {
|
||||||
record database.Source
|
record domain.SourceEntity
|
||||||
|
|
||||||
// internal variables at time of collection
|
// internal variables at time of collection
|
||||||
channelID string
|
channelID string
|
||||||
@ -37,7 +36,7 @@ var (
|
|||||||
|
|
||||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||||
|
|
||||||
func NewYoutubeClient(Record database.Source) YoutubeClient {
|
func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
|
||||||
yc := YoutubeClient{
|
yc := YoutubeClient{
|
||||||
record: Record,
|
record: Record,
|
||||||
cacheGroup: "youtube",
|
cacheGroup: "youtube",
|
||||||
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CheckSource will go and run all the commands needed to process a source.
|
// CheckSource will go and run all the commands needed to process a source.
|
||||||
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||||
var items []database.Article
|
var items []domain.ArticleEntity
|
||||||
docParser, err := yc.GetParser(yc.record.Url)
|
docParser, err := yc.GetParser(yc.record.Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return items, err
|
return items, err
|
||||||
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
|
||||||
parser, err := yc.GetParser(item.Link)
|
parser, err := yc.GetParser(item.Link)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||||
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
|||||||
log.Printf("[YouTube] %v", msg)
|
log.Printf("[YouTube] %v", msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
var article = database.Article{
|
var article = domain.ArticleEntity{
|
||||||
Sourceid: yc.record.ID,
|
SourceID: yc.record.ID,
|
||||||
Tags: tags,
|
Tags: tags,
|
||||||
Title: item.Title,
|
Title: item.Title,
|
||||||
Url: item.Link,
|
Url: item.Link,
|
||||||
Pubdate: *item.PublishedParsed,
|
PubDate: *item.PublishedParsed,
|
||||||
Thumbnail: thumb,
|
Thumbnail: thumb,
|
||||||
Description: item.Description,
|
Description: item.Description,
|
||||||
Authorname: sql.NullString{String: item.Author.Name},
|
AuthorName: item.Author.Name,
|
||||||
Authorimage: sql.NullString{String: yc.avatarUri},
|
AuthorImageUrl: yc.avatarUri,
|
||||||
}
|
}
|
||||||
return article
|
return article
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,17 +3,15 @@ package input_test
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||||
"github.com/google/uuid"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var YouTubeRecord database.Source = database.Source{
|
var YouTubeRecord domain.SourceEntity = domain.SourceEntity{
|
||||||
ID: uuid.New(),
|
ID: 999,
|
||||||
Name: "dadjokes",
|
DisplayName: "dadjokes",
|
||||||
Source: "reddit",
|
Source: domain.SourceCollectorYoutube,
|
||||||
Site: "reddit",
|
Url: "https://youtube.com/gamegrumps",
|
||||||
Url: "https://youtube.com/gamegrumps",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetPageParser(t *testing.T) {
|
func TestGetPageParser(t *testing.T) {
|
||||||
|
|||||||
26
internal/services/queue.go
Normal file
26
internal/services/queue.go
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
package services
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
|
||||||
|
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||||
|
"github.com/maragudk/goqite"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
type queues struct {
|
||||||
|
repos RepositoryService
|
||||||
|
|
||||||
|
RssCollection goqite.Queue
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewQueues(conn *sql.DB) queues {
|
||||||
|
return queues{
|
||||||
|
repos: NewRepositoryService(conn),
|
||||||
|
RssCollection: *goqite.New(goqite.NewOpts{
|
||||||
|
DB: conn,
|
||||||
|
Name: domain.QueueRssCollection,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
4
makefile
4
makefile
@ -4,8 +4,8 @@ help: ## Shows this help command
|
|||||||
|
|
||||||
build: ## builds the application with the current go runtime
|
build: ## builds the application with the current go runtime
|
||||||
~/go/bin/swag f
|
~/go/bin/swag f
|
||||||
~/go/bin/swag i
|
~/go/bin/swag init -g cmd/server.go
|
||||||
go build .
|
go build cmd/server.go
|
||||||
|
|
||||||
docker-build: ## Generates the docker image
|
docker-build: ## Generates the docker image
|
||||||
docker build -t "newsbot.collector.api" .
|
docker build -t "newsbot.collector.api" .
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user