Compare commits
6 Commits
main
...
features/h
| Author | SHA1 | Date | |
|---|---|---|---|
| fbed111fbb | |||
| 43a086cc02 | |||
| 2afad5d761 | |||
| fcf80ec5fe | |||
| ce45b509d3 | |||
| e48b64bbaa |
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,7 +2,6 @@
|
||||
dev.session.sql
|
||||
__debug_bin
|
||||
server
|
||||
.vscode
|
||||
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
@ -11,6 +10,7 @@ server
|
||||
*.so
|
||||
*.dylib
|
||||
collector
|
||||
*.db
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@ -9,7 +9,7 @@
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"program": "."
|
||||
"program": "./cmd/server.go"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -10,8 +10,7 @@ import (
|
||||
"github.com/pressly/goose/v3"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
)
|
||||
@ -42,12 +41,15 @@ func main() {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
queries := database.New(db)
|
||||
//queues := services.NewQueues(db)
|
||||
//queues.RssIndex.Send(ctx, goqite.Message{
|
||||
// Body: []byte("hello world"),
|
||||
//})
|
||||
|
||||
c := cron.NewScheduler(ctx)
|
||||
c := cron.NewScheduler(ctx, db)
|
||||
c.Start()
|
||||
|
||||
server := v1.NewServer(ctx, queries, configs, db)
|
||||
server := v1.NewServer(ctx, configs, db)
|
||||
|
||||
fmt.Println("API is online and waiting for requests.")
|
||||
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)
|
||||
|
||||
386
docs/docs.go
386
docs/docs.go
@ -449,25 +449,6 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/queue/discord/webhooks": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Queue"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/settings/{key}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
@ -875,167 +856,6 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Unable to reach SQL.",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to process data from SQL.",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/SourceId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "id",
|
||||
"name": "id",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/discordId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "id",
|
||||
"name": "id",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Unable to reach SQL or Data problems",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Data problems",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/details": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 50 entries with full deatils on the source and output.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptionDetails"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/delete": {
|
||||
"delete": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "id",
|
||||
"name": "id",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "discordWebHookId",
|
||||
"name": "discordWebHookId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "sourceId",
|
||||
"name": "sourceId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
@ -1192,212 +1012,6 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.ArticleDetailsDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"authorImage": {
|
||||
"type": "string"
|
||||
},
|
||||
"authorName": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"pubdate": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/definitions/models.SourceDto"
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"thumbnail": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"video": {
|
||||
"type": "string"
|
||||
},
|
||||
"videoHeight": {
|
||||
"type": "integer"
|
||||
},
|
||||
"videoWidth": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.DiscordQueueDetailsDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"article": {
|
||||
"$ref": "#/definitions/models.ArticleDetailsDto"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.DiscordWebHooksDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ID": {
|
||||
"type": "string"
|
||||
},
|
||||
"channel": {
|
||||
"type": "string"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"server": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.SourceDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"site": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"type": "string"
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.SubscriptionDetailsDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"discordwebhook": {
|
||||
"$ref": "#/definitions/models.DiscordWebHooksDto"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/definitions/models.SourceDto"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.SubscriptionDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"discordwebhookid": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"sourceid": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ApiError": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ListDiscordWebHooksQueueResults": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"payload": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ListSubscriptionDetails": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"payload": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/models.SubscriptionDetailsDto"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ListSubscriptions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"payload": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/models.SubscriptionDto"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
@ -440,25 +440,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/queue/discord/webhooks": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Queue"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/settings/{key}": {
|
||||
"get": {
|
||||
"produces": [
|
||||
@ -866,167 +847,6 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Unable to reach SQL.",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to process data from SQL.",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/SourceId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "id",
|
||||
"name": "id",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/by/discordId": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 100 entries from the queue to be processed.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "id",
|
||||
"name": "id",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptions"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Unable to reach SQL or Data problems",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Data problems",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ApiError"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/details": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Returns the top 50 entries with full deatils on the source and output.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "ok",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/v1.ListSubscriptionDetails"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/delete": {
|
||||
"delete": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "id",
|
||||
"name": "id",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
},
|
||||
"/subscriptions/discord/webhook/new": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Subscription"
|
||||
],
|
||||
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "discordWebHookId",
|
||||
"name": "discordWebHookId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "sourceId",
|
||||
"name": "sourceId",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
@ -1183,212 +1003,6 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.ArticleDetailsDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"authorImage": {
|
||||
"type": "string"
|
||||
},
|
||||
"authorName": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"pubdate": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/definitions/models.SourceDto"
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"thumbnail": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"video": {
|
||||
"type": "string"
|
||||
},
|
||||
"videoHeight": {
|
||||
"type": "integer"
|
||||
},
|
||||
"videoWidth": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.DiscordQueueDetailsDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"article": {
|
||||
"$ref": "#/definitions/models.ArticleDetailsDto"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.DiscordWebHooksDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ID": {
|
||||
"type": "string"
|
||||
},
|
||||
"channel": {
|
||||
"type": "string"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"server": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.SourceDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"site": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"type": "string"
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.SubscriptionDetailsDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"discordwebhook": {
|
||||
"$ref": "#/definitions/models.DiscordWebHooksDto"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/definitions/models.SourceDto"
|
||||
}
|
||||
}
|
||||
},
|
||||
"models.SubscriptionDto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"discordwebhookid": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"sourceid": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ApiError": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ListDiscordWebHooksQueueResults": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"payload": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ListSubscriptionDetails": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"payload": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/models.SubscriptionDetailsDto"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v1.ListSubscriptions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"payload": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/models.SubscriptionDto"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -102,140 +102,6 @@ definitions:
|
||||
$ref: '#/definitions/domain.SourceDto'
|
||||
type: array
|
||||
type: object
|
||||
models.ArticleDetailsDto:
|
||||
properties:
|
||||
authorImage:
|
||||
type: string
|
||||
authorName:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
id:
|
||||
type: string
|
||||
pubdate:
|
||||
type: string
|
||||
source:
|
||||
$ref: '#/definitions/models.SourceDto'
|
||||
tags:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
thumbnail:
|
||||
type: string
|
||||
title:
|
||||
type: string
|
||||
url:
|
||||
type: string
|
||||
video:
|
||||
type: string
|
||||
videoHeight:
|
||||
type: integer
|
||||
videoWidth:
|
||||
type: integer
|
||||
type: object
|
||||
models.DiscordQueueDetailsDto:
|
||||
properties:
|
||||
article:
|
||||
$ref: '#/definitions/models.ArticleDetailsDto'
|
||||
id:
|
||||
type: string
|
||||
type: object
|
||||
models.DiscordWebHooksDto:
|
||||
properties:
|
||||
ID:
|
||||
type: string
|
||||
channel:
|
||||
type: string
|
||||
enabled:
|
||||
type: boolean
|
||||
server:
|
||||
type: string
|
||||
url:
|
||||
type: string
|
||||
type: object
|
||||
models.SourceDto:
|
||||
properties:
|
||||
deleted:
|
||||
type: boolean
|
||||
enabled:
|
||||
type: boolean
|
||||
id:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
site:
|
||||
type: string
|
||||
source:
|
||||
type: string
|
||||
tags:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
type:
|
||||
type: string
|
||||
url:
|
||||
type: string
|
||||
value:
|
||||
type: string
|
||||
type: object
|
||||
models.SubscriptionDetailsDto:
|
||||
properties:
|
||||
discordwebhook:
|
||||
$ref: '#/definitions/models.DiscordWebHooksDto'
|
||||
id:
|
||||
type: string
|
||||
source:
|
||||
$ref: '#/definitions/models.SourceDto'
|
||||
type: object
|
||||
models.SubscriptionDto:
|
||||
properties:
|
||||
discordwebhookid:
|
||||
type: string
|
||||
id:
|
||||
type: string
|
||||
sourceid:
|
||||
type: string
|
||||
type: object
|
||||
v1.ApiError:
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
status:
|
||||
type: integer
|
||||
type: object
|
||||
v1.ListDiscordWebHooksQueueResults:
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
payload:
|
||||
items:
|
||||
$ref: '#/definitions/models.DiscordQueueDetailsDto'
|
||||
type: array
|
||||
status:
|
||||
type: integer
|
||||
type: object
|
||||
v1.ListSubscriptionDetails:
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
payload:
|
||||
items:
|
||||
$ref: '#/definitions/models.SubscriptionDetailsDto'
|
||||
type: array
|
||||
status:
|
||||
type: integer
|
||||
type: object
|
||||
v1.ListSubscriptions:
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
payload:
|
||||
items:
|
||||
$ref: '#/definitions/models.SubscriptionDto'
|
||||
type: array
|
||||
status:
|
||||
type: integer
|
||||
type: object
|
||||
info:
|
||||
contact: {}
|
||||
title: NewsBot collector
|
||||
@ -527,18 +393,6 @@ paths:
|
||||
tags:
|
||||
- Discord
|
||||
- Webhook
|
||||
/queue/discord/webhooks:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: ok
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ListDiscordWebHooksQueueResults'
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Queue
|
||||
/settings/{key}:
|
||||
get:
|
||||
parameters:
|
||||
@ -807,109 +661,4 @@ paths:
|
||||
summary: Creates a new youtube source to monitor.
|
||||
tags:
|
||||
- Source
|
||||
/subscriptions:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: ok
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ListSubscriptions'
|
||||
"400":
|
||||
description: Unable to reach SQL.
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ApiError'
|
||||
"500":
|
||||
description: Failed to process data from SQL.
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ApiError'
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/by/SourceId:
|
||||
get:
|
||||
parameters:
|
||||
- description: id
|
||||
in: query
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: ok
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ListSubscriptions'
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/by/discordId:
|
||||
get:
|
||||
parameters:
|
||||
- description: id
|
||||
in: query
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: ok
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ListSubscriptions'
|
||||
"400":
|
||||
description: Unable to reach SQL or Data problems
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ApiError'
|
||||
"500":
|
||||
description: Data problems
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ApiError'
|
||||
summary: Returns the top 100 entries from the queue to be processed.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/details:
|
||||
get:
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: ok
|
||||
schema:
|
||||
$ref: '#/definitions/v1.ListSubscriptionDetails'
|
||||
summary: Returns the top 50 entries with full deatils on the source and output.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/discord/webhook/delete:
|
||||
delete:
|
||||
parameters:
|
||||
- description: id
|
||||
in: query
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
responses: {}
|
||||
summary: Removes a Discord WebHook Subscription based on the Subscription ID.
|
||||
tags:
|
||||
- Subscription
|
||||
/subscriptions/discord/webhook/new:
|
||||
post:
|
||||
parameters:
|
||||
- description: discordWebHookId
|
||||
in: query
|
||||
name: discordWebHookId
|
||||
required: true
|
||||
type: string
|
||||
- description: sourceId
|
||||
in: query
|
||||
name: sourceId
|
||||
required: true
|
||||
type: string
|
||||
responses: {}
|
||||
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
|
||||
tags:
|
||||
- Subscription
|
||||
swagger: "2.0"
|
||||
|
||||
2
go.mod
2
go.mod
@ -5,12 +5,12 @@ go 1.22
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0
|
||||
github.com/glebarez/go-sqlite v1.22.0
|
||||
github.com/go-chi/chi/v5 v5.0.7
|
||||
github.com/go-rod/rod v0.107.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/huandu/go-sqlbuilder v1.27.1
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/labstack/echo/v4 v4.12.0
|
||||
github.com/maragudk/goqite v0.1.0
|
||||
github.com/mmcdole/gofeed v1.1.3
|
||||
github.com/nicklaw5/helix/v2 v2.4.0
|
||||
github.com/pressly/goose/v3 v3.20.0
|
||||
|
||||
8
go.sum
8
go.sum
@ -18,8 +18,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
||||
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
||||
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
||||
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
@ -71,11 +69,17 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/maragudk/goqite v0.1.0 h1:k/GgnJU9DBMVAx2DxBgmB+UKQVzJNjO1JPqmUJFH4+8=
|
||||
github.com/maragudk/goqite v0.1.0/go.mod h1:5430TCLkycUeLE314c9fifTrTbwcJqJXdU3iyEiF6hM=
|
||||
github.com/maragudk/is v0.1.0 h1:obq9anZNmOYcaNbeT0LMyjIexdNeYTw/TLAPD/BnZHA=
|
||||
github.com/maragudk/is v0.1.0/go.mod h1:W/r6+TpnISu+a88OLXQy5JQGCOhXQXXLD2e5b4xMn5c=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||
|
||||
25
internal/database/migrations/20240428201449_goqite.sql
Normal file
25
internal/database/migrations/20240428201449_goqite.sql
Normal file
@ -0,0 +1,25 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
create table goqite (
|
||||
id text primary key default ('m_' || lower(hex(randomblob(16)))),
|
||||
created text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
||||
updated text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
||||
queue text not null,
|
||||
body blob not null,
|
||||
timeout text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
|
||||
received integer not null default 0
|
||||
) strict;
|
||||
|
||||
create trigger goqite_updated_timestamp after update on goqite begin
|
||||
update goqite set updated = strftime('%Y-%m-%dT%H:%M:%fZ') where id = old.id;
|
||||
end;
|
||||
|
||||
create index goqite_queue_created_idx on goqite (queue, created);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE goqite
|
||||
DROP INDEX goqite_queue_created_idx
|
||||
DROP TRIGGER goqite_updated_timestamp
|
||||
-- +goose StatementEnd
|
||||
9
internal/domain/events.go
Normal file
9
internal/domain/events.go
Normal file
@ -0,0 +1,9 @@
|
||||
package domain
|
||||
|
||||
const (
|
||||
QueueRssCollection = "RssCollection"
|
||||
)
|
||||
|
||||
type RssCollectionEvent struct {
|
||||
|
||||
}
|
||||
@ -11,13 +11,12 @@ import (
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Router *echo.Echo
|
||||
Db *database.Queries
|
||||
dto *dto.DtoClient
|
||||
//dto *dto.DtoClient
|
||||
config services.Configs
|
||||
repo services.RepositoryService
|
||||
}
|
||||
@ -39,10 +38,10 @@ var (
|
||||
ErrUnableToConvertToJson string = "Unable to convert to json"
|
||||
)
|
||||
|
||||
func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler {
|
||||
func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
|
||||
s := &Handler{
|
||||
Db: db,
|
||||
dto: dto.NewDtoClient(db),
|
||||
//Db: db,
|
||||
//dto: dto.NewDtoClient(db),
|
||||
config: configs,
|
||||
repo: services.NewRepositoryService(conn),
|
||||
}
|
||||
@ -85,13 +84,13 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
|
||||
sources.POST("/:ID/disable", s.disableSource)
|
||||
sources.POST("/:ID/enable", s.enableSource)
|
||||
|
||||
subs := v1.Group("/subscriptions")
|
||||
subs.GET("/", s.ListSubscriptions)
|
||||
subs.GET("/details", s.ListSubscriptionDetails)
|
||||
subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
|
||||
subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
|
||||
subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
||||
subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
||||
//subs := v1.Group("/subscriptions")
|
||||
//subs.GET("/", s.ListSubscriptions)
|
||||
//subs.GET("/details", s.ListSubscriptionDetails)
|
||||
//subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
|
||||
//subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
|
||||
//subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
|
||||
//subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
@ -1,11 +1,7 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
type ListDiscordWebHooksQueueResults struct {
|
||||
@ -19,22 +15,22 @@ type ListDiscordWebHooksQueueResults struct {
|
||||
// @Tags Queue
|
||||
// @Router /queue/discord/webhooks [get]
|
||||
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
|
||||
func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
||||
p := ListDiscordWebHooksQueueResults{
|
||||
ApiStatusModel: ApiStatusModel{
|
||||
Message: "OK",
|
||||
StatusCode: http.StatusOK,
|
||||
},
|
||||
}
|
||||
|
||||
// Get the raw resp from sql
|
||||
res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
||||
if err != nil {
|
||||
return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
p.Payload = res
|
||||
return c.JSON(http.StatusOK, p)
|
||||
}
|
||||
//func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
|
||||
// p := ListDiscordWebHooksQueueResults{
|
||||
// ApiStatusModel: ApiStatusModel{
|
||||
// Message: "OK",
|
||||
// StatusCode: http.StatusOK,
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// // Get the raw resp from sql
|
||||
// res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
|
||||
// if err != nil {
|
||||
// return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
|
||||
// Message: err.Error(),
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// p.Payload = res
|
||||
// return c.JSON(http.StatusOK, p)
|
||||
//}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package v1
|
||||
|
||||
/*
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
@ -228,3 +229,4 @@ func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error {
|
||||
|
||||
return c.JSON(http.StatusOK, nil)
|
||||
}
|
||||
*/
|
||||
30
internal/services/cron/ffxivJob.go
Normal file
30
internal/services/cron/ffxivJob.go
Normal file
@ -0,0 +1,30 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
)
|
||||
|
||||
func (c *Cron) CheckFfxiv() {
|
||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 10, domain.SourceCollectorFfxiv)
|
||||
if err != nil {
|
||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
fc := input.NewFFXIVClient(source)
|
||||
items, err := fc.CheckSource()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
c.SaveNewArticles(items, domain.SourceCollectorFfxiv)
|
||||
}
|
||||
log.Printf("[FFXIV Done!]")
|
||||
}
|
||||
33
internal/services/cron/redditJob.go
Normal file
33
internal/services/cron/redditJob.go
Normal file
@ -0,0 +1,33 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
)
|
||||
|
||||
// This is the main entry point to query all the reddit services
|
||||
func (c *Cron) CheckReddit() {
|
||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorReddit)
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
|
||||
rc := input.NewRedditClient(source)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
|
||||
c.SaveNewArticles(redditArticles, domain.SourceCollectorReddit)
|
||||
}
|
||||
log.Print("[Reddit] Done!")
|
||||
}
|
||||
34
internal/services/cron/rss.go
Normal file
34
internal/services/cron/rss.go
Normal file
@ -0,0 +1,34 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
func (c *Cron) LoadRssQueue() {
|
||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
|
||||
var records []domain.SourceEntity
|
||||
|
||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
|
||||
if err != nil {
|
||||
return records, err
|
||||
}
|
||||
|
||||
return sources, nil
|
||||
}
|
||||
@ -3,83 +3,57 @@ package cron
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
|
||||
)
|
||||
|
||||
type Cron struct {
|
||||
Db *database.Queries
|
||||
ctx *context.Context
|
||||
timer *cron.Cron
|
||||
repo services.RepositoryService
|
||||
//queues services
|
||||
}
|
||||
|
||||
func openDatabase() (*database.Queries, error) {
|
||||
_env := services.NewConfig()
|
||||
connString := _env.GetConfig(services.Sql_Connection_String)
|
||||
if connString == "" {
|
||||
panic("Connection String is null!")
|
||||
}
|
||||
db, err := sql.Open("postgres", connString)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
queries := database.New(db)
|
||||
return queries, err
|
||||
}
|
||||
|
||||
func NewScheduler(ctx context.Context) *Cron {
|
||||
func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
|
||||
c := &Cron{
|
||||
ctx: &ctx,
|
||||
ctx: &ctx,
|
||||
repo: services.NewRepositoryService(conn),
|
||||
}
|
||||
|
||||
timer := cron.New()
|
||||
queries, err := openDatabase()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
c.Db = queries
|
||||
|
||||
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||
features := services.NewConfig()
|
||||
features := services.GetEnvConfig()
|
||||
|
||||
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||
if res {
|
||||
if features.RedditEnabled {
|
||||
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
||||
log.Print("[Input] Reddit backend was enabled")
|
||||
//go c.CheckReddit()
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||
if res {
|
||||
if features.YoutubeEnabled {
|
||||
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
||||
log.Print("[Input] YouTube backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||
if res {
|
||||
if features.FfxivEnabled {
|
||||
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
||||
log.Print("[Input] FFXIV backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||
if res {
|
||||
if features.TwitchEnabled {
|
||||
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
||||
log.Print("[Input] Twitch backend was enabled")
|
||||
}
|
||||
|
||||
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
||||
log.Print("[Output] Discord Output was enabled")
|
||||
//timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
|
||||
//log.Print("[Output] Discord Output was enabled")
|
||||
|
||||
c.timer = timer
|
||||
return c
|
||||
@ -93,104 +67,7 @@ func (c *Cron) Stop() {
|
||||
c.timer.Stop()
|
||||
}
|
||||
|
||||
// This is the main entry point to query all the reddit services
|
||||
func (c *Cron) CheckReddit() {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[Reddit] Checking '%v'...", source.Name)
|
||||
rc := input.NewRedditClient(source)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
redditArticles := rc.ConvertToArticles(raw)
|
||||
c.checkPosts(redditArticles, "Reddit")
|
||||
}
|
||||
log.Print("[Reddit] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckYoutube() {
|
||||
// Add call to the db to request youtube sources.
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
|
||||
if err != nil {
|
||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[YouTube] Checking '%v'...", source.Name)
|
||||
yc := input.NewYoutubeClient(source)
|
||||
raw, err := yc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(raw, "YouTube")
|
||||
}
|
||||
log.Print("[YouTube] Done!")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckFfxiv() {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
|
||||
if err != nil {
|
||||
log.Printf("[FFXIV] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
fc := input.NewFFXIVClient(source)
|
||||
items, err := fc.CheckSource()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(items, "FFXIV")
|
||||
}
|
||||
log.Printf("[FFXIV Done!]")
|
||||
}
|
||||
|
||||
func (c *Cron) CheckTwitch() error {
|
||||
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
|
||||
if err != nil {
|
||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[Twitch] Checking '%v'...", source.Name)
|
||||
tc.ReplaceSourceRecord(source)
|
||||
items, err := tc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.checkPosts(items, "Twitch")
|
||||
}
|
||||
|
||||
log.Print("[Twitch] Done!")
|
||||
return nil
|
||||
}
|
||||
|
||||
/* TODO move to the sqlite queue
|
||||
func (c *Cron) CheckDiscordQueue() error {
|
||||
// Get items from the table
|
||||
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
|
||||
@ -260,55 +137,26 @@ func (c *Cron) CheckDiscordQueue() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
*/
|
||||
|
||||
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error {
|
||||
func (c Cron) SaveNewArticles(posts []domain.ArticleEntity, sourceName string) error {
|
||||
for _, item := range posts {
|
||||
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url)
|
||||
_, err := c.repo.Articles.GetByUrl(*c.ctx, item.Url)
|
||||
if err == nil {
|
||||
// This url is already known, so skip it
|
||||
continue
|
||||
}
|
||||
|
||||
// Load the new article in the repository
|
||||
rows, err := c.repo.Articles.Create(*c.ctx, item.SourceID, item.Tags, item.Title, item.Url, item.Thumbnail, item.Description, item.AuthorName, item.AuthorImageUrl, item.PubDate, item.IsVideo)
|
||||
if err != nil {
|
||||
id := uuid.New()
|
||||
|
||||
err := c.postArticle(id, item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
|
||||
}
|
||||
|
||||
err = c.addToDiscordQueue(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if rows != 1 {
|
||||
return errors.New("failed to create a new record for some reason")
|
||||
}
|
||||
}
|
||||
time.Sleep(30 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
|
||||
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
|
||||
ID: id,
|
||||
Sourceid: item.Sourceid,
|
||||
Tags: item.Tags,
|
||||
Title: item.Title,
|
||||
Url: item.Url,
|
||||
Pubdate: item.Pubdate,
|
||||
Video: item.Video,
|
||||
Videoheight: item.Videoheight,
|
||||
Videowidth: item.Videowidth,
|
||||
Thumbnail: item.Thumbnail,
|
||||
Description: item.Description,
|
||||
Authorname: item.Authorname,
|
||||
Authorimage: item.Authorimage,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
|
||||
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
|
||||
ID: uuid.New(),
|
||||
Articleid: Id,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -2,33 +2,70 @@ package cron_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
_ "github.com/glebarez/go-sqlite"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func TestInvokeTwitch(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// TODO add database mocks but not sure how to do that yet.
|
||||
func TestCheckReddit(t *testing.T) {
|
||||
db, err := setupInMemoryDb()
|
||||
if err != nil {
|
||||
t.Log(err)
|
||||
t.FailNow()
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
c := cron.NewScheduler(ctx)
|
||||
c := cron.NewScheduler(ctx, db)
|
||||
c.CheckReddit()
|
||||
}
|
||||
|
||||
func TestCheckYouTube(t *testing.T) {
|
||||
db, err := setupInMemoryDb()
|
||||
if err != nil {
|
||||
t.Log(err)
|
||||
t.FailNow()
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
c := cron.NewScheduler(ctx)
|
||||
c := cron.NewScheduler(ctx, db)
|
||||
c.CheckYoutube()
|
||||
}
|
||||
|
||||
func TestCheckTwitch(t *testing.T) {
|
||||
db, err := setupInMemoryDb()
|
||||
if err != nil {
|
||||
t.Log(err)
|
||||
t.FailNow()
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
c := cron.NewScheduler(ctx)
|
||||
err := c.CheckTwitch()
|
||||
c := cron.NewScheduler(ctx, db)
|
||||
err = c.CheckTwitch()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setupInMemoryDb() (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = goose.SetDialect("sqlite3")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = goose.Up(db, "../database/migrations")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
41
internal/services/cron/twitchJob.go
Normal file
41
internal/services/cron/twitchJob.go
Normal file
@ -0,0 +1,41 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
)
|
||||
|
||||
func (c *Cron) CheckTwitch() error {
|
||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorTwitch)
|
||||
if err != nil {
|
||||
log.Printf("[Twitch] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
tc, err := input.NewTwitchClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
|
||||
tc.ReplaceSourceRecord(source)
|
||||
items, err := tc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.SaveNewArticles(items, domain.SourceCollectorTwitch)
|
||||
}
|
||||
|
||||
log.Print("[Twitch] Done!")
|
||||
return nil
|
||||
}
|
||||
34
internal/services/cron/youtube.go
Normal file
34
internal/services/cron/youtube.go
Normal file
@ -0,0 +1,34 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
)
|
||||
|
||||
func (c *Cron) CheckYoutube() {
|
||||
// Add call to the db to request youtube sources.
|
||||
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorYoutube)
|
||||
if err != nil {
|
||||
log.Printf("[Youtube] No sources found to query - %v\r", err)
|
||||
}
|
||||
|
||||
for _, source := range sources {
|
||||
if !source.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
|
||||
|
||||
yc := input.NewYoutubeClient(source)
|
||||
raw, err := yc.GetContent()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
c.SaveNewArticles(raw, domain.SourceCollectorYoutube)
|
||||
}
|
||||
log.Print("[YouTube] Done!")
|
||||
}
|
||||
|
||||
|
||||
@ -1,140 +0,0 @@
|
||||
// The converter package lives between the database calls and the API calls.
|
||||
// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type DtoClient struct {
|
||||
db *database.Queries
|
||||
}
|
||||
|
||||
func NewDtoClient(db *database.Queries) *DtoClient {
|
||||
return &DtoClient{
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
|
||||
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, article := range a {
|
||||
res = append(res, c.convertArticle(article))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
|
||||
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
|
||||
Limit: limit,
|
||||
Offset: page * limit,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, article := range a {
|
||||
res = append(res, c.convertArticle(article))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) {
|
||||
a, err := c.db.GetArticleByID(ctx, ID)
|
||||
if err != nil {
|
||||
return models.ArticleDto{}, err
|
||||
}
|
||||
|
||||
return c.convertArticle(a), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) {
|
||||
a, err := c.db.GetArticleByID(ctx, ID)
|
||||
if err != nil {
|
||||
return models.ArticleDetailsDto{}, err
|
||||
}
|
||||
|
||||
s, err := c.db.GetSourceByID(ctx, a.Sourceid)
|
||||
if err != nil {
|
||||
return models.ArticleDetailsDto{}, err
|
||||
}
|
||||
|
||||
res := c.convertArticleDetails(a, s)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
|
||||
Sourceid: SourceID,
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, article := range a {
|
||||
res = append(res, c.convertArticle(article))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto {
|
||||
return models.ArticleDto{
|
||||
ID: i.ID,
|
||||
Source: i.Sourceid,
|
||||
Tags: c.SplitTags(i.Tags),
|
||||
Title: i.Title,
|
||||
Url: i.Url,
|
||||
Pubdate: i.Pubdate,
|
||||
Video: i.Video.String,
|
||||
Videoheight: i.Videoheight,
|
||||
Videowidth: i.Videoheight,
|
||||
Thumbnail: i.Thumbnail,
|
||||
Description: i.Description,
|
||||
Authorname: i.Authorname.String,
|
||||
Authorimage: i.Authorimage.String,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto {
|
||||
return models.ArticleDetailsDto{
|
||||
ID: i.ID,
|
||||
Source: c.ConvertToSource(s),
|
||||
Tags: c.SplitTags(i.Tags),
|
||||
Title: i.Title,
|
||||
Url: i.Url,
|
||||
Pubdate: i.Pubdate,
|
||||
Video: i.Video.String,
|
||||
Videoheight: i.Videoheight,
|
||||
Videowidth: i.Videoheight,
|
||||
Thumbnail: i.Thumbnail,
|
||||
Description: i.Description,
|
||||
Authorname: i.Authorname.String,
|
||||
Authorimage: i.Authorimage.String,
|
||||
}
|
||||
}
|
||||
|
||||
func (c DtoClient) SplitTags(t string) []string {
|
||||
return strings.Split(t, ", ")
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
|
||||
var res []models.DiscordWebHooksDto
|
||||
|
||||
items, err := c.db.ListDiscordWebhooks(ctx, total)
|
||||
if err != nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertDiscordWebhook(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) {
|
||||
var res models.DiscordWebHooksDto
|
||||
|
||||
item, err := c.db.GetDiscordWebHooksByID(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return c.ConvertDiscordWebhook(item), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) {
|
||||
var res []models.DiscordWebHooksDto
|
||||
|
||||
items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{
|
||||
Server: server,
|
||||
Channel: channel,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertDiscordWebhook(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto {
|
||||
return models.DiscordWebHooksDto{
|
||||
ID: i.ID,
|
||||
Url: i.Url,
|
||||
Server: i.Server,
|
||||
Channel: i.Channel,
|
||||
Enabled: i.Enabled,
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
|
||||
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) {
|
||||
var res []models.DiscordQueueDetailsDto
|
||||
|
||||
items, err := c.db.ListDiscordQueueItems(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
article, err := c.GetArticleDetails(ctx, item.ID)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res = append(res, models.DiscordQueueDetailsDto{
|
||||
ID: item.ID,
|
||||
Article: article,
|
||||
})
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto {
|
||||
return models.DiscordQueueDto{
|
||||
ID: i.ID,
|
||||
Articleid: i.Articleid,
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
|
||||
var res []models.SourceDto
|
||||
|
||||
items, err := c.db.ListSources(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertToSource(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) {
|
||||
var res []models.SourceDto
|
||||
|
||||
items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName))
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertToSource(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) {
|
||||
var res models.SourceDto
|
||||
|
||||
item, err := c.db.GetSourceByID(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return c.ConvertToSource(item), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) {
|
||||
var res models.SourceDto
|
||||
|
||||
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
|
||||
Name: name,
|
||||
Source: source,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return c.ConvertToSource(item), nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto {
|
||||
var deleted bool
|
||||
if !i.Deleted.Valid {
|
||||
deleted = true
|
||||
}
|
||||
|
||||
return models.SourceDto{
|
||||
ID: i.ID,
|
||||
Site: i.Site,
|
||||
Name: i.Name,
|
||||
Source: i.Source,
|
||||
Type: i.Type,
|
||||
Value: i.Value.String,
|
||||
Enabled: i.Enabled,
|
||||
Url: i.Url,
|
||||
Tags: c.SplitTags(i.Tags),
|
||||
Deleted: deleted,
|
||||
}
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
|
||||
var res []models.SubscriptionDto
|
||||
|
||||
items, err := c.db.ListSubscriptions(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertSubscription(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) {
|
||||
var res []models.SubscriptionDetailsDto
|
||||
|
||||
items, err := c.ListSubscriptions(ctx, limit)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
source, err := c.GetSourceById(ctx, item.SourceId)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res = append(res, models.SubscriptionDetailsDto{
|
||||
ID: item.ID,
|
||||
Source: source,
|
||||
DiscordWebHook: dwh,
|
||||
})
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
||||
var res []models.SubscriptionDto
|
||||
|
||||
items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertSubscription(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
|
||||
var res []models.SubscriptionDto
|
||||
|
||||
items, err := c.db.GetSubscriptionsBySourceID(ctx, id)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
res = append(res, c.ConvertSubscription(item))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto {
|
||||
return models.SubscriptionDto{
|
||||
ID: i.ID,
|
||||
DiscordWebhookId: i.Discordwebhookid,
|
||||
SourceId: i.Sourceid,
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
@ -13,7 +12,7 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
)
|
||||
|
||||
@ -25,7 +24,7 @@ const (
|
||||
)
|
||||
|
||||
type FFXIVClient struct {
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
//SourceID uint
|
||||
//Url string
|
||||
//Region string
|
||||
@ -33,15 +32,15 @@ type FFXIVClient struct {
|
||||
cacheGroup string
|
||||
}
|
||||
|
||||
func NewFFXIVClient(Record database.Source) FFXIVClient {
|
||||
func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
|
||||
return FFXIVClient{
|
||||
record: Record,
|
||||
cacheGroup: "ffxiv",
|
||||
}
|
||||
}
|
||||
|
||||
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
var articles []database.Article
|
||||
func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
|
||||
var articles []domain.ArticleEntity
|
||||
|
||||
parser := fc.GetBrowser()
|
||||
defer parser.Close()
|
||||
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
|
||||
return articles, err
|
||||
}
|
||||
|
||||
article := database.Article{
|
||||
Sourceid: fc.record.ID,
|
||||
Tags: tags,
|
||||
Title: title,
|
||||
Url: link,
|
||||
Pubdate: pubDate,
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Thumbnail: thumb,
|
||||
Description: description,
|
||||
Authorname: sql.NullString{String: authorName},
|
||||
Authorimage: sql.NullString{String: authorImage},
|
||||
article := domain.ArticleEntity{
|
||||
SourceID: fc.record.ID,
|
||||
Tags: tags,
|
||||
Title: title,
|
||||
Url: link,
|
||||
PubDate: pubDate,
|
||||
Thumbnail: thumb,
|
||||
Description: description,
|
||||
AuthorName: authorName,
|
||||
AuthorImageUrl: authorImage,
|
||||
}
|
||||
log.Printf("Collected '%v' from '%v'", article.Title, article.Url)
|
||||
|
||||
|
||||
@ -3,18 +3,16 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var FFXIVRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Site: "ffxiv",
|
||||
Name: "Final Fantasy XIV - NA",
|
||||
Source: "ffxiv",
|
||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||
var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 997,
|
||||
DisplayName: "Final Fantasy XIV - NA",
|
||||
Source: domain.SourceCollectorFfxiv,
|
||||
Url: "https://na.finalfantasyxiv.com/lodestone/",
|
||||
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
|
||||
}
|
||||
|
||||
func TestFfxivGetParser(t *testing.T) {
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -9,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/go-rod/rod"
|
||||
@ -17,8 +15,8 @@ import (
|
||||
)
|
||||
|
||||
type RedditClient struct {
|
||||
config RedditConfig
|
||||
record database.Source
|
||||
config services.Configs
|
||||
record domain.SourceEntity
|
||||
}
|
||||
|
||||
type RedditConfig struct {
|
||||
@ -27,14 +25,11 @@ type RedditConfig struct {
|
||||
PullNSFW string
|
||||
}
|
||||
|
||||
func NewRedditClient(Record database.Source) *RedditClient {
|
||||
func NewRedditClient(record domain.SourceEntity) *RedditClient {
|
||||
rc := RedditClient{
|
||||
record: Record,
|
||||
record: record,
|
||||
config: services.GetEnvConfig(),
|
||||
}
|
||||
cc := services.NewConfig()
|
||||
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
|
||||
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
|
||||
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
|
||||
|
||||
//rc.disableHttp2Client()
|
||||
|
||||
@ -71,7 +66,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
// TODO Wire this to support the config options
|
||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name)
|
||||
log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
|
||||
|
||||
content, err := getHttpContent(Url)
|
||||
if err != nil {
|
||||
@ -88,24 +83,28 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
|
||||
var redditArticles []database.Article
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
|
||||
var redditArticles []domain.ArticleEntity
|
||||
|
||||
for _, item := range items.Data.Children {
|
||||
var article database.Article
|
||||
var article domain.ArticleEntity
|
||||
article, err := rc.convertToArticle(item.Data)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("[Reddit] %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
redditArticles = append(redditArticles, article)
|
||||
}
|
||||
|
||||
return redditArticles
|
||||
}
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
|
||||
var item database.Article
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
|
||||
var item domain.ArticleEntity
|
||||
|
||||
if source.Content == "" && source.Url != "" {
|
||||
item = rc.convertPicturePost(source)
|
||||
@ -131,65 +130,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Title: source.Title,
|
||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Pubdate: time.Now(),
|
||||
Video: sql.NullString{String: "null"},
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
Authorimage: sql.NullString{String: "null"},
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Title: source.Title,
|
||||
Tags: fmt.Sprintf("%v", rc.record.Tags),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
PubDate: time.Now(),
|
||||
IsVideo: false,
|
||||
Thumbnail: source.Thumbnail,
|
||||
Description: source.Content,
|
||||
AuthorName: source.Author,
|
||||
AuthorImageUrl: "",
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.Content,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.Media.RedditVideo.FallBackUrl,
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// This post is nothing more then a redirect to another location.
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
|
||||
var item = domain.ArticleEntity{
|
||||
SourceID: rc.record.ID,
|
||||
Tags: "a",
|
||||
Title: source.Title,
|
||||
Pubdate: time.Now(),
|
||||
PubDate: time.Now(),
|
||||
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
|
||||
Videoheight: 0,
|
||||
Videowidth: 0,
|
||||
Authorname: sql.NullString{String: source.Author},
|
||||
AuthorName: source.Author,
|
||||
Description: source.UrlOverriddenByDest,
|
||||
}
|
||||
return item
|
||||
|
||||
@ -3,18 +3,16 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var RedditRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
Url: "https://reddit.com/r/dadjokes",
|
||||
Tags: "reddit, dadjokes",
|
||||
var RedditRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 999,
|
||||
DisplayName: "dadjokes",
|
||||
Source: domain.SourceCollectorReddit,
|
||||
Url: "https://reddit.com/r/dadjokes",
|
||||
Tags: "reddit, dadjokes",
|
||||
}
|
||||
|
||||
func TestGetContent(t *testing.T) {
|
||||
|
||||
@ -8,9 +8,10 @@ import (
|
||||
)
|
||||
|
||||
var rssRecord = domain.SourceEntity{
|
||||
ID: 1,
|
||||
ID: 1,
|
||||
DisplayName: "ArsTechnica",
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||
Source: domain.SourceCollectorRss,
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
||||
}
|
||||
|
||||
func TestRssClientConstructor(t *testing.T) {
|
||||
|
||||
@ -1,19 +1,18 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/nicklaw5/helix/v2"
|
||||
)
|
||||
|
||||
type TwitchClient struct {
|
||||
SourceRecord database.Source
|
||||
SourceRecord domain.SourceEntity
|
||||
|
||||
// config
|
||||
monitorClips string
|
||||
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
||||
}
|
||||
|
||||
// This will let you replace the bound source record to keep the same session alive.
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) {
|
||||
func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
|
||||
tc.SourceRecord = source
|
||||
}
|
||||
|
||||
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||
var items []domain.ArticleEntity
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
|
||||
}
|
||||
|
||||
for _, video := range posts {
|
||||
var article database.Article
|
||||
var article domain.ArticleEntity
|
||||
|
||||
AuthorName, err := tc.ExtractAuthor(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
article.Authorname = sql.NullString{String: AuthorName}
|
||||
article.AuthorName = AuthorName
|
||||
|
||||
Authorimage, err := tc.ExtractAuthorImage(user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
article.Authorimage = sql.NullString{String: Authorimage}
|
||||
article.AuthorImageUrl = Authorimage
|
||||
|
||||
article.Description, err = tc.ExtractDescription(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
article.Pubdate, err = tc.ExtractPubDate(video)
|
||||
article.PubDate, err = tc.ExtractPubDate(video)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
article.Sourceid = tc.SourceRecord.ID
|
||||
article.SourceID = tc.SourceRecord.ID
|
||||
article.Tags, err = tc.ExtractTags(video, user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
var blank helix.User
|
||||
|
||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||
Logins: []string{tc.SourceRecord.Name},
|
||||
Logins: []string{tc.SourceRecord.DisplayName},
|
||||
})
|
||||
if err != nil {
|
||||
return blank, err
|
||||
|
||||
@ -4,21 +4,20 @@ import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var TwitchSourceRecord = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "nintendo",
|
||||
Source: "Twitch",
|
||||
var TwitchSourceRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "nintendo",
|
||||
Source: domain.SourceCollectorTwitch,
|
||||
}
|
||||
|
||||
var TwitchInvalidRecord = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "EvilNintendo",
|
||||
Source: "Twitch",
|
||||
var TwitchInvalidRecord = domain.SourceEntity{
|
||||
ID: 9999,
|
||||
DisplayName: "EvilNintendo",
|
||||
Source: domain.SourceCollectorTwitch,
|
||||
}
|
||||
|
||||
func TestTwitchLogin(t *testing.T) {
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package input
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -12,11 +11,11 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
type YoutubeClient struct {
|
||||
record database.Source
|
||||
record domain.SourceEntity
|
||||
|
||||
// internal variables at time of collection
|
||||
channelID string
|
||||
@ -37,7 +36,7 @@ var (
|
||||
|
||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||
|
||||
func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
|
||||
yc := YoutubeClient{
|
||||
record: Record,
|
||||
cacheGroup: "youtube",
|
||||
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
|
||||
}
|
||||
|
||||
// CheckSource will go and run all the commands needed to process a source.
|
||||
func (yc *YoutubeClient) GetContent() ([]database.Article, error) {
|
||||
var items []database.Article
|
||||
func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
|
||||
var items []domain.ArticleEntity
|
||||
docParser, err := yc.GetParser(yc.record.Url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
|
||||
parser, err := yc.GetParser(item.Link)
|
||||
if err != nil {
|
||||
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
|
||||
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
|
||||
log.Printf("[YouTube] %v", msg)
|
||||
}
|
||||
|
||||
var article = database.Article{
|
||||
Sourceid: yc.record.ID,
|
||||
Tags: tags,
|
||||
Title: item.Title,
|
||||
Url: item.Link,
|
||||
Pubdate: *item.PublishedParsed,
|
||||
Thumbnail: thumb,
|
||||
Description: item.Description,
|
||||
Authorname: sql.NullString{String: item.Author.Name},
|
||||
Authorimage: sql.NullString{String: yc.avatarUri},
|
||||
var article = domain.ArticleEntity{
|
||||
SourceID: yc.record.ID,
|
||||
Tags: tags,
|
||||
Title: item.Title,
|
||||
Url: item.Link,
|
||||
PubDate: *item.PublishedParsed,
|
||||
Thumbnail: thumb,
|
||||
Description: item.Description,
|
||||
AuthorName: item.Author.Name,
|
||||
AuthorImageUrl: yc.avatarUri,
|
||||
}
|
||||
return article
|
||||
}
|
||||
|
||||
@ -3,17 +3,15 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var YouTubeRecord database.Source = database.Source{
|
||||
ID: uuid.New(),
|
||||
Name: "dadjokes",
|
||||
Source: "reddit",
|
||||
Site: "reddit",
|
||||
Url: "https://youtube.com/gamegrumps",
|
||||
var YouTubeRecord domain.SourceEntity = domain.SourceEntity{
|
||||
ID: 999,
|
||||
DisplayName: "dadjokes",
|
||||
Source: domain.SourceCollectorYoutube,
|
||||
Url: "https://youtube.com/gamegrumps",
|
||||
}
|
||||
|
||||
func TestGetPageParser(t *testing.T) {
|
||||
|
||||
26
internal/services/queue.go
Normal file
26
internal/services/queue.go
Normal file
@ -0,0 +1,26 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"github.com/maragudk/goqite"
|
||||
)
|
||||
|
||||
|
||||
|
||||
type queues struct {
|
||||
repos RepositoryService
|
||||
|
||||
RssCollection goqite.Queue
|
||||
}
|
||||
|
||||
func NewQueues(conn *sql.DB) queues {
|
||||
return queues{
|
||||
repos: NewRepositoryService(conn),
|
||||
RssCollection: *goqite.New(goqite.NewOpts{
|
||||
DB: conn,
|
||||
Name: domain.QueueRssCollection,
|
||||
}),
|
||||
}
|
||||
}
|
||||
4
makefile
4
makefile
@ -4,8 +4,8 @@ help: ## Shows this help command
|
||||
|
||||
build: ## builds the application with the current go runtime
|
||||
~/go/bin/swag f
|
||||
~/go/bin/swag i
|
||||
go build .
|
||||
~/go/bin/swag init -g cmd/server.go
|
||||
go build cmd/server.go
|
||||
|
||||
docker-build: ## Generates the docker image
|
||||
docker build -t "newsbot.collector.api" .
|
||||
|
||||
Loading…
Reference in New Issue
Block a user