Compare commits

..

6 Commits

36 changed files with 474 additions and 1818 deletions

2
.gitignore vendored
View File

@ -2,7 +2,6 @@
dev.session.sql dev.session.sql
__debug_bin __debug_bin
server server
.vscode
# Binaries for programs and plugins # Binaries for programs and plugins
*.exe *.exe
@ -11,6 +10,7 @@ server
*.so *.so
*.dylib *.dylib
collector collector
*.db
# Test binary, built with `go test -c` # Test binary, built with `go test -c`
*.test *.test

2
.vscode/launch.json vendored
View File

@ -9,7 +9,7 @@
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "auto", "mode": "auto",
"program": "." "program": "./cmd/server.go"
} }
] ]
} }

View File

@ -10,8 +10,7 @@ import (
"github.com/pressly/goose/v3" "github.com/pressly/goose/v3"
"git.jamestombleson.com/jtom38/newsbot-api/docs" "git.jamestombleson.com/jtom38/newsbot-api/docs"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
) )
@ -42,12 +41,15 @@ func main() {
panic(err) panic(err)
} }
queries := database.New(db) //queues := services.NewQueues(db)
//queues.RssIndex.Send(ctx, goqite.Message{
// Body: []byte("hello world"),
//})
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx, db)
c.Start() c.Start()
server := v1.NewServer(ctx, queries, configs, db) server := v1.NewServer(ctx, configs, db)
fmt.Println("API is online and waiting for requests.") fmt.Println("API is online and waiting for requests.")
fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress) fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress)

View File

@ -449,25 +449,6 @@ const docTemplate = `{
} }
} }
}, },
"/queue/discord/webhooks": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Queue"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
}
}
}
}
},
"/settings/{key}": { "/settings/{key}": {
"get": { "get": {
"produces": [ "produces": [
@ -875,167 +856,6 @@ const docTemplate = `{
} }
} }
} }
},
"/subscriptions": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptions"
}
},
"400": {
"description": "Unable to reach SQL.",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
},
"500": {
"description": "Failed to process data from SQL.",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
}
}
}
},
"/subscriptions/by/SourceId": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "query",
"required": true
}
],
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptions"
}
}
}
}
},
"/subscriptions/by/discordId": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "query",
"required": true
}
],
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptions"
}
},
"400": {
"description": "Unable to reach SQL or Data problems",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
},
"500": {
"description": "Data problems",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
}
}
}
},
"/subscriptions/details": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 50 entries with full deatils on the source and output.",
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptionDetails"
}
}
}
}
},
"/subscriptions/discord/webhook/delete": {
"delete": {
"tags": [
"Subscription"
],
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/subscriptions/discord/webhook/new": {
"post": {
"tags": [
"Subscription"
],
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
"parameters": [
{
"type": "string",
"description": "discordWebHookId",
"name": "discordWebHookId",
"in": "query",
"required": true
},
{
"type": "string",
"description": "sourceId",
"name": "sourceId",
"in": "query",
"required": true
}
],
"responses": {}
}
} }
}, },
"definitions": { "definitions": {
@ -1192,212 +1012,6 @@ const docTemplate = `{
} }
} }
} }
},
"models.ArticleDetailsDto": {
"type": "object",
"properties": {
"authorImage": {
"type": "string"
},
"authorName": {
"type": "string"
},
"description": {
"type": "string"
},
"id": {
"type": "string"
},
"pubdate": {
"type": "string"
},
"source": {
"$ref": "#/definitions/models.SourceDto"
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"thumbnail": {
"type": "string"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
},
"video": {
"type": "string"
},
"videoHeight": {
"type": "integer"
},
"videoWidth": {
"type": "integer"
}
}
},
"models.DiscordQueueDetailsDto": {
"type": "object",
"properties": {
"article": {
"$ref": "#/definitions/models.ArticleDetailsDto"
},
"id": {
"type": "string"
}
}
},
"models.DiscordWebHooksDto": {
"type": "object",
"properties": {
"ID": {
"type": "string"
},
"channel": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"server": {
"type": "string"
},
"url": {
"type": "string"
}
}
},
"models.SourceDto": {
"type": "object",
"properties": {
"deleted": {
"type": "boolean"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"site": {
"type": "string"
},
"source": {
"type": "string"
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"type": {
"type": "string"
},
"url": {
"type": "string"
},
"value": {
"type": "string"
}
}
},
"models.SubscriptionDetailsDto": {
"type": "object",
"properties": {
"discordwebhook": {
"$ref": "#/definitions/models.DiscordWebHooksDto"
},
"id": {
"type": "string"
},
"source": {
"$ref": "#/definitions/models.SourceDto"
}
}
},
"models.SubscriptionDto": {
"type": "object",
"properties": {
"discordwebhookid": {
"type": "string"
},
"id": {
"type": "string"
},
"sourceid": {
"type": "string"
}
}
},
"v1.ApiError": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"status": {
"type": "integer"
}
}
},
"v1.ListDiscordWebHooksQueueResults": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"payload": {
"type": "array",
"items": {
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
}
},
"status": {
"type": "integer"
}
}
},
"v1.ListSubscriptionDetails": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"payload": {
"type": "array",
"items": {
"$ref": "#/definitions/models.SubscriptionDetailsDto"
}
},
"status": {
"type": "integer"
}
}
},
"v1.ListSubscriptions": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"payload": {
"type": "array",
"items": {
"$ref": "#/definitions/models.SubscriptionDto"
}
},
"status": {
"type": "integer"
}
}
} }
} }
}` }`

View File

@ -440,25 +440,6 @@
} }
} }
}, },
"/queue/discord/webhooks": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Queue"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListDiscordWebHooksQueueResults"
}
}
}
}
},
"/settings/{key}": { "/settings/{key}": {
"get": { "get": {
"produces": [ "produces": [
@ -866,167 +847,6 @@
} }
} }
} }
},
"/subscriptions": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptions"
}
},
"400": {
"description": "Unable to reach SQL.",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
},
"500": {
"description": "Failed to process data from SQL.",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
}
}
}
},
"/subscriptions/by/SourceId": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "query",
"required": true
}
],
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptions"
}
}
}
}
},
"/subscriptions/by/discordId": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 100 entries from the queue to be processed.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "query",
"required": true
}
],
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptions"
}
},
"400": {
"description": "Unable to reach SQL or Data problems",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
},
"500": {
"description": "Data problems",
"schema": {
"$ref": "#/definitions/v1.ApiError"
}
}
}
}
},
"/subscriptions/details": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Subscription"
],
"summary": "Returns the top 50 entries with full deatils on the source and output.",
"responses": {
"200": {
"description": "ok",
"schema": {
"$ref": "#/definitions/v1.ListSubscriptionDetails"
}
}
}
}
},
"/subscriptions/discord/webhook/delete": {
"delete": {
"tags": [
"Subscription"
],
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/subscriptions/discord/webhook/new": {
"post": {
"tags": [
"Subscription"
],
"summary": "Creates a new subscription to link a post from a Source to a DiscordWebHook.",
"parameters": [
{
"type": "string",
"description": "discordWebHookId",
"name": "discordWebHookId",
"in": "query",
"required": true
},
{
"type": "string",
"description": "sourceId",
"name": "sourceId",
"in": "query",
"required": true
}
],
"responses": {}
}
} }
}, },
"definitions": { "definitions": {
@ -1183,212 +1003,6 @@
} }
} }
} }
},
"models.ArticleDetailsDto": {
"type": "object",
"properties": {
"authorImage": {
"type": "string"
},
"authorName": {
"type": "string"
},
"description": {
"type": "string"
},
"id": {
"type": "string"
},
"pubdate": {
"type": "string"
},
"source": {
"$ref": "#/definitions/models.SourceDto"
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"thumbnail": {
"type": "string"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
},
"video": {
"type": "string"
},
"videoHeight": {
"type": "integer"
},
"videoWidth": {
"type": "integer"
}
}
},
"models.DiscordQueueDetailsDto": {
"type": "object",
"properties": {
"article": {
"$ref": "#/definitions/models.ArticleDetailsDto"
},
"id": {
"type": "string"
}
}
},
"models.DiscordWebHooksDto": {
"type": "object",
"properties": {
"ID": {
"type": "string"
},
"channel": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"server": {
"type": "string"
},
"url": {
"type": "string"
}
}
},
"models.SourceDto": {
"type": "object",
"properties": {
"deleted": {
"type": "boolean"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"site": {
"type": "string"
},
"source": {
"type": "string"
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"type": {
"type": "string"
},
"url": {
"type": "string"
},
"value": {
"type": "string"
}
}
},
"models.SubscriptionDetailsDto": {
"type": "object",
"properties": {
"discordwebhook": {
"$ref": "#/definitions/models.DiscordWebHooksDto"
},
"id": {
"type": "string"
},
"source": {
"$ref": "#/definitions/models.SourceDto"
}
}
},
"models.SubscriptionDto": {
"type": "object",
"properties": {
"discordwebhookid": {
"type": "string"
},
"id": {
"type": "string"
},
"sourceid": {
"type": "string"
}
}
},
"v1.ApiError": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"status": {
"type": "integer"
}
}
},
"v1.ListDiscordWebHooksQueueResults": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"payload": {
"type": "array",
"items": {
"$ref": "#/definitions/models.DiscordQueueDetailsDto"
}
},
"status": {
"type": "integer"
}
}
},
"v1.ListSubscriptionDetails": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"payload": {
"type": "array",
"items": {
"$ref": "#/definitions/models.SubscriptionDetailsDto"
}
},
"status": {
"type": "integer"
}
}
},
"v1.ListSubscriptions": {
"type": "object",
"properties": {
"message": {
"type": "string"
},
"payload": {
"type": "array",
"items": {
"$ref": "#/definitions/models.SubscriptionDto"
}
},
"status": {
"type": "integer"
}
}
} }
} }
} }

View File

@ -102,140 +102,6 @@ definitions:
$ref: '#/definitions/domain.SourceDto' $ref: '#/definitions/domain.SourceDto'
type: array type: array
type: object type: object
models.ArticleDetailsDto:
properties:
authorImage:
type: string
authorName:
type: string
description:
type: string
id:
type: string
pubdate:
type: string
source:
$ref: '#/definitions/models.SourceDto'
tags:
items:
type: string
type: array
thumbnail:
type: string
title:
type: string
url:
type: string
video:
type: string
videoHeight:
type: integer
videoWidth:
type: integer
type: object
models.DiscordQueueDetailsDto:
properties:
article:
$ref: '#/definitions/models.ArticleDetailsDto'
id:
type: string
type: object
models.DiscordWebHooksDto:
properties:
ID:
type: string
channel:
type: string
enabled:
type: boolean
server:
type: string
url:
type: string
type: object
models.SourceDto:
properties:
deleted:
type: boolean
enabled:
type: boolean
id:
type: string
name:
type: string
site:
type: string
source:
type: string
tags:
items:
type: string
type: array
type:
type: string
url:
type: string
value:
type: string
type: object
models.SubscriptionDetailsDto:
properties:
discordwebhook:
$ref: '#/definitions/models.DiscordWebHooksDto'
id:
type: string
source:
$ref: '#/definitions/models.SourceDto'
type: object
models.SubscriptionDto:
properties:
discordwebhookid:
type: string
id:
type: string
sourceid:
type: string
type: object
v1.ApiError:
properties:
message:
type: string
status:
type: integer
type: object
v1.ListDiscordWebHooksQueueResults:
properties:
message:
type: string
payload:
items:
$ref: '#/definitions/models.DiscordQueueDetailsDto'
type: array
status:
type: integer
type: object
v1.ListSubscriptionDetails:
properties:
message:
type: string
payload:
items:
$ref: '#/definitions/models.SubscriptionDetailsDto'
type: array
status:
type: integer
type: object
v1.ListSubscriptions:
properties:
message:
type: string
payload:
items:
$ref: '#/definitions/models.SubscriptionDto'
type: array
status:
type: integer
type: object
info: info:
contact: {} contact: {}
title: NewsBot collector title: NewsBot collector
@ -527,18 +393,6 @@ paths:
tags: tags:
- Discord - Discord
- Webhook - Webhook
/queue/discord/webhooks:
get:
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/v1.ListDiscordWebHooksQueueResults'
summary: Returns the top 100 entries from the queue to be processed.
tags:
- Queue
/settings/{key}: /settings/{key}:
get: get:
parameters: parameters:
@ -807,109 +661,4 @@ paths:
summary: Creates a new youtube source to monitor. summary: Creates a new youtube source to monitor.
tags: tags:
- Source - Source
/subscriptions:
get:
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/v1.ListSubscriptions'
"400":
description: Unable to reach SQL.
schema:
$ref: '#/definitions/v1.ApiError'
"500":
description: Failed to process data from SQL.
schema:
$ref: '#/definitions/v1.ApiError'
summary: Returns the top 100 entries from the queue to be processed.
tags:
- Subscription
/subscriptions/by/SourceId:
get:
parameters:
- description: id
in: query
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/v1.ListSubscriptions'
summary: Returns the top 100 entries from the queue to be processed.
tags:
- Subscription
/subscriptions/by/discordId:
get:
parameters:
- description: id
in: query
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/v1.ListSubscriptions'
"400":
description: Unable to reach SQL or Data problems
schema:
$ref: '#/definitions/v1.ApiError'
"500":
description: Data problems
schema:
$ref: '#/definitions/v1.ApiError'
summary: Returns the top 100 entries from the queue to be processed.
tags:
- Subscription
/subscriptions/details:
get:
produces:
- application/json
responses:
"200":
description: ok
schema:
$ref: '#/definitions/v1.ListSubscriptionDetails'
summary: Returns the top 50 entries with full deatils on the source and output.
tags:
- Subscription
/subscriptions/discord/webhook/delete:
delete:
parameters:
- description: id
in: query
name: id
required: true
type: string
responses: {}
summary: Removes a Discord WebHook Subscription based on the Subscription ID.
tags:
- Subscription
/subscriptions/discord/webhook/new:
post:
parameters:
- description: discordWebHookId
in: query
name: discordWebHookId
required: true
type: string
- description: sourceId
in: query
name: sourceId
required: true
type: string
responses: {}
summary: Creates a new subscription to link a post from a Source to a DiscordWebHook.
tags:
- Subscription
swagger: "2.0" swagger: "2.0"

2
go.mod
View File

@ -5,12 +5,12 @@ go 1.22
require ( require (
github.com/PuerkitoBio/goquery v1.8.0 github.com/PuerkitoBio/goquery v1.8.0
github.com/glebarez/go-sqlite v1.22.0 github.com/glebarez/go-sqlite v1.22.0
github.com/go-chi/chi/v5 v5.0.7
github.com/go-rod/rod v0.107.1 github.com/go-rod/rod v0.107.1
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/huandu/go-sqlbuilder v1.27.1 github.com/huandu/go-sqlbuilder v1.27.1
github.com/joho/godotenv v1.4.0 github.com/joho/godotenv v1.4.0
github.com/labstack/echo/v4 v4.12.0 github.com/labstack/echo/v4 v4.12.0
github.com/maragudk/goqite v0.1.0
github.com/mmcdole/gofeed v1.1.3 github.com/mmcdole/gofeed v1.1.3
github.com/nicklaw5/helix/v2 v2.4.0 github.com/nicklaw5/helix/v2 v2.4.0
github.com/pressly/goose/v3 v3.20.0 github.com/pressly/goose/v3 v3.20.0

8
go.sum
View File

@ -18,8 +18,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@ -71,11 +69,17 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/maragudk/goqite v0.1.0 h1:k/GgnJU9DBMVAx2DxBgmB+UKQVzJNjO1JPqmUJFH4+8=
github.com/maragudk/goqite v0.1.0/go.mod h1:5430TCLkycUeLE314c9fifTrTbwcJqJXdU3iyEiF6hM=
github.com/maragudk/is v0.1.0 h1:obq9anZNmOYcaNbeT0LMyjIexdNeYTw/TLAPD/BnZHA=
github.com/maragudk/is v0.1.0/go.mod h1:W/r6+TpnISu+a88OLXQy5JQGCOhXQXXLD2e5b4xMn5c=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=

View File

@ -0,0 +1,25 @@
-- +goose Up
-- +goose StatementBegin
create table goqite (
id text primary key default ('m_' || lower(hex(randomblob(16)))),
created text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
updated text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
queue text not null,
body blob not null,
timeout text not null default (strftime('%Y-%m-%dT%H:%M:%fZ')),
received integer not null default 0
) strict;
create trigger goqite_updated_timestamp after update on goqite begin
update goqite set updated = strftime('%Y-%m-%dT%H:%M:%fZ') where id = old.id;
end;
create index goqite_queue_created_idx on goqite (queue, created);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE goqite
DROP INDEX goqite_queue_created_idx
DROP TRIGGER goqite_updated_timestamp
-- +goose StatementEnd

View File

@ -0,0 +1,9 @@
package domain
const (
QueueRssCollection = "RssCollection"
)
type RssCollectionEvent struct {
}

View File

@ -11,13 +11,12 @@ import (
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
) )
type Handler struct { type Handler struct {
Router *echo.Echo Router *echo.Echo
Db *database.Queries Db *database.Queries
dto *dto.DtoClient //dto *dto.DtoClient
config services.Configs config services.Configs
repo services.RepositoryService repo services.RepositoryService
} }
@ -39,10 +38,10 @@ var (
ErrUnableToConvertToJson string = "Unable to convert to json" ErrUnableToConvertToJson string = "Unable to convert to json"
) )
func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler { func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler {
s := &Handler{ s := &Handler{
Db: db, //Db: db,
dto: dto.NewDtoClient(db), //dto: dto.NewDtoClient(db),
config: configs, config: configs,
repo: services.NewRepositoryService(conn), repo: services.NewRepositoryService(conn),
} }
@ -85,13 +84,13 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi
sources.POST("/:ID/disable", s.disableSource) sources.POST("/:ID/disable", s.disableSource)
sources.POST("/:ID/enable", s.enableSource) sources.POST("/:ID/enable", s.enableSource)
subs := v1.Group("/subscriptions") //subs := v1.Group("/subscriptions")
subs.GET("/", s.ListSubscriptions) //subs.GET("/", s.ListSubscriptions)
subs.GET("/details", s.ListSubscriptionDetails) //subs.GET("/details", s.ListSubscriptionDetails)
subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId) //subs.GET("/by/discordId", s.GetSubscriptionsByDiscordId)
subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId) //subs.GET("/by/sourceId", s.GetSubscriptionsBySourceId)
subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription) //subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription)
subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription) //subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
return s return s
} }

View File

@ -1,11 +1,7 @@
package v1 package v1
import ( import (
"net/http"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/labstack/echo/v4"
) )
type ListDiscordWebHooksQueueResults struct { type ListDiscordWebHooksQueueResults struct {
@ -19,22 +15,22 @@ type ListDiscordWebHooksQueueResults struct {
// @Tags Queue // @Tags Queue
// @Router /queue/discord/webhooks [get] // @Router /queue/discord/webhooks [get]
// @Success 200 {object} ListDiscordWebHooksQueueResults "ok" // @Success 200 {object} ListDiscordWebHooksQueueResults "ok"
func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error { //func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error {
p := ListDiscordWebHooksQueueResults{ // p := ListDiscordWebHooksQueueResults{
ApiStatusModel: ApiStatusModel{ // ApiStatusModel: ApiStatusModel{
Message: "OK", // Message: "OK",
StatusCode: http.StatusOK, // StatusCode: http.StatusOK,
}, // },
} // }
//
// Get the raw resp from sql // // Get the raw resp from sql
res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50) // res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50)
if err != nil { // if err != nil {
return c.JSON(http.StatusInternalServerError, domain.BaseResponse{ // return c.JSON(http.StatusInternalServerError, domain.BaseResponse{
Message: err.Error(), // Message: err.Error(),
}) // })
} // }
//
p.Payload = res // p.Payload = res
return c.JSON(http.StatusOK, p) // return c.JSON(http.StatusOK, p)
} //}

View File

@ -1,5 +1,6 @@
package v1 package v1
/*
import ( import (
"context" "context"
"encoding/json" "encoding/json"
@ -228,3 +229,4 @@ func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error {
return c.JSON(http.StatusOK, nil) return c.JSON(http.StatusOK, nil)
} }
*/

View File

@ -0,0 +1,30 @@
package cron
import (
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CheckFfxiv() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 10, domain.SourceCollectorFfxiv)
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
c.SaveNewArticles(items, domain.SourceCollectorFfxiv)
}
log.Printf("[FFXIV Done!]")
}

View File

@ -0,0 +1,33 @@
package cron
import (
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
// This is the main entry point to query all the reddit services
func (c *Cron) CheckReddit() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorReddit)
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.DisplayName)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
c.SaveNewArticles(redditArticles, domain.SourceCollectorReddit)
}
log.Print("[Reddit] Done!")
}

View File

@ -0,0 +1,34 @@
package cron
import (
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
)
func (c *Cron) LoadRssQueue() {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss)
if err != nil {
log.Println(err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
}
}
func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) {
var records []domain.SourceEntity
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType)
if err != nil {
return records, err
}
return sources, nil
}

View File

@ -3,83 +3,57 @@ package cron
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt" "errors"
"log" "log"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
) )
type Cron struct { type Cron struct {
Db *database.Queries
ctx *context.Context ctx *context.Context
timer *cron.Cron timer *cron.Cron
repo services.RepositoryService
//queues services
} }
func openDatabase() (*database.Queries, error) { func NewScheduler(ctx context.Context, conn *sql.DB) *Cron {
_env := services.NewConfig()
connString := _env.GetConfig(services.Sql_Connection_String)
if connString == "" {
panic("Connection String is null!")
}
db, err := sql.Open("postgres", connString)
if err != nil {
panic(err)
}
queries := database.New(db)
return queries, err
}
func NewScheduler(ctx context.Context) *Cron {
c := &Cron{ c := &Cron{
ctx: &ctx, ctx: &ctx,
repo: services.NewRepositoryService(conn),
} }
timer := cron.New() timer := cron.New()
queries, err := openDatabase()
if err != nil {
panic(err)
}
c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
features := services.NewConfig() features := services.GetEnvConfig()
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND) if features.RedditEnabled {
if res {
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("[Input] Reddit backend was enabled") log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit() //go c.CheckReddit()
} }
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND) if features.YoutubeEnabled {
if res {
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled") log.Print("[Input] YouTube backend was enabled")
} }
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND) if features.FfxivEnabled {
if res {
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled") log.Print("[Input] FFXIV backend was enabled")
} }
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND) if features.TwitchEnabled {
if res {
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled") log.Print("[Input] Twitch backend was enabled")
} }
timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() }) //timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() })
log.Print("[Output] Discord Output was enabled") //log.Print("[Output] Discord Output was enabled")
c.timer = timer c.timer = timer
return c return c
@ -93,104 +67,7 @@ func (c *Cron) Stop() {
c.timer.Stop() c.timer.Stop()
} }
// This is the main entry point to query all the reddit services /* TODO move to the sqlite queue
func (c *Cron) CheckReddit() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit")
if err != nil {
log.Printf("[Reddit] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Reddit] Checking '%v'...", source.Name)
rc := input.NewRedditClient(source)
raw, err := rc.GetContent()
if err != nil {
log.Println(err)
}
redditArticles := rc.ConvertToArticles(raw)
c.checkPosts(redditArticles, "Reddit")
}
log.Print("[Reddit] Done!")
}
func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources.
sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube")
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.Name)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(raw, "YouTube")
}
log.Print("[YouTube] Done!")
}
func (c *Cron) CheckFfxiv() {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv")
if err != nil {
log.Printf("[FFXIV] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
fc := input.NewFFXIVClient(source)
items, err := fc.CheckSource()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "FFXIV")
}
log.Printf("[FFXIV Done!]")
}
func (c *Cron) CheckTwitch() error {
sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch")
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
return err
}
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.Name)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
c.checkPosts(items, "Twitch")
}
log.Print("[Twitch] Done!")
return nil
}
func (c *Cron) CheckDiscordQueue() error { func (c *Cron) CheckDiscordQueue() error {
// Get items from the table // Get items from the table
queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50) queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50)
@ -260,55 +137,26 @@ func (c *Cron) CheckDiscordQueue() error {
return nil return nil
} }
*/
func (c *Cron) checkPosts(posts []database.Article, sourceName string) error { func (c Cron) SaveNewArticles(posts []domain.ArticleEntity, sourceName string) error {
for _, item := range posts { for _, item := range posts {
_, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) _, err := c.repo.Articles.GetByUrl(*c.ctx, item.Url)
if err == nil {
// This url is already known, so skip it
continue
}
// Load the new article in the repository
rows, err := c.repo.Articles.Create(*c.ctx, item.SourceID, item.Tags, item.Title, item.Url, item.Thumbnail, item.Description, item.AuthorName, item.AuthorImageUrl, item.PubDate, item.IsVideo)
if err != nil { if err != nil {
id := uuid.New() return err
}
err := c.postArticle(id, item)
if err != nil {
return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err)
}
err = c.addToDiscordQueue(id)
if err != nil {
return err
}
if rows != 1 {
return errors.New("failed to create a new record for some reason")
} }
} }
time.Sleep(30 * time.Second)
return nil
}
func (c *Cron) postArticle(id uuid.UUID, item database.Article) error {
err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{
ID: id,
Sourceid: item.Sourceid,
Tags: item.Tags,
Title: item.Title,
Url: item.Url,
Pubdate: item.Pubdate,
Video: item.Video,
Videoheight: item.Videoheight,
Videowidth: item.Videowidth,
Thumbnail: item.Thumbnail,
Description: item.Description,
Authorname: item.Authorname,
Authorimage: item.Authorimage,
})
return err
}
func (c *Cron) addToDiscordQueue(Id uuid.UUID) error {
err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{
ID: uuid.New(),
Articleid: Id,
})
if err != nil {
return err
}
return nil return nil
} }

View File

@ -2,33 +2,70 @@ package cron_test
import ( import (
"context" "context"
"database/sql"
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
_ "github.com/glebarez/go-sqlite"
"github.com/pressly/goose/v3"
) )
func TestInvokeTwitch(t *testing.T) {
}
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) { func TestCheckReddit(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background() ctx := context.Background()
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx, db)
c.CheckReddit() c.CheckReddit()
} }
func TestCheckYouTube(t *testing.T) { func TestCheckYouTube(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background() ctx := context.Background()
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx, db)
c.CheckYoutube() c.CheckYoutube()
} }
func TestCheckTwitch(t *testing.T) { func TestCheckTwitch(t *testing.T) {
db, err := setupInMemoryDb()
if err != nil {
t.Log(err)
t.FailNow()
}
defer db.Close()
ctx := context.Background() ctx := context.Background()
c := cron.NewScheduler(ctx) c := cron.NewScheduler(ctx, db)
err := c.CheckTwitch() err = c.CheckTwitch()
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
} }
func setupInMemoryDb() (*sql.DB, error) {
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
return nil, err
}
err = goose.SetDialect("sqlite3")
if err != nil {
return nil, err
}
err = goose.Up(db, "../database/migrations")
if err != nil {
return nil, err
}
return db, nil
}

View File

@ -0,0 +1,41 @@
package cron
import (
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CheckTwitch() error {
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorTwitch)
if err != nil {
log.Printf("[Twitch] No sources found to query - %v\r", err)
}
tc, err := input.NewTwitchClient()
if err != nil {
return err
}
err = tc.Login()
if err != nil {
return err
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[Twitch] Checking '%v'...", source.DisplayName)
tc.ReplaceSourceRecord(source)
items, err := tc.GetContent()
if err != nil {
log.Println(err)
}
c.SaveNewArticles(items, domain.SourceCollectorTwitch)
}
log.Print("[Twitch] Done!")
return nil
}

View File

@ -0,0 +1,34 @@
package cron
import (
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
)
func (c *Cron) CheckYoutube() {
// Add call to the db to request youtube sources.
sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 100, domain.SourceCollectorYoutube)
if err != nil {
log.Printf("[Youtube] No sources found to query - %v\r", err)
}
for _, source := range sources {
if !source.Enabled {
continue
}
log.Printf("[YouTube] Checking '%v'...", source.DisplayName)
yc := input.NewYoutubeClient(source)
raw, err := yc.GetContent()
if err != nil {
log.Println(err)
}
c.SaveNewArticles(raw, domain.SourceCollectorYoutube)
}
log.Print("[YouTube] Done!")
}

View File

@ -1,140 +0,0 @@
// The converter package lives between the database calls and the API calls.
// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much
package dto
import (
"context"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
type DtoClient struct {
db *database.Queries
}
func NewDtoClient(db *database.Queries) *DtoClient {
return &DtoClient{
db: db,
}
}
func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
Limit: int32(limit),
Offset: int32(limit * page),
})
if err != nil {
return res, err
}
for _, article := range a {
res = append(res, c.convertArticle(article))
}
return res, nil
}
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
Limit: limit,
Offset: page * limit,
})
if err != nil {
return res, err
}
for _, article := range a {
res = append(res, c.convertArticle(article))
}
return res, nil
}
func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) {
a, err := c.db.GetArticleByID(ctx, ID)
if err != nil {
return models.ArticleDto{}, err
}
return c.convertArticle(a), nil
}
func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) {
a, err := c.db.GetArticleByID(ctx, ID)
if err != nil {
return models.ArticleDetailsDto{}, err
}
s, err := c.db.GetSourceByID(ctx, a.Sourceid)
if err != nil {
return models.ArticleDetailsDto{}, err
}
res := c.convertArticleDetails(a, s)
return res, nil
}
func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
Sourceid: SourceID,
Limit: int32(limit),
Offset: int32(limit * page),
})
if err != nil {
return res, err
}
for _, article := range a {
res = append(res, c.convertArticle(article))
}
return res, nil
}
func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto {
return models.ArticleDto{
ID: i.ID,
Source: i.Sourceid,
Tags: c.SplitTags(i.Tags),
Title: i.Title,
Url: i.Url,
Pubdate: i.Pubdate,
Video: i.Video.String,
Videoheight: i.Videoheight,
Videowidth: i.Videoheight,
Thumbnail: i.Thumbnail,
Description: i.Description,
Authorname: i.Authorname.String,
Authorimage: i.Authorimage.String,
}
}
func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto {
return models.ArticleDetailsDto{
ID: i.ID,
Source: c.ConvertToSource(s),
Tags: c.SplitTags(i.Tags),
Title: i.Title,
Url: i.Url,
Pubdate: i.Pubdate,
Video: i.Video.String,
Videoheight: i.Videoheight,
Videowidth: i.Videoheight,
Thumbnail: i.Thumbnail,
Description: i.Description,
Authorname: i.Authorname.String,
Authorimage: i.Authorimage.String,
}
}
func (c DtoClient) SplitTags(t string) []string {
return strings.Split(t, ", ")
}

View File

@ -1,63 +0,0 @@
package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
var res []models.DiscordWebHooksDto
items, err := c.db.ListDiscordWebhooks(ctx, total)
if err != nil {
return res, nil
}
for _, item := range items {
res = append(res, c.ConvertDiscordWebhook(item))
}
return res, nil
}
func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) {
var res models.DiscordWebHooksDto
item, err := c.db.GetDiscordWebHooksByID(ctx, id)
if err != nil {
return res, err
}
return c.ConvertDiscordWebhook(item), nil
}
func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) {
var res []models.DiscordWebHooksDto
items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{
Server: server,
Channel: channel,
})
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertDiscordWebhook(item))
}
return res, nil
}
func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto {
return models.DiscordWebHooksDto{
ID: i.ID,
Url: i.Url,
Server: i.Server,
Channel: i.Channel,
Enabled: i.Enabled,
}
}

View File

@ -1,42 +0,0 @@
package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
)
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
}
func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) {
var res []models.DiscordQueueDetailsDto
items, err := c.db.ListDiscordQueueItems(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
article, err := c.GetArticleDetails(ctx, item.ID)
if err != nil {
return res, err
}
res = append(res, models.DiscordQueueDetailsDto{
ID: item.ID,
Article: article,
})
}
return res, nil
}
func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto {
return models.DiscordQueueDto{
ID: i.ID,
Articleid: i.Articleid,
}
}

View File

@ -1,85 +0,0 @@
package dto
import (
"context"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
var res []models.SourceDto
items, err := c.db.ListSources(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertToSource(item))
}
return res, nil
}
func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) {
var res []models.SourceDto
items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName))
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertToSource(item))
}
return res, nil
}
func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) {
var res models.SourceDto
item, err := c.db.GetSourceByID(ctx, id)
if err != nil {
return res, err
}
return c.ConvertToSource(item), nil
}
func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) {
var res models.SourceDto
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
Name: name,
Source: source,
})
if err != nil {
return res, err
}
return c.ConvertToSource(item), nil
}
func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto {
var deleted bool
if !i.Deleted.Valid {
deleted = true
}
return models.SourceDto{
ID: i.ID,
Site: i.Site,
Name: i.Name,
Source: i.Source,
Type: i.Type,
Value: i.Value.String,
Enabled: i.Enabled,
Url: i.Url,
Tags: c.SplitTags(i.Tags),
Deleted: deleted,
}
}

View File

@ -1,91 +0,0 @@
package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
)
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
var res []models.SubscriptionDto
items, err := c.db.ListSubscriptions(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertSubscription(item))
}
return res, nil
}
func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) {
var res []models.SubscriptionDetailsDto
items, err := c.ListSubscriptions(ctx, limit)
if err != nil {
return res, err
}
for _, item := range items {
dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId)
if err != nil {
return res, err
}
source, err := c.GetSourceById(ctx, item.SourceId)
if err != nil {
return res, err
}
res = append(res, models.SubscriptionDetailsDto{
ID: item.ID,
Source: source,
DiscordWebHook: dwh,
})
}
return res, nil
}
func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
var res []models.SubscriptionDto
items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertSubscription(item))
}
return res, nil
}
func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) {
var res []models.SubscriptionDto
items, err := c.db.GetSubscriptionsBySourceID(ctx, id)
if err != nil {
return res, err
}
for _, item := range items {
res = append(res, c.ConvertSubscription(item))
}
return res, nil
}
func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto {
return models.SubscriptionDto{
ID: i.ID,
DiscordWebhookId: i.Discordwebhookid,
SourceId: i.Sourceid,
}
}

View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"log" "log"
"net/http" "net/http"
@ -13,7 +12,7 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid" "github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
@ -25,7 +24,7 @@ const (
) )
type FFXIVClient struct { type FFXIVClient struct {
record database.Source record domain.SourceEntity
//SourceID uint //SourceID uint
//Url string //Url string
//Region string //Region string
@ -33,15 +32,15 @@ type FFXIVClient struct {
cacheGroup string cacheGroup string
} }
func NewFFXIVClient(Record database.Source) FFXIVClient { func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient {
return FFXIVClient{ return FFXIVClient{
record: Record, record: Record,
cacheGroup: "ffxiv", cacheGroup: "ffxiv",
} }
} }
func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) {
var articles []database.Article var articles []domain.ArticleEntity
parser := fc.GetBrowser() parser := fc.GetBrowser()
defer parser.Close() defer parser.Close()
@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
return articles, err return articles, err
} }
article := database.Article{ article := domain.ArticleEntity{
Sourceid: fc.record.ID, SourceID: fc.record.ID,
Tags: tags, Tags: tags,
Title: title, Title: title,
Url: link, Url: link,
Pubdate: pubDate, PubDate: pubDate,
Videoheight: 0, Thumbnail: thumb,
Videowidth: 0, Description: description,
Thumbnail: thumb, AuthorName: authorName,
Description: description, AuthorImageUrl: authorImage,
Authorname: sql.NullString{String: authorName},
Authorimage: sql.NullString{String: authorImage},
} }
log.Printf("Collected '%v' from '%v'", article.Title, article.Url) log.Printf("Collected '%v' from '%v'", article.Title, article.Url)

View File

@ -3,18 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var FFXIVRecord database.Source = database.Source{ var FFXIVRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 997,
Site: "ffxiv", DisplayName: "Final Fantasy XIV - NA",
Name: "Final Fantasy XIV - NA", Source: domain.SourceCollectorFfxiv,
Source: "ffxiv", Url: "https://na.finalfantasyxiv.com/lodestone/",
Url: "https://na.finalfantasyxiv.com/lodestone/", Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
Tags: "ffxiv, final, fantasy, xiv, na, lodestone",
} }
func TestFfxivGetParser(t *testing.T) { func TestFfxivGetParser(t *testing.T) {

View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -9,7 +8,6 @@ import (
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod" "github.com/go-rod/rod"
@ -17,8 +15,8 @@ import (
) )
type RedditClient struct { type RedditClient struct {
config RedditConfig config services.Configs
record database.Source record domain.SourceEntity
} }
type RedditConfig struct { type RedditConfig struct {
@ -27,14 +25,11 @@ type RedditConfig struct {
PullNSFW string PullNSFW string
} }
func NewRedditClient(Record database.Source) *RedditClient { func NewRedditClient(record domain.SourceEntity) *RedditClient {
rc := RedditClient{ rc := RedditClient{
record: Record, record: record,
config: services.GetEnvConfig(),
} }
cc := services.NewConfig()
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
//rc.disableHttp2Client() //rc.disableHttp2Client()
@ -71,7 +66,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
// TODO Wire this to support the config options // TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url) Url := fmt.Sprintf("%v.json", rc.record.Url)
log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name) log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName)
content, err := getHttpContent(Url) content, err := getHttpContent(Url)
if err != nil { if err != nil {
@ -88,24 +83,28 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
return items, nil return items, nil
} }
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article { func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity {
var redditArticles []database.Article var redditArticles []domain.ArticleEntity
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article database.Article var article domain.ArticleEntity
article, err := rc.convertToArticle(item.Data) article, err := rc.convertToArticle(item.Data)
if err != nil { if err != nil {
log.Printf("[Reddit] %v", err) log.Printf("[Reddit] %v", err)
continue continue
} }
redditArticles = append(redditArticles, article) redditArticles = append(redditArticles, article)
} }
return redditArticles return redditArticles
} }
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) { func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) {
var item database.Article var item domain.ArticleEntity
if source.Content == "" && source.Url != "" { if source.Content == "" && source.Url != "" {
item = rc.convertPicturePost(source) item = rc.convertPicturePost(source)
@ -131,65 +130,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
return item, nil return item, nil
} }
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Title: source.Title, Title: source.Title,
Tags: fmt.Sprintf("%v", rc.record.Tags), Tags: fmt.Sprintf("%v", rc.record.Tags),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Pubdate: time.Now(), PubDate: time.Now(),
Video: sql.NullString{String: "null"}, IsVideo: false,
Videoheight: 0, Thumbnail: source.Thumbnail,
Videowidth: 0, Description: source.Content,
Thumbnail: source.Thumbnail, AuthorName: source.Author,
Description: source.Content, AuthorImageUrl: "",
Authorname: sql.NullString{String: source.Author},
Authorimage: sql.NullString{String: "null"},
} }
return item return item
} }
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Videoheight: 0,
Videowidth: 0,
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Authorname: sql.NullString{String: source.Author}, AuthorName: source.Author,
Description: source.Content, Description: source.Content,
} }
return item return item
} }
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0, AuthorName: source.Author,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.Media.RedditVideo.FallBackUrl, Description: source.Media.RedditVideo.FallBackUrl,
} }
return item return item
} }
// This post is nothing more then a redirect to another location. // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article { func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity {
var item = database.Article{ var item = domain.ArticleEntity{
Sourceid: rc.record.ID, SourceID: rc.record.ID,
Tags: "a", Tags: "a",
Title: source.Title, Title: source.Title,
Pubdate: time.Now(), PubDate: time.Now(),
Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink),
Videoheight: 0, AuthorName: source.Author,
Videowidth: 0,
Authorname: sql.NullString{String: source.Author},
Description: source.UrlOverriddenByDest, Description: source.UrlOverriddenByDest,
} }
return item return item

View File

@ -3,18 +3,16 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var RedditRecord database.Source = database.Source{ var RedditRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 999,
Name: "dadjokes", DisplayName: "dadjokes",
Source: "reddit", Source: domain.SourceCollectorReddit,
Site: "reddit", Url: "https://reddit.com/r/dadjokes",
Url: "https://reddit.com/r/dadjokes", Tags: "reddit, dadjokes",
Tags: "reddit, dadjokes",
} }
func TestGetContent(t *testing.T) { func TestGetContent(t *testing.T) {

View File

@ -8,9 +8,10 @@ import (
) )
var rssRecord = domain.SourceEntity{ var rssRecord = domain.SourceEntity{
ID: 1, ID: 1,
DisplayName: "ArsTechnica", DisplayName: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index", Source: domain.SourceCollectorRss,
Url: "https://feeds.arstechnica.com/arstechnica/index",
} }
func TestRssClientConstructor(t *testing.T) { func TestRssClientConstructor(t *testing.T) {

View File

@ -1,19 +1,18 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/nicklaw5/helix/v2" "github.com/nicklaw5/helix/v2"
) )
type TwitchClient struct { type TwitchClient struct {
SourceRecord database.Source SourceRecord domain.SourceEntity
// config // config
monitorClips string monitorClips string
@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
} }
// This will let you replace the bound source record to keep the same session alive. // This will let you replace the bound source record to keep the same session alive.
func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) {
tc.SourceRecord = source tc.SourceRecord = source
} }
@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error {
return nil return nil
} }
func (tc *TwitchClient) GetContent() ([]database.Article, error) { func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) {
var items []database.Article var items []domain.ArticleEntity
user, err := tc.GetUserDetails() user, err := tc.GetUserDetails()
if err != nil { if err != nil {
@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) {
} }
for _, video := range posts { for _, video := range posts {
var article database.Article var article domain.ArticleEntity
AuthorName, err := tc.ExtractAuthor(video) AuthorName, err := tc.ExtractAuthor(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Authorname = sql.NullString{String: AuthorName} article.AuthorName = AuthorName
Authorimage, err := tc.ExtractAuthorImage(user) Authorimage, err := tc.ExtractAuthorImage(user)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Authorimage = sql.NullString{String: Authorimage} article.AuthorImageUrl = Authorimage
article.Description, err = tc.ExtractDescription(video) article.Description, err = tc.ExtractDescription(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Pubdate, err = tc.ExtractPubDate(video) article.PubDate, err = tc.ExtractPubDate(video)
if err != nil { if err != nil {
return items, err return items, err
} }
article.Sourceid = tc.SourceRecord.ID article.SourceID = tc.SourceRecord.ID
article.Tags, err = tc.ExtractTags(video, user) article.Tags, err = tc.ExtractTags(video, user)
if err != nil { if err != nil {
return items, err return items, err
@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) {
var blank helix.User var blank helix.User
users, err := tc.api.GetUsers(&helix.UsersParams{ users, err := tc.api.GetUsers(&helix.UsersParams{
Logins: []string{tc.SourceRecord.Name}, Logins: []string{tc.SourceRecord.DisplayName},
}) })
if err != nil { if err != nil {
return blank, err return blank, err

View File

@ -4,21 +4,20 @@ import (
"log" "log"
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var TwitchSourceRecord = database.Source{ var TwitchSourceRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "nintendo", DisplayName: "nintendo",
Source: "Twitch", Source: domain.SourceCollectorTwitch,
} }
var TwitchInvalidRecord = database.Source{ var TwitchInvalidRecord = domain.SourceEntity{
ID: uuid.New(), ID: 9999,
Name: "EvilNintendo", DisplayName: "EvilNintendo",
Source: "Twitch", Source: domain.SourceCollectorTwitch,
} }
func TestTwitchLogin(t *testing.T) { func TestTwitchLogin(t *testing.T) {

View File

@ -1,7 +1,6 @@
package input package input
import ( import (
"database/sql"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@ -12,11 +11,11 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
) )
type YoutubeClient struct { type YoutubeClient struct {
record database.Source record domain.SourceEntity
// internal variables at time of collection // internal variables at time of collection
channelID string channelID string
@ -37,7 +36,7 @@ var (
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
func NewYoutubeClient(Record database.Source) YoutubeClient { func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient {
yc := YoutubeClient{ yc := YoutubeClient{
record: Record, record: Record,
cacheGroup: "youtube", cacheGroup: "youtube",
@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient {
} }
// CheckSource will go and run all the commands needed to process a source. // CheckSource will go and run all the commands needed to process a source.
func (yc *YoutubeClient) GetContent() ([]database.Article, error) { func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) {
var items []database.Article var items []domain.ArticleEntity
docParser, err := yc.GetParser(yc.record.Url) docParser, err := yc.GetParser(yc.record.Url)
if err != nil { if err != nil {
return items, err return items, err
@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool {
return false return false
} }
func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity {
parser, err := yc.GetParser(item.Link) parser, err := yc.GetParser(item.Link)
if err != nil { if err != nil {
log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link)
@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article {
log.Printf("[YouTube] %v", msg) log.Printf("[YouTube] %v", msg)
} }
var article = database.Article{ var article = domain.ArticleEntity{
Sourceid: yc.record.ID, SourceID: yc.record.ID,
Tags: tags, Tags: tags,
Title: item.Title, Title: item.Title,
Url: item.Link, Url: item.Link,
Pubdate: *item.PublishedParsed, PubDate: *item.PublishedParsed,
Thumbnail: thumb, Thumbnail: thumb,
Description: item.Description, Description: item.Description,
Authorname: sql.NullString{String: item.Author.Name}, AuthorName: item.Author.Name,
Authorimage: sql.NullString{String: yc.avatarUri}, AuthorImageUrl: yc.avatarUri,
} }
return article return article
} }

View File

@ -3,17 +3,15 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
) )
var YouTubeRecord database.Source = database.Source{ var YouTubeRecord domain.SourceEntity = domain.SourceEntity{
ID: uuid.New(), ID: 999,
Name: "dadjokes", DisplayName: "dadjokes",
Source: "reddit", Source: domain.SourceCollectorYoutube,
Site: "reddit", Url: "https://youtube.com/gamegrumps",
Url: "https://youtube.com/gamegrumps",
} }
func TestGetPageParser(t *testing.T) { func TestGetPageParser(t *testing.T) {

View File

@ -0,0 +1,26 @@
package services
import (
"database/sql"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/maragudk/goqite"
)
type queues struct {
repos RepositoryService
RssCollection goqite.Queue
}
func NewQueues(conn *sql.DB) queues {
return queues{
repos: NewRepositoryService(conn),
RssCollection: *goqite.New(goqite.NewOpts{
DB: conn,
Name: domain.QueueRssCollection,
}),
}
}

View File

@ -4,8 +4,8 @@ help: ## Shows this help command
build: ## builds the application with the current go runtime build: ## builds the application with the current go runtime
~/go/bin/swag f ~/go/bin/swag f
~/go/bin/swag i ~/go/bin/swag init -g cmd/server.go
go build . go build cmd/server.go
docker-build: ## Generates the docker image docker-build: ## Generates the docker image
docker build -t "newsbot.collector.api" . docker build -t "newsbot.collector.api" .