Compare commits

..

2 Commits

60 changed files with 158 additions and 250 deletions

View File

@ -1,64 +0,0 @@
name: Docker
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
on:
#schedule:
# - cron: '21 19 * * *'
push:
branches: [ master ]
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
#pull_request:
# branches: [ master ]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
#images: ${{ env.REGISTRY }}/newsbot.worker
# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

View File

@ -1,23 +0,0 @@
name: Go
on:
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.18
- name: Build
run: go build -v ./...
#- name: Test
# run: go test -v ./...

View File

@ -6,33 +6,33 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/docs"
"github.com/jtom38/newsbot/collector/docs" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/routes" v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"github.com/jtom38/newsbot/collector/services/config" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/jtom38/newsbot/collector/services/cron" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
) )
// @title NewsBot collector // @title NewsBot collector
// @version 0.1 // @version 0.1
// @BasePath /api // @BasePath /api
func main() { func main() {
cfg := config.New() cfg := services.NewConfig()
address := cfg.GetConfig(config.ServerAddress) address := cfg.GetConfig(services.ServerAddress)
docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address) docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
ctx := context.Background() ctx := context.Background()
db, err := sql.Open("postgres", cfg.GetConfig(config.Sql_Connection_String)) db, err := sql.Open("postgres", cfg.GetConfig(services.Sql_Connection_String))
if err != nil { if err != nil {
panic(err) panic(err)
} }
queries := database.New(db) queries := database.New(db)
c := cron.New(ctx) c := cron.NewScheduler(ctx)
c.Start() c.Start()
server := routes.NewServer(ctx, queries) server := v1.NewServer(ctx, queries)
fmt.Println("API is online and waiting for requests.") fmt.Println("API is online and waiting for requests.")
fmt.Printf("API: http://%v:8081/api\r\n", address) fmt.Printf("API: http://%v:8081/api\r\n", address)

4
go.mod
View File

@ -1,6 +1,6 @@
module github.com/jtom38/newsbot/collector module git.jamestombleson.com/jtom38/newsbot-api
go 1.18 go 1.22
require ( require (
github.com/PuerkitoBio/goquery v1.8.0 github.com/PuerkitoBio/goquery v1.8.0

1
internal/domain/dto.go Normal file
View File

@ -0,0 +1 @@
package domain

View File

@ -1,4 +1,4 @@
package models package domain
import ( import (
"time" "time"

View File

@ -1,8 +1,6 @@
package models package domain
import ( import "time"
"time"
)
type CacheItem struct { type CacheItem struct {
Key string Key string

View File

@ -6,7 +6,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
) )
type ArticleDto struct { type ArticleDto struct {

View File

@ -1,4 +1,4 @@
package models package domain
// This is the root Json object. It does not contain data that we care about though. // This is the root Json object. It does not contain data that we care about though.
type RedditJsonContent struct { type RedditJsonContent struct {

View File

@ -1,12 +1,12 @@
package routes package v1
import ( import (
"net/http" "net/http"
"strconv" "strconv"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/domain/models"
) )
func (s *Server) GetArticleRouter() http.Handler { func (s *Server) GetArticleRouter() http.Handler {
@ -168,12 +168,11 @@ func (s *Server) ListArticlesBySourceId(w http.ResponseWriter, r *http.Request)
query := r.URL.Query() query := r.URL.Query()
_id := query["id"][0] _id := query["id"][0]
uuid, err := uuid.Parse(_id) uuid, err := uuid.Parse(_id)
if err != nil { if err != nil {
s.WriteError(w, err.Error(), http.StatusBadRequest) s.WriteError(w, err.Error(), http.StatusBadRequest)
return return
} }
// if a page number was sent, process it // if a page number was sent, process it
if len(query["page"]) >= 1 { if len(query["page"]) >= 1 {

View File

@ -1,4 +1,4 @@
package routes package v1
import ( import (
"encoding/json" "encoding/json"
@ -6,10 +6,10 @@ import (
"net/http" "net/http"
"strings" "strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
type ListDiscordWebhooks struct { type ListDiscordWebhooks struct {

View File

@ -1,10 +1,10 @@
package routes package v1
import ( import (
"net/http" "net/http"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/jtom38/newsbot/collector/domain/models"
) )
type ListDiscordWebHooksQueueResults struct { type ListDiscordWebHooksQueueResults struct {

View File

@ -0,0 +1 @@
package v1_test

View File

@ -1,4 +1,4 @@
package routes package v1
import ( import (
"context" "context"
@ -11,9 +11,9 @@ import (
_ "github.com/lib/pq" _ "github.com/lib/pq"
httpSwagger "github.com/swaggo/http-swagger" httpSwagger "github.com/swaggo/http-swagger"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/dto" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/jtom38/newsbot/collector/services/config" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
) )
type Server struct { type Server struct {
@ -56,8 +56,8 @@ func NewServer(ctx context.Context, db *database.Queries) *Server {
} }
func openDatabase(ctx context.Context) (*database.Queries, error) { func openDatabase(ctx context.Context) (*database.Queries, error) {
_env := config.New() _env := services.NewConfig()
connString := _env.GetConfig(config.Sql_Connection_String) connString := _env.GetConfig(services.Sql_Connection_String)
db, err := sql.Open("postgres", connString) db, err := sql.Open("postgres", connString)
if err != nil { if err != nil {
panic(err) panic(err)
@ -81,7 +81,7 @@ func (s *Server) MountRoutes() {
s.Router.Mount("/api/articles", s.GetArticleRouter()) s.Router.Mount("/api/articles", s.GetArticleRouter())
s.Router.Mount("/api/queue", s.GetQueueRouter()) s.Router.Mount("/api/queue", s.GetQueueRouter())
s.Router.Mount("/api/discord/webhooks", s.DiscordWebHookRouter()) s.Router.Mount("/api/discord/webhooks", s.DiscordWebHookRouter())
//s.Router.Get("/api/settings", s.getSettings) //s.Router.Get("/api/settings", s.getSettings)
s.Router.Mount("/api/sources", s.GetSourcesRouter()) s.Router.Mount("/api/sources", s.GetSourcesRouter())
@ -116,7 +116,7 @@ func (s *Server) WriteError(w http.ResponseWriter, errMessage string, HttpStatus
func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) { func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) {
w.Header().Set(HeaderContentType, ApplicationJson) w.Header().Set(HeaderContentType, ApplicationJson)
bres, err := json.Marshal(model) bres, err := json.Marshal(model)
if err != nil { if err != nil {
s.WriteError(w, err.Error(), http.StatusInternalServerError) s.WriteError(w, err.Error(), http.StatusInternalServerError)
@ -124,4 +124,4 @@ func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) {
} }
w.Write(bres) w.Write(bres)
} }

View File

@ -1,4 +1,4 @@
package routes package v1
import ( import (
"encoding/json" "encoding/json"

View File

@ -1,4 +1,4 @@
package routes package v1
import ( import (
"encoding/json" "encoding/json"
@ -6,10 +6,10 @@ import (
"net/http" "net/http"
"strings" "strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
func (s *Server) GetSourcesRouter() http.Handler { func (s *Server) GetSourcesRouter() http.Handler {

View File

@ -1,14 +1,14 @@
package routes package v1
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"net/http" "net/http"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
func (s *Server) GetSubscriptionsRouter() http.Handler { func (s *Server) GetSubscriptionsRouter() http.Handler {

View File

@ -3,7 +3,7 @@ package cache
import ( import (
"time" "time"
"github.com/jtom38/newsbot/collector/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
) )
type CacheClient struct { type CacheClient struct {
@ -19,7 +19,7 @@ func NewCacheClient(group string) CacheClient {
} }
func (cc *CacheClient) Insert(key string, value string) { func (cc *CacheClient) Insert(key string, value string) {
item := models.CacheItem{ item := domain.CacheItem{
Key: key, Key: key,
Value: value, Value: value,
Group: cc.group, Group: cc.group,
@ -29,7 +29,7 @@ func (cc *CacheClient) Insert(key string, value string) {
cacheStorage = append(cacheStorage, &item) cacheStorage = append(cacheStorage, &item)
} }
func (cc *CacheClient) FindByKey(key string) (*models.CacheItem, error) { func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
for _, item := range cacheStorage { for _, item := range cacheStorage {
if item.Group != cc.group { if item.Group != cc.group {
continue continue
@ -46,10 +46,10 @@ func (cc *CacheClient) FindByKey(key string) (*models.CacheItem, error) {
return item, nil return item, nil
} }
return &models.CacheItem{}, ErrCacheRecordMissing return &domain.CacheItem{}, ErrCacheRecordMissing
} }
func (cc *CacheClient) FindByValue(value string) (*models.CacheItem, error) { func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
for _, item := range cacheStorage { for _, item := range cacheStorage {
if item.Group != cc.group { if item.Group != cc.group {
continue continue
@ -65,5 +65,5 @@ func (cc *CacheClient) FindByValue(value string) (*models.CacheItem, error) {
} }
return item, nil return item, nil
} }
return &models.CacheItem{}, ErrCacheRecordMissing return &domain.CacheItem{}, ErrCacheRecordMissing
} }

View File

@ -3,7 +3,7 @@ package cache_test
import ( import (
"testing" "testing"
"github.com/jtom38/newsbot/collector/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
func TestNewCacheClient(t *testing.T) { func TestNewCacheClient(t *testing.T) {

View File

@ -3,11 +3,11 @@ package cache
import ( import (
"errors" "errors"
"github.com/jtom38/newsbot/collector/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
) )
var ( var (
cacheStorage []*models.CacheItem cacheStorage []*domain.CacheItem
ErrCacheRecordMissing = errors.New("unable to find the requested record") ErrCacheRecordMissing = errors.New("unable to find the requested record")
) )

View File

@ -3,7 +3,7 @@ package cache
import ( import (
"time" "time"
"github.com/jtom38/newsbot/collector/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
) )
// When a record becomes tainted, it needs to be renewed or it will be dropped from the cache. // When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
@ -36,8 +36,8 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
} }
// This creates a new slice and skips over the item that needs to be dropped // This creates a new slice and skips over the item that needs to be dropped
func (cam CacheAgeMonitor) removeEntry(index int) []*models.CacheItem { func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
var temp []*models.CacheItem var temp []*domain.CacheItem
for i, item := range cacheStorage { for i, item := range cacheStorage {
if i != index { if i != index {
temp = append(temp, item) temp = append(temp, item)

View File

@ -3,7 +3,7 @@ package cache_test
import ( import (
"testing" "testing"
"github.com/jtom38/newsbot/collector/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
func TestCacheTaintItem(t *testing.T) { func TestCacheTaintItem(t *testing.T) {

View File

@ -1,4 +1,4 @@
package config package services
import ( import (
"errors" "errors"
@ -34,7 +34,7 @@ const (
type ConfigClient struct{} type ConfigClient struct{}
func New() ConfigClient { func NewConfig() ConfigClient {
c := ConfigClient{} c := ConfigClient{}
c.RefreshEnv() c.RefreshEnv()

View File

@ -0,0 +1,22 @@
package services_test
import (
"os"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
)
func TestNewClient(t *testing.T) {
services.NewConfig()
}
func TestGetConfigExpectNull(t *testing.T) {
cc := services.NewConfig()
os.Setenv(services.REDDIT_PULL_HOT, "")
res := cc.GetConfig(services.REDDIT_PULL_HOT)
if res != "" {
panic("expected blank")
}
}

View File

@ -11,10 +11,10 @@ import (
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/services/config" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/jtom38/newsbot/collector/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/jtom38/newsbot/collector/services/output" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
) )
type Cron struct { type Cron struct {
@ -24,8 +24,8 @@ type Cron struct {
} }
func openDatabase() (*database.Queries, error) { func openDatabase() (*database.Queries, error) {
_env := config.New() _env := services.NewConfig()
connString := _env.GetConfig(config.Sql_Connection_String) connString := _env.GetConfig(services.Sql_Connection_String)
if connString == "" { if connString == "" {
panic("Connection String is null!") panic("Connection String is null!")
} }
@ -38,7 +38,7 @@ func openDatabase() (*database.Queries, error) {
return queries, err return queries, err
} }
func New(ctx context.Context) *Cron { func NewScheduler(ctx context.Context) *Cron {
c := &Cron{ c := &Cron{
ctx: &ctx, ctx: &ctx,
} }
@ -51,28 +51,28 @@ func New(ctx context.Context) *Cron {
c.Db = queries c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
features := config.New() features := services.NewConfig()
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND) res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
if res { if res {
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("[Input] Reddit backend was enabled") log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit() //go c.CheckReddit()
} }
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND) res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res { if res {
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled") log.Print("[Input] YouTube backend was enabled")
} }
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND) res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
if res { if res {
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled") log.Print("[Input] FFXIV backend was enabled")
} }
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND) res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
if res { if res {
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled") log.Print("[Input] Twitch backend was enabled")

View File

@ -4,7 +4,7 @@ import (
"context" "context"
"testing" "testing"
"github.com/jtom38/newsbot/collector/services/cron" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
) )
func TestInvokeTwitch(t *testing.T) { func TestInvokeTwitch(t *testing.T) {
@ -14,19 +14,19 @@ func TestInvokeTwitch(t *testing.T) {
// TODO add database mocks but not sure how to do that yet. // TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) { func TestCheckReddit(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.New(ctx) c := cron.NewScheduler(ctx)
c.CheckReddit() c.CheckReddit()
} }
func TestCheckYouTube(t *testing.T) { func TestCheckYouTube(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.New(ctx) c := cron.NewScheduler(ctx)
c.CheckYoutube() c.CheckYoutube()
} }
func TestCheckTwitch(t *testing.T) { func TestCheckTwitch(t *testing.T) {
ctx := context.Background() ctx := context.Background()
c := cron.New(ctx) c := cron.NewScheduler(ctx)
err := c.CheckTwitch() err := c.CheckTwitch()
if err != nil { if err != nil {
t.Error(err) t.Error(err)

View File

@ -6,9 +6,9 @@ import (
"context" "context"
"strings" "strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
type DtoClient struct { type DtoClient struct {
@ -25,7 +25,7 @@ func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models
var res []models.ArticleDto var res []models.ArticleDto
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{ a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
Limit: int32(limit), Limit: int32(limit),
Offset: int32(limit * page), Offset: int32(limit * page),
}) })
if err != nil { if err != nil {
@ -38,11 +38,11 @@ func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models
return res, nil return res, nil
} }
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32 ) ([]models.ArticleDto, error) { func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
var res []models.ArticleDto var res []models.ArticleDto
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{ a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
Limit: limit, Limit: limit,
Offset: page * limit, Offset: page * limit,
}) })
if err != nil { if err != nil {
@ -85,8 +85,8 @@ func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid
var res []models.ArticleDto var res []models.ArticleDto
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{ a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
Sourceid: SourceID, Sourceid: SourceID,
Limit: int32(limit), Limit: int32(limit),
Offset: int32(limit * page), Offset: int32(limit * page),
}) })
if err != nil { if err != nil {
return res, err return res, err

View File

@ -3,9 +3,9 @@ package dto
import ( import (
"context" "context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) { func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {

View File

@ -3,8 +3,8 @@ package dto
import ( import (
"context" "context"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
) )
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) { func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {

View File

@ -4,9 +4,9 @@ import (
"context" "context"
"strings" "strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) { func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
@ -54,7 +54,7 @@ func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source s
var res models.SourceDto var res models.SourceDto
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{ item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
Name: name, Name: name,
Source: source, Source: source,
}) })
if err != nil { if err != nil {

View File

@ -3,9 +3,9 @@ package dto
import ( import (
"context" "context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
) )
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) { func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {

View File

@ -13,8 +13,8 @@ import (
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
) )
const ( const (

View File

@ -3,9 +3,9 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services/input"
) )
var FFXIVRecord database.Source = database.Source{ var FFXIVRecord database.Source = database.Source{

View File

@ -9,11 +9,11 @@ import (
"strings" "strings"
"time" "time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod" "github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
"github.com/jtom38/newsbot/collector/services/config"
) )
type RedditClient struct { type RedditClient struct {
@ -31,10 +31,10 @@ func NewRedditClient(Record database.Source) *RedditClient {
rc := RedditClient{ rc := RedditClient{
record: Record, record: Record,
} }
cc := config.New() cc := services.NewConfig()
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT) rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW) rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP) rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
//rc.disableHttp2Client() //rc.disableHttp2Client()
@ -65,8 +65,8 @@ func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
// GetContent() reaches out to Reddit and pulls the Json data. // GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct. // It will then convert the data to a struct and return the struct.
func (rc *RedditClient) GetContent() (models.RedditJsonContent, error) { func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
var items models.RedditJsonContent = models.RedditJsonContent{} var items domain.RedditJsonContent = domain.RedditJsonContent{}
// TODO Wire this to support the config options // TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url) Url := fmt.Sprintf("%v.json", rc.record.Url)
@ -88,7 +88,7 @@ func (rc *RedditClient) GetContent() (models.RedditJsonContent, error) {
return items, nil return items, nil
} }
func (rc *RedditClient) ConvertToArticles(items models.RedditJsonContent) []database.Article { func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
var redditArticles []database.Article var redditArticles []database.Article
for _, item := range items.Data.Children { for _, item := range items.Data.Children {
var article database.Article var article database.Article
@ -104,7 +104,7 @@ func (rc *RedditClient) ConvertToArticles(items models.RedditJsonContent) []data
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database. // This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source models.RedditPost) (database.Article, error) { func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
var item database.Article var item database.Article
if source.Content == "" && source.Url != "" { if source.Content == "" && source.Url != "" {
@ -131,7 +131,7 @@ func (rc *RedditClient) convertToArticle(source models.RedditPost) (database.Art
return item, nil return item, nil
} }
func (rc *RedditClient) convertPicturePost(source models.RedditPost) database.Article { func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Title: source.Title, Title: source.Title,
@ -149,7 +149,7 @@ func (rc *RedditClient) convertPicturePost(source models.RedditPost) database.Ar
return item return item
} }
func (rc *RedditClient) convertTextPost(source models.RedditPost) database.Article { func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
@ -164,7 +164,7 @@ func (rc *RedditClient) convertTextPost(source models.RedditPost) database.Artic
return item return item
} }
func (rc *RedditClient) convertVideoPost(source models.RedditPost) database.Article { func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",
@ -180,7 +180,7 @@ func (rc *RedditClient) convertVideoPost(source models.RedditPost) database.Arti
} }
// This post is nothing more then a redirect to another location. // This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source models.RedditPost) database.Article { func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
var item = database.Article{ var item = database.Article{
Sourceid: rc.record.ID, Sourceid: rc.record.ID,
Tags: "a", Tags: "a",

View File

@ -3,9 +3,9 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
) )
var RedditRecord database.Source = database.Source{ var RedditRecord database.Source = database.Source{

View File

@ -4,16 +4,16 @@ import (
"fmt" "fmt"
"log" "log"
"github.com/jtom38/newsbot/collector/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/services/cache" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
) )
type rssClient struct { type rssClient struct {
SourceRecord models.Sources SourceRecord domain.Sources
} }
func NewRssClient(sourceRecord models.Sources) rssClient { func NewRssClient(sourceRecord domain.Sources) rssClient {
client := rssClient{ client := rssClient{
SourceRecord: sourceRecord, SourceRecord: sourceRecord,
} }

View File

@ -3,11 +3,11 @@ package input_test
import ( import (
"testing" "testing"
"github.com/jtom38/newsbot/collector/domain/models" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/services/input" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
) )
var rssRecord = models.Sources{ var rssRecord = domain.Sources{
ID: 1, ID: 1,
Name: "ArsTechnica", Name: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index", Url: "https://feeds.arstechnica.com/arstechnica/index",

View File

@ -7,8 +7,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/services/config" "git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/nicklaw5/helix/v2" "github.com/nicklaw5/helix/v2"
) )
@ -31,14 +31,14 @@ var (
) )
func NewTwitchClient() (TwitchClient, error) { func NewTwitchClient() (TwitchClient, error) {
c := config.New() c := services.NewConfig()
id := c.GetConfig(config.TWITCH_CLIENT_ID) id := c.GetConfig(services.TWITCH_CLIENT_ID)
if id == "" { if id == "" {
return TwitchClient{}, ErrTwitchClientIdMissing return TwitchClient{}, ErrTwitchClientIdMissing
} }
secret := c.GetConfig(config.TWITCH_CLIENT_SECRET) secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
if secret == "" { if secret == "" {
return TwitchClient{}, ErrTwitchClientSecretMissing return TwitchClient{}, ErrTwitchClientSecretMissing
} }
@ -50,8 +50,8 @@ func NewTwitchClient() (TwitchClient, error) {
client := TwitchClient{ client := TwitchClient{
//SourceRecord: &source, //SourceRecord: &source,
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS), monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD), monitorVod: c.GetConfig(services.TWITCH_MONITOR_VOD),
api: &api, api: &api,
} }

View File

@ -4,9 +4,9 @@ import (
"log" "log"
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
) )
var TwitchSourceRecord = database.Source{ var TwitchSourceRecord = database.Source{

View File

@ -7,14 +7,12 @@ import (
"log" "log"
"net/http" "net/http"
//"strconv"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/go-rod/rod" "github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
) )
type YoutubeClient struct { type YoutubeClient struct {
@ -110,7 +108,7 @@ func (yc *YoutubeClient) GetBrowser() (*rod.Browser, error) {
if err != nil { if err != nil {
return browser, err return browser, err
} }
browser = rod.New().ControlURL(u).MustConnect() browser = rod.New().ControlURL(u).MustConnect()
} }
return browser, nil return browser, nil

View File

@ -3,9 +3,9 @@ package input_test
import ( import (
"testing" "testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
) )
var YouTubeRecord database.Source = database.Source{ var YouTubeRecord database.Source = database.Source{

View File

@ -8,7 +8,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"github.com/jtom38/newsbot/collector/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/database"
) )
type discordField struct { type discordField struct {

View File

@ -4,12 +4,11 @@ import (
"os" "os"
"strings" "strings"
"testing" "testing"
//"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/joho/godotenv" "github.com/joho/godotenv"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/output"
) )
var ( var (

View File

@ -13,10 +13,10 @@ docker-build: ## Generates the docker image
docker image ls | grep newsbot.collector.api docker image ls | grep newsbot.collector.api
migrate-dev: ## Apply sql migrations to dev db migrate-dev: ## Apply sql migrations to dev db
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
migrate-dev-down: ## revert sql migrations to dev db migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
swag: ## Generates the swagger documentation with the swag tool swag: ## Generates the swagger documentation with the swag tool
~/go/bin/swag f ~/go/bin/swag f

View File

@ -1 +0,0 @@
package routes_test

View File

@ -1,22 +0,0 @@
package config_test
import (
"os"
"testing"
"github.com/jtom38/newsbot/collector/services/config"
)
func TestNewClient(t *testing.T) {
config.New()
}
func TestGetConfigExpectNull(t *testing.T) {
cc := config.New()
os.Setenv(config.REDDIT_PULL_HOT, "")
res := cc.GetConfig(config.REDDIT_PULL_HOT)
if res != "" {
panic("expected blank")
}
}