Merge pull request 'moving code into the internal package' (#1) from features/reorganize-project into master
Reviewed-on: #1
This commit is contained in:
commit
feb9895eb7
64
.github/workflows/docker.build.yaml
vendored
64
.github/workflows/docker.build.yaml
vendored
@ -1,64 +0,0 @@
|
||||
name: Docker
|
||||
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
on:
|
||||
#schedule:
|
||||
# - cron: '21 19 * * *'
|
||||
push:
|
||||
branches: [ master ]
|
||||
# Publish semver tags as releases.
|
||||
tags: [ 'v*.*.*' ]
|
||||
#pull_request:
|
||||
# branches: [ master ]
|
||||
|
||||
env:
|
||||
# Use docker.io for Docker Hub if empty
|
||||
REGISTRY: ghcr.io
|
||||
# github.repository as <account>/<repo>
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into registry ${{ env.REGISTRY }}
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Extract metadata (tags, labels) for Docker
|
||||
# https://github.com/docker/metadata-action
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
#images: ${{ env.REGISTRY }}/newsbot.worker
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
|
||||
with:
|
||||
context: .
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
23
.github/workflows/go-build.yml
vendored
23
.github/workflows/go-build.yml
vendored
@ -1,23 +0,0 @@
|
||||
name: Go
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.18
|
||||
|
||||
- name: Build
|
||||
run: go build -v ./...
|
||||
|
||||
#- name: Test
|
||||
# run: go test -v ./...
|
@ -6,33 +6,33 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/docs"
|
||||
"github.com/jtom38/newsbot/collector/routes"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"github.com/jtom38/newsbot/collector/services/cron"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/docs"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
)
|
||||
|
||||
// @title NewsBot collector
|
||||
// @version 0.1
|
||||
// @BasePath /api
|
||||
func main() {
|
||||
cfg := config.New()
|
||||
address := cfg.GetConfig(config.ServerAddress)
|
||||
cfg := services.NewConfig()
|
||||
address := cfg.GetConfig(services.ServerAddress)
|
||||
docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
|
||||
|
||||
ctx := context.Background()
|
||||
db, err := sql.Open("postgres", cfg.GetConfig(config.Sql_Connection_String))
|
||||
db, err := sql.Open("postgres", cfg.GetConfig(services.Sql_Connection_String))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
queries := database.New(db)
|
||||
|
||||
c := cron.New(ctx)
|
||||
c := cron.NewScheduler(ctx)
|
||||
c.Start()
|
||||
|
||||
server := routes.NewServer(ctx, queries)
|
||||
server := v1.NewServer(ctx, queries)
|
||||
|
||||
fmt.Println("API is online and waiting for requests.")
|
||||
fmt.Printf("API: http://%v:8081/api\r\n", address)
|
4
go.mod
4
go.mod
@ -1,6 +1,6 @@
|
||||
module github.com/jtom38/newsbot/collector
|
||||
module git.jamestombleson.com/jtom38/newsbot-api
|
||||
|
||||
go 1.18
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0
|
||||
|
1
internal/domain/dto.go
Normal file
1
internal/domain/dto.go
Normal file
@ -0,0 +1 @@
|
||||
package domain
|
@ -1,4 +1,4 @@
|
||||
package models
|
||||
package domain
|
||||
|
||||
import (
|
||||
"time"
|
@ -1,8 +1,6 @@
|
||||
package models
|
||||
package domain
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
import "time"
|
||||
|
||||
type CacheItem struct {
|
||||
Key string
|
@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
)
|
||||
|
||||
type ArticleDto struct {
|
@ -1,4 +1,4 @@
|
||||
package models
|
||||
package domain
|
||||
|
||||
// This is the root Json object. It does not contain data that we care about though.
|
||||
type RedditJsonContent struct {
|
@ -1,12 +1,12 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
func (s *Server) GetArticleRouter() http.Handler {
|
||||
@ -168,12 +168,11 @@ func (s *Server) ListArticlesBySourceId(w http.ResponseWriter, r *http.Request)
|
||||
query := r.URL.Query()
|
||||
_id := query["id"][0]
|
||||
|
||||
|
||||
uuid, err := uuid.Parse(_id)
|
||||
if err != nil {
|
||||
s.WriteError(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// if a page number was sent, process it
|
||||
if len(query["page"]) >= 1 {
|
@ -1,4 +1,4 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@ -6,10 +6,10 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
type ListDiscordWebhooks struct {
|
@ -1,10 +1,10 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
type ListDiscordWebHooksQueueResults struct {
|
1
internal/handler/v1/root_test.go
Normal file
1
internal/handler/v1/root_test.go
Normal file
@ -0,0 +1 @@
|
||||
package v1_test
|
@ -1,4 +1,4 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -11,9 +11,9 @@ import (
|
||||
_ "github.com/lib/pq"
|
||||
httpSwagger "github.com/swaggo/http-swagger"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/dto"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
@ -56,8 +56,8 @@ func NewServer(ctx context.Context, db *database.Queries) *Server {
|
||||
}
|
||||
|
||||
func openDatabase(ctx context.Context) (*database.Queries, error) {
|
||||
_env := config.New()
|
||||
connString := _env.GetConfig(config.Sql_Connection_String)
|
||||
_env := services.NewConfig()
|
||||
connString := _env.GetConfig(services.Sql_Connection_String)
|
||||
db, err := sql.Open("postgres", connString)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@ -81,7 +81,7 @@ func (s *Server) MountRoutes() {
|
||||
s.Router.Mount("/api/articles", s.GetArticleRouter())
|
||||
s.Router.Mount("/api/queue", s.GetQueueRouter())
|
||||
s.Router.Mount("/api/discord/webhooks", s.DiscordWebHookRouter())
|
||||
|
||||
|
||||
//s.Router.Get("/api/settings", s.getSettings)
|
||||
|
||||
s.Router.Mount("/api/sources", s.GetSourcesRouter())
|
||||
@ -116,7 +116,7 @@ func (s *Server) WriteError(w http.ResponseWriter, errMessage string, HttpStatus
|
||||
|
||||
func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) {
|
||||
w.Header().Set(HeaderContentType, ApplicationJson)
|
||||
|
||||
|
||||
bres, err := json.Marshal(model)
|
||||
if err != nil {
|
||||
s.WriteError(w, err.Error(), http.StatusInternalServerError)
|
||||
@ -124,4 +124,4 @@ func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) {
|
||||
}
|
||||
|
||||
w.Write(bres)
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
@ -1,4 +1,4 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@ -6,10 +6,10 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
func (s *Server) GetSourcesRouter() http.Handler {
|
@ -1,14 +1,14 @@
|
||||
package routes
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
func (s *Server) GetSubscriptionsRouter() http.Handler {
|
@ -3,7 +3,7 @@ package cache
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
type CacheClient struct {
|
||||
@ -19,7 +19,7 @@ func NewCacheClient(group string) CacheClient {
|
||||
}
|
||||
|
||||
func (cc *CacheClient) Insert(key string, value string) {
|
||||
item := models.CacheItem{
|
||||
item := domain.CacheItem{
|
||||
Key: key,
|
||||
Value: value,
|
||||
Group: cc.group,
|
||||
@ -29,7 +29,7 @@ func (cc *CacheClient) Insert(key string, value string) {
|
||||
cacheStorage = append(cacheStorage, &item)
|
||||
}
|
||||
|
||||
func (cc *CacheClient) FindByKey(key string) (*models.CacheItem, error) {
|
||||
func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
|
||||
for _, item := range cacheStorage {
|
||||
if item.Group != cc.group {
|
||||
continue
|
||||
@ -46,10 +46,10 @@ func (cc *CacheClient) FindByKey(key string) (*models.CacheItem, error) {
|
||||
return item, nil
|
||||
}
|
||||
|
||||
return &models.CacheItem{}, ErrCacheRecordMissing
|
||||
return &domain.CacheItem{}, ErrCacheRecordMissing
|
||||
}
|
||||
|
||||
func (cc *CacheClient) FindByValue(value string) (*models.CacheItem, error) {
|
||||
func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
|
||||
for _, item := range cacheStorage {
|
||||
if item.Group != cc.group {
|
||||
continue
|
||||
@ -65,5 +65,5 @@ func (cc *CacheClient) FindByValue(value string) (*models.CacheItem, error) {
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
return &models.CacheItem{}, ErrCacheRecordMissing
|
||||
return &domain.CacheItem{}, ErrCacheRecordMissing
|
||||
}
|
@ -3,7 +3,7 @@ package cache_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services/cache"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
)
|
||||
|
||||
func TestNewCacheClient(t *testing.T) {
|
@ -3,11 +3,11 @@ package cache
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
var (
|
||||
cacheStorage []*models.CacheItem
|
||||
cacheStorage []*domain.CacheItem
|
||||
|
||||
ErrCacheRecordMissing = errors.New("unable to find the requested record")
|
||||
)
|
@ -3,7 +3,7 @@ package cache
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
)
|
||||
|
||||
// When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
|
||||
@ -36,8 +36,8 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
|
||||
}
|
||||
|
||||
// This creates a new slice and skips over the item that needs to be dropped
|
||||
func (cam CacheAgeMonitor) removeEntry(index int) []*models.CacheItem {
|
||||
var temp []*models.CacheItem
|
||||
func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
|
||||
var temp []*domain.CacheItem
|
||||
for i, item := range cacheStorage {
|
||||
if i != index {
|
||||
temp = append(temp, item)
|
@ -3,7 +3,7 @@ package cache_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services/cache"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
)
|
||||
|
||||
func TestCacheTaintItem(t *testing.T) {
|
@ -1,4 +1,4 @@
|
||||
package config
|
||||
package services
|
||||
|
||||
import (
|
||||
"errors"
|
||||
@ -34,7 +34,7 @@ const (
|
||||
|
||||
type ConfigClient struct{}
|
||||
|
||||
func New() ConfigClient {
|
||||
func NewConfig() ConfigClient {
|
||||
c := ConfigClient{}
|
||||
c.RefreshEnv()
|
||||
|
22
internal/services/config_test.go
Normal file
22
internal/services/config_test.go
Normal file
@ -0,0 +1,22 @@
|
||||
package services_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
)
|
||||
|
||||
func TestNewClient(t *testing.T) {
|
||||
services.NewConfig()
|
||||
}
|
||||
|
||||
func TestGetConfigExpectNull(t *testing.T) {
|
||||
cc := services.NewConfig()
|
||||
os.Setenv(services.REDDIT_PULL_HOT, "")
|
||||
res := cc.GetConfig(services.REDDIT_PULL_HOT)
|
||||
if res != "" {
|
||||
panic("expected blank")
|
||||
}
|
||||
|
||||
}
|
@ -11,10 +11,10 @@ import (
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
"github.com/jtom38/newsbot/collector/services/output"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
|
||||
)
|
||||
|
||||
type Cron struct {
|
||||
@ -24,8 +24,8 @@ type Cron struct {
|
||||
}
|
||||
|
||||
func openDatabase() (*database.Queries, error) {
|
||||
_env := config.New()
|
||||
connString := _env.GetConfig(config.Sql_Connection_String)
|
||||
_env := services.NewConfig()
|
||||
connString := _env.GetConfig(services.Sql_Connection_String)
|
||||
if connString == "" {
|
||||
panic("Connection String is null!")
|
||||
}
|
||||
@ -38,7 +38,7 @@ func openDatabase() (*database.Queries, error) {
|
||||
return queries, err
|
||||
}
|
||||
|
||||
func New(ctx context.Context) *Cron {
|
||||
func NewScheduler(ctx context.Context) *Cron {
|
||||
c := &Cron{
|
||||
ctx: &ctx,
|
||||
}
|
||||
@ -51,28 +51,28 @@ func New(ctx context.Context) *Cron {
|
||||
c.Db = queries
|
||||
|
||||
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||
features := config.New()
|
||||
features := services.NewConfig()
|
||||
|
||||
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
|
||||
log.Print("[Input] Reddit backend was enabled")
|
||||
//go c.CheckReddit()
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
|
||||
log.Print("[Input] YouTube backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
|
||||
log.Print("[Input] FFXIV backend was enabled")
|
||||
}
|
||||
|
||||
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
|
||||
if res {
|
||||
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
|
||||
log.Print("[Input] Twitch backend was enabled")
|
@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services/cron"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
|
||||
)
|
||||
|
||||
func TestInvokeTwitch(t *testing.T) {
|
||||
@ -14,19 +14,19 @@ func TestInvokeTwitch(t *testing.T) {
|
||||
// TODO add database mocks but not sure how to do that yet.
|
||||
func TestCheckReddit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.New(ctx)
|
||||
c := cron.NewScheduler(ctx)
|
||||
c.CheckReddit()
|
||||
}
|
||||
|
||||
func TestCheckYouTube(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.New(ctx)
|
||||
c := cron.NewScheduler(ctx)
|
||||
c.CheckYoutube()
|
||||
}
|
||||
|
||||
func TestCheckTwitch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := cron.New(ctx)
|
||||
c := cron.NewScheduler(ctx)
|
||||
err := c.CheckTwitch()
|
||||
if err != nil {
|
||||
t.Error(err)
|
@ -6,9 +6,9 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
type DtoClient struct {
|
||||
@ -25,7 +25,7 @@ func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models
|
||||
var res []models.ArticleDto
|
||||
|
||||
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
|
||||
Limit: int32(limit),
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
})
|
||||
if err != nil {
|
||||
@ -38,11 +38,11 @@ func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32 ) ([]models.ArticleDto, error) {
|
||||
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
|
||||
var res []models.ArticleDto
|
||||
|
||||
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
|
||||
Limit: limit,
|
||||
Limit: limit,
|
||||
Offset: page * limit,
|
||||
})
|
||||
if err != nil {
|
||||
@ -85,8 +85,8 @@ func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid
|
||||
var res []models.ArticleDto
|
||||
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
|
||||
Sourceid: SourceID,
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
Limit: int32(limit),
|
||||
Offset: int32(limit * page),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
@ -3,9 +3,9 @@ package dto
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {
|
@ -3,8 +3,8 @@ package dto
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {
|
@ -4,9 +4,9 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
|
||||
@ -54,7 +54,7 @@ func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source s
|
||||
var res models.SourceDto
|
||||
|
||||
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
|
||||
Name: name,
|
||||
Name: name,
|
||||
Source: source,
|
||||
})
|
||||
if err != nil {
|
@ -3,9 +3,9 @@ package dto
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
)
|
||||
|
||||
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {
|
@ -13,8 +13,8 @@ import (
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/cache"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
)
|
||||
|
||||
const (
|
@ -3,9 +3,9 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
ffxiv "github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var FFXIVRecord database.Source = database.Source{
|
@ -9,11 +9,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
)
|
||||
|
||||
type RedditClient struct {
|
||||
@ -31,10 +31,10 @@ func NewRedditClient(Record database.Source) *RedditClient {
|
||||
rc := RedditClient{
|
||||
record: Record,
|
||||
}
|
||||
cc := config.New()
|
||||
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
|
||||
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
|
||||
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
|
||||
cc := services.NewConfig()
|
||||
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
|
||||
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
|
||||
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
|
||||
|
||||
//rc.disableHttp2Client()
|
||||
|
||||
@ -65,8 +65,8 @@ func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
|
||||
|
||||
// GetContent() reaches out to Reddit and pulls the Json data.
|
||||
// It will then convert the data to a struct and return the struct.
|
||||
func (rc *RedditClient) GetContent() (models.RedditJsonContent, error) {
|
||||
var items models.RedditJsonContent = models.RedditJsonContent{}
|
||||
func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
|
||||
var items domain.RedditJsonContent = domain.RedditJsonContent{}
|
||||
|
||||
// TODO Wire this to support the config options
|
||||
Url := fmt.Sprintf("%v.json", rc.record.Url)
|
||||
@ -88,7 +88,7 @@ func (rc *RedditClient) GetContent() (models.RedditJsonContent, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) ConvertToArticles(items models.RedditJsonContent) []database.Article {
|
||||
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
|
||||
var redditArticles []database.Article
|
||||
for _, item := range items.Data.Children {
|
||||
var article database.Article
|
||||
@ -104,7 +104,7 @@ func (rc *RedditClient) ConvertToArticles(items models.RedditJsonContent) []data
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc *RedditClient) convertToArticle(source models.RedditPost) (database.Article, error) {
|
||||
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
|
||||
var item database.Article
|
||||
|
||||
if source.Content == "" && source.Url != "" {
|
||||
@ -131,7 +131,7 @@ func (rc *RedditClient) convertToArticle(source models.RedditPost) (database.Art
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertPicturePost(source models.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Title: source.Title,
|
||||
@ -149,7 +149,7 @@ func (rc *RedditClient) convertPicturePost(source models.RedditPost) database.Ar
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertTextPost(source models.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
@ -164,7 +164,7 @@ func (rc *RedditClient) convertTextPost(source models.RedditPost) database.Artic
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc *RedditClient) convertVideoPost(source models.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
||||
@ -180,7 +180,7 @@ func (rc *RedditClient) convertVideoPost(source models.RedditPost) database.Arti
|
||||
}
|
||||
|
||||
// This post is nothing more then a redirect to another location.
|
||||
func (rc *RedditClient) convertRedirectPost(source models.RedditPost) database.Article {
|
||||
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
|
||||
var item = database.Article{
|
||||
Sourceid: rc.record.ID,
|
||||
Tags: "a",
|
@ -3,9 +3,9 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var RedditRecord database.Source = database.Source{
|
@ -4,16 +4,16 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"github.com/jtom38/newsbot/collector/services/cache"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
type rssClient struct {
|
||||
SourceRecord models.Sources
|
||||
SourceRecord domain.Sources
|
||||
}
|
||||
|
||||
func NewRssClient(sourceRecord models.Sources) rssClient {
|
||||
func NewRssClient(sourceRecord domain.Sources) rssClient {
|
||||
client := rssClient{
|
||||
SourceRecord: sourceRecord,
|
||||
}
|
@ -3,11 +3,11 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/models"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
)
|
||||
|
||||
var rssRecord = models.Sources{
|
||||
var rssRecord = domain.Sources{
|
||||
ID: 1,
|
||||
Name: "ArsTechnica",
|
||||
Url: "https://feeds.arstechnica.com/arstechnica/index",
|
@ -7,8 +7,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
|
||||
"github.com/nicklaw5/helix/v2"
|
||||
)
|
||||
|
||||
@ -31,14 +31,14 @@ var (
|
||||
)
|
||||
|
||||
func NewTwitchClient() (TwitchClient, error) {
|
||||
c := config.New()
|
||||
c := services.NewConfig()
|
||||
|
||||
id := c.GetConfig(config.TWITCH_CLIENT_ID)
|
||||
id := c.GetConfig(services.TWITCH_CLIENT_ID)
|
||||
if id == "" {
|
||||
return TwitchClient{}, ErrTwitchClientIdMissing
|
||||
}
|
||||
|
||||
secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
|
||||
secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
|
||||
if secret == "" {
|
||||
return TwitchClient{}, ErrTwitchClientSecretMissing
|
||||
}
|
||||
@ -50,8 +50,8 @@ func NewTwitchClient() (TwitchClient, error) {
|
||||
|
||||
client := TwitchClient{
|
||||
//SourceRecord: &source,
|
||||
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
|
||||
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
|
||||
monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
|
||||
monitorVod: c.GetConfig(services.TWITCH_MONITOR_VOD),
|
||||
api: &api,
|
||||
}
|
||||
|
@ -4,9 +4,9 @@ import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var TwitchSourceRecord = database.Source{
|
@ -7,14 +7,12 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
//"strconv"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
)
|
||||
|
||||
type YoutubeClient struct {
|
||||
@ -110,7 +108,7 @@ func (yc *YoutubeClient) GetBrowser() (*rod.Browser, error) {
|
||||
if err != nil {
|
||||
return browser, err
|
||||
}
|
||||
|
||||
|
||||
browser = rod.New().ControlURL(u).MustConnect()
|
||||
}
|
||||
return browser, nil
|
@ -3,9 +3,9 @@ package input_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/input"
|
||||
)
|
||||
|
||||
var YouTubeRecord database.Source = database.Source{
|
@ -8,7 +8,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
)
|
||||
|
||||
type discordField struct {
|
@ -4,12 +4,11 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
//"time"
|
||||
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
|
||||
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
|
||||
"github.com/google/uuid"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services/output"
|
||||
)
|
||||
|
||||
var (
|
4
makefile
4
makefile
@ -13,10 +13,10 @@ docker-build: ## Generates the docker image
|
||||
docker image ls | grep newsbot.collector.api
|
||||
|
||||
migrate-dev: ## Apply sql migrations to dev db
|
||||
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
|
||||
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
|
||||
|
||||
migrate-dev-down: ## revert sql migrations to dev db
|
||||
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
|
||||
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
|
||||
|
||||
swag: ## Generates the swagger documentation with the swag tool
|
||||
~/go/bin/swag f
|
||||
|
@ -1 +0,0 @@
|
||||
package routes_test
|
@ -1,22 +0,0 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
)
|
||||
|
||||
func TestNewClient(t *testing.T) {
|
||||
config.New()
|
||||
}
|
||||
|
||||
func TestGetConfigExpectNull(t *testing.T) {
|
||||
cc := config.New()
|
||||
os.Setenv(config.REDDIT_PULL_HOT, "")
|
||||
res := cc.GetConfig(config.REDDIT_PULL_HOT)
|
||||
if res != "" {
|
||||
panic("expected blank")
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user