Compare commits

..

No commits in common. "feb9895eb7e78429e4f5e080a1d19af0c31857e3" and "b0790359d5f953adb3cbc2754b3f20202bb6498c" have entirely different histories.

60 changed files with 250 additions and 158 deletions

64
.github/workflows/docker.build.yaml vendored Normal file
View File

@ -0,0 +1,64 @@
name: Docker
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
on:
#schedule:
# - cron: '21 19 * * *'
push:
branches: [ master ]
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
#pull_request:
# branches: [ master ]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
#images: ${{ env.REGISTRY }}/newsbot.worker
# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

23
.github/workflows/go-build.yml vendored Normal file
View File

@ -0,0 +1,23 @@
name: Go
on:
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.18
- name: Build
run: go build -v ./...
#- name: Test
# run: go test -v ./...

View File

@ -1,6 +1,8 @@
package domain
package models
import "time"
import (
"time"
)
type CacheItem struct {
Key string

View File

@ -1,4 +1,4 @@
package domain
package models
import (
"time"

View File

@ -6,7 +6,7 @@ import (
"github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/database"
)
type ArticleDto struct {

View File

@ -1,4 +1,4 @@
package domain
package models
// This is the root Json object. It does not contain data that we care about though.
type RedditJsonContent struct {

View File

@ -6,9 +6,9 @@ import (
"context"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
type DtoClient struct {
@ -25,7 +25,7 @@ func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models
var res []models.ArticleDto
a, err := c.db.ListArticles(ctx, database.ListArticlesParams{
Limit: int32(limit),
Limit: int32(limit),
Offset: int32(limit * page),
})
if err != nil {
@ -38,11 +38,11 @@ func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models
return res, nil
}
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) {
func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32 ) ([]models.ArticleDto, error) {
var res []models.ArticleDto
a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{
Limit: limit,
Limit: limit,
Offset: page * limit,
})
if err != nil {
@ -85,8 +85,8 @@ func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid
var res []models.ArticleDto
a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{
Sourceid: SourceID,
Limit: int32(limit),
Offset: int32(limit * page),
Limit: int32(limit),
Offset: int32(limit * page),
})
if err != nil {
return res, err

View File

@ -3,9 +3,9 @@ package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) {

View File

@ -3,8 +3,8 @@ package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) {

View File

@ -4,9 +4,9 @@ import (
"context"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) {
@ -54,7 +54,7 @@ func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source s
var res models.SourceDto
item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{
Name: name,
Name: name,
Source: source,
})
if err != nil {

View File

@ -3,9 +3,9 @@ package dto
import (
"context"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) {

4
go.mod
View File

@ -1,6 +1,6 @@
module git.jamestombleson.com/jtom38/newsbot-api
module github.com/jtom38/newsbot/collector
go 1.22
go 1.18
require (
github.com/PuerkitoBio/goquery v1.8.0

View File

@ -1 +0,0 @@
package domain

View File

@ -1 +0,0 @@
package v1_test

View File

@ -1,22 +0,0 @@
package services_test
import (
"os"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
)
func TestNewClient(t *testing.T) {
services.NewConfig()
}
func TestGetConfigExpectNull(t *testing.T) {
cc := services.NewConfig()
os.Setenv(services.REDDIT_PULL_HOT, "")
res := cc.GetConfig(services.REDDIT_PULL_HOT)
if res != "" {
panic("expected blank")
}
}

View File

@ -6,33 +6,33 @@ import (
"fmt"
"net/http"
"git.jamestombleson.com/jtom38/newsbot-api/docs"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/docs"
"github.com/jtom38/newsbot/collector/routes"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/jtom38/newsbot/collector/services/cron"
)
// @title NewsBot collector
// @version 0.1
// @BasePath /api
func main() {
cfg := services.NewConfig()
address := cfg.GetConfig(services.ServerAddress)
cfg := config.New()
address := cfg.GetConfig(config.ServerAddress)
docs.SwaggerInfo.Host = fmt.Sprintf("%v:8081", address)
ctx := context.Background()
db, err := sql.Open("postgres", cfg.GetConfig(services.Sql_Connection_String))
db, err := sql.Open("postgres", cfg.GetConfig(config.Sql_Connection_String))
if err != nil {
panic(err)
}
queries := database.New(db)
c := cron.NewScheduler(ctx)
c := cron.New(ctx)
c.Start()
server := v1.NewServer(ctx, queries)
server := routes.NewServer(ctx, queries)
fmt.Println("API is online and waiting for requests.")
fmt.Printf("API: http://%v:8081/api\r\n", address)

View File

@ -13,10 +13,10 @@ docker-build: ## Generates the docker image
docker image ls | grep newsbot.collector.api
migrate-dev: ## Apply sql migrations to dev db
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up
migrate-dev-down: ## revert sql migrations to dev db
goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
goose -dir "./database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down
swag: ## Generates the swagger documentation with the swag tool
~/go/bin/swag f

View File

@ -1,12 +1,12 @@
package v1
package routes
import (
"net/http"
"strconv"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (s *Server) GetArticleRouter() http.Handler {
@ -168,11 +168,12 @@ func (s *Server) ListArticlesBySourceId(w http.ResponseWriter, r *http.Request)
query := r.URL.Query()
_id := query["id"][0]
uuid, err := uuid.Parse(_id)
if err != nil {
s.WriteError(w, err.Error(), http.StatusBadRequest)
return
}
}
// if a page number was sent, process it
if len(query["page"]) >= 1 {

View File

@ -1,4 +1,4 @@
package v1
package routes
import (
"encoding/json"
@ -6,10 +6,10 @@ import (
"net/http"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
type ListDiscordWebhooks struct {

View File

@ -1,10 +1,10 @@
package v1
package routes
import (
"net/http"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5"
"github.com/jtom38/newsbot/collector/domain/models"
)
type ListDiscordWebHooksQueueResults struct {

1
routes/root_test.go Normal file
View File

@ -0,0 +1 @@
package routes_test

View File

@ -1,4 +1,4 @@
package v1
package routes
import (
"context"
@ -11,9 +11,9 @@ import (
_ "github.com/lib/pq"
httpSwagger "github.com/swaggo/http-swagger"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/dto"
"github.com/jtom38/newsbot/collector/services/config"
)
type Server struct {
@ -56,8 +56,8 @@ func NewServer(ctx context.Context, db *database.Queries) *Server {
}
func openDatabase(ctx context.Context) (*database.Queries, error) {
_env := services.NewConfig()
connString := _env.GetConfig(services.Sql_Connection_String)
_env := config.New()
connString := _env.GetConfig(config.Sql_Connection_String)
db, err := sql.Open("postgres", connString)
if err != nil {
panic(err)
@ -81,7 +81,7 @@ func (s *Server) MountRoutes() {
s.Router.Mount("/api/articles", s.GetArticleRouter())
s.Router.Mount("/api/queue", s.GetQueueRouter())
s.Router.Mount("/api/discord/webhooks", s.DiscordWebHookRouter())
//s.Router.Get("/api/settings", s.getSettings)
s.Router.Mount("/api/sources", s.GetSourcesRouter())
@ -116,7 +116,7 @@ func (s *Server) WriteError(w http.ResponseWriter, errMessage string, HttpStatus
func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) {
w.Header().Set(HeaderContentType, ApplicationJson)
bres, err := json.Marshal(model)
if err != nil {
s.WriteError(w, err.Error(), http.StatusInternalServerError)
@ -124,4 +124,4 @@ func (s *Server) WriteJson(w http.ResponseWriter, model interface{}) {
}
w.Write(bres)
}
}

View File

@ -1,4 +1,4 @@
package v1
package routes
import (
"encoding/json"

View File

@ -1,4 +1,4 @@
package v1
package routes
import (
"encoding/json"
@ -6,10 +6,10 @@ import (
"net/http"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (s *Server) GetSourcesRouter() http.Handler {

View File

@ -1,14 +1,14 @@
package v1
package routes
import (
"context"
"encoding/json"
"net/http"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
)
func (s *Server) GetSubscriptionsRouter() http.Handler {

View File

@ -3,7 +3,7 @@ package cache
import (
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/domain/models"
)
type CacheClient struct {
@ -19,7 +19,7 @@ func NewCacheClient(group string) CacheClient {
}
func (cc *CacheClient) Insert(key string, value string) {
item := domain.CacheItem{
item := models.CacheItem{
Key: key,
Value: value,
Group: cc.group,
@ -29,7 +29,7 @@ func (cc *CacheClient) Insert(key string, value string) {
cacheStorage = append(cacheStorage, &item)
}
func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
func (cc *CacheClient) FindByKey(key string) (*models.CacheItem, error) {
for _, item := range cacheStorage {
if item.Group != cc.group {
continue
@ -46,10 +46,10 @@ func (cc *CacheClient) FindByKey(key string) (*domain.CacheItem, error) {
return item, nil
}
return &domain.CacheItem{}, ErrCacheRecordMissing
return &models.CacheItem{}, ErrCacheRecordMissing
}
func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
func (cc *CacheClient) FindByValue(value string) (*models.CacheItem, error) {
for _, item := range cacheStorage {
if item.Group != cc.group {
continue
@ -65,5 +65,5 @@ func (cc *CacheClient) FindByValue(value string) (*domain.CacheItem, error) {
}
return item, nil
}
return &domain.CacheItem{}, ErrCacheRecordMissing
return &models.CacheItem{}, ErrCacheRecordMissing
}

View File

@ -3,7 +3,7 @@ package cache_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/services/cache"
)
func TestNewCacheClient(t *testing.T) {

View File

@ -3,11 +3,11 @@ package cache
import (
"errors"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/domain/models"
)
var (
cacheStorage []*domain.CacheItem
cacheStorage []*models.CacheItem
ErrCacheRecordMissing = errors.New("unable to find the requested record")
)

View File

@ -3,7 +3,7 @@ package cache
import (
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"github.com/jtom38/newsbot/collector/domain/models"
)
// When a record becomes tainted, it needs to be renewed or it will be dropped from the cache.
@ -36,8 +36,8 @@ func (cam CacheAgeMonitor) CheckExpiredEntries() {
}
// This creates a new slice and skips over the item that needs to be dropped
func (cam CacheAgeMonitor) removeEntry(index int) []*domain.CacheItem {
var temp []*domain.CacheItem
func (cam CacheAgeMonitor) removeEntry(index int) []*models.CacheItem {
var temp []*models.CacheItem
for i, item := range cacheStorage {
if i != index {
temp = append(temp, item)

View File

@ -3,7 +3,7 @@ package cache_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/services/cache"
)
func TestCacheTaintItem(t *testing.T) {

View File

@ -1,4 +1,4 @@
package services
package config
import (
"errors"
@ -34,7 +34,7 @@ const (
type ConfigClient struct{}
func NewConfig() ConfigClient {
func New() ConfigClient {
c := ConfigClient{}
c.RefreshEnv()

View File

@ -0,0 +1,22 @@
package config_test
import (
"os"
"testing"
"github.com/jtom38/newsbot/collector/services/config"
)
func TestNewClient(t *testing.T) {
config.New()
}
func TestGetConfigExpectNull(t *testing.T) {
cc := config.New()
os.Setenv(config.REDDIT_PULL_HOT, "")
res := cc.GetConfig(config.REDDIT_PULL_HOT)
if res != "" {
panic("expected blank")
}
}

View File

@ -11,10 +11,10 @@ import (
_ "github.com/lib/pq"
"github.com/robfig/cron/v3"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/jtom38/newsbot/collector/services/input"
"github.com/jtom38/newsbot/collector/services/output"
)
type Cron struct {
@ -24,8 +24,8 @@ type Cron struct {
}
func openDatabase() (*database.Queries, error) {
_env := services.NewConfig()
connString := _env.GetConfig(services.Sql_Connection_String)
_env := config.New()
connString := _env.GetConfig(config.Sql_Connection_String)
if connString == "" {
panic("Connection String is null!")
}
@ -38,7 +38,7 @@ func openDatabase() (*database.Queries, error) {
return queries, err
}
func NewScheduler(ctx context.Context) *Cron {
func New(ctx context.Context) *Cron {
c := &Cron{
ctx: &ctx,
}
@ -51,28 +51,28 @@ func NewScheduler(ctx context.Context) *Cron {
c.Db = queries
//timer.AddFunc("*/5 * * * *", func() { go CheckCache() })
features := services.NewConfig()
features := config.New()
res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND)
res, _ := features.GetFeature(config.FEATURE_ENABLE_REDDIT_BACKEND)
if res {
timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() })
log.Print("[Input] Reddit backend was enabled")
//go c.CheckReddit()
}
res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND)
res, _ = features.GetFeature(config.FEATURE_ENABLE_YOUTUBE_BACKEND)
if res {
timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() })
log.Print("[Input] YouTube backend was enabled")
}
res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND)
res, _ = features.GetFeature(config.FEATURE_ENABLE_FFXIV_BACKEND)
if res {
timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() })
log.Print("[Input] FFXIV backend was enabled")
}
res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND)
res, _ = features.GetFeature(config.FEATURE_ENABLE_TWITCH_BACKEND)
if res {
timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() })
log.Print("[Input] Twitch backend was enabled")

View File

@ -4,7 +4,7 @@ import (
"context"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron"
"github.com/jtom38/newsbot/collector/services/cron"
)
func TestInvokeTwitch(t *testing.T) {
@ -14,19 +14,19 @@ func TestInvokeTwitch(t *testing.T) {
// TODO add database mocks but not sure how to do that yet.
func TestCheckReddit(t *testing.T) {
ctx := context.Background()
c := cron.NewScheduler(ctx)
c := cron.New(ctx)
c.CheckReddit()
}
func TestCheckYouTube(t *testing.T) {
ctx := context.Background()
c := cron.NewScheduler(ctx)
c := cron.New(ctx)
c.CheckYoutube()
}
func TestCheckTwitch(t *testing.T) {
ctx := context.Background()
c := cron.NewScheduler(ctx)
c := cron.New(ctx)
err := c.CheckTwitch()
if err != nil {
t.Error(err)

View File

@ -13,8 +13,8 @@ import (
"github.com/go-rod/rod/lib/launcher"
"github.com/google/uuid"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/cache"
)
const (

View File

@ -3,9 +3,9 @@ package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
ffxiv "github.com/jtom38/newsbot/collector/services/input"
)
var FFXIVRecord database.Source = database.Source{

View File

@ -9,11 +9,11 @@ import (
"strings"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/domain/models"
"github.com/jtom38/newsbot/collector/services/config"
)
type RedditClient struct {
@ -31,10 +31,10 @@ func NewRedditClient(Record database.Source) *RedditClient {
rc := RedditClient{
record: Record,
}
cc := services.NewConfig()
rc.config.PullHot = cc.GetConfig(services.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(services.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(services.REDDIT_PULL_TOP)
cc := config.New()
rc.config.PullHot = cc.GetConfig(config.REDDIT_PULL_HOT)
rc.config.PullNSFW = cc.GetConfig(config.REDDIT_PULL_NSFW)
rc.config.PullTop = cc.GetConfig(config.REDDIT_PULL_TOP)
//rc.disableHttp2Client()
@ -65,8 +65,8 @@ func (rc *RedditClient) GetPage(parser *rod.Browser, url string) *rod.Page {
// GetContent() reaches out to Reddit and pulls the Json data.
// It will then convert the data to a struct and return the struct.
func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
var items domain.RedditJsonContent = domain.RedditJsonContent{}
func (rc *RedditClient) GetContent() (models.RedditJsonContent, error) {
var items models.RedditJsonContent = models.RedditJsonContent{}
// TODO Wire this to support the config options
Url := fmt.Sprintf("%v.json", rc.record.Url)
@ -88,7 +88,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) {
return items, nil
}
func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article {
func (rc *RedditClient) ConvertToArticles(items models.RedditJsonContent) []database.Article {
var redditArticles []database.Article
for _, item := range items.Data.Children {
var article database.Article
@ -104,7 +104,7 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []data
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
// This data can be passed to the database.
func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) {
func (rc *RedditClient) convertToArticle(source models.RedditPost) (database.Article, error) {
var item database.Article
if source.Content == "" && source.Url != "" {
@ -131,7 +131,7 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art
return item, nil
}
func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article {
func (rc *RedditClient) convertPicturePost(source models.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Title: source.Title,
@ -149,7 +149,7 @@ func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Ar
return item
}
func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article {
func (rc *RedditClient) convertTextPost(source models.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
@ -164,7 +164,7 @@ func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Artic
return item
}
func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article {
func (rc *RedditClient) convertVideoPost(source models.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",
@ -180,7 +180,7 @@ func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Arti
}
// This post is nothing more then a redirect to another location.
func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article {
func (rc *RedditClient) convertRedirectPost(source models.RedditPost) database.Article {
var item = database.Article{
Sourceid: rc.record.ID,
Tags: "a",

View File

@ -3,9 +3,9 @@ package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
)
var RedditRecord database.Source = database.Source{

View File

@ -4,16 +4,16 @@ import (
"fmt"
"log"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache"
"github.com/jtom38/newsbot/collector/domain/models"
"github.com/jtom38/newsbot/collector/services/cache"
"github.com/mmcdole/gofeed"
)
type rssClient struct {
SourceRecord domain.Sources
SourceRecord models.Sources
}
func NewRssClient(sourceRecord domain.Sources) rssClient {
func NewRssClient(sourceRecord models.Sources) rssClient {
client := rssClient{
SourceRecord: sourceRecord,
}

View File

@ -3,11 +3,11 @@ package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/domain"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/jtom38/newsbot/collector/domain/models"
"github.com/jtom38/newsbot/collector/services/input"
)
var rssRecord = domain.Sources{
var rssRecord = models.Sources{
ID: 1,
Name: "ArsTechnica",
Url: "https://feeds.arstechnica.com/arstechnica/index",

View File

@ -7,8 +7,8 @@ import (
"strings"
"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/config"
"github.com/nicklaw5/helix/v2"
)
@ -31,14 +31,14 @@ var (
)
func NewTwitchClient() (TwitchClient, error) {
c := services.NewConfig()
c := config.New()
id := c.GetConfig(services.TWITCH_CLIENT_ID)
id := c.GetConfig(config.TWITCH_CLIENT_ID)
if id == "" {
return TwitchClient{}, ErrTwitchClientIdMissing
}
secret := c.GetConfig(services.TWITCH_CLIENT_SECRET)
secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
if secret == "" {
return TwitchClient{}, ErrTwitchClientSecretMissing
}
@ -50,8 +50,8 @@ func NewTwitchClient() (TwitchClient, error) {
client := TwitchClient{
//SourceRecord: &source,
monitorClips: c.GetConfig(services.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(services.TWITCH_MONITOR_VOD),
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
api: &api,
}

View File

@ -4,9 +4,9 @@ import (
"log"
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
)
var TwitchSourceRecord = database.Source{

View File

@ -7,12 +7,14 @@ import (
"log"
"net/http"
//"strconv"
"github.com/PuerkitoBio/goquery"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/mmcdole/gofeed"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/database"
)
type YoutubeClient struct {
@ -108,7 +110,7 @@ func (yc *YoutubeClient) GetBrowser() (*rod.Browser, error) {
if err != nil {
return browser, err
}
browser = rod.New().ControlURL(u).MustConnect()
}
return browser, nil

View File

@ -3,9 +3,9 @@ package input_test
import (
"testing"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/input"
"github.com/google/uuid"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/input"
)
var YouTubeRecord database.Source = database.Source{

View File

@ -8,7 +8,7 @@ import (
"net/http"
"strings"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"github.com/jtom38/newsbot/collector/database"
)
type discordField struct {

View File

@ -4,11 +4,12 @@ import (
"os"
"strings"
"testing"
//"time"
"git.jamestombleson.com/jtom38/newsbot-api/internal/database"
"git.jamestombleson.com/jtom38/newsbot-api/internal/services/output"
"github.com/google/uuid"
"github.com/joho/godotenv"
"github.com/jtom38/newsbot/collector/database"
"github.com/jtom38/newsbot/collector/services/output"
)
var (