Feature/deployment update (#13)
* updated to go 1.18.4 and added alpine dep. Image tested locally * added the first example of how to run the app * added deployment notes * updated bootup logic around the env and how to handle things * swagger update
This commit is contained in:
parent
a1324ee1c1
commit
65f4281f92
10
Dockerfile
10
Dockerfile
@ -1,16 +1,16 @@
|
||||
FROM golang:1.18.3 as build
|
||||
FROM golang:1.18.4 as build
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
RUN go build .
|
||||
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
|
||||
FROM alpine
|
||||
FROM alpine:latest as app
|
||||
|
||||
RUN mkdir /app && \
|
||||
mkdir /app/migrations
|
||||
RUN apk --no-cache add bash libc6-compat
|
||||
RUN mkdir /app && mkdir /app/migrations
|
||||
COPY --from=build /app/collector /app
|
||||
COPY --from=build /go/bin/goose /app
|
||||
COPY ./database/migrations/ /app/migrations
|
||||
|
||||
ENTRYPOINT [ "/app/collector" ]
|
||||
CMD [ "/app/collector" ]
|
22
README.md
22
README.md
@ -1,2 +1,24 @@
|
||||
# newsbot.collector.api
|
||||
|
||||
This is the collection service of newsbot to pull articles from the web.
|
||||
|
||||
## Deployment
|
||||
|
||||
1. Create a copy of the docker compose file and make it local
|
||||
2. Update the `docker-compose.yaml` with your secrets
|
||||
3. Run migrations
|
||||
2. `docker compose run api /app/goose -dir "/app/migrations" up`
|
||||
4. Run app
|
||||
1. `docker compose up -d`
|
||||
5. Once the app is running go to the swagger page and validate that you see the seeded sources.
|
||||
1. `http://localhost:8081/swagger/index.html#/Source/get_config_sources`
|
||||
2. `curl -X 'GET' 'http://localhost:8081/api/config/sources' -H 'accept: application/json'`
|
||||
6. Add any new sources
|
||||
7. Add a Discord Web Hook
|
||||
8. Create your subscription links
|
||||
1. This is a link between a source and a discord web hook. Without this, the app will not send a notification about new posts.
|
||||
|
||||
### Errors
|
||||
|
||||
- pq: permission denied to create extension "uuid-ossp"
|
||||
- Might need to grant your account `ALTER USER root WITH SUPERUSER;` to create the 'uuid-ossp' for uuid creations
|
||||
|
@ -1,100 +0,0 @@
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
|
||||
// Generate this struct fr
|
||||
type ArticlesClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) List() ([]model.Articles, error) {
|
||||
var items []model.Articles
|
||||
url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
|
||||
resp, err := getContent(url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(resp, &items)
|
||||
if err != nil {
|
||||
return []model.Articles{}, err
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
|
||||
var items model.Articles
|
||||
url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
|
||||
resp, err := getContent(url)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(resp, &items)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
|
||||
var item model.Articles
|
||||
get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
|
||||
resp, err := getContent(get)
|
||||
if err != nil {
|
||||
return item, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(resp, &item)
|
||||
if err != nil {
|
||||
return item, err
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Delete(id int32) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Add(item model.Articles) error {
|
||||
//return errors.New("not implemented")
|
||||
url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
|
||||
|
||||
bItem, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return errors.New("failed to post to the DB")
|
||||
}
|
||||
|
||||
return nil
|
||||
//body, err := ioutil.ReadAll(resp.Body)
|
||||
//if err != nil { return err }
|
||||
|
||||
}
|
@ -1,74 +0,0 @@
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
)
|
||||
|
||||
type DatabaseClient struct {
|
||||
Diagnosis DiagnosisClient
|
||||
|
||||
Articles ArticlesClient
|
||||
Sources SourcesClient
|
||||
}
|
||||
|
||||
// This will generate a new client to interface with the API Database.
|
||||
func NewDatabaseClient() DatabaseClient {
|
||||
cc := config.New()
|
||||
dbUri := cc.GetConfig(config.DB_URI)
|
||||
|
||||
var client = DatabaseClient{}
|
||||
client.Diagnosis.rootUri = dbUri
|
||||
client.Sources.rootUri = dbUri
|
||||
client.Articles.rootUri = dbUri
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func getContent(url string) ([]byte, error) {
|
||||
client := &http.Client{}
|
||||
var blank []byte
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil { return blank, err }
|
||||
|
||||
// set the user agent header to avoid kick backs.. as much
|
||||
req.Header.Set("User-Agent", getUserAgent())
|
||||
|
||||
log.Printf("Requesting content from %v\n", url)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil { return blank, err }
|
||||
if resp.StatusCode == 404 {
|
||||
err = errors.New("404 not found")
|
||||
return blank, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil { return blank, err }
|
||||
|
||||
//log.Println(string(body))
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func httpDelete(url string) error {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil { return err }
|
||||
|
||||
//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
||||
req.Header.Set("User-Agent", getUserAgent())
|
||||
|
||||
_, err = client.Do(req)
|
||||
if err != nil { return err }
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getUserAgent() string {
|
||||
return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
|
||||
}
|
@ -1,24 +0,0 @@
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
//"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
type DiagnosisClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (dc *DiagnosisClient) Ping() error {
|
||||
dbPing := fmt.Sprintf("%v/ping", dc.rootUri)
|
||||
resp, err := http.Get(dbPing)
|
||||
if err != nil { return err }
|
||||
|
||||
_, err = ioutil.ReadAll(resp.Body)
|
||||
if err != nil { return err }
|
||||
return nil
|
||||
}
|
||||
|
@ -1,38 +0,0 @@
|
||||
package databaseRest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
|
||||
type SourcesClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (sb *SourcesClient) List() ([]model.Sources, error) {
|
||||
var items []model.Sources
|
||||
url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
|
||||
resp, err := getContent(url)
|
||||
if err != nil { return items, err }
|
||||
|
||||
err = json.Unmarshal(resp, &items)
|
||||
if err != nil { return []model.Sources{}, err }
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) {
|
||||
items, err := sb.List()
|
||||
if err != nil { log.Panicln(err) }
|
||||
|
||||
var res []model.Sources
|
||||
for _, item := range(items) {
|
||||
if item.Source == SourceType {
|
||||
res = append(res, item)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
33
docker-compose.yaml
Normal file
33
docker-compose.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
version: "3"
|
||||
|
||||
networks:
|
||||
newsbot:
|
||||
|
||||
services:
|
||||
api:
|
||||
image: ghcr.io/jtom38/newsbot.collector.api:master
|
||||
environment:
|
||||
SQL_CONNECTION_STRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
|
||||
|
||||
# Used for database migrations
|
||||
GOOSE_DRIVER: "postgres"
|
||||
|
||||
# Connection String to Postgresql
|
||||
GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
|
||||
|
||||
# Enable/Disable Reddit monitoring
|
||||
FEATURE_ENABLE_REDDIT_BACKEND: true
|
||||
|
||||
# Enable/Disable YouTube monitoring
|
||||
FEATURE_ENABLE_YOUTUBE_BACKEND: false
|
||||
|
||||
# Set your Twitch Developer ID and Secrets here and they will be used to collect updates.
|
||||
TWITCH_CLIENT_ID: ""
|
||||
TWITCH_CLIENT_SECRET: ""
|
||||
|
||||
# If you want to collect news on Final Fantasy XIV, set this to true
|
||||
FEATURE_ENABLE_FFXIV_BACKEND: false
|
||||
ports:
|
||||
- 8081:8081
|
||||
networks:
|
||||
- newsbot
|
@ -40,8 +40,8 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Source ID UUID",
|
||||
"name": "id",
|
||||
"description": "Tag name",
|
||||
"name": "Tag",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
|
@ -31,8 +31,8 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Source ID UUID",
|
||||
"name": "id",
|
||||
"description": "Tag name",
|
||||
"name": "Tag",
|
||||
"in": "query",
|
||||
"required": true
|
||||
}
|
||||
|
@ -29,9 +29,9 @@ paths:
|
||||
/articles/by/sourceid:
|
||||
get:
|
||||
parameters:
|
||||
- description: Source ID UUID
|
||||
- description: Tag name
|
||||
in: query
|
||||
name: id
|
||||
name: Tag
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
|
@ -11,37 +11,34 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
DB_URI string = "DB_URI"
|
||||
|
||||
DB_URI string = "DB_URI"
|
||||
|
||||
Sql_Connection_String string = "SQL_CONNECTION_STRING"
|
||||
|
||||
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
|
||||
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
|
||||
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
|
||||
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||
|
||||
FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
|
||||
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
|
||||
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
|
||||
|
||||
FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
|
||||
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
|
||||
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
|
||||
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
|
||||
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
|
||||
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
|
||||
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
|
||||
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
|
||||
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
|
||||
|
||||
FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
|
||||
|
||||
)
|
||||
|
||||
type ConfigClient struct {}
|
||||
type ConfigClient struct{}
|
||||
|
||||
func New() ConfigClient {
|
||||
_, err := os.Open(".env")
|
||||
if err == nil {
|
||||
loadEnvFile()
|
||||
}
|
||||
c := ConfigClient{}
|
||||
c.RefreshEnv()
|
||||
|
||||
return ConfigClient{}
|
||||
return c
|
||||
}
|
||||
|
||||
func (cc *ConfigClient) GetConfig(key string) string {
|
||||
@ -70,7 +67,16 @@ func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
|
||||
|
||||
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
|
||||
func (cc *ConfigClient) RefreshEnv() {
|
||||
loadEnvFile()
|
||||
// Check to see if we have the env file on the system
|
||||
_, err := os.Stat(".env")
|
||||
|
||||
// We have the file, load it.
|
||||
if err == nil {
|
||||
_, err := os.Open(".env")
|
||||
if err == nil {
|
||||
loadEnvFile()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func loadEnvFile() {
|
||||
|
@ -26,6 +26,9 @@ type Cron struct {
|
||||
func openDatabase() (*database.Queries, error) {
|
||||
_env := config.New()
|
||||
connString := _env.GetConfig(config.Sql_Connection_String)
|
||||
if connString == "" {
|
||||
panic("Connection String is null!")
|
||||
}
|
||||
db, err := sql.Open("postgres", connString)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
Loading…
Reference in New Issue
Block a user