From 65f4281f929ddb50b0fc6e67d55740a404dacf51 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 13 Jul 2022 15:52:25 -0700 Subject: [PATCH] Feature/deployment update (#13) * updated to go 1.18.4 and added alpine dep. Image tested locally * added the first example of how to run the app * added deployment notes * updated bootup logic around the env and how to handle things * swagger update --- Dockerfile | 10 ++-- README.md | 22 ++++++++ databaseRest/articles.go | 100 ------------------------------------- databaseRest/common.go | 74 --------------------------- databaseRest/diagnosis.go | 24 --------- databaseRest/sources.go | 38 -------------- docker-compose.yaml | 33 ++++++++++++ docs/docs.go | 4 +- docs/swagger.json | 4 +- docs/swagger.yaml | 4 +- services/config/config.go | 42 +++++++++------- services/cron/scheduler.go | 3 ++ 12 files changed, 93 insertions(+), 265 deletions(-) delete mode 100644 databaseRest/articles.go delete mode 100644 databaseRest/common.go delete mode 100644 databaseRest/diagnosis.go delete mode 100644 databaseRest/sources.go create mode 100644 docker-compose.yaml diff --git a/Dockerfile b/Dockerfile index 29ea3ba..b80dec1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ -FROM golang:1.18.3 as build +FROM golang:1.18.4 as build COPY . /app WORKDIR /app RUN go build . RUN go install github.com/pressly/goose/v3/cmd/goose@latest -FROM alpine +FROM alpine:latest as app -RUN mkdir /app && \ - mkdir /app/migrations +RUN apk --no-cache add bash libc6-compat +RUN mkdir /app && mkdir /app/migrations COPY --from=build /app/collector /app COPY --from=build /go/bin/goose /app COPY ./database/migrations/ /app/migrations -ENTRYPOINT [ "/app/collector" ] \ No newline at end of file +CMD [ "/app/collector" ] \ No newline at end of file diff --git a/README.md b/README.md index 94a2b98..2b4ab82 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ # newsbot.collector.api + This is the collection service of newsbot to pull articles from the web. + +## Deployment + +1. Create a copy of the docker compose file and make it local +2. Update the `docker-compose.yaml` with your secrets +3. Run migrations + 2. `docker compose run api /app/goose -dir "/app/migrations" up` +4. Run app + 1. `docker compose up -d` +5. Once the app is running go to the swagger page and validate that you see the seeded sources. + 1. `http://localhost:8081/swagger/index.html#/Source/get_config_sources` + 2. `curl -X 'GET' 'http://localhost:8081/api/config/sources' -H 'accept: application/json'` +6. Add any new sources +7. Add a Discord Web Hook +8. Create your subscription links + 1. This is a link between a source and a discord web hook. Without this, the app will not send a notification about new posts. + +### Errors + +- pq: permission denied to create extension "uuid-ossp" + - Might need to grant your account `ALTER USER root WITH SUPERUSER;` to create the 'uuid-ossp' for uuid creations diff --git a/databaseRest/articles.go b/databaseRest/articles.go deleted file mode 100644 index 35c8ed6..0000000 --- a/databaseRest/articles.go +++ /dev/null @@ -1,100 +0,0 @@ -package databaseRest - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "net/http" - - "github.com/jtom38/newsbot/collector/domain/model" -) - -// Generate this struct fr -type ArticlesClient struct { - rootUri string -} - -func (ac *ArticlesClient) List() ([]model.Articles, error) { - var items []model.Articles - url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri) - resp, err := getContent(url) - if err != nil { - return items, err - } - - err = json.Unmarshal(resp, &items) - if err != nil { - return []model.Articles{}, err - } - - return items, nil -} - -func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) { - var items model.Articles - url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID) - resp, err := getContent(url) - if err != nil { - return items, err - } - - err = json.Unmarshal(resp, &items) - if err != nil { - return items, err - } - - return items, nil -} - -func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) { - var item model.Articles - get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url) - resp, err := getContent(get) - if err != nil { - return item, err - } - - err = json.Unmarshal(resp, &item) - if err != nil { - return item, err - } - - return item, nil -} - -func (ac *ArticlesClient) Delete(id int32) error { - return errors.New("not implemented") -} - -func (ac *ArticlesClient) Add(item model.Articles) error { - //return errors.New("not implemented") - url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri) - - bItem, err := json.Marshal(item) - if err != nil { - return err - } - - client := &http.Client{} - req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem)) - if err != nil { - return err - } - - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return errors.New("failed to post to the DB") - } - - return nil - //body, err := ioutil.ReadAll(resp.Body) - //if err != nil { return err } - -} diff --git a/databaseRest/common.go b/databaseRest/common.go deleted file mode 100644 index 01be22b..0000000 --- a/databaseRest/common.go +++ /dev/null @@ -1,74 +0,0 @@ -package databaseRest - -import ( - "errors" - "io/ioutil" - "log" - "net/http" - - "github.com/jtom38/newsbot/collector/services/config" -) - -type DatabaseClient struct { - Diagnosis DiagnosisClient - - Articles ArticlesClient - Sources SourcesClient -} - -// This will generate a new client to interface with the API Database. -func NewDatabaseClient() DatabaseClient { - cc := config.New() - dbUri := cc.GetConfig(config.DB_URI) - - var client = DatabaseClient{} - client.Diagnosis.rootUri = dbUri - client.Sources.rootUri = dbUri - client.Articles.rootUri = dbUri - - return client -} - -func getContent(url string) ([]byte, error) { - client := &http.Client{} - var blank []byte - - req, err := http.NewRequest("GET", url, nil) - if err != nil { return blank, err } - - // set the user agent header to avoid kick backs.. as much - req.Header.Set("User-Agent", getUserAgent()) - - log.Printf("Requesting content from %v\n", url) - resp, err := client.Do(req) - if err != nil { return blank, err } - if resp.StatusCode == 404 { - err = errors.New("404 not found") - return blank, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { return blank, err } - - //log.Println(string(body)) - return body, nil -} - -func httpDelete(url string) error { - client := &http.Client{} - req, err := http.NewRequest("DELETE", url, nil) - if err != nil { return err } - - //req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0") - req.Header.Set("User-Agent", getUserAgent()) - - _, err = client.Do(req) - if err != nil { return err } - - return nil -} - -func getUserAgent() string { - return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0" -} \ No newline at end of file diff --git a/databaseRest/diagnosis.go b/databaseRest/diagnosis.go deleted file mode 100644 index 8b838dc..0000000 --- a/databaseRest/diagnosis.go +++ /dev/null @@ -1,24 +0,0 @@ -package databaseRest - -import ( - "fmt" - "io/ioutil" - "net/http" - - //"github.com/jtom38/newsbot/collector/services" -) - -type DiagnosisClient struct { - rootUri string -} - -func (dc *DiagnosisClient) Ping() error { - dbPing := fmt.Sprintf("%v/ping", dc.rootUri) - resp, err := http.Get(dbPing) - if err != nil { return err } - - _, err = ioutil.ReadAll(resp.Body) - if err != nil { return err } - return nil -} - diff --git a/databaseRest/sources.go b/databaseRest/sources.go deleted file mode 100644 index 37cf189..0000000 --- a/databaseRest/sources.go +++ /dev/null @@ -1,38 +0,0 @@ -package databaseRest - -import ( - "encoding/json" - "fmt" - "log" - - "github.com/jtom38/newsbot/collector/domain/model" -) - -type SourcesClient struct { - rootUri string -} - -func (sb *SourcesClient) List() ([]model.Sources, error) { - var items []model.Sources - url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri) - resp, err := getContent(url) - if err != nil { return items, err } - - err = json.Unmarshal(resp, &items) - if err != nil { return []model.Sources{}, err } - - return items, nil -} - -func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) { - items, err := sb.List() - if err != nil { log.Panicln(err) } - - var res []model.Sources - for _, item := range(items) { - if item.Source == SourceType { - res = append(res, item) - } - } - return res, nil -} \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..97883bc --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,33 @@ +version: "3" + +networks: + newsbot: + +services: + api: + image: ghcr.io/jtom38/newsbot.collector.api:master + environment: + SQL_CONNECTION_STRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" + + # Used for database migrations + GOOSE_DRIVER: "postgres" + + # Connection String to Postgresql + GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable" + + # Enable/Disable Reddit monitoring + FEATURE_ENABLE_REDDIT_BACKEND: true + + # Enable/Disable YouTube monitoring + FEATURE_ENABLE_YOUTUBE_BACKEND: false + + # Set your Twitch Developer ID and Secrets here and they will be used to collect updates. + TWITCH_CLIENT_ID: "" + TWITCH_CLIENT_SECRET: "" + + # If you want to collect news on Final Fantasy XIV, set this to true + FEATURE_ENABLE_FFXIV_BACKEND: false + ports: + - 8081:8081 + networks: + - newsbot \ No newline at end of file diff --git a/docs/docs.go b/docs/docs.go index 40c1c45..df00253 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -40,8 +40,8 @@ const docTemplate = `{ "parameters": [ { "type": "string", - "description": "Source ID UUID", - "name": "id", + "description": "Tag name", + "name": "Tag", "in": "query", "required": true } diff --git a/docs/swagger.json b/docs/swagger.json index 6087439..ccecfb2 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -31,8 +31,8 @@ "parameters": [ { "type": "string", - "description": "Source ID UUID", - "name": "id", + "description": "Tag name", + "name": "Tag", "in": "query", "required": true } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 01b599b..8474a8c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -29,9 +29,9 @@ paths: /articles/by/sourceid: get: parameters: - - description: Source ID UUID + - description: Tag name in: query - name: id + name: Tag required: true type: string produces: diff --git a/services/config/config.go b/services/config/config.go index 240aff0..016e38a 100644 --- a/services/config/config.go +++ b/services/config/config.go @@ -11,37 +11,34 @@ import ( ) const ( - DB_URI string = "DB_URI" - + DB_URI string = "DB_URI" + Sql_Connection_String string = "SQL_CONNECTION_STRING" FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND" - REDDIT_PULL_TOP = "REDDIT_PULL_TOP" - REDDIT_PULL_HOT = "REDDIT_PULL_HOT" - REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW" + REDDIT_PULL_TOP = "REDDIT_PULL_TOP" + REDDIT_PULL_HOT = "REDDIT_PULL_HOT" + REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW" FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND" - YOUTUBE_DEBUG = "YOUTUBE_DEBUG" + YOUTUBE_DEBUG = "YOUTUBE_DEBUG" FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND" - TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID" - TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET" - TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS" - TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD" + TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID" + TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET" + TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS" + TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD" FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND" - ) -type ConfigClient struct {} +type ConfigClient struct{} func New() ConfigClient { - _, err := os.Open(".env") - if err == nil { - loadEnvFile() - } + c := ConfigClient{} + c.RefreshEnv() - return ConfigClient{} + return c } func (cc *ConfigClient) GetConfig(key string) string { @@ -70,7 +67,16 @@ func (cc *ConfigClient) GetFeature(flag string) (bool, error) { // Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes. func (cc *ConfigClient) RefreshEnv() { - loadEnvFile() + // Check to see if we have the env file on the system + _, err := os.Stat(".env") + + // We have the file, load it. + if err == nil { + _, err := os.Open(".env") + if err == nil { + loadEnvFile() + } + } } func loadEnvFile() { diff --git a/services/cron/scheduler.go b/services/cron/scheduler.go index 7f7cd36..369909f 100644 --- a/services/cron/scheduler.go +++ b/services/cron/scheduler.go @@ -26,6 +26,9 @@ type Cron struct { func openDatabase() (*database.Queries, error) { _env := config.New() connString := _env.GetConfig(config.Sql_Connection_String) + if connString == "" { + panic("Connection String is null!") + } db, err := sql.Open("postgres", connString) if err != nil { panic(err)