jtom38
/
newsbot-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature/deployment update (#13) 

* updated to go 1.18.4 and added alpine dep.  Image tested locally

* added the first example of how to run the app

* added deployment notes

* updated bootup logic around the env and how to handle things

* swagger update
This commit is contained in:
James Tombleson 2022-07-13 15:52:25 -07:00 committed by GitHub
parent a1324ee1c1
commit 65f4281f92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 93 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Dockerfile
View File

@ -1,16 +1,16 @@
FROM golang:1.18.3 as build
FROM golang:1.18.4 as build
COPY . /app
WORKDIR /app
RUN go build .
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
FROM alpine
FROM alpine:latest as app
RUN mkdir /app && \
mkdir /app/migrations
RUN apk --no-cache add bash libc6-compat
RUN mkdir /app && mkdir /app/migrations
COPY --from=build /app/collector /app
COPY --from=build /go/bin/goose /app
COPY ./database/migrations/ /app/migrations
ENTRYPOINT [ "/app/collector" ]
CMD [ "/app/collector" ]

22
README.md
View File

@ -1,2 +1,24 @@
# newsbot.collector.api
This is the collection service of newsbot to pull articles from the web.
## Deployment
1. Create a copy of the docker compose file and make it local
2. Update the `docker-compose.yaml` with your secrets
3. Run migrations
2. `docker compose run api /app/goose -dir "/app/migrations" up`
4. Run app
1. `docker compose up -d`
5. Once the app is running go to the swagger page and validate that you see the seeded sources.
1. `http://localhost:8081/swagger/index.html#/Source/get_config_sources`
2. `curl -X 'GET' 'http://localhost:8081/api/config/sources' -H 'accept: application/json'`
6. Add any new sources
7. Add a Discord Web Hook
8. Create your subscription links
1. This is a link between a source and a discord web hook. Without this, the app will not send a notification about new posts.
### Errors
- pq: permission denied to create extension "uuid-ossp"
- Might need to grant your account `ALTER USER root WITH SUPERUSER;` to create the 'uuid-ossp' for uuid creations

100
databaseRest/articles.go
View File

@ -1,100 +0,0 @@
package databaseRest
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/jtom38/newsbot/collector/domain/model"
)
// Generate this struct fr
type ArticlesClient struct {
rootUri string
}
func (ac *ArticlesClient) List() ([]model.Articles, error) {
var items []model.Articles
url := fmt.Sprintf("%v/api/v1/articles", ac.rootUri)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return []model.Articles{}, err
}
return items, nil
}
func (ac *ArticlesClient) FindByID(ID uint) (model.Articles, error) {
var items model.Articles
url := fmt.Sprintf("%v/api/v1/articles/%v", ac.rootUri, ID)
resp, err := getContent(url)
if err != nil {
return items, err
}
err = json.Unmarshal(resp, &items)
if err != nil {
return items, err
}
return items, nil
}
func (ac *ArticlesClient) FindByUrl(url string) (model.Articles, error) {
var item model.Articles
get := fmt.Sprintf("%v/api/v1/articles/url/%v", ac.rootUri, url)
resp, err := getContent(get)
if err != nil {
return item, err
}
err = json.Unmarshal(resp, &item)
if err != nil {
return item, err
}
return item, nil
}
func (ac *ArticlesClient) Delete(id int32) error {
return errors.New("not implemented")
}
func (ac *ArticlesClient) Add(item model.Articles) error {
//return errors.New("not implemented")
url := fmt.Sprintf("%v/api/v1/articles/", ac.rootUri)
bItem, err := json.Marshal(item)
if err != nil {
return err
}
client := &http.Client{}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bItem))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.New("failed to post to the DB")
}
return nil
//body, err := ioutil.ReadAll(resp.Body)
//if err != nil { return err }
}

74
databaseRest/common.go
View File

@ -1,74 +0,0 @@
package databaseRest
import (
"errors"
"io/ioutil"
"log"
"net/http"
"github.com/jtom38/newsbot/collector/services/config"
)
type DatabaseClient struct {
Diagnosis DiagnosisClient
Articles ArticlesClient
Sources SourcesClient
}
// This will generate a new client to interface with the API Database.
func NewDatabaseClient() DatabaseClient {
cc := config.New()
dbUri := cc.GetConfig(config.DB_URI)
var client = DatabaseClient{}
client.Diagnosis.rootUri = dbUri
client.Sources.rootUri = dbUri
client.Articles.rootUri = dbUri
return client
}
func getContent(url string) ([]byte, error) {
client := &http.Client{}
var blank []byte
req, err := http.NewRequest("GET", url, nil)
if err != nil { return blank, err }
// set the user agent header to avoid kick backs.. as much
req.Header.Set("User-Agent", getUserAgent())
log.Printf("Requesting content from %v\n", url)
resp, err := client.Do(req)
if err != nil { return blank, err }
if resp.StatusCode == 404 {
err = errors.New("404 not found")
return blank, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil { return blank, err }
//log.Println(string(body))
return body, nil
}
func httpDelete(url string) error {
client := &http.Client{}
req, err := http.NewRequest("DELETE", url, nil)
if err != nil { return err }
//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
req.Header.Set("User-Agent", getUserAgent())
_, err = client.Do(req)
if err != nil { return err }
return nil
}
func getUserAgent() string {
return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
}

24
databaseRest/diagnosis.go
View File

@ -1,24 +0,0 @@
package databaseRest
import (
"fmt"
"io/ioutil"
"net/http"
//"github.com/jtom38/newsbot/collector/services"
)
type DiagnosisClient struct {
rootUri string
}
func (dc *DiagnosisClient) Ping() error {
dbPing := fmt.Sprintf("%v/ping", dc.rootUri)
resp, err := http.Get(dbPing)
if err != nil { return err }
_, err = ioutil.ReadAll(resp.Body)
if err != nil { return err }
return nil
}

38
databaseRest/sources.go
View File

@ -1,38 +0,0 @@
package databaseRest
import (
"encoding/json"
"fmt"
"log"
"github.com/jtom38/newsbot/collector/domain/model"
)
type SourcesClient struct {
rootUri string
}
func (sb *SourcesClient) List() ([]model.Sources, error) {
var items []model.Sources
url := fmt.Sprintf("%v/api/v1/sources", sb.rootUri)
resp, err := getContent(url)
if err != nil { return items, err }
err = json.Unmarshal(resp, &items)
if err != nil { return []model.Sources{}, err }
return items, nil
}
func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) {
items, err := sb.List()
if err != nil { log.Panicln(err) }
var res []model.Sources
for _, item := range(items) {
if item.Source == SourceType {
res = append(res, item)
}
}
return res, nil
}

33
docker-compose.yaml Normal file
View File

@ -0,0 +1,33 @@
version: "3"
networks:
newsbot:
services:
api:
image: ghcr.io/jtom38/newsbot.collector.api:master
environment:
SQL_CONNECTION_STRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
# Used for database migrations
GOOSE_DRIVER: "postgres"
# Connection String to Postgresql
GOOSE_DBSTRING: "host=localhost user=postgres password=postgres dbname=postgres sslmode=disable"
# Enable/Disable Reddit monitoring
FEATURE_ENABLE_REDDIT_BACKEND: true
# Enable/Disable YouTube monitoring
FEATURE_ENABLE_YOUTUBE_BACKEND: false
# Set your Twitch Developer ID and Secrets here and they will be used to collect updates.
TWITCH_CLIENT_ID: ""
TWITCH_CLIENT_SECRET: ""
# If you want to collect news on Final Fantasy XIV, set this to true
FEATURE_ENABLE_FFXIV_BACKEND: false
ports:
- 8081:8081
networks:
- newsbot

4
docs/docs.go
View File

@ -40,8 +40,8 @@ const docTemplate = `{
"parameters": [
{
"type": "string",
"description": "Source ID UUID",
"name": "id",
"description": "Tag name",
"name": "Tag",
"in": "query",
"required": true
}

4
docs/swagger.json
View File

@ -31,8 +31,8 @@
"parameters": [
{
"type": "string",
"description": "Source ID UUID",
"name": "id",
"description": "Tag name",
"name": "Tag",
"in": "query",
"required": true
}

4
docs/swagger.yaml
View File

@ -29,9 +29,9 @@ paths:
/articles/by/sourceid:
get:
parameters:
- description: Source ID UUID
- description: Tag name
in: query
name: id
name: Tag
required: true
type: string
produces:

42
services/config/config.go
View File

@ -11,37 +11,34 @@ import (
)
const (
DB_URI string = "DB_URI"
DB_URI string = "DB_URI"
Sql_Connection_String string = "SQL_CONNECTION_STRING"
FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
FEATURE_ENABLE_YOUTUBE_BACKEND = "FEATURE_ENABLE_YOUTUBE_BACKEND"
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
FEATURE_ENABLE_TWITCH_BACKEND = "FEATURE_ENABLE_TWITCH_BACKEND"
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
FEATURE_ENABLE_FFXIV_BACKEND = "FEATURE_ENABLE_FFXIV_BACKEND"
)
type ConfigClient struct {}
type ConfigClient struct{}
func New() ConfigClient {
_, err := os.Open(".env")
if err == nil {
loadEnvFile()
}
c := ConfigClient{}
c.RefreshEnv()
return ConfigClient{}
return c
}
func (cc *ConfigClient) GetConfig(key string) string {
@ -70,7 +67,16 @@ func (cc *ConfigClient) GetFeature(flag string) (bool, error) {
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
func (cc *ConfigClient) RefreshEnv() {
loadEnvFile()
// Check to see if we have the env file on the system
_, err := os.Stat(".env")
// We have the file, load it.
if err == nil {
_, err := os.Open(".env")
if err == nil {
loadEnvFile()
}
}
}
func loadEnvFile() {

3
services/cron/scheduler.go
View File

@ -26,6 +26,9 @@ type Cron struct {
func openDatabase() (*database.Queries, error) {
_env := config.New()
connString := _env.GetConfig(config.Sql_Connection_String)
if connString == "" {
panic("Connection String is null!")
}
db, err := sql.Open("postgres", connString)
if err != nil {
panic(err)