From d8e85b1bd7aa17deffd9fdf0234bd991911996c7 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Sat, 2 Apr 2022 12:05:32 -0700 Subject: [PATCH] Getting the basics setup for the collector. It can pull reddit json and almost post the db api --- .github/workflows/docker.build.yaml | 64 +++++++++++ .gitignore | 3 + .vscode/launch.json | 15 +++ Dockerfile | 10 ++ database/articles.go | 34 ++++++ database/common.go | 67 ++++++++++++ database/diagnosis.go | 24 +++++ database/sources.go | 37 +++++++ domain/model/database.go | 98 +++++++++++++++++ domain/model/reddit.go | 51 +++++++++ go.mod | 9 ++ go.sum | 159 ++++++++++++++++++++++++++++ main.go | 58 ++++++++++ routes/root.go | 25 +++++ scheduler.go | 19 ++++ services/config.go | 47 ++++++++ services/httpClient.go | 29 +++++ services/reddit.go | 90 ++++++++++++++++ 18 files changed, 839 insertions(+) create mode 100644 .github/workflows/docker.build.yaml create mode 100644 .vscode/launch.json create mode 100644 Dockerfile create mode 100644 database/articles.go create mode 100644 database/common.go create mode 100644 database/diagnosis.go create mode 100644 database/sources.go create mode 100644 domain/model/database.go create mode 100644 domain/model/reddit.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 routes/root.go create mode 100644 scheduler.go create mode 100644 services/config.go create mode 100644 services/httpClient.go create mode 100644 services/reddit.go diff --git a/.github/workflows/docker.build.yaml b/.github/workflows/docker.build.yaml new file mode 100644 index 0000000..7b4e750 --- /dev/null +++ b/.github/workflows/docker.build.yaml @@ -0,0 +1,64 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + #schedule: + # - cron: '21 19 * * *' + push: + branches: [ master ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + #pull_request: + # branches: [ master ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + #images: ${{ env.REGISTRY }}/newsbot.worker + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 66fd13c..9b6d9fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,12 @@ +.env + # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib +collector # Test binary, built with `go test -c` *.test diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..f944eb2 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "." + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..559dfed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.16 as build + +COPY . /app +WORKDIR /app +RUN go build . + +FROM alpine + +COPY --from=build /app/db /app +ENTRYPOINT [ "/app/db" ] \ No newline at end of file diff --git a/database/articles.go b/database/articles.go new file mode 100644 index 0000000..bd1754a --- /dev/null +++ b/database/articles.go @@ -0,0 +1,34 @@ +package database + +import ( + "errors" + + "github.com/jtom38/newsbot/collector/domain/model" +) + +// Generate this struct fr +type ArticlesClient struct { + rootUri string +} + +func (ac *ArticlesClient) List() []model.Articles { + var items []model.Articles + return items +} + +func (ac *ArticlesClient) Find() []model.Articles { + var items []model.Articles + return items +} + +func (ac *ArticlesClient) FindByUrl(url string) model.Articles { + return model.Articles{} +} + +func (ac *ArticlesClient) Delete(id int32) error { + return errors.New("not implemented") +} + +func (ac *ArticlesClient) Add() error { + return errors.New("not implemented") +} diff --git a/database/common.go b/database/common.go new file mode 100644 index 0000000..eaaf8b6 --- /dev/null +++ b/database/common.go @@ -0,0 +1,67 @@ +package database + +import ( + "log" + "io/ioutil" + "net/http" + + "github.com/jtom38/newsbot/collector/services" +) + +type DatabaseClient struct { + Diagnosis DiagnosisClient + + Articles ArticlesClient + Sources SourcesClient +} + +// This will generate a new client to interface with the API Database. +func NewDatabaseClient() DatabaseClient { + var dbUri = services.NewConfigClient().GetConfig(services.DB_URI) + + var client = DatabaseClient{} + client.Diagnosis.rootUri = dbUri + client.Sources.rootUri = dbUri + client.Articles.rootUri = dbUri + + return client +} + +func getContent(url string) []byte { + client := &http.Client{} + + req, err := http.NewRequest("GET", url, nil) + if err != nil { log.Fatalln(err) } + + // set the user agent header to avoid kick backs.. as much + req.Header.Set("User-Agent", getUserAgent()) + + log.Printf("Requesting content from %v\n", url) + resp, err := client.Do(req) + if err != nil { log.Fatalln(err) } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { log.Fatalln(err) } + + //log.Println(string(body)) + return body +} + +func httpDelete(url string) error { + client := &http.Client{} + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { return err } + + //req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0") + req.Header.Set("User-Agent", getUserAgent()) + + _, err = client.Do(req) + if err != nil { return err } + + return nil +} + +func getUserAgent() string { + return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0" +} \ No newline at end of file diff --git a/database/diagnosis.go b/database/diagnosis.go new file mode 100644 index 0000000..d0333b8 --- /dev/null +++ b/database/diagnosis.go @@ -0,0 +1,24 @@ +package database + +import ( + "fmt" + "io/ioutil" + "net/http" + + //"github.com/jtom38/newsbot/collector/services" +) + +type DiagnosisClient struct { + rootUri string +} + +func (dc *DiagnosisClient) Ping() error { + dbPing := fmt.Sprintf("%v/ping", dc.rootUri) + resp, err := http.Get(dbPing) + if err != nil { return err } + + _, err = ioutil.ReadAll(resp.Body) + if err != nil { return err } + return nil +} + diff --git a/database/sources.go b/database/sources.go new file mode 100644 index 0000000..bffad50 --- /dev/null +++ b/database/sources.go @@ -0,0 +1,37 @@ +package database + +import ( + "encoding/json" + "fmt" + "log" + + "github.com/jtom38/newsbot/collector/domain/model" +) + +type SourcesClient struct { + rootUri string +} + +func (sb *SourcesClient) List() ([]model.Sources, error) { + var items []model.Sources + url := fmt.Sprintf("%v/v1/sources", sb.rootUri) + resp := getContent(url) + + err := json.Unmarshal(resp, &items) + if err != nil { return []model.Sources{}, err } + + return items, nil +} + +func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) { + items, err := sb.List() + if err != nil { log.Panicln(err) } + + var res []model.Sources + for _, item := range(items) { + if item.Source == SourceType { + res = append(res, item) + } + } + return res, nil +} \ No newline at end of file diff --git a/domain/model/database.go b/domain/model/database.go new file mode 100644 index 0000000..21d4a11 --- /dev/null +++ b/domain/model/database.go @@ -0,0 +1,98 @@ +package model + +import ( + "time" +) + +// Articles represents the model for an Article +type Articles struct { + ID int64 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + + SourceId int32 `json:"sourceId"` + Tags string `json:"tags"` + Title string `json:"title"` + Url string `json:"url"` + PubDate time.Time `json:"pubdate"` + Video string `json:"video"` + VideoHeight int16 `json:"videoHeight"` + VideoWidth int16 `json:"videoWidth"` + Thumbnail string `json:"thumbnail"` + Description string `json:"description"` + AuthorName string `json:"authorName"` + AuthorImage string `json:"authorImage"` +} + +type DiscordQueue struct { + ID int64 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + ArticleId string `json:"articleId"` +} + +type DiscordWebHooks struct { + ID int32 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + + Name string `json:"name"` + Key string `json:"key"` + Url string `json:"url"` + Server string `json:"server"` + Channel string `json:"channel"` + Enabled bool `json:"enabled"` +} + +type Icons struct { + ID int32 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + + FileName string `json:"fileName"` + Site string `json:"site"` +} + +type Settings struct { + ID int16 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + + Key string `json:"key"` + Value string `json:"value"` + Options string `json:"options"` +} + +type Sources struct { + ID int32 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + + Site string `json:"site"` + Name string `json:"name"` + Source string `json:"source"` + Type string `json:"type"` + Value string `json:"value"` + Enabled bool `json:"enabled"` + Url string `json:"url"` + Tags string `json:"tags"` +} + +type SourceLinks struct { + ID int32 `json:"ID"` + CreatedAt time.Time `json:"CreatedAt"` + UpdatedAt time.Time `json:"UpdatedAt"` + DeletedAt time.Time `json:"DeletedAt"` + + SourceID string `json:"sourceId"` + SourceType string `json:"sourceType"` + SourceName string `json:"sourceName"` + DiscordID string `json:"discordId"` + DiscordName string `json:"discordName"` +} diff --git a/domain/model/reddit.go b/domain/model/reddit.go new file mode 100644 index 0000000..b82ca57 --- /dev/null +++ b/domain/model/reddit.go @@ -0,0 +1,51 @@ +package model + +// This is the root Json object. It does not contain data that we care about though. +type RedditJsonContent struct { + Kind string `json:"kind"` + Data RedditJsonContentData `json:"data"` +} + +type RedditJsonContentData struct { + After string `json:"after"` + Dist int `json:"dist"` + Modhash string `json:"modhash"` + Children []RedditJsonContentChildren `json:"children"` +} + +type RedditJsonContentChildren struct { + Kind string `json:"kind"` + Data RedditPost `json:"data"` +} + +// RedditPost contains the information that was posted by a user. +type RedditPost struct { + Subreddit string `json:"subreddit"` + Title string `json:"title"` + Content string `json:"selftext"` + ContentHtml string `json:"selftext_html"` + Author string `json:"author"` + Permalink string `json:"permalink"` + IsVideo bool `json:"is_video"` + Media RedditPostMedia `json:"media"` + Url string `json:"url"` +} + +// RedditPostMedia defines if the post contains a video that is hosted on Reddit. +type RedditPostMedia struct { + RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"` +} + +// RedditVideo contains information about the video in the post. +type RedditPostMediaRedditVideo struct { + BitrateKbps int `json:"bitrate_kpbs"` + FallBackUrl string `json:"fallback_url"` + Height int `json:"height"` + Width int `json:"width"` + ScrubberMediaUrl string `json:"scrubber_media_url"` + DashUrl string `json:"dash_url"` + Duration int `json:"duration"` + HlsUrl string `json:"hls_url"` + IsGif bool `json:"is_gif"` + TranscodingStatus string `json:"transcoding_status"` +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d18423e --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/jtom38/newsbot/collector + +go 1.16 + +require ( + github.com/go-chi/chi/v5 v5.0.7 // indirect + github.com/joho/godotenv v1.4.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..00c4e81 --- /dev/null +++ b/go.sum @@ -0,0 +1,159 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/czasg/go-queue v0.0.0-20211206021309-e3b3e4c4ae3b/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4= +github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8 h1:N1zVvx6PqlP5W99D4wmvEoEf9z7dczVICL/oW/g66vc= +github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4= +github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c h1:UPAcTyzz3s5GPRb/inSVniQQ4tM1vv2ZcpRIUjJqleM= +github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c/go.mod h1:MKIE5kTpr1G+ONHemz6SAAOwnzFPDMln5GXYhmMJue8= +github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e h1:q3yK8340BwYlDIEnFNxEwUQ2Hh9C3EEtDVglaeAG0fs= +github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e/go.mod h1:sRIiPHI++Op2UGzIH7TeZemadsaj00XSN6KwUDVfw0o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= +github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso= +github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= +mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= diff --git a/main.go b/main.go new file mode 100644 index 0000000..1a352bb --- /dev/null +++ b/main.go @@ -0,0 +1,58 @@ +package main + +import ( + //"fmt" + "log" + "net/http" + + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + + "github.com/jtom38/newsbot/collector/routes" + "github.com/jtom38/newsbot/collector/database" + "github.com/jtom38/newsbot/collector/domain/model" + "github.com/jtom38/newsbot/collector/services" +) + +func main() { + //EnableScheduler() + dc := database.NewDatabaseClient() + err := dc.Diagnosis.Ping() + if err != nil { log.Fatalln(err) } + + CheckReddit() + + app := chi.NewRouter() + app.Use(middleware.Logger) + app.Use(middleware.Recoverer) + + //app.Mount("/swagger", httpSwagger.WrapHandler) + app.Mount("/api", routes.RootRoutes()) + + log.Println("API is online and waiting for requests.") + log.Println("API: http://localhost:8081/api") + //log.Println("Swagger: http://localhost:8080/swagger/index.html") + err = http.ListenAndServe(":8081", app) + if err != nil { log.Fatalln(err) } +} + +func CheckReddit() { + dc := database.NewDatabaseClient() + dc.Articles.List() + sources, err := dc.Sources.FindBySource("reddit") + if err != nil { log.Println(err) } + + rc := services.NewReddit(sources[0].Name, sources[0].ID) + raw, err := rc.GetContent() + if err != nil { log.Println(err) } + + var redditArticles []model.Articles + for _, item := range raw.Data.Children { + var article model.Articles + article, err = rc.ConvertToArticle(item.Data) + redditArticles = append(redditArticles, article) + } + dc.Articles.Add() + +} \ No newline at end of file diff --git a/routes/root.go b/routes/root.go new file mode 100644 index 0000000..4eeae7d --- /dev/null +++ b/routes/root.go @@ -0,0 +1,25 @@ +package routes + +import ( + "net/http" + "fmt" + "github.com/go-chi/chi/v5" +) + +func RootRoutes() chi.Router { + app := chi.NewRouter() + app.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Hello World!")) + }) + + app.Get("/ping", func(w http.ResponseWriter, r *http.Request) { + msg := "pong" + w.Write([]byte(msg)) + }) + + app.Get("/hello/{world}", func(w http.ResponseWriter, r *http.Request) { + msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "world")) + w.Write([]byte(msg)) + }) + return app +} \ No newline at end of file diff --git a/scheduler.go b/scheduler.go new file mode 100644 index 0000000..a6b2f1d --- /dev/null +++ b/scheduler.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + + "github.com/robfig/cron/v3" +) + +func Hello(t string) { + fmt.Println("hello " + t) +} + +func EnableScheduler() { + c := cron.New() + c.AddFunc("*/1 * * * *", func() { + go Hello("new world order") + }) + c.Start() +} \ No newline at end of file diff --git a/services/config.go b/services/config.go new file mode 100644 index 0000000..5681a50 --- /dev/null +++ b/services/config.go @@ -0,0 +1,47 @@ +package services + +import ( + "os" + "log" + + "github.com/joho/godotenv" +) + +const ( + DB_URI string = "DB_URI" + + REDDIT_PULL_TOP = "REDDIT_PULL_TOP" + REDDIT_PULL_HOT = "REDDIT_PULL_HOT" + REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW" +) + +type ConfigClient struct {} + +func NewConfigClient() ConfigClient { + _, err := os.Open(".env") + if err == nil { + loadEnvFile() + } + + return ConfigClient{} +} + +func (cc ConfigClient) GetConfig(key string) string { + res, filled := os.LookupEnv(key) + if !filled { + log.Printf("Missing the a value for '%v'. Could generate errors.", key) + } + return res +} + +// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes. +func (cc ConfigClient) RefreshEnv() { + loadEnvFile() +} + +func loadEnvFile() { + err := godotenv.Load() + if err != nil { + log.Fatalln(err) + } +} diff --git a/services/httpClient.go b/services/httpClient.go new file mode 100644 index 0000000..cd59fdc --- /dev/null +++ b/services/httpClient.go @@ -0,0 +1,29 @@ +package services + +import ( + "net/http" + "log" + "io/ioutil" +) + +// This will use the net/http client reach out to a site and collect the content. +func getHttpContent(uri string) ([]byte, error) { + + client := &http.Client{} + + req, err := http.NewRequest("GET", uri, nil) + if err != nil { return nil, err } + + // set the user agent header to avoid kick backs.. as much + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0") + + log.Printf("Requesting content from %v\n", uri) + resp, err := client.Do(req) + if err != nil { log.Fatalln(err) } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { return nil, err } + + return body, nil +} \ No newline at end of file diff --git a/services/reddit.go b/services/reddit.go new file mode 100644 index 0000000..be62aa8 --- /dev/null +++ b/services/reddit.go @@ -0,0 +1,90 @@ +package services + +import ( + "encoding/json" + "fmt" + "log" + "errors" + + "github.com/jtom38/newsbot/collector/domain/model" +) + +type RedditClient struct { + subreddit string + url string + sourceId int32 +} + +var ( + PULLTOP string + PULLHOT string + PULLNSFW string +) + +func init() { + cc := NewConfigClient() + PULLTOP = cc.GetConfig(REDDIT_PULL_TOP) + PULLHOT = cc.GetConfig(REDDIT_PULL_HOT) + PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW) +} + +func NewReddit(subreddit string, sourceID int32) RedditClient { + rc := RedditClient{ + subreddit: subreddit, + url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit), + sourceId: sourceID, + } + return rc +} + +// GetContent() reaches out to Reddit and pulls the Json data. +// It will then convert the data to a struct and return the struct. +func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) { + var items model.RedditJsonContent = model.RedditJsonContent{} + + log.Printf("Collecting results on '%v'", rc.subreddit) + content, err := getHttpContent(rc.url) + if err != nil { return items, err } + + json.Unmarshal(content, &items) + + return items, nil +} + +// ConvertToArticle() will take the reddit model struct and convert them over to Article structs. +// This data can be passed to the database. +func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) { + var item model.Articles + + + if source.Content == "" && source.Url != ""{ + item = rc.convertPicturePost(source) + } + + if item.Description == "" { + var err = errors.New("reddit post failed to parse correctly") + return item, err + } + + return item, nil +} + +func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles { + var item = model.Articles{ + SourceId: rc.sourceId, + Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink), + Title: source.Title, + AuthorName: source.Author, + Description: source.Content, + + } + return item +} + +func (rc RedditClient) isTextPost(source model.RedditPost) { + +} + +func (rc RedditClient) isVideoPost(source model.RedditPost) { + +} \ No newline at end of file