Getting the basics setup for the collector. It can pull reddit json and almost post the db api
This commit is contained in:
parent
c6620e3c88
commit
d8e85b1bd7
|
|
@ -0,0 +1,64 @@
|
|||
name: Docker
|
||||
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
on:
|
||||
#schedule:
|
||||
# - cron: '21 19 * * *'
|
||||
push:
|
||||
branches: [ master ]
|
||||
# Publish semver tags as releases.
|
||||
tags: [ 'v*.*.*' ]
|
||||
#pull_request:
|
||||
# branches: [ master ]
|
||||
|
||||
env:
|
||||
# Use docker.io for Docker Hub if empty
|
||||
REGISTRY: ghcr.io
|
||||
# github.repository as <account>/<repo>
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into registry ${{ env.REGISTRY }}
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Extract metadata (tags, labels) for Docker
|
||||
# https://github.com/docker/metadata-action
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
#images: ${{ env.REGISTRY }}/newsbot.worker
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
|
||||
with:
|
||||
context: .
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
|
@ -1,9 +1,12 @@
|
|||
.env
|
||||
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
collector
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Launch Package",
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"program": "."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
FROM golang:1.16 as build
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
RUN go build .
|
||||
|
||||
FROM alpine
|
||||
|
||||
COPY --from=build /app/db /app
|
||||
ENTRYPOINT [ "/app/db" ]
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
|
||||
// Generate this struct fr
|
||||
type ArticlesClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) List() []model.Articles {
|
||||
var items []model.Articles
|
||||
return items
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Find() []model.Articles {
|
||||
var items []model.Articles
|
||||
return items
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) FindByUrl(url string) model.Articles {
|
||||
return model.Articles{}
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Delete(id int32) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (ac *ArticlesClient) Add() error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"log"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
type DatabaseClient struct {
|
||||
Diagnosis DiagnosisClient
|
||||
|
||||
Articles ArticlesClient
|
||||
Sources SourcesClient
|
||||
}
|
||||
|
||||
// This will generate a new client to interface with the API Database.
|
||||
func NewDatabaseClient() DatabaseClient {
|
||||
var dbUri = services.NewConfigClient().GetConfig(services.DB_URI)
|
||||
|
||||
var client = DatabaseClient{}
|
||||
client.Diagnosis.rootUri = dbUri
|
||||
client.Sources.rootUri = dbUri
|
||||
client.Articles.rootUri = dbUri
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func getContent(url string) []byte {
|
||||
client := &http.Client{}
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
|
||||
// set the user agent header to avoid kick backs.. as much
|
||||
req.Header.Set("User-Agent", getUserAgent())
|
||||
|
||||
log.Printf("Requesting content from %v\n", url)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
|
||||
//log.Println(string(body))
|
||||
return body
|
||||
}
|
||||
|
||||
func httpDelete(url string) error {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil { return err }
|
||||
|
||||
//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
||||
req.Header.Set("User-Agent", getUserAgent())
|
||||
|
||||
_, err = client.Do(req)
|
||||
if err != nil { return err }
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getUserAgent() string {
|
||||
return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
//"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
type DiagnosisClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (dc *DiagnosisClient) Ping() error {
|
||||
dbPing := fmt.Sprintf("%v/ping", dc.rootUri)
|
||||
resp, err := http.Get(dbPing)
|
||||
if err != nil { return err }
|
||||
|
||||
_, err = ioutil.ReadAll(resp.Body)
|
||||
if err != nil { return err }
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
|
||||
type SourcesClient struct {
|
||||
rootUri string
|
||||
}
|
||||
|
||||
func (sb *SourcesClient) List() ([]model.Sources, error) {
|
||||
var items []model.Sources
|
||||
url := fmt.Sprintf("%v/v1/sources", sb.rootUri)
|
||||
resp := getContent(url)
|
||||
|
||||
err := json.Unmarshal(resp, &items)
|
||||
if err != nil { return []model.Sources{}, err }
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) {
|
||||
items, err := sb.List()
|
||||
if err != nil { log.Panicln(err) }
|
||||
|
||||
var res []model.Sources
|
||||
for _, item := range(items) {
|
||||
if item.Source == SourceType {
|
||||
res = append(res, item)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Articles represents the model for an Article
|
||||
type Articles struct {
|
||||
ID int64 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
SourceId int32 `json:"sourceId"`
|
||||
Tags string `json:"tags"`
|
||||
Title string `json:"title"`
|
||||
Url string `json:"url"`
|
||||
PubDate time.Time `json:"pubdate"`
|
||||
Video string `json:"video"`
|
||||
VideoHeight int16 `json:"videoHeight"`
|
||||
VideoWidth int16 `json:"videoWidth"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
Description string `json:"description"`
|
||||
AuthorName string `json:"authorName"`
|
||||
AuthorImage string `json:"authorImage"`
|
||||
}
|
||||
|
||||
type DiscordQueue struct {
|
||||
ID int64 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
ArticleId string `json:"articleId"`
|
||||
}
|
||||
|
||||
type DiscordWebHooks struct {
|
||||
ID int32 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
Name string `json:"name"`
|
||||
Key string `json:"key"`
|
||||
Url string `json:"url"`
|
||||
Server string `json:"server"`
|
||||
Channel string `json:"channel"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type Icons struct {
|
||||
ID int32 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
FileName string `json:"fileName"`
|
||||
Site string `json:"site"`
|
||||
}
|
||||
|
||||
type Settings struct {
|
||||
ID int16 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Options string `json:"options"`
|
||||
}
|
||||
|
||||
type Sources struct {
|
||||
ID int32 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
Site string `json:"site"`
|
||||
Name string `json:"name"`
|
||||
Source string `json:"source"`
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Url string `json:"url"`
|
||||
Tags string `json:"tags"`
|
||||
}
|
||||
|
||||
type SourceLinks struct {
|
||||
ID int32 `json:"ID"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||
DeletedAt time.Time `json:"DeletedAt"`
|
||||
|
||||
SourceID string `json:"sourceId"`
|
||||
SourceType string `json:"sourceType"`
|
||||
SourceName string `json:"sourceName"`
|
||||
DiscordID string `json:"discordId"`
|
||||
DiscordName string `json:"discordName"`
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package model
|
||||
|
||||
// This is the root Json object. It does not contain data that we care about though.
|
||||
type RedditJsonContent struct {
|
||||
Kind string `json:"kind"`
|
||||
Data RedditJsonContentData `json:"data"`
|
||||
}
|
||||
|
||||
type RedditJsonContentData struct {
|
||||
After string `json:"after"`
|
||||
Dist int `json:"dist"`
|
||||
Modhash string `json:"modhash"`
|
||||
Children []RedditJsonContentChildren `json:"children"`
|
||||
}
|
||||
|
||||
type RedditJsonContentChildren struct {
|
||||
Kind string `json:"kind"`
|
||||
Data RedditPost `json:"data"`
|
||||
}
|
||||
|
||||
// RedditPost contains the information that was posted by a user.
|
||||
type RedditPost struct {
|
||||
Subreddit string `json:"subreddit"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"selftext"`
|
||||
ContentHtml string `json:"selftext_html"`
|
||||
Author string `json:"author"`
|
||||
Permalink string `json:"permalink"`
|
||||
IsVideo bool `json:"is_video"`
|
||||
Media RedditPostMedia `json:"media"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
|
||||
// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
|
||||
type RedditPostMedia struct {
|
||||
RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
|
||||
}
|
||||
|
||||
// RedditVideo contains information about the video in the post.
|
||||
type RedditPostMediaRedditVideo struct {
|
||||
BitrateKbps int `json:"bitrate_kpbs"`
|
||||
FallBackUrl string `json:"fallback_url"`
|
||||
Height int `json:"height"`
|
||||
Width int `json:"width"`
|
||||
ScrubberMediaUrl string `json:"scrubber_media_url"`
|
||||
DashUrl string `json:"dash_url"`
|
||||
Duration int `json:"duration"`
|
||||
HlsUrl string `json:"hls_url"`
|
||||
IsGif bool `json:"is_gif"`
|
||||
TranscodingStatus string `json:"transcoding_status"`
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
module github.com/jtom38/newsbot/collector
|
||||
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/go-chi/chi/v5 v5.0.7 // indirect
|
||||
github.com/joho/godotenv v1.4.0 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/czasg/go-queue v0.0.0-20211206021309-e3b3e4c4ae3b/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4=
|
||||
github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8 h1:N1zVvx6PqlP5W99D4wmvEoEf9z7dczVICL/oW/g66vc=
|
||||
github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4=
|
||||
github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c h1:UPAcTyzz3s5GPRb/inSVniQQ4tM1vv2ZcpRIUjJqleM=
|
||||
github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c/go.mod h1:MKIE5kTpr1G+ONHemz6SAAOwnzFPDMln5GXYhmMJue8=
|
||||
github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e h1:q3yK8340BwYlDIEnFNxEwUQ2Hh9C3EEtDVglaeAG0fs=
|
||||
github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e/go.mod h1:sRIiPHI++Op2UGzIH7TeZemadsaj00XSN6KwUDVfw0o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
||||
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso=
|
||||
github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg=
|
||||
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
|
||||
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
|
||||
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
|
||||
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
|
||||
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
|
||||
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w=
|
||||
mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ=
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
//"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/routes"
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//EnableScheduler()
|
||||
dc := database.NewDatabaseClient()
|
||||
err := dc.Diagnosis.Ping()
|
||||
if err != nil { log.Fatalln(err) }
|
||||
|
||||
CheckReddit()
|
||||
|
||||
app := chi.NewRouter()
|
||||
app.Use(middleware.Logger)
|
||||
app.Use(middleware.Recoverer)
|
||||
|
||||
//app.Mount("/swagger", httpSwagger.WrapHandler)
|
||||
app.Mount("/api", routes.RootRoutes())
|
||||
|
||||
log.Println("API is online and waiting for requests.")
|
||||
log.Println("API: http://localhost:8081/api")
|
||||
//log.Println("Swagger: http://localhost:8080/swagger/index.html")
|
||||
err = http.ListenAndServe(":8081", app)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
}
|
||||
|
||||
func CheckReddit() {
|
||||
dc := database.NewDatabaseClient()
|
||||
dc.Articles.List()
|
||||
sources, err := dc.Sources.FindBySource("reddit")
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
rc := services.NewReddit(sources[0].Name, sources[0].ID)
|
||||
raw, err := rc.GetContent()
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
var redditArticles []model.Articles
|
||||
for _, item := range raw.Data.Children {
|
||||
var article model.Articles
|
||||
article, err = rc.ConvertToArticle(item.Data)
|
||||
redditArticles = append(redditArticles, article)
|
||||
}
|
||||
dc.Articles.Add()
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package routes
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"fmt"
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
func RootRoutes() chi.Router {
|
||||
app := chi.NewRouter()
|
||||
app.Get("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("Hello World!"))
|
||||
})
|
||||
|
||||
app.Get("/ping", func(w http.ResponseWriter, r *http.Request) {
|
||||
msg := "pong"
|
||||
w.Write([]byte(msg))
|
||||
})
|
||||
|
||||
app.Get("/hello/{world}", func(w http.ResponseWriter, r *http.Request) {
|
||||
msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "world"))
|
||||
w.Write([]byte(msg))
|
||||
})
|
||||
return app
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
func Hello(t string) {
|
||||
fmt.Println("hello " + t)
|
||||
}
|
||||
|
||||
func EnableScheduler() {
|
||||
c := cron.New()
|
||||
c.AddFunc("*/1 * * * *", func() {
|
||||
go Hello("new world order")
|
||||
})
|
||||
c.Start()
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"os"
|
||||
"log"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
const (
|
||||
DB_URI string = "DB_URI"
|
||||
|
||||
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
|
||||
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||
)
|
||||
|
||||
type ConfigClient struct {}
|
||||
|
||||
func NewConfigClient() ConfigClient {
|
||||
_, err := os.Open(".env")
|
||||
if err == nil {
|
||||
loadEnvFile()
|
||||
}
|
||||
|
||||
return ConfigClient{}
|
||||
}
|
||||
|
||||
func (cc ConfigClient) GetConfig(key string) string {
|
||||
res, filled := os.LookupEnv(key)
|
||||
if !filled {
|
||||
log.Printf("Missing the a value for '%v'. Could generate errors.", key)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
|
||||
func (cc ConfigClient) RefreshEnv() {
|
||||
loadEnvFile()
|
||||
}
|
||||
|
||||
func loadEnvFile() {
|
||||
err := godotenv.Load()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
// This will use the net/http client reach out to a site and collect the content.
|
||||
func getHttpContent(uri string) ([]byte, error) {
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
req, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil { return nil, err }
|
||||
|
||||
// set the user agent header to avoid kick backs.. as much
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
||||
|
||||
log.Printf("Requesting content from %v\n", uri)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil { log.Fatalln(err) }
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil { return nil, err }
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"errors"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
)
|
||||
|
||||
type RedditClient struct {
|
||||
subreddit string
|
||||
url string
|
||||
sourceId int32
|
||||
}
|
||||
|
||||
var (
|
||||
PULLTOP string
|
||||
PULLHOT string
|
||||
PULLNSFW string
|
||||
)
|
||||
|
||||
func init() {
|
||||
cc := NewConfigClient()
|
||||
PULLTOP = cc.GetConfig(REDDIT_PULL_TOP)
|
||||
PULLHOT = cc.GetConfig(REDDIT_PULL_HOT)
|
||||
PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW)
|
||||
}
|
||||
|
||||
func NewReddit(subreddit string, sourceID int32) RedditClient {
|
||||
rc := RedditClient{
|
||||
subreddit: subreddit,
|
||||
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
|
||||
sourceId: sourceID,
|
||||
}
|
||||
return rc
|
||||
}
|
||||
|
||||
// GetContent() reaches out to Reddit and pulls the Json data.
|
||||
// It will then convert the data to a struct and return the struct.
|
||||
func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
|
||||
var items model.RedditJsonContent = model.RedditJsonContent{}
|
||||
|
||||
log.Printf("Collecting results on '%v'", rc.subreddit)
|
||||
content, err := getHttpContent(rc.url)
|
||||
if err != nil { return items, err }
|
||||
|
||||
json.Unmarshal(content, &items)
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||
// This data can be passed to the database.
|
||||
func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) {
|
||||
var item model.Articles
|
||||
|
||||
|
||||
if source.Content == "" && source.Url != ""{
|
||||
item = rc.convertPicturePost(source)
|
||||
}
|
||||
|
||||
if item.Description == "" {
|
||||
var err = errors.New("reddit post failed to parse correctly")
|
||||
return item, err
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
|
||||
var item = model.Articles{
|
||||
SourceId: rc.sourceId,
|
||||
Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink),
|
||||
Title: source.Title,
|
||||
AuthorName: source.Author,
|
||||
Description: source.Content,
|
||||
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func (rc RedditClient) isTextPost(source model.RedditPost) {
|
||||
|
||||
}
|
||||
|
||||
func (rc RedditClient) isVideoPost(source model.RedditPost) {
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue