Getting the basics setup for the collector. It can pull reddit json and almost post the db api
This commit is contained in:
parent
c6620e3c88
commit
d8e85b1bd7
64
.github/workflows/docker.build.yaml
vendored
Normal file
64
.github/workflows/docker.build.yaml
vendored
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
name: Docker
|
||||||
|
|
||||||
|
# This workflow uses actions that are not certified by GitHub.
|
||||||
|
# They are provided by a third-party and are governed by
|
||||||
|
# separate terms of service, privacy policy, and support
|
||||||
|
# documentation.
|
||||||
|
|
||||||
|
on:
|
||||||
|
#schedule:
|
||||||
|
# - cron: '21 19 * * *'
|
||||||
|
push:
|
||||||
|
branches: [ master ]
|
||||||
|
# Publish semver tags as releases.
|
||||||
|
tags: [ 'v*.*.*' ]
|
||||||
|
#pull_request:
|
||||||
|
# branches: [ master ]
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Use docker.io for Docker Hub if empty
|
||||||
|
REGISTRY: ghcr.io
|
||||||
|
# github.repository as <account>/<repo>
|
||||||
|
IMAGE_NAME: ${{ github.repository }}
|
||||||
|
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
# Login against a Docker registry except on PR
|
||||||
|
# https://github.com/docker/login-action
|
||||||
|
- name: Log into registry ${{ env.REGISTRY }}
|
||||||
|
if: github.event_name != 'pull_request'
|
||||||
|
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
|
||||||
|
with:
|
||||||
|
registry: ${{ env.REGISTRY }}
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
# Extract metadata (tags, labels) for Docker
|
||||||
|
# https://github.com/docker/metadata-action
|
||||||
|
- name: Extract Docker metadata
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
|
||||||
|
with:
|
||||||
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
|
#images: ${{ env.REGISTRY }}/newsbot.worker
|
||||||
|
|
||||||
|
# Build and push Docker image with Buildx (don't push on PR)
|
||||||
|
# https://github.com/docker/build-push-action
|
||||||
|
- name: Build and push Docker image
|
||||||
|
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
push: ${{ github.event_name != 'pull_request' }}
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,9 +1,12 @@
|
|||||||
|
.env
|
||||||
|
|
||||||
# Binaries for programs and plugins
|
# Binaries for programs and plugins
|
||||||
*.exe
|
*.exe
|
||||||
*.exe~
|
*.exe~
|
||||||
*.dll
|
*.dll
|
||||||
*.so
|
*.so
|
||||||
*.dylib
|
*.dylib
|
||||||
|
collector
|
||||||
|
|
||||||
# Test binary, built with `go test -c`
|
# Test binary, built with `go test -c`
|
||||||
*.test
|
*.test
|
||||||
|
|||||||
15
.vscode/launch.json
vendored
Normal file
15
.vscode/launch.json
vendored
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
// Use IntelliSense to learn about possible attributes.
|
||||||
|
// Hover to view descriptions of existing attributes.
|
||||||
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Launch Package",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "auto",
|
||||||
|
"program": "."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
10
Dockerfile
Normal file
10
Dockerfile
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
FROM golang:1.16 as build
|
||||||
|
|
||||||
|
COPY . /app
|
||||||
|
WORKDIR /app
|
||||||
|
RUN go build .
|
||||||
|
|
||||||
|
FROM alpine
|
||||||
|
|
||||||
|
COPY --from=build /app/db /app
|
||||||
|
ENTRYPOINT [ "/app/db" ]
|
||||||
34
database/articles.go
Normal file
34
database/articles.go
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"github.com/jtom38/newsbot/collector/domain/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Generate this struct fr
|
||||||
|
type ArticlesClient struct {
|
||||||
|
rootUri string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *ArticlesClient) List() []model.Articles {
|
||||||
|
var items []model.Articles
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *ArticlesClient) Find() []model.Articles {
|
||||||
|
var items []model.Articles
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *ArticlesClient) FindByUrl(url string) model.Articles {
|
||||||
|
return model.Articles{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *ArticlesClient) Delete(id int32) error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *ArticlesClient) Add() error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
67
database/common.go
Normal file
67
database/common.go
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/jtom38/newsbot/collector/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DatabaseClient struct {
|
||||||
|
Diagnosis DiagnosisClient
|
||||||
|
|
||||||
|
Articles ArticlesClient
|
||||||
|
Sources SourcesClient
|
||||||
|
}
|
||||||
|
|
||||||
|
// This will generate a new client to interface with the API Database.
|
||||||
|
func NewDatabaseClient() DatabaseClient {
|
||||||
|
var dbUri = services.NewConfigClient().GetConfig(services.DB_URI)
|
||||||
|
|
||||||
|
var client = DatabaseClient{}
|
||||||
|
client.Diagnosis.rootUri = dbUri
|
||||||
|
client.Sources.rootUri = dbUri
|
||||||
|
client.Articles.rootUri = dbUri
|
||||||
|
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
func getContent(url string) []byte {
|
||||||
|
client := &http.Client{}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
|
if err != nil { log.Fatalln(err) }
|
||||||
|
|
||||||
|
// set the user agent header to avoid kick backs.. as much
|
||||||
|
req.Header.Set("User-Agent", getUserAgent())
|
||||||
|
|
||||||
|
log.Printf("Requesting content from %v\n", url)
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil { log.Fatalln(err) }
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil { log.Fatalln(err) }
|
||||||
|
|
||||||
|
//log.Println(string(body))
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
func httpDelete(url string) error {
|
||||||
|
client := &http.Client{}
|
||||||
|
req, err := http.NewRequest("DELETE", url, nil)
|
||||||
|
if err != nil { return err }
|
||||||
|
|
||||||
|
//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
||||||
|
req.Header.Set("User-Agent", getUserAgent())
|
||||||
|
|
||||||
|
_, err = client.Do(req)
|
||||||
|
if err != nil { return err }
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getUserAgent() string {
|
||||||
|
return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
|
||||||
|
}
|
||||||
24
database/diagnosis.go
Normal file
24
database/diagnosis.go
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
//"github.com/jtom38/newsbot/collector/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DiagnosisClient struct {
|
||||||
|
rootUri string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (dc *DiagnosisClient) Ping() error {
|
||||||
|
dbPing := fmt.Sprintf("%v/ping", dc.rootUri)
|
||||||
|
resp, err := http.Get(dbPing)
|
||||||
|
if err != nil { return err }
|
||||||
|
|
||||||
|
_, err = ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil { return err }
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
37
database/sources.go
Normal file
37
database/sources.go
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"github.com/jtom38/newsbot/collector/domain/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type SourcesClient struct {
|
||||||
|
rootUri string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *SourcesClient) List() ([]model.Sources, error) {
|
||||||
|
var items []model.Sources
|
||||||
|
url := fmt.Sprintf("%v/v1/sources", sb.rootUri)
|
||||||
|
resp := getContent(url)
|
||||||
|
|
||||||
|
err := json.Unmarshal(resp, &items)
|
||||||
|
if err != nil { return []model.Sources{}, err }
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *SourcesClient) FindBySource(SourceType string) ([]model.Sources, error) {
|
||||||
|
items, err := sb.List()
|
||||||
|
if err != nil { log.Panicln(err) }
|
||||||
|
|
||||||
|
var res []model.Sources
|
||||||
|
for _, item := range(items) {
|
||||||
|
if item.Source == SourceType {
|
||||||
|
res = append(res, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
98
domain/model/database.go
Normal file
98
domain/model/database.go
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Articles represents the model for an Article
|
||||||
|
type Articles struct {
|
||||||
|
ID int64 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
|
SourceId int32 `json:"sourceId"`
|
||||||
|
Tags string `json:"tags"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Url string `json:"url"`
|
||||||
|
PubDate time.Time `json:"pubdate"`
|
||||||
|
Video string `json:"video"`
|
||||||
|
VideoHeight int16 `json:"videoHeight"`
|
||||||
|
VideoWidth int16 `json:"videoWidth"`
|
||||||
|
Thumbnail string `json:"thumbnail"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
AuthorName string `json:"authorName"`
|
||||||
|
AuthorImage string `json:"authorImage"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type DiscordQueue struct {
|
||||||
|
ID int64 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
ArticleId string `json:"articleId"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type DiscordWebHooks struct {
|
||||||
|
ID int32 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
|
Name string `json:"name"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
Url string `json:"url"`
|
||||||
|
Server string `json:"server"`
|
||||||
|
Channel string `json:"channel"`
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Icons struct {
|
||||||
|
ID int32 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
|
FileName string `json:"fileName"`
|
||||||
|
Site string `json:"site"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Settings struct {
|
||||||
|
ID int16 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
|
Key string `json:"key"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Options string `json:"options"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Sources struct {
|
||||||
|
ID int32 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
|
Site string `json:"site"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
Url string `json:"url"`
|
||||||
|
Tags string `json:"tags"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type SourceLinks struct {
|
||||||
|
ID int32 `json:"ID"`
|
||||||
|
CreatedAt time.Time `json:"CreatedAt"`
|
||||||
|
UpdatedAt time.Time `json:"UpdatedAt"`
|
||||||
|
DeletedAt time.Time `json:"DeletedAt"`
|
||||||
|
|
||||||
|
SourceID string `json:"sourceId"`
|
||||||
|
SourceType string `json:"sourceType"`
|
||||||
|
SourceName string `json:"sourceName"`
|
||||||
|
DiscordID string `json:"discordId"`
|
||||||
|
DiscordName string `json:"discordName"`
|
||||||
|
}
|
||||||
51
domain/model/reddit.go
Normal file
51
domain/model/reddit.go
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
// This is the root Json object. It does not contain data that we care about though.
|
||||||
|
type RedditJsonContent struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Data RedditJsonContentData `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RedditJsonContentData struct {
|
||||||
|
After string `json:"after"`
|
||||||
|
Dist int `json:"dist"`
|
||||||
|
Modhash string `json:"modhash"`
|
||||||
|
Children []RedditJsonContentChildren `json:"children"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RedditJsonContentChildren struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Data RedditPost `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RedditPost contains the information that was posted by a user.
|
||||||
|
type RedditPost struct {
|
||||||
|
Subreddit string `json:"subreddit"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Content string `json:"selftext"`
|
||||||
|
ContentHtml string `json:"selftext_html"`
|
||||||
|
Author string `json:"author"`
|
||||||
|
Permalink string `json:"permalink"`
|
||||||
|
IsVideo bool `json:"is_video"`
|
||||||
|
Media RedditPostMedia `json:"media"`
|
||||||
|
Url string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RedditPostMedia defines if the post contains a video that is hosted on Reddit.
|
||||||
|
type RedditPostMedia struct {
|
||||||
|
RedditVideo RedditPostMediaRedditVideo `json:"reddit_video"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RedditVideo contains information about the video in the post.
|
||||||
|
type RedditPostMediaRedditVideo struct {
|
||||||
|
BitrateKbps int `json:"bitrate_kpbs"`
|
||||||
|
FallBackUrl string `json:"fallback_url"`
|
||||||
|
Height int `json:"height"`
|
||||||
|
Width int `json:"width"`
|
||||||
|
ScrubberMediaUrl string `json:"scrubber_media_url"`
|
||||||
|
DashUrl string `json:"dash_url"`
|
||||||
|
Duration int `json:"duration"`
|
||||||
|
HlsUrl string `json:"hls_url"`
|
||||||
|
IsGif bool `json:"is_gif"`
|
||||||
|
TranscodingStatus string `json:"transcoding_status"`
|
||||||
|
}
|
||||||
9
go.mod
Normal file
9
go.mod
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
module github.com/jtom38/newsbot/collector
|
||||||
|
|
||||||
|
go 1.16
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/go-chi/chi/v5 v5.0.7 // indirect
|
||||||
|
github.com/joho/godotenv v1.4.0 // indirect
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||||
|
)
|
||||||
159
go.sum
Normal file
159
go.sum
Normal file
@ -0,0 +1,159 @@
|
|||||||
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||||
|
github.com/czasg/go-queue v0.0.0-20211206021309-e3b3e4c4ae3b/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4=
|
||||||
|
github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8 h1:N1zVvx6PqlP5W99D4wmvEoEf9z7dczVICL/oW/g66vc=
|
||||||
|
github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4=
|
||||||
|
github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c h1:UPAcTyzz3s5GPRb/inSVniQQ4tM1vv2ZcpRIUjJqleM=
|
||||||
|
github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c/go.mod h1:MKIE5kTpr1G+ONHemz6SAAOwnzFPDMln5GXYhmMJue8=
|
||||||
|
github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e h1:q3yK8340BwYlDIEnFNxEwUQ2Hh9C3EEtDVglaeAG0fs=
|
||||||
|
github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e/go.mod h1:sRIiPHI++Op2UGzIH7TeZemadsaj00XSN6KwUDVfw0o=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||||
|
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||||
|
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
||||||
|
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||||
|
github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso=
|
||||||
|
github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg=
|
||||||
|
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
|
||||||
|
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
|
||||||
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||||
|
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||||
|
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||||
|
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||||
|
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||||
|
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||||
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
|
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
|
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||||
|
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||||
|
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||||
|
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||||
|
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||||
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||||
|
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||||
|
github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||||
|
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||||
|
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||||
|
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
|
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
|
||||||
|
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||||
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||||
|
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||||
|
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
|
||||||
|
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
|
||||||
|
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||||
|
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
|
||||||
|
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||||
|
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
|
||||||
|
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
||||||
|
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||||
|
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||||
|
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
|
||||||
|
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
|
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||||
|
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
|
||||||
|
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||||
|
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||||
|
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||||
|
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||||
|
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||||
|
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||||
|
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||||
|
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||||
|
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||||
|
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w=
|
||||||
|
mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ=
|
||||||
58
main.go
Normal file
58
main.go
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
//"fmt"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
|
||||||
|
"github.com/jtom38/newsbot/collector/routes"
|
||||||
|
"github.com/jtom38/newsbot/collector/database"
|
||||||
|
"github.com/jtom38/newsbot/collector/domain/model"
|
||||||
|
"github.com/jtom38/newsbot/collector/services"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
//EnableScheduler()
|
||||||
|
dc := database.NewDatabaseClient()
|
||||||
|
err := dc.Diagnosis.Ping()
|
||||||
|
if err != nil { log.Fatalln(err) }
|
||||||
|
|
||||||
|
CheckReddit()
|
||||||
|
|
||||||
|
app := chi.NewRouter()
|
||||||
|
app.Use(middleware.Logger)
|
||||||
|
app.Use(middleware.Recoverer)
|
||||||
|
|
||||||
|
//app.Mount("/swagger", httpSwagger.WrapHandler)
|
||||||
|
app.Mount("/api", routes.RootRoutes())
|
||||||
|
|
||||||
|
log.Println("API is online and waiting for requests.")
|
||||||
|
log.Println("API: http://localhost:8081/api")
|
||||||
|
//log.Println("Swagger: http://localhost:8080/swagger/index.html")
|
||||||
|
err = http.ListenAndServe(":8081", app)
|
||||||
|
if err != nil { log.Fatalln(err) }
|
||||||
|
}
|
||||||
|
|
||||||
|
func CheckReddit() {
|
||||||
|
dc := database.NewDatabaseClient()
|
||||||
|
dc.Articles.List()
|
||||||
|
sources, err := dc.Sources.FindBySource("reddit")
|
||||||
|
if err != nil { log.Println(err) }
|
||||||
|
|
||||||
|
rc := services.NewReddit(sources[0].Name, sources[0].ID)
|
||||||
|
raw, err := rc.GetContent()
|
||||||
|
if err != nil { log.Println(err) }
|
||||||
|
|
||||||
|
var redditArticles []model.Articles
|
||||||
|
for _, item := range raw.Data.Children {
|
||||||
|
var article model.Articles
|
||||||
|
article, err = rc.ConvertToArticle(item.Data)
|
||||||
|
redditArticles = append(redditArticles, article)
|
||||||
|
}
|
||||||
|
dc.Articles.Add()
|
||||||
|
|
||||||
|
}
|
||||||
25
routes/root.go
Normal file
25
routes/root.go
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
package routes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"fmt"
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
func RootRoutes() chi.Router {
|
||||||
|
app := chi.NewRouter()
|
||||||
|
app.Get("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Write([]byte("Hello World!"))
|
||||||
|
})
|
||||||
|
|
||||||
|
app.Get("/ping", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
msg := "pong"
|
||||||
|
w.Write([]byte(msg))
|
||||||
|
})
|
||||||
|
|
||||||
|
app.Get("/hello/{world}", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
msg := fmt.Sprintf("Hello %v", chi.URLParam(r, "world"))
|
||||||
|
w.Write([]byte(msg))
|
||||||
|
})
|
||||||
|
return app
|
||||||
|
}
|
||||||
19
scheduler.go
Normal file
19
scheduler.go
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Hello(t string) {
|
||||||
|
fmt.Println("hello " + t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func EnableScheduler() {
|
||||||
|
c := cron.New()
|
||||||
|
c.AddFunc("*/1 * * * *", func() {
|
||||||
|
go Hello("new world order")
|
||||||
|
})
|
||||||
|
c.Start()
|
||||||
|
}
|
||||||
47
services/config.go
Normal file
47
services/config.go
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
package services
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"github.com/joho/godotenv"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
DB_URI string = "DB_URI"
|
||||||
|
|
||||||
|
REDDIT_PULL_TOP = "REDDIT_PULL_TOP"
|
||||||
|
REDDIT_PULL_HOT = "REDDIT_PULL_HOT"
|
||||||
|
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ConfigClient struct {}
|
||||||
|
|
||||||
|
func NewConfigClient() ConfigClient {
|
||||||
|
_, err := os.Open(".env")
|
||||||
|
if err == nil {
|
||||||
|
loadEnvFile()
|
||||||
|
}
|
||||||
|
|
||||||
|
return ConfigClient{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cc ConfigClient) GetConfig(key string) string {
|
||||||
|
res, filled := os.LookupEnv(key)
|
||||||
|
if !filled {
|
||||||
|
log.Printf("Missing the a value for '%v'. Could generate errors.", key)
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use this when your ConfigClient has been opened for awhile and you want to ensure you have the most recent env changes.
|
||||||
|
func (cc ConfigClient) RefreshEnv() {
|
||||||
|
loadEnvFile()
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadEnvFile() {
|
||||||
|
err := godotenv.Load()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
29
services/httpClient.go
Normal file
29
services/httpClient.go
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
package services
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"log"
|
||||||
|
"io/ioutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This will use the net/http client reach out to a site and collect the content.
|
||||||
|
func getHttpContent(uri string) ([]byte, error) {
|
||||||
|
|
||||||
|
client := &http.Client{}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", uri, nil)
|
||||||
|
if err != nil { return nil, err }
|
||||||
|
|
||||||
|
// set the user agent header to avoid kick backs.. as much
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
||||||
|
|
||||||
|
log.Printf("Requesting content from %v\n", uri)
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil { log.Fatalln(err) }
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil { return nil, err }
|
||||||
|
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
90
services/reddit.go
Normal file
90
services/reddit.go
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
package services
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"github.com/jtom38/newsbot/collector/domain/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RedditClient struct {
|
||||||
|
subreddit string
|
||||||
|
url string
|
||||||
|
sourceId int32
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
PULLTOP string
|
||||||
|
PULLHOT string
|
||||||
|
PULLNSFW string
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cc := NewConfigClient()
|
||||||
|
PULLTOP = cc.GetConfig(REDDIT_PULL_TOP)
|
||||||
|
PULLHOT = cc.GetConfig(REDDIT_PULL_HOT)
|
||||||
|
PULLNSFW = cc.GetConfig(REDDIT_PULL_NSFW)
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewReddit(subreddit string, sourceID int32) RedditClient {
|
||||||
|
rc := RedditClient{
|
||||||
|
subreddit: subreddit,
|
||||||
|
url: fmt.Sprintf("https://www.reddit.com/r/%v.json", subreddit),
|
||||||
|
sourceId: sourceID,
|
||||||
|
}
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetContent() reaches out to Reddit and pulls the Json data.
|
||||||
|
// It will then convert the data to a struct and return the struct.
|
||||||
|
func (rc RedditClient) GetContent() (model.RedditJsonContent, error ) {
|
||||||
|
var items model.RedditJsonContent = model.RedditJsonContent{}
|
||||||
|
|
||||||
|
log.Printf("Collecting results on '%v'", rc.subreddit)
|
||||||
|
content, err := getHttpContent(rc.url)
|
||||||
|
if err != nil { return items, err }
|
||||||
|
|
||||||
|
json.Unmarshal(content, &items)
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConvertToArticle() will take the reddit model struct and convert them over to Article structs.
|
||||||
|
// This data can be passed to the database.
|
||||||
|
func (rc RedditClient) ConvertToArticle(source model.RedditPost) (model.Articles, error) {
|
||||||
|
var item model.Articles
|
||||||
|
|
||||||
|
|
||||||
|
if source.Content == "" && source.Url != ""{
|
||||||
|
item = rc.convertPicturePost(source)
|
||||||
|
}
|
||||||
|
|
||||||
|
if item.Description == "" {
|
||||||
|
var err = errors.New("reddit post failed to parse correctly")
|
||||||
|
return item, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rc RedditClient) convertPicturePost(source model.RedditPost) model.Articles {
|
||||||
|
var item = model.Articles{
|
||||||
|
SourceId: rc.sourceId,
|
||||||
|
Url: fmt.Sprintf("https://www.reddit.com/%v", source.Permalink),
|
||||||
|
Title: source.Title,
|
||||||
|
AuthorName: source.Author,
|
||||||
|
Description: source.Content,
|
||||||
|
|
||||||
|
}
|
||||||
|
return item
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rc RedditClient) isTextPost(source model.RedditPost) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rc RedditClient) isVideoPost(source model.RedditPost) {
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user