Features/twitch (#7)
* ran go mod tity * updated formatting and moved to using some common errors * some common source errors * bumped go version * a bulk of the work has been completed to parse twitch videos/users * Added twitch config values * added common errors * moved the scheduler to its own package to get it out of main and also start adding tests * added a new err for invalid author images * Updated an error used * tests updated to avoid name duplication * new func to replace the cached source recor and updated getcontent to return collected posts * updated scheduler ref * moved to services/cron
This commit is contained in:
GitHub
parent
11892b9a7b
commit
333a4f5345
@ -1,4 +1,4 @@
|
||||
FROM golang:1.16 as build
|
||||
FROM golang:1.18.2 as build
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
31
go.mod
31
go.mod
@ -1,13 +1,28 @@
|
||||
module github.com/jtom38/newsbot/collector
|
||||
|
||||
go 1.16
|
||||
go 1.18
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.7 // indirect
|
||||
github.com/go-rod/rod v0.105.1 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/joho/godotenv v1.4.0 // indirect
|
||||
github.com/mmcdole/gofeed v1.1.3 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.8.0
|
||||
github.com/go-chi/chi/v5 v5.0.7
|
||||
github.com/go-rod/rod v0.105.1
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/mmcdole/gofeed v1.1.3
|
||||
github.com/nicklaw5/helix/v2 v2.4.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
|
||||
github.com/json-iterator/go v1.1.10 // indirect
|
||||
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/gson v0.7.1 // indirect
|
||||
github.com/ysmood/leakless v0.7.0 // indirect
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
)
|
||||
|
||||
146
go.sum
146
go.sum
@ -1,4 +1,3 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
@ -6,61 +5,23 @@ github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejn
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/czasg/go-queue v0.0.0-20211206021309-e3b3e4c4ae3b/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4=
|
||||
github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8 h1:N1zVvx6PqlP5W99D4wmvEoEf9z7dczVICL/oW/g66vc=
|
||||
github.com/czasg/go-queue v0.0.0-20211206102528-0d03e5c8ace8/go.mod h1:Myb/1g8zHhdc13TwwkSuJ5wShsub1yoPrT+ta8isZQ4=
|
||||
github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c h1:UPAcTyzz3s5GPRb/inSVniQQ4tM1vv2ZcpRIUjJqleM=
|
||||
github.com/czasg/go-sche v0.0.0-20211207124133-70e74bf90c4c/go.mod h1:MKIE5kTpr1G+ONHemz6SAAOwnzFPDMln5GXYhmMJue8=
|
||||
github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e h1:q3yK8340BwYlDIEnFNxEwUQ2Hh9C3EEtDVglaeAG0fs=
|
||||
github.com/czasg/gonal v0.0.0-20211207122219-7f69affd4b3e/go.mod h1:sRIiPHI++Op2UGzIH7TeZemadsaj00XSN6KwUDVfw0o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
|
||||
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso=
|
||||
github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg=
|
||||
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
|
||||
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
|
||||
github.com/go-rod/rod v0.105.1 h1:r0bNmO9siOe13lG6Vbkaak11u48rYmWGl/Hk4MJdOiE=
|
||||
github.com/go-rod/rod v0.105.1/go.mod h1:Wrnn6HokFHskwaIVke3ML1y/NBVp7XPIeB8eDzR9vuw=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
|
||||
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
|
||||
@ -69,131 +30,42 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OH
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
|
||||
github.com/nicklaw5/helix/v2 v2.4.0 h1:ZvqCKVqza1eJYyqgTRrZ/xjDq0w/EQVFNkN067Utls0=
|
||||
github.com/nicklaw5/helix/v2 v2.4.0/go.mod h1:0ONzvVi1cH+k3a7EDIFNNqxfW0podhf+CqlmFvuexq8=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
|
||||
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
|
||||
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
|
||||
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/got v0.23.2 h1:U2U0vyQ/gDaawkKJZK/hyza8UUXbWCurbmazK7AcAfY=
|
||||
github.com/ysmood/got v0.23.2/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
|
||||
github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
|
||||
github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
|
||||
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w=
|
||||
mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ=
|
||||
|
||||
4
main.go
4
main.go
@ -8,14 +8,14 @@ import (
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/routes"
|
||||
"github.com/jtom38/newsbot/collector/services/cron"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//dc := database.NewDatabaseClient()
|
||||
//err := dc.Diagnosis.Ping()
|
||||
//if err != nil { log.Fatalln(err) }
|
||||
|
||||
EnableScheduler()
|
||||
cron.EnableScheduler()
|
||||
|
||||
app := chi.NewRouter()
|
||||
app.Use(middleware.Logger)
|
||||
|
||||
17
services/common.go
Normal file
17
services/common.go
Normal file
@ -0,0 +1,17 @@
|
||||
package services
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrMissingAuthorName = errors.New("unable to find the post author name")
|
||||
ErrMissingAuthorImage = errors.New("unable to find the post author image")
|
||||
ErrMissingThumbnail = errors.New("unable to find the post thumbnail url")
|
||||
ErrMissingPublishDate = errors.New("unable to find the post publish date")
|
||||
ErrMissingTags = errors.New("unable to find the post tags")
|
||||
ErrMissingDescription = errors.New("unable to find the post description")
|
||||
ErrMissingUrl = errors.New("unable to find the post url")
|
||||
|
||||
ErrInvalidAuthorImage = errors.New("expected value looks to be wrong, something is missing")
|
||||
)
|
||||
|
||||
const DATETIME_FORMAT string = "1/2/2006 3:4 PM"
|
||||
@ -15,6 +15,11 @@ const (
|
||||
REDDIT_PULL_NSFW = "REDDIT_PULL_NSFW"
|
||||
|
||||
YOUTUBE_DEBUG = "YOUTUBE_DEBUG"
|
||||
|
||||
TWITCH_CLIENT_ID = "TWITCH_CLIENT_ID"
|
||||
TWITCH_CLIENT_SECRET = "TWITCH_CLIENT_SECRET"
|
||||
TWITCH_MONITOR_CLIPS = "TWITCH_MONITOR_CLIPS"
|
||||
TWITCH_MONITOR_VOD = "TWITCH_MONITOR_VOD"
|
||||
)
|
||||
|
||||
type ConfigClient struct {}
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package main
|
||||
package cron
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
@ -9,20 +8,16 @@ import (
|
||||
"github.com/jtom38/newsbot/collector/database"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
//"github.com/jtom38/newsbot/collector/services/cache"
|
||||
|
||||
)
|
||||
|
||||
func Hello(t string) {
|
||||
fmt.Println("hello " + t)
|
||||
}
|
||||
|
||||
func EnableScheduler() {
|
||||
c := cron.New()
|
||||
|
||||
//c.AddFunc("*/5 * * * *", func() { go CheckCache() })
|
||||
c.AddFunc("*/30 * * * *", func() { go CheckReddit() })
|
||||
c.AddFunc("*/30 * * * *", func() { go CheckYoutube() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckReddit() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckYoutube() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckFfxiv() })
|
||||
c.AddFunc("* */1 * * *", func() { go CheckTwitch() })
|
||||
|
||||
c.Start()
|
||||
}
|
||||
@ -77,3 +72,35 @@ func CheckFfxiv() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CheckTwitch() error {
|
||||
// TODO Wire this for the DB
|
||||
// just a mock object for now
|
||||
dc := database.NewDatabaseClient()
|
||||
|
||||
sources, err := dc.Sources.FindBySource("Twitch")
|
||||
if err != nil { return err }
|
||||
|
||||
client, err := services.NewTwitchClient(sources[0])
|
||||
if err != nil { log.Println(err) }
|
||||
|
||||
err = client.Login()
|
||||
if err != nil { return err }
|
||||
|
||||
for _, source := range sources {
|
||||
client.ReplaceSourceRecord(source)
|
||||
|
||||
posts, err := client.GetContent()
|
||||
if err != nil { return err }
|
||||
|
||||
for _, item := range posts {
|
||||
_, err = dc.Articles.FindByUrl(item.Url)
|
||||
if err != nil {
|
||||
err = dc.Articles.Add(item)
|
||||
if err != nil { log.Println("Failed to post article.")}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
7
services/cron/scheduler_test.go
Normal file
7
services/cron/scheduler_test.go
Normal file
@ -0,0 +1,7 @@
|
||||
package cron_test
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestInvokeTwitch(t *testing.T) {
|
||||
|
||||
}
|
||||
230
services/twitch.go
Normal file
230
services/twitch.go
Normal file
@ -0,0 +1,230 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
//"log"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/services/config"
|
||||
"github.com/nicklaw5/helix/v2"
|
||||
)
|
||||
|
||||
type TwitchClient struct {
|
||||
SourceRecord model.Sources
|
||||
|
||||
// config
|
||||
monitorClips string
|
||||
monitorVod string
|
||||
|
||||
// API Connection
|
||||
api *helix.Client
|
||||
}
|
||||
|
||||
var (
|
||||
ErrTwitchClientIdMissing = errors.New("unable to find the client id for auth")
|
||||
ErrTwitchClientSecretMissing = errors.New("unable to find the client secret for auth")
|
||||
|
||||
twitchScopes = "user:read:email"
|
||||
)
|
||||
|
||||
func NewTwitchClient(source model.Sources) (TwitchClient, error) {
|
||||
c := config.New()
|
||||
|
||||
id := c.GetConfig(config.TWITCH_CLIENT_ID)
|
||||
if id == "" {
|
||||
return TwitchClient{}, ErrTwitchClientIdMissing
|
||||
}
|
||||
|
||||
secret := c.GetConfig(config.TWITCH_CLIENT_SECRET)
|
||||
if secret == "" {
|
||||
return TwitchClient{}, ErrTwitchClientSecretMissing
|
||||
}
|
||||
|
||||
api, err := initTwitchApi(id, secret)
|
||||
if err != nil {
|
||||
return TwitchClient{}, nil
|
||||
}
|
||||
|
||||
client := TwitchClient{
|
||||
SourceRecord: source,
|
||||
monitorClips: c.GetConfig(config.TWITCH_MONITOR_CLIPS),
|
||||
monitorVod: c.GetConfig(config.TWITCH_MONITOR_VOD),
|
||||
api: &api,
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Sets up the API connection to Twitch.
|
||||
func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) {
|
||||
api, err := helix.NewClient(&helix.Options{
|
||||
ClientID: ClientId,
|
||||
ClientSecret: ClientSecret,
|
||||
})
|
||||
if err != nil {
|
||||
return helix.Client{}, err
|
||||
}
|
||||
|
||||
return *api, nil
|
||||
}
|
||||
|
||||
// This will let you replace the bound source record to keep the same session alive.
|
||||
func (tc TwitchClient) ReplaceSourceRecord(source model.Sources) {
|
||||
tc.SourceRecord = source
|
||||
}
|
||||
|
||||
// Invokes Logon request to the API
|
||||
func (tc TwitchClient) Login() error {
|
||||
token, err := tc.api.RequestAppAccessToken([]string{twitchScopes})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tc.api.SetAppAccessToken(token.Data.AccessToken)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) GetContent() ([]model.Articles, error) {
|
||||
var items []model.Articles
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil {
|
||||
return items, err
|
||||
}
|
||||
|
||||
for _, video := range posts {
|
||||
article := model.Articles{}
|
||||
|
||||
article.AuthorName, err = tc.ExtractAuthor(video)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.AuthorImage, err = tc.ExtractAuthorImage(user)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.Description, err = tc.ExtractDescription(video)
|
||||
if err != nil {return items, err }
|
||||
|
||||
article.PubDate, err = tc.ExtractPubDate(video)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.SourceID = tc.SourceRecord.ID
|
||||
article.Tags, err = tc.ExtractTags(video, user)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.Thumbnail, err = tc.ExtractThumbnail(video)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.Title, err = tc.ExtractTitle(video)
|
||||
if err != nil { return items, err }
|
||||
|
||||
article.Url, err = tc.ExtractUrl(video)
|
||||
if err != nil { return items, err }
|
||||
|
||||
items = append(items, article)
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) GetUserDetails() (helix.User, error) {
|
||||
var blank helix.User
|
||||
|
||||
users, err := tc.api.GetUsers(&helix.UsersParams{
|
||||
Logins: []string{tc.SourceRecord.Name},
|
||||
})
|
||||
if err != nil {
|
||||
return blank, err
|
||||
}
|
||||
return users.Data.Users[0], nil
|
||||
}
|
||||
|
||||
// This will reach out and collect the posts made by the user.
|
||||
func (tc TwitchClient) GetPosts(user helix.User) ([]helix.Video, error) {
|
||||
var blank []helix.Video
|
||||
|
||||
videos, err := tc.api.GetVideos(&helix.VideosParams{
|
||||
UserID: user.ID,
|
||||
})
|
||||
if err != nil {
|
||||
return blank, err
|
||||
}
|
||||
|
||||
//log.Println(videos.Data.Videos)
|
||||
return videos.Data.Videos, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractAuthor(post helix.Video) (string, error) {
|
||||
if post.UserName == "" {
|
||||
return "", ErrMissingAuthorName
|
||||
}
|
||||
return post.UserName, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractThumbnail(post helix.Video) (string, error) {
|
||||
if post.ThumbnailURL == "" {
|
||||
return "", ErrMissingThumbnail
|
||||
}
|
||||
var thumb = post.ThumbnailURL
|
||||
thumb = strings.Replace(thumb, "%{width}", "600", -1)
|
||||
thumb = strings.Replace(thumb, "%{height}", "400", -1)
|
||||
return thumb, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractPubDate(post helix.Video) (time.Time, error) {
|
||||
if post.PublishedAt == "" {
|
||||
return time.Now(), ErrMissingPublishDate
|
||||
}
|
||||
pubDate, err := time.Parse("2006-01-02T15:04:05Z", post.PublishedAt)
|
||||
if err != nil {
|
||||
return time.Now(), err
|
||||
}
|
||||
return pubDate, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractDescription(post helix.Video) (string, error) {
|
||||
// Check if the description is null but we have a title.
|
||||
// The poster didnt add a description but this isnt an error.
|
||||
if post.Description == "" && post.Title == "" {
|
||||
return "", ErrMissingDescription
|
||||
}
|
||||
if post.Description == "" {
|
||||
return "No description was given", nil
|
||||
}
|
||||
return post.Description, nil
|
||||
}
|
||||
|
||||
// Extracts the avatar of the author with some validation.
|
||||
func (tc TwitchClient) ExtractAuthorImage(user helix.User) (string, error) {
|
||||
if user.ProfileImageURL == "" { return "", ErrMissingAuthorImage }
|
||||
if !strings.Contains(user.ProfileImageURL, "-profile_image-") { return "", ErrInvalidAuthorImage }
|
||||
return user.ProfileImageURL, nil
|
||||
}
|
||||
|
||||
// Generate tags based on the video metadata.
|
||||
// TODO Figure out how to query what game is played
|
||||
func (tc TwitchClient) ExtractTags(post helix.Video, user helix.User) (string, error) {
|
||||
res := fmt.Sprintf("twitch,%v,%v", post.Title, user.DisplayName)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Extracts the title from a post with some validation.
|
||||
func (tc TwitchClient) ExtractTitle(post helix.Video) (string, error) {
|
||||
if post.Title == "" {
|
||||
return "", errors.New("unable to find the title on the requested post")
|
||||
}
|
||||
return post.Title, nil
|
||||
}
|
||||
|
||||
func (tc TwitchClient) ExtractUrl(post helix.Video) (string, error) {
|
||||
if post.URL == "" { return "", ErrMissingUrl }
|
||||
return post.URL, nil
|
||||
}
|
||||
265
services/twitch_test.go
Normal file
265
services/twitch_test.go
Normal file
@ -0,0 +1,265 @@
|
||||
package services_test
|
||||
|
||||
import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/jtom38/newsbot/collector/domain/model"
|
||||
"github.com/jtom38/newsbot/collector/services"
|
||||
)
|
||||
|
||||
var sourceRecord = model.Sources{
|
||||
ID: 1,
|
||||
Name: "nintendo",
|
||||
Source: "Twitch",
|
||||
}
|
||||
|
||||
var invalidRecord = model.Sources{
|
||||
ID: 1,
|
||||
Name: "EvilNintendo",
|
||||
Source: "Twitch",
|
||||
}
|
||||
|
||||
func TestTwitchLogin(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// reach out and confirms that the API returns posts made by the user.
|
||||
func TestTwitchReturnsUserPosts(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(posts) == 0 {
|
||||
t.Error("expected videos but got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwitchReturnsNothingDueToInvalidUserName(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(invalidRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(posts) != 0 {
|
||||
t.Error("expected videos but got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwitchReturnsVideoAuthor(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if posts[0].UserName == "" {
|
||||
t.Error("uable to parse username")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwitchReturnsThumbnail(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {t.Error(err) }
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
value, err := tc.ExtractThumbnail(posts[0])
|
||||
if err != nil { t.Error(err) }
|
||||
if value == "" { t.Error("uable to parse username") }
|
||||
}
|
||||
|
||||
func TestTwitchReturnsPubDate(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
date, err := tc.ExtractPubDate(posts[0])
|
||||
log.Println(date)
|
||||
if err != nil { t.Error(err) }
|
||||
}
|
||||
|
||||
func TestTwitchReturnsDescription(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
_, err = tc.ExtractDescription(posts[0])
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwitchReturnsAuthorImage(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {t.Error(err) }
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {t.Error(err) }
|
||||
|
||||
_, err = tc.ExtractAuthorImage(user)
|
||||
if err != nil { t.Error(err) }
|
||||
}
|
||||
|
||||
func TestTwitchReturnsTags(t *testing.T) {
|
||||
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
_, err = tc.ExtractTags(posts[0], user)
|
||||
if err != nil { t.Error(err) }
|
||||
}
|
||||
|
||||
func TestTwitchReturnsTitle(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
res, err := tc.ExtractTitle(posts[0])
|
||||
if err != nil { t.Error(err) }
|
||||
if res == "" { t.Error("expected a filled string but got nil")}
|
||||
}
|
||||
|
||||
func TestTwitchReturnsUrl(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
user, err := tc.GetUserDetails()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
posts, err := tc.GetPosts(user)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
res, err := tc.ExtractUrl(posts[0])
|
||||
if err != nil { t.Error(err) }
|
||||
if res == "" { t.Error("expected a filled string but got nil")}
|
||||
}
|
||||
|
||||
func TestTwitchGetContent(t *testing.T) {
|
||||
tc, err := services.NewTwitchClient(sourceRecord)
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
err = tc.Login()
|
||||
if err != nil { t.Error(err) }
|
||||
|
||||
posts, err := tc.GetContent()
|
||||
if err != nil {t.Error(err) }
|
||||
if len(posts) == 0 { t.Error("posts came back with 0 posts") }
|
||||
if len(posts) != 20 { t.Error("expected 20 posts") }
|
||||
}
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
//"strconv"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
@ -19,39 +20,34 @@ type YoutubeClient struct {
|
||||
Url string
|
||||
ChannelID string
|
||||
AvatarUri string
|
||||
Config YoutubeConfig
|
||||
|
||||
// config
|
||||
//debug bool
|
||||
|
||||
// cache config
|
||||
cacheGroup string
|
||||
}
|
||||
|
||||
type YoutubeConfig struct {
|
||||
Debug bool
|
||||
}
|
||||
|
||||
var (
|
||||
// This is a local slice to store what URI's have been seen to remove extra calls to the DB
|
||||
YoutubeUriCache []*string
|
||||
|
||||
ErrThumbnailMissing = errors.New("unable to find the video thumbnail on a youtube video")
|
||||
ErrTagsMissing = errors.New("unable to find the tags on the video")
|
||||
ErrAvatarMissing = errors.New("unable to find the avatar image on the page")
|
||||
ErrChannelIdMissing = errors.New("unable to find the channelId on the requested page")
|
||||
ErrYoutubeChannelIdMissing = errors.New("unable to find the channelId on the requested page")
|
||||
)
|
||||
|
||||
const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id="
|
||||
|
||||
func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
|
||||
yc := YoutubeClient{
|
||||
SourceID: SourceID,
|
||||
Url: Url,
|
||||
SourceID: SourceID,
|
||||
Url: Url,
|
||||
cacheGroup: "youtube",
|
||||
}
|
||||
/*
|
||||
cc := NewConfigClient()
|
||||
cc := NewConfigClient()
|
||||
|
||||
debug, err := strconv.ParseBool(cc.GetConfig(YOUTUBE_DEBUG))
|
||||
if err != nil { panic("'YOUTUBE_DEBUG' was not a bool value")}
|
||||
yc.Config.Debug = debug
|
||||
debug, err := strconv.ParseBool(cc.GetConfig(YOUTUBE_DEBUG))
|
||||
if err != nil { panic("'YOUTUBE_DEBUG' was not a bool value")}
|
||||
yc.Config.Debug = debug
|
||||
*/
|
||||
return yc
|
||||
}
|
||||
@ -59,28 +55,42 @@ func NewYoutubeClient(SourceID uint, Url string) YoutubeClient {
|
||||
// CheckSource will go and run all the commands needed to process a source.
|
||||
func (yc *YoutubeClient) CheckSource() error {
|
||||
docParser, err := yc.GetParser(yc.Url)
|
||||
if err != nil { return err }
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check cache/db for existing value
|
||||
// If we have the value, skip
|
||||
//channelId, err := yc.extractChannelId()
|
||||
channelId, err := yc.GetChannelId(docParser)
|
||||
if err != nil { return err }
|
||||
if channelId == "" { return ErrChannelIdMissing }
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if channelId == "" {
|
||||
return ErrYoutubeChannelIdMissing
|
||||
}
|
||||
yc.ChannelID = channelId
|
||||
|
||||
// Check the cache/db forthe value.
|
||||
// if we have the value, skip
|
||||
avatar, err := yc.GetAvatarUri()
|
||||
if err != nil { return err }
|
||||
if avatar == "" { return ErrAvatarMissing }
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if avatar == "" {
|
||||
return ErrMissingAuthorImage
|
||||
}
|
||||
yc.AvatarUri = avatar
|
||||
|
||||
feed, err := yc.PullFeed()
|
||||
if err != nil { return err }
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newPosts, err := yc.CheckForNewPosts(feed)
|
||||
if err != nil { return err }
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//TODO post to the API
|
||||
for _, item := range newPosts {
|
||||
@ -131,7 +141,7 @@ func (yc *YoutubeClient) GetChannelId(doc *goquery.Document) (string, error) {
|
||||
return yc.ChannelID, nil
|
||||
}
|
||||
}
|
||||
return "", ErrChannelIdMissing
|
||||
return "", ErrYoutubeChannelIdMissing
|
||||
}
|
||||
|
||||
// This pulls the youtube page and finds the ChannelID.
|
||||
@ -150,7 +160,7 @@ func (yc *YoutubeClient) GetAvatarUri() (string, error) {
|
||||
res := page.MustElement("#channel-header-container > yt-img-shadow:nth-child(1) > img:nth-child(1)").MustAttribute("src")
|
||||
|
||||
if *res == "" || res == nil {
|
||||
return AvatarUri, ErrAvatarMissing
|
||||
return AvatarUri, ErrMissingAuthorImage
|
||||
}
|
||||
|
||||
AvatarUri = *res
|
||||
@ -170,7 +180,7 @@ func (yc *YoutubeClient) GetTags(parser *goquery.Document) (string, error) {
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
return "", ErrTagsMissing
|
||||
return "", ErrMissingTags
|
||||
}
|
||||
|
||||
func (yc *YoutubeClient) GetVideoThumbnail(parser *goquery.Document) (string, error) {
|
||||
@ -182,7 +192,7 @@ func (yc *YoutubeClient) GetVideoThumbnail(parser *goquery.Document) (string, er
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
return "", ErrThumbnailMissing
|
||||
return "", ErrMissingThumbnail
|
||||
}
|
||||
|
||||
// This will pull the RSS feed items and return the results
|
||||
@ -205,7 +215,9 @@ func (yc *YoutubeClient) CheckForNewPosts(feed *gofeed.Feed) ([]*gofeed.Item, er
|
||||
|
||||
// Check the cache/db to see if this URI has been seen already
|
||||
uriExists := yc.CheckUriCache(&item.Link)
|
||||
if uriExists { continue }
|
||||
if uriExists {
|
||||
continue
|
||||
}
|
||||
|
||||
//TODO Check the DB if the cache is not aware
|
||||
//TODO If the db knew about it, append it to the local cache
|
||||
@ -235,13 +247,13 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) model.Articles {
|
||||
|
||||
tags, err := yc.GetTags(parser)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("%v. %v", ErrTagsMissing, item.Link)
|
||||
msg := fmt.Sprintf("%v. %v", err, item.Link)
|
||||
log.Println(msg)
|
||||
}
|
||||
|
||||
thumb, err := yc.GetVideoThumbnail(parser)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("%v. %v", ErrThumbnailMissing, item.Link)
|
||||
msg := fmt.Sprintf("%v. %v", err, item.Link)
|
||||
log.Println(msg)
|
||||
}
|
||||
|
||||
|
||||
@ -49,7 +49,7 @@ func TestGetAvatarUri(t *testing.T) {
|
||||
)
|
||||
res, err := yc.GetAvatarUri()
|
||||
if err != nil { panic(err) }
|
||||
if res == "" { panic(services.ErrAvatarMissing)}
|
||||
if res == "" { panic(services.ErrMissingAuthorImage)}
|
||||
}
|
||||
|
||||
func TestGetVideoTags(t *testing.T) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user