James Tombleson
eba63c27ef
* added extra packages to help with parsing * getting the core built for Youtube collection. The feed can be pulled and starting to build the article object * added some tests, reddit will need more love but youtube is starting off with more tests. Starting to add Rod to pull missing values from the site * Added rod to work with browser automation * Moved the config inside the client as they can change within runtime and need to be refreshed on each client creation * added more features to collect data per video and tests to support them. * adding the cache and tests * moved errors to var values at the top * youtube is now getting collected and tests have been setup * resolved an issue with the config * setting up the first interface, its not used yet * more updates to the cache service. Not finished yet and could see rework * added logic to monitor Youtube. Still basic logic that needs to be wired up to the database
74 lines
1.7 KiB
Go
74 lines
1.7 KiB
Go
package database
|
|
|
|
import (
|
|
"errors"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
|
|
"github.com/jtom38/newsbot/collector/services"
|
|
)
|
|
|
|
type DatabaseClient struct {
|
|
Diagnosis DiagnosisClient
|
|
|
|
Articles ArticlesClient
|
|
Sources SourcesClient
|
|
}
|
|
|
|
// This will generate a new client to interface with the API Database.
|
|
func NewDatabaseClient() DatabaseClient {
|
|
cc := services.NewConfigClient()
|
|
dbUri := cc.GetConfig(services.DB_URI)
|
|
|
|
var client = DatabaseClient{}
|
|
client.Diagnosis.rootUri = dbUri
|
|
client.Sources.rootUri = dbUri
|
|
client.Articles.rootUri = dbUri
|
|
|
|
return client
|
|
}
|
|
|
|
func getContent(url string) ([]byte, error) {
|
|
client := &http.Client{}
|
|
var blank []byte
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil { return blank, err }
|
|
|
|
// set the user agent header to avoid kick backs.. as much
|
|
req.Header.Set("User-Agent", getUserAgent())
|
|
|
|
log.Printf("Requesting content from %v\n", url)
|
|
resp, err := client.Do(req)
|
|
if err != nil { return blank, err }
|
|
if resp.StatusCode == 404 {
|
|
err = errors.New("404 not found")
|
|
return blank, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil { return blank, err }
|
|
|
|
//log.Println(string(body))
|
|
return body, nil
|
|
}
|
|
|
|
func httpDelete(url string) error {
|
|
client := &http.Client{}
|
|
req, err := http.NewRequest("DELETE", url, nil)
|
|
if err != nil { return err }
|
|
|
|
//req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0")
|
|
req.Header.Set("User-Agent", getUserAgent())
|
|
|
|
_, err = client.Do(req)
|
|
if err != nil { return err }
|
|
|
|
return nil
|
|
}
|
|
|
|
func getUserAgent() string {
|
|
return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0"
|
|
} |