New endpoints for the portal to use (#31)

* added a route to delete subscriptions based on the ID given

* added a new route to find a record based on the name and source

* added a route to query Discord Web Hooks by Server and Channel names

* tested the endpoints and they seem good to test more
This commit is contained in:
James Tombleson 2022-11-30 21:43:53 -08:00 committed by GitHub
parent 94da578c82
commit c161658487
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 574 additions and 75 deletions

View File

@ -582,6 +582,45 @@ func (q *Queries) GetDiscordWebHooksByID(ctx context.Context, id uuid.UUID) (Dis
return i, err
}
const getDiscordWebHooksByServerAndChannel = `-- name: GetDiscordWebHooksByServerAndChannel :many
SELECT id, url, server, channel, enabled FROM DiscordWebHooks
WHERE Server = $1 and Channel = $2
`
type GetDiscordWebHooksByServerAndChannelParams struct {
Server string
Channel string
}
func (q *Queries) GetDiscordWebHooksByServerAndChannel(ctx context.Context, arg GetDiscordWebHooksByServerAndChannelParams) ([]Discordwebhook, error) {
rows, err := q.db.QueryContext(ctx, getDiscordWebHooksByServerAndChannel, arg.Server, arg.Channel)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Discordwebhook
for rows.Next() {
var i Discordwebhook
if err := rows.Scan(
&i.ID,
&i.Url,
&i.Server,
&i.Channel,
&i.Enabled,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getIconByID = `-- name: GetIconByID :one
Select id, filename, site FROM Icons
Where ID = $1 Limit 1
@ -742,6 +781,32 @@ func (q *Queries) GetSourceByName(ctx context.Context, name string) (Source, err
return i, err
}
const getSourceByNameAndSource = `-- name: GetSourceByNameAndSource :one
Select id, site, name, source, type, value, enabled, url, tags from Sources WHERE name = $1 and source = $2
`
type GetSourceByNameAndSourceParams struct {
Name string
Source string
}
func (q *Queries) GetSourceByNameAndSource(ctx context.Context, arg GetSourceByNameAndSourceParams) (Source, error) {
row := q.db.QueryRowContext(ctx, getSourceByNameAndSource, arg.Name, arg.Source)
var i Source
err := row.Scan(
&i.ID,
&i.Site,
&i.Name,
&i.Source,
&i.Type,
&i.Value,
&i.Enabled,
&i.Url,
&i.Tags,
)
return i, err
}
const getSubscriptionsByDiscordWebHookId = `-- name: GetSubscriptionsByDiscordWebHookId :many
Select id, discordwebhookid, sourceid from subscriptions Where discordwebhookid = $1
`

View File

@ -74,6 +74,10 @@ Where ID = $1 LIMIT 1;
Select * From DiscordWebHooks
Where Server = $1;
-- name: GetDiscordWebHooksByServerAndChannel :many
SELECT * FROM DiscordWebHooks
WHERE Server = $1 and Channel = $2;
-- name: GetDiscordWebHookByUrl :one
Select * From DiscordWebHooks Where url = $1;
@ -146,6 +150,9 @@ Select * From Sources where ID = $1 Limit 1;
-- name: GetSourceByName :one
Select * from Sources where name = $1 Limit 1;
-- name: GetSourceByNameAndSource :one
Select * from Sources WHERE name = $1 and source = $2;
-- name: ListSources :many
Select * From Sources Limit $1;

View File

@ -126,6 +126,35 @@ const docTemplate = `{
"responses": {}
}
},
"/config/sources/by/sourceAndName": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Config",
"Source"
],
"summary": "Returns a single entity by ID",
"parameters": [
{
"type": "string",
"description": "dadjokes",
"name": "name",
"in": "query",
"required": true
},
{
"type": "string",
"description": "reddit",
"name": "source",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/config/sources/new/reddit": {
"post": {
"tags": [
@ -305,6 +334,36 @@ const docTemplate = `{
"responses": {}
}
},
"/discord/webhooks/by/serverAndChannel": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Config",
"Discord",
"Webhook"
],
"summary": "Returns all the known web hooks based on the Server and Channel given.",
"parameters": [
{
"type": "string",
"description": "Fancy Server",
"name": "server",
"in": "query",
"required": true
},
{
"type": "string",
"description": "memes",
"name": "channel",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/discord/webhooks/new": {
"post": {
"tags": [
@ -378,6 +437,24 @@ const docTemplate = `{
}
],
"responses": {}
},
"patch": {
"tags": [
"Config",
"Discord",
"Webhook"
],
"summary": "Updates a valid discord webhook ID based on the body given.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {}
}
},
"/discord/webhooks/{id}/disable": {
@ -543,6 +620,27 @@ const docTemplate = `{
"responses": {}
}
},
"/subscriptions/discord/webhook/delete": {
"delete": {
"tags": [
"Config",
"Source",
"Discord",
"Subscription"
],
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
"parameters": [
{
"type": "string",
"description": "Id",
"name": "Id",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/subscriptions/new/discordwebhook": {
"post": {
"tags": [

View File

@ -117,6 +117,35 @@
"responses": {}
}
},
"/config/sources/by/sourceAndName": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Config",
"Source"
],
"summary": "Returns a single entity by ID",
"parameters": [
{
"type": "string",
"description": "dadjokes",
"name": "name",
"in": "query",
"required": true
},
{
"type": "string",
"description": "reddit",
"name": "source",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/config/sources/new/reddit": {
"post": {
"tags": [
@ -296,6 +325,36 @@
"responses": {}
}
},
"/discord/webhooks/by/serverAndChannel": {
"get": {
"produces": [
"application/json"
],
"tags": [
"Config",
"Discord",
"Webhook"
],
"summary": "Returns all the known web hooks based on the Server and Channel given.",
"parameters": [
{
"type": "string",
"description": "Fancy Server",
"name": "server",
"in": "query",
"required": true
},
{
"type": "string",
"description": "memes",
"name": "channel",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/discord/webhooks/new": {
"post": {
"tags": [
@ -369,6 +428,24 @@
}
],
"responses": {}
},
"patch": {
"tags": [
"Config",
"Discord",
"Webhook"
],
"summary": "Updates a valid discord webhook ID based on the body given.",
"parameters": [
{
"type": "string",
"description": "id",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {}
}
},
"/discord/webhooks/{id}/disable": {
@ -534,6 +611,27 @@
"responses": {}
}
},
"/subscriptions/discord/webhook/delete": {
"delete": {
"tags": [
"Config",
"Source",
"Discord",
"Subscription"
],
"summary": "Removes a Discord WebHook Subscription based on the Subscription ID.",
"parameters": [
{
"type": "string",
"description": "Id",
"name": "Id",
"in": "query",
"required": true
}
],
"responses": {}
}
},
"/subscriptions/new/discordwebhook": {
"post": {
"tags": [

View File

@ -133,6 +133,26 @@ paths:
tags:
- Config
- Source
/config/sources/by/sourceAndName:
get:
parameters:
- description: dadjokes
in: query
name: name
required: true
type: string
- description: reddit
in: query
name: source
required: true
type: string
produces:
- application/json
responses: {}
summary: Returns a single entity by ID
tags:
- Config
- Source
/config/sources/new/reddit:
post:
parameters:
@ -234,6 +254,19 @@ paths:
- Config
- Discord
- Webhook
patch:
parameters:
- description: id
in: path
name: id
required: true
type: string
responses: {}
summary: Updates a valid discord webhook ID based on the body given.
tags:
- Config
- Discord
- Webhook
/discord/webhooks/{id}/disable:
post:
parameters:
@ -262,6 +295,27 @@ paths:
- Config
- Discord
- Webhook
/discord/webhooks/by/serverAndChannel:
get:
parameters:
- description: Fancy Server
in: query
name: server
required: true
type: string
- description: memes
in: query
name: channel
required: true
type: string
produces:
- application/json
responses: {}
summary: Returns all the known web hooks based on the Server and Channel given.
tags:
- Config
- Discord
- Webhook
/discord/webhooks/new:
post:
parameters:
@ -369,6 +423,21 @@ paths:
tags:
- Config
- Subscription
/subscriptions/discord/webhook/delete:
delete:
parameters:
- description: Id
in: query
name: Id
required: true
type: string
responses: {}
summary: Removes a Discord WebHook Subscription based on the Subscription ID.
tags:
- Config
- Source
- Discord
- Subscription
/subscriptions/new/discordwebhook:
post:
parameters:

View File

@ -1,6 +1,7 @@
package routes
import (
"context"
"encoding/json"
"log"
"net/http"
@ -71,6 +72,48 @@ func (s *Server) GetDiscordWebHooksById(w http.ResponseWriter, r *http.Request)
w.Write(bres)
}
// GetDiscordWebHookByServerAndChannel
// @Summary Returns all the known web hooks based on the Server and Channel given.
// @Produce application/json
// @Param server query string true "Fancy Server"
// @Param channel query string true "memes"
// @Tags Config, Discord, Webhook
// @Router /discord/webhooks/by/serverAndChannel [get]
func (s *Server) GetDiscordWebHooksByServerAndChannel(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
query := r.URL.Query()
_server := query["server"][0]
if _server == "" {
http.Error(w, "ID is missing", http.StatusInternalServerError)
return
}
_channel := query["channel"][0]
if _channel == "" {
http.Error(w, "Channel is missing", http.StatusInternalServerError)
return
}
res, err := s.Db.GetDiscordWebHooksByServerAndChannel(context.Background(), database.GetDiscordWebHooksByServerAndChannelParams{
Server: _server,
Channel: _channel,
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
bres, err := json.Marshal(res)
if err != nil {
http.Error(w, "unable to convert to json", http.StatusInternalServerError)
panic(err)
}
w.Write(bres)
}
// NewDiscordWebHook
// @Summary Creates a new record for a discord web hook to post data to.
// @Param url query string true "url"
@ -194,7 +237,7 @@ func (s *Server) deleteDiscordWebHook(w http.ResponseWriter, r *http.Request) {
// @Summary Updates a valid discord webhook ID based on the body given.
// @Param id path string true "id"
// @Tags Config, Discord, Webhook
// @Router /discord/webhooks/{id} [delete]
// @Router /discord/webhooks/{id} [patch]
func (s *Server) UpdateDiscordWebHook(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "ID")

View File

@ -87,6 +87,8 @@ func (s *Server) MountRoutes() {
s.Router.Post("/api/discord/webhooks/new", s.NewDiscordWebHook)
s.Router.Get("/api/discord/webhooks", s.GetDiscordWebHooks)
//s.Router.Get("/api/discord/webhooks/byId", s.GetDiscordWebHooksById)
s.Router.Get("/api/discord/webhooks/by/serverAndChannel", s.GetDiscordWebHooksByServerAndChannel)
s.Router.Route("/api/discord/webhooks/{ID}", func(r chi.Router) {
r.Get("/", s.GetDiscordWebHooksById)
r.Delete("/", s.deleteDiscordWebHook)
@ -113,10 +115,12 @@ func (s *Server) MountRoutes() {
r.Post("/disable", s.disableSource)
r.Post("/enable", s.enableSource)
})
s.Router.Get("/api/config/sources/by/sourceAndName", s.GetSourceBySourceAndName)
/* Subscriptions */
s.Router.Get("/api/subscriptions", s.ListSubscriptions)
s.Router.Get("/api/subscriptions/byDiscordId", s.GetSubscriptionsByDiscordId)
s.Router.Get("/api/subscriptions/bySourceId", s.GetSubscriptionsBySourceId)
s.Router.Post("/api/subscriptions/new/discordwebhook", s.newDiscordWebHookSubscription)
s.Router.Delete("/api/subscriptions/discord/webhook/delete", s.DeleteDiscordWebHookSubscription)
}

View File

@ -1,6 +1,7 @@
package routes
import (
"context"
"encoding/json"
"fmt"
"log"
@ -111,6 +112,45 @@ func (s *Server) getSources(w http.ResponseWriter, r *http.Request) {
w.Write(bResult)
}
// GetSourceByNameAndSource
// @Summary Returns a single entity by ID
// @Param name query string true "dadjokes"
// @Param source query string true "reddit"
// @Produce application/json
// @Tags Config, Source
// @Router /config/sources/by/sourceAndName [get]
func (s *Server) GetSourceBySourceAndName(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
name := query["name"][0]
if name == "" {
http.Error(w, "Parameter 'name' was missing in the query.", http.StatusInternalServerError)
return
}
source := query["source"][0]
if source == "" {
http.Error(w, "The parameter 'source' was missing in the query.", http.StatusInternalServerError)
return
}
item, err := s.Db.GetSourceByNameAndSource(context.Background(), database.GetSourceByNameAndSourceParams{
Name: name,
Source: source,
})
if err != nil {
http.Error(w, "Unable to find the requested record.", http.StatusInternalServerError)
}
bResult, err := json.Marshal(item)
if err != nil {
log.Panicln(err)
}
w.Header().Set("Content-Type", "application/json")
w.Write(bResult)
}
// NewRedditSource
// @Summary Creates a new reddit source to monitor.
// @Param name query string true "name"

View File

@ -1,6 +1,7 @@
package routes
import (
"context"
"encoding/json"
"net/http"
@ -171,3 +172,25 @@ func (s *Server) newDiscordWebHookSubscription(w http.ResponseWriter, r *http.Re
w.Header().Set("Content-Type", "application/json")
w.Write(bJson)
}
// DeleteDiscordWebHookSubscription
// @Summary Removes a Discord WebHook Subscription based on the Subscription ID.
// @Param Id query string true "Id"
// @Tags Config, Source, Discord, Subscription
// @Router /subscriptions/discord/webhook/delete [delete]
func (s *Server) DeleteDiscordWebHookSubscription(w http.ResponseWriter, r *http.Request) {
var ErrMissingSubscriptionID string = "Request was missing a 'Id' or was a invalid UUID."
query := r.URL.Query()
uid, err := uuid.Parse(query["Id"][0])
if err != nil {
http.Error(w, ErrMissingSubscriptionID, http.StatusBadRequest)
return
}
err = s.Db.DeleteSubscription(context.Background(), uid)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}

View File

@ -47,38 +47,55 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
defer parser.Close()
links, err := fc.PullFeed(parser)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
cache := cache.NewCacheClient(fc.cacheGroup)
for _, link := range links {
// Check cache/db if this link has been seen already, skip
_, err := cache.FindByValue(link)
if err == nil { continue }
if err == nil {
continue
}
page := fc.GetPage(parser, link)
title, err := fc.ExtractTitle(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
thumb, err := fc.ExtractThumbnail(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
pubDate, err := fc.ExtractPubDate(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
description, err := fc.ExtractDescription(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
authorName, err := fc.ExtractAuthor(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
authorImage, err := fc.ExtractAuthorImage(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
tags, err := fc.ExtractTags(page)
if err != nil { return articles, err }
if err != nil {
return articles, err
}
article := database.Article{
Sourceid: fc.record.ID,
@ -105,15 +122,19 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) {
func (fc *FFXIVClient) GetParser() (*goquery.Document, error) {
html, err := http.Get(fc.record.Url)
if err != nil { return nil, err }
if err != nil {
return nil, err
}
defer html.Body.Close()
doc, err := goquery.NewDocumentFromReader(html.Body)
if err != nil { return nil, err }
if err != nil {
return nil, err
}
return doc, nil
}
func (fc *FFXIVClient) GetBrowser() (*rod.Browser) {
func (fc *FFXIVClient) GetBrowser() *rod.Browser {
var browser *rod.Browser
if path, exists := launcher.LookPath(); exists {
u := launcher.New().Bin(path).MustLaunch()
@ -166,7 +187,9 @@ func (rc *FFXIVClient) GetPage(parser *rod.Browser, url string) *rod.Page {
func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
thumbnail := page.MustElementX("/html/body/div[3]/div[2]/div[1]/article/div[1]/img").MustProperty("src").String()
if thumbnail == "" { return "", errors.New("unable to find thumbnail")}
if thumbnail == "" {
return "", errors.New("unable to find thumbnail")
}
title := page.MustElement(".news__header > h1:nth-child(2)").MustText()
log.Println(title)
@ -176,17 +199,23 @@ func (fc *FFXIVClient) ExtractThumbnail(page *rod.Page) (string, error) {
func (fc *FFXIVClient) ExtractPubDate(page *rod.Page) (time.Time, error) {
stringDate := page.MustElement(".news__ic--topics").MustText()
if stringDate == "" { return time.Now(), errors.New("unable to locate the publish date on the post")}
if stringDate == "" {
return time.Now(), errors.New("unable to locate the publish date on the post")
}
PubDate, err := time.Parse(FFXIV_TIME_FORMAT, stringDate)
if err != nil { return time.Now(), err }
if err != nil {
return time.Now(), err
}
return PubDate, nil
}
func (fc *FFXIVClient) ExtractDescription(page *rod.Page) (string, error) {
res := page.MustElement(".news__detail__wrapper").MustText()
if res == "" { return "", errors.New("unable to locate the description on the post")}
if res == "" {
return "", errors.New("unable to locate the description on the post")
}
return res, nil
}
@ -195,11 +224,17 @@ func (fc *FFXIVClient) ExtractAuthor(page *rod.Page) (string, error) {
meta := page.MustElements("head > meta")
for _, item := range meta {
name, err := item.Property("name")
if err != nil { return "", err }
if err != nil {
return "", err
}
if name.String() != "author" { continue }
if name.String() != "author" {
continue
}
content, err := item.Property("content")
if err != nil { return "", err }
if err != nil {
return "", err
}
return content.String(), nil
}
@ -211,11 +246,17 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
meta := page.MustElements("head > meta")
for _, item := range meta {
name, err := item.Property("name")
if err != nil { return "", err }
if err != nil {
return "", err
}
if name.String() != "keywords" { continue }
if name.String() != "keywords" {
continue
}
content, err := item.Property("content")
if err != nil { return "", err }
if err != nil {
return "", err
}
return content.String(), nil
}
@ -225,12 +266,18 @@ func (fc *FFXIVClient) ExtractTags(page *rod.Page) (string, error) {
func (fc *FFXIVClient) ExtractTitle(page *rod.Page) (string, error) {
title, err := page.MustElement("head > title").Text()
if err != nil { return "", err }
if err != nil {
return "", err
}
if !strings.Contains(title, "|") { return "", errors.New("unable to split the title, missing | in the string")}
if !strings.Contains(title, "|") {
return "", errors.New("unable to split the title, missing | in the string")
}
res := strings.Split(title, "|")
if title != "" { return res[0], nil }
if title != "" {
return res[0], nil
}
//log.Println(meta)
return "", errors.New("unable to find the author on the page")
@ -240,15 +287,20 @@ func (fc *FFXIVClient) ExtractAuthorImage(page *rod.Page) (string, error) {
meta := page.MustElements("head > link")
for _, item := range meta {
name, err := item.Property("rel")
if err != nil { return "", err }
if err != nil {
return "", err
}
if name.String() != "apple-touch-icon-precomposed" { continue }
if name.String() != "apple-touch-icon-precomposed" {
continue
}
content, err := item.Property("href")
if err != nil { return "", err }
if err != nil {
return "", err
}
return content.String(), nil
}
//log.Println(meta)
return "", errors.New("unable to find the author image on the page")
}