From bfa0f1023d88c8603180836fc9fa8cc6c0258b8a Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 1 May 2024 17:49:38 -0700 Subject: [PATCH 01/13] got the rss job setup and disabled the others for now --- internal/repository/article.go | 17 ++++++ internal/services/cron/rss.go | 59 +++++++++++++++++++++ internal/services/cron/rss_test.go | 43 +++++++++++++++ internal/services/cron/scheduler.go | 21 +++++++- internal/services/cron/scheduler_test.go | 18 +++++++ internal/services/input/common.go | 2 +- internal/services/input/rss.go | 67 ++++++++++++------------ internal/services/input/rss_test.go | 17 ++++-- 8 files changed, 205 insertions(+), 39 deletions(-) create mode 100644 internal/services/cron/rss.go create mode 100644 internal/services/cron/rss_test.go diff --git a/internal/repository/article.go b/internal/repository/article.go index 0652ba6..928a957 100644 --- a/internal/repository/article.go +++ b/internal/repository/article.go @@ -24,6 +24,7 @@ type ArticlesRepo interface { ListByPublishDate(ctx context.Context, page, limit int, orderBy string) ([]domain.ArticleEntity, error) ListBySource(ctx context.Context, page, limit, sourceId int, orderBy string) ([]domain.ArticleEntity, error) Create(ctx context.Context, sourceId int64, tags, title, url, thumbnailUrl, description, authorName, authorImageUrl string, pubDate time.Time, isVideo bool) (int64, error) + CreateFromEntity(ctx context.Context, entity domain.ArticleEntity) (int64, error) } type ArticleRepository struct { @@ -192,6 +193,22 @@ func (ar ArticleRepository) Create(ctx context.Context, sourceId int64, tags, ti return 1, nil } +func (ar ArticleRepository) CreateFromEntity(ctx context.Context, entity domain.ArticleEntity) (int64, error) { + dt := time.Now() + queryBuilder := sqlbuilder.NewInsertBuilder() + queryBuilder.InsertInto("articles") + queryBuilder.Cols("UpdatedAt", "CreatedAt", "DeletedAt", "SourceId", "Tags", "Title", "Url", "PubDate", "IsVideo", "ThumbnailUrl", "Description", "AuthorName", "AuthorImageUrl") + queryBuilder.Values(dt, dt, timeZero, entity.SourceID, entity.Tags, entity.Title, entity.Url, entity.PubDate, entity.IsVideo, entity.Thumbnail, entity.Description, entity.AuthorName, entity.AuthorImageUrl) + query, args := queryBuilder.Build() + + _, err := ar.conn.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + + return 1, nil +} + func (ur ArticleRepository) processRows(rows *sql.Rows) []domain.ArticleEntity { items := []domain.ArticleEntity{} diff --git a/internal/services/cron/rss.go b/internal/services/cron/rss.go new file mode 100644 index 0000000..fce1a0d --- /dev/null +++ b/internal/services/cron/rss.go @@ -0,0 +1,59 @@ +package cron + +import ( + "log" + "time" + + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" +) + +func (c *Cron) CheckRssSources() { + log.Println("Starting ") + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss) + if err != nil { + log.Println(err) + } + + for sourceIndex, source := range sources { + if !source.Enabled { + continue + } + + rssClient := input.NewRssClient(source) + articles, err := rssClient.GetArticles() + if err != nil { + log.Println(err) + } + + for _, article := range articles { + _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + if err == nil { + continue + } + + rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + if err != nil { + log.Println(err) + } + if rowsCreated != 1 { + log.Println("Got back the wrong number of rows") + } + } + + if sourceIndex != len(sources) { + time.Sleep(time.Second * 30) + } + } +} + +func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) { + var records []domain.SourceEntity + + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType) + if err != nil { + return records, err + } + + return sources, nil +} diff --git a/internal/services/cron/rss_test.go b/internal/services/cron/rss_test.go new file mode 100644 index 0000000..4ab8da7 --- /dev/null +++ b/internal/services/cron/rss_test.go @@ -0,0 +1,43 @@ +package cron_test + +import ( + "context" + "testing" + + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/services" + "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" +) + +func TestRssPullsCorrectly(t *testing.T) { + conn, err := setupInMemoryDb() + if err != nil { + t.Error(err) + t.FailNow() + } + defer conn.Close() + + ctx := context.Background() + db := services.NewRepositoryService(conn) + rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api",true) + if err != nil { + t.Error(err) + t.FailNow() + } + + if rowsCreated != 1 { + t.Error("failed to create the source record") + t.FailNow() + } + + client := cron.NewScheduler(ctx, conn) + client.CheckRssSources() + + articles, err := db.Articles.ListByPage(ctx, 0, 100) + if err != nil { + t.Error(err) + t.FailNow() + } + + t.Log(len(articles)) +} diff --git a/internal/services/cron/scheduler.go b/internal/services/cron/scheduler.go index edb89a0..f2f978c 100644 --- a/internal/services/cron/scheduler.go +++ b/internal/services/cron/scheduler.go @@ -42,7 +42,6 @@ func NewScheduler(ctx context.Context) *Cron { c := &Cron{ ctx: &ctx, } - timer := cron.New() queries, err := openDatabase() if err != nil { @@ -51,8 +50,16 @@ func NewScheduler(ctx context.Context) *Cron { c.Db = queries //timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) + //features := services.GetEnvConfig() features := services.NewConfig() + timer.AddFunc("5 * * * *", c.CheckRssSources) + + //if features.RedditEnabled { + // timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) + // log.Print("[Input] Reddit backend was enabled") + // //go c.CheckReddit() + //} res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND) if res { timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) @@ -60,18 +67,30 @@ func NewScheduler(ctx context.Context) *Cron { //go c.CheckReddit() } + //if features.YoutubeEnabled { + // timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) + // log.Print("[Input] YouTube backend was enabled") + //} res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND) if res { timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) log.Print("[Input] YouTube backend was enabled") } + //if features.FfxivEnabled { + // timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) + // log.Print("[Input] FFXIV backend was enabled") + //} res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND) if res { timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) log.Print("[Input] FFXIV backend was enabled") } + //if features.TwitchEnabled { + // timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) + // log.Print("[Input] Twitch backend was enabled") + //} res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND) if res { timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) diff --git a/internal/services/cron/scheduler_test.go b/internal/services/cron/scheduler_test.go index 14bbd38..a0dda86 100644 --- a/internal/services/cron/scheduler_test.go +++ b/internal/services/cron/scheduler_test.go @@ -32,3 +32,21 @@ func TestCheckTwitch(t *testing.T) { t.Error(err) } } + +func setupInMemoryDb() (*sql.DB, error) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + return nil, err + } + + err = goose.SetDialect("sqlite3") + if err != nil { + return nil, err + } + + err = goose.Up(db, "../../database/migrations") + if err != nil { + return nil, err + } + return db, nil +} diff --git a/internal/services/input/common.go b/internal/services/input/common.go index 6cae09a..4775e7a 100644 --- a/internal/services/input/common.go +++ b/internal/services/input/common.go @@ -14,4 +14,4 @@ var ( ErrInvalidAuthorImage = errors.New("expected value looks to be wrong, something is missing") ) -const DATETIME_FORMAT string = "1/2/2006 3:4 PM" +const DATETIME_FORMAT string = "1/2/2006 3:4 PM" \ No newline at end of file diff --git a/internal/services/input/rss.go b/internal/services/input/rss.go index 3d2833b..1faa40c 100644 --- a/internal/services/input/rss.go +++ b/internal/services/input/rss.go @@ -1,14 +1,16 @@ package input import ( - "fmt" - "log" + "strings" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" - "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" "github.com/mmcdole/gofeed" ) +type FeedInput interface { + GetArticles() (domain.ArticleEntity, error) +} + type rssClient struct { SourceRecord domain.SourceEntity } @@ -21,39 +23,36 @@ func NewRssClient(sourceRecord domain.SourceEntity) rssClient { return client } -//func (rc rssClient) ReplaceSourceRecord(source model.Sources) { -//rc.SourceRecord = source -//} - -func (rc rssClient) getCacheGroup() string { - return fmt.Sprintf("rss-%v", rc.SourceRecord.DisplayName) -} - -func (rc rssClient) GetContent() error { - feed, err := rc.PullFeed() - if err != nil { - return err - } - - cacheClient := cache.NewCacheClient(rc.getCacheGroup()) - - for _, item := range feed.Items { - log.Println(item) - - cacheClient.FindByValue(item.Link) - - } - - return nil -} - -func (rc rssClient) PullFeed() (*gofeed.Feed, error) { - feedUri := fmt.Sprintf("%v", rc.SourceRecord.Url) - fp := gofeed.NewParser() - feed, err := fp.ParseURL(feedUri) +func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) { + parser := gofeed.NewParser() + feed, err := parser.ParseURL(rc.SourceRecord.Url) if err != nil { return nil, err } - return feed, nil + sourceTags := strings.Split(rc.SourceRecord.Tags, ",") + var articles []domain.ArticleEntity + for _, post := range feed.Items { + article := domain.ArticleEntity{ + SourceID: rc.SourceRecord.ID, + Title: post.Title, + Description: post.Content, + Url: post.Link, + PubDate: *post.PublishedParsed, + AuthorName: post.Author.Email, + } + + var postTags []string + postTags = append(postTags, sourceTags...) + postTags = append(postTags, post.Categories...) + article.Tags = strings.Join(postTags, ",") + + if post.Image == nil { + article.Thumbnail = "" + } + + articles = append(articles, article) + } + + return articles, nil } diff --git a/internal/services/input/rss_test.go b/internal/services/input/rss_test.go index 728bd85..0b9f3c0 100644 --- a/internal/services/input/rss_test.go +++ b/internal/services/input/rss_test.go @@ -19,12 +19,23 @@ func TestRssClientConstructor(t *testing.T) { func TestRssGetFeed(t *testing.T) { client := input.NewRssClient(rssRecord) - feed, err := client.PullFeed() + _, err := client.GetArticles() if err != nil { t.Error(err) } - if len(feed.Items) >= 0 { - t.Error("failed to collect items from the fees") +} + +func TestRssAgainstGita(t *testing.T) { + client := input.NewRssClient(domain.SourceEntity{ + ID: 2, + DisplayName: "Gitea - Newsbot-api", + Source: domain.SourceCollectorRss, + Url: "https://git.jamestombleson.com/jtom38/newsbot-api.rss", + Tags: "rss,gitea,newsbot-api", + }) + _, err := client.GetArticles() + if err != nil { + t.Error(err) } } -- 2.45.2 From 77b5c2bfa4de33e3a49916265cd0b69032f8ddd4 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 1 May 2024 18:26:14 -0700 Subject: [PATCH 02/13] inputs now use the domain package --- internal/services/input/ffxiv.go | 33 +++++----- internal/services/input/ffxiv_test.go | 16 +++-- internal/services/input/reddit.go | 80 +++++++++++-------------- internal/services/input/reddit_test.go | 16 +++-- internal/services/input/rss_test.go | 5 +- internal/services/input/twitch.go | 23 ++++--- internal/services/input/twitch_test.go | 19 +++--- internal/services/input/youtube.go | 33 +++++----- internal/services/input/youtube_test.go | 14 ++--- 9 files changed, 109 insertions(+), 130 deletions(-) diff --git a/internal/services/input/ffxiv.go b/internal/services/input/ffxiv.go index 581eeb9..7072548 100644 --- a/internal/services/input/ffxiv.go +++ b/internal/services/input/ffxiv.go @@ -1,7 +1,6 @@ package input import ( - "database/sql" "errors" "log" "net/http" @@ -13,7 +12,7 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/google/uuid" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cache" ) @@ -25,7 +24,7 @@ const ( ) type FFXIVClient struct { - record database.Source + record domain.SourceEntity //SourceID uint //Url string //Region string @@ -33,15 +32,15 @@ type FFXIVClient struct { cacheGroup string } -func NewFFXIVClient(Record database.Source) FFXIVClient { +func NewFFXIVClient(Record domain.SourceEntity) FFXIVClient { return FFXIVClient{ record: Record, cacheGroup: "ffxiv", } } -func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { - var articles []database.Article +func (fc *FFXIVClient) CheckSource() ([]domain.ArticleEntity, error) { + var articles []domain.ArticleEntity parser := fc.GetBrowser() defer parser.Close() @@ -97,18 +96,16 @@ func (fc *FFXIVClient) CheckSource() ([]database.Article, error) { return articles, err } - article := database.Article{ - Sourceid: fc.record.ID, - Tags: tags, - Title: title, - Url: link, - Pubdate: pubDate, - Videoheight: 0, - Videowidth: 0, - Thumbnail: thumb, - Description: description, - Authorname: sql.NullString{String: authorName}, - Authorimage: sql.NullString{String: authorImage}, + article := domain.ArticleEntity{ + SourceID: fc.record.ID, + Tags: tags, + Title: title, + Url: link, + PubDate: pubDate, + Thumbnail: thumb, + Description: description, + AuthorName: authorName, + AuthorImageUrl: authorImage, } log.Printf("Collected '%v' from '%v'", article.Title, article.Url) diff --git a/internal/services/input/ffxiv_test.go b/internal/services/input/ffxiv_test.go index bb29542..f2fde75 100644 --- a/internal/services/input/ffxiv_test.go +++ b/internal/services/input/ffxiv_test.go @@ -3,18 +3,16 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" ffxiv "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var FFXIVRecord database.Source = database.Source{ - ID: uuid.New(), - Site: "ffxiv", - Name: "Final Fantasy XIV - NA", - Source: "ffxiv", - Url: "https://na.finalfantasyxiv.com/lodestone/", - Tags: "ffxiv, final, fantasy, xiv, na, lodestone", +var FFXIVRecord domain.SourceEntity = domain.SourceEntity{ + ID: 9999, + DisplayName: "Final Fantasy XIV - NA", + Source: domain.SourceCollectorFfxiv, + Url: "https://na.finalfantasyxiv.com/lodestone/", + Tags: "ffxiv, final, fantasy, xiv, na, lodestone", } func TestFfxivGetParser(t *testing.T) { diff --git a/internal/services/input/reddit.go b/internal/services/input/reddit.go index 9f6c455..0a6aab7 100644 --- a/internal/services/input/reddit.go +++ b/internal/services/input/reddit.go @@ -1,7 +1,6 @@ package input import ( - "database/sql" "encoding/json" "errors" "fmt" @@ -9,7 +8,6 @@ import ( "strings" "time" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "github.com/go-rod/rod" @@ -18,7 +16,7 @@ import ( type RedditClient struct { config RedditConfig - record database.Source + record domain.SourceEntity } type RedditConfig struct { @@ -27,7 +25,7 @@ type RedditConfig struct { PullNSFW string } -func NewRedditClient(Record database.Source) *RedditClient { +func NewRedditClient(Record domain.SourceEntity) *RedditClient { rc := RedditClient{ record: Record, } @@ -71,7 +69,7 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) { // TODO Wire this to support the config options Url := fmt.Sprintf("%v.json", rc.record.Url) - log.Printf("[Reddit] Collecting results on '%v'", rc.record.Name) + log.Printf("[Reddit] Collecting results on '%v'", rc.record.DisplayName) content, err := getHttpContent(Url) if err != nil { @@ -88,10 +86,10 @@ func (rc *RedditClient) GetContent() (domain.RedditJsonContent, error) { return items, nil } -func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []database.Article { - var redditArticles []database.Article +func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []domain.ArticleEntity { + var redditArticles []domain.ArticleEntity for _, item := range items.Data.Children { - var article database.Article + var article domain.ArticleEntity article, err := rc.convertToArticle(item.Data) if err != nil { log.Printf("[Reddit] %v", err) @@ -104,8 +102,8 @@ func (rc *RedditClient) ConvertToArticles(items domain.RedditJsonContent) []data // ConvertToArticle() will take the reddit model struct and convert them over to Article structs. // This data can be passed to the database. -func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Article, error) { - var item database.Article +func (rc *RedditClient) convertToArticle(source domain.RedditPost) (domain.ArticleEntity, error) { + var item domain.ArticleEntity if source.Content == "" && source.Url != "" { item = rc.convertPicturePost(source) @@ -131,65 +129,57 @@ func (rc *RedditClient) convertToArticle(source domain.RedditPost) (database.Art return item, nil } -func (rc *RedditClient) convertPicturePost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, - Title: source.Title, - Tags: fmt.Sprintf("%v", rc.record.Tags), - Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Pubdate: time.Now(), - Video: sql.NullString{String: "null"}, - Videoheight: 0, - Videowidth: 0, - Thumbnail: source.Thumbnail, - Description: source.Content, - Authorname: sql.NullString{String: source.Author}, - Authorimage: sql.NullString{String: "null"}, +func (rc *RedditClient) convertPicturePost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, + Title: source.Title, + Tags: fmt.Sprintf("%v", rc.record.Tags), + Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), + PubDate: time.Now(), + IsVideo: false, + Thumbnail: source.Thumbnail, + Description: source.Content, + AuthorName: source.Author, + AuthorImageUrl: "null", } return item } -func (rc *RedditClient) convertTextPost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, +func (rc *RedditClient) convertTextPost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, Tags: "a", Title: source.Title, - Pubdate: time.Now(), - Videoheight: 0, - Videowidth: 0, + PubDate: time.Now(), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Authorname: sql.NullString{String: source.Author}, + AuthorName: source.Author, Description: source.Content, } return item } -func (rc *RedditClient) convertVideoPost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, +func (rc *RedditClient) convertVideoPost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, Tags: "a", Title: source.Title, - Pubdate: time.Now(), + PubDate: time.Now(), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Videoheight: 0, - Videowidth: 0, - Authorname: sql.NullString{String: source.Author}, + AuthorName: source.Author, Description: source.Media.RedditVideo.FallBackUrl, } return item } // This post is nothing more then a redirect to another location. -func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) database.Article { - var item = database.Article{ - Sourceid: rc.record.ID, +func (rc *RedditClient) convertRedirectPost(source domain.RedditPost) domain.ArticleEntity { + var item = domain.ArticleEntity{ + SourceID: rc.record.ID, Tags: "a", Title: source.Title, - Pubdate: time.Now(), + PubDate: time.Now(), Url: fmt.Sprintf("https://www.reddit.com%v", source.Permalink), - Videoheight: 0, - Videowidth: 0, - Authorname: sql.NullString{String: source.Author}, + AuthorName: source.Author, Description: source.UrlOverriddenByDest, } return item diff --git a/internal/services/input/reddit_test.go b/internal/services/input/reddit_test.go index a87af6d..bb7eb76 100644 --- a/internal/services/input/reddit_test.go +++ b/internal/services/input/reddit_test.go @@ -3,18 +3,16 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var RedditRecord database.Source = database.Source{ - ID: uuid.New(), - Name: "dadjokes", - Source: "reddit", - Site: "reddit", - Url: "https://reddit.com/r/dadjokes", - Tags: "reddit, dadjokes", +var RedditRecord domain.SourceEntity = domain.SourceEntity{ + ID: 9999, + DisplayName: "dadjokes", + Source: domain.SourceCollectorRss, + Url: "https://reddit.com/r/dadjokes", + Tags: "reddit, dadjokes", } func TestGetContent(t *testing.T) { diff --git a/internal/services/input/rss_test.go b/internal/services/input/rss_test.go index 0b9f3c0..b03e798 100644 --- a/internal/services/input/rss_test.go +++ b/internal/services/input/rss_test.go @@ -8,9 +8,10 @@ import ( ) var rssRecord = domain.SourceEntity{ - ID: 1, + ID: 1, DisplayName: "ArsTechnica", - Url: "https://feeds.arstechnica.com/arstechnica/index", + Url: "https://feeds.arstechnica.com/arstechnica/index", + Source: domain.SourceCollectorRss, } func TestRssClientConstructor(t *testing.T) { diff --git a/internal/services/input/twitch.go b/internal/services/input/twitch.go index 566b8c4..317db56 100644 --- a/internal/services/input/twitch.go +++ b/internal/services/input/twitch.go @@ -1,19 +1,18 @@ package input import ( - "database/sql" "errors" "fmt" "strings" "time" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "github.com/nicklaw5/helix/v2" ) type TwitchClient struct { - SourceRecord database.Source + SourceRecord domain.SourceEntity // config monitorClips string @@ -72,7 +71,7 @@ func initTwitchApi(ClientId string, ClientSecret string) (helix.Client, error) { } // This will let you replace the bound source record to keep the same session alive. -func (tc *TwitchClient) ReplaceSourceRecord(source database.Source) { +func (tc *TwitchClient) ReplaceSourceRecord(source domain.SourceEntity) { tc.SourceRecord = source } @@ -87,8 +86,8 @@ func (tc *TwitchClient) Login() error { return nil } -func (tc *TwitchClient) GetContent() ([]database.Article, error) { - var items []database.Article +func (tc *TwitchClient) GetContent() ([]domain.ArticleEntity, error) { + var items []domain.ArticleEntity user, err := tc.GetUserDetails() if err != nil { @@ -101,31 +100,31 @@ func (tc *TwitchClient) GetContent() ([]database.Article, error) { } for _, video := range posts { - var article database.Article + var article domain.ArticleEntity AuthorName, err := tc.ExtractAuthor(video) if err != nil { return items, err } - article.Authorname = sql.NullString{String: AuthorName} + article.AuthorName = AuthorName Authorimage, err := tc.ExtractAuthorImage(user) if err != nil { return items, err } - article.Authorimage = sql.NullString{String: Authorimage} + article.AuthorImageUrl = Authorimage article.Description, err = tc.ExtractDescription(video) if err != nil { return items, err } - article.Pubdate, err = tc.ExtractPubDate(video) + article.PubDate, err = tc.ExtractPubDate(video) if err != nil { return items, err } - article.Sourceid = tc.SourceRecord.ID + article.SourceID = tc.SourceRecord.ID article.Tags, err = tc.ExtractTags(video, user) if err != nil { return items, err @@ -156,7 +155,7 @@ func (tc *TwitchClient) GetUserDetails() (helix.User, error) { var blank helix.User users, err := tc.api.GetUsers(&helix.UsersParams{ - Logins: []string{tc.SourceRecord.Name}, + Logins: []string{tc.SourceRecord.DisplayName}, }) if err != nil { return blank, err diff --git a/internal/services/input/twitch_test.go b/internal/services/input/twitch_test.go index 887bb1e..df06ad3 100644 --- a/internal/services/input/twitch_test.go +++ b/internal/services/input/twitch_test.go @@ -4,21 +4,20 @@ import ( "log" "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var TwitchSourceRecord = database.Source{ - ID: uuid.New(), - Name: "nintendo", - Source: "Twitch", +var TwitchSourceRecord = domain.SourceEntity{ + ID: 9999, + DisplayName: "nintendo", + Source: domain.SourceCollectorTwitch, } -var TwitchInvalidRecord = database.Source{ - ID: uuid.New(), - Name: "EvilNintendo", - Source: "Twitch", +var TwitchInvalidRecord = domain.SourceEntity{ + ID: 9999, + DisplayName: "EvilNintendo", + Source: domain.SourceCollectorTwitch, } func TestTwitchLogin(t *testing.T) { diff --git a/internal/services/input/youtube.go b/internal/services/input/youtube.go index 379e7b9..1bfdfe2 100644 --- a/internal/services/input/youtube.go +++ b/internal/services/input/youtube.go @@ -1,7 +1,6 @@ package input import ( - "database/sql" "errors" "fmt" "log" @@ -12,11 +11,11 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/mmcdole/gofeed" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" ) type YoutubeClient struct { - record database.Source + record domain.SourceEntity // internal variables at time of collection channelID string @@ -37,7 +36,7 @@ var ( const YOUTUBE_FEED_URL string = "https://www.youtube.com/feeds/videos.xml?channel_id=" -func NewYoutubeClient(Record database.Source) YoutubeClient { +func NewYoutubeClient(Record domain.SourceEntity) YoutubeClient { yc := YoutubeClient{ record: Record, cacheGroup: "youtube", @@ -46,8 +45,8 @@ func NewYoutubeClient(Record database.Source) YoutubeClient { } // CheckSource will go and run all the commands needed to process a source. -func (yc *YoutubeClient) GetContent() ([]database.Article, error) { - var items []database.Article +func (yc *YoutubeClient) GetContent() ([]domain.ArticleEntity, error) { + var items []domain.ArticleEntity docParser, err := yc.GetParser(yc.record.Url) if err != nil { return items, err @@ -247,7 +246,7 @@ func (yc *YoutubeClient) CheckUriCache(uri *string) bool { return false } -func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { +func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) domain.ArticleEntity { parser, err := yc.GetParser(item.Link) if err != nil { log.Printf("[YouTube] Unable to process %v, submit this link as an issue.\n", item.Link) @@ -265,16 +264,16 @@ func (yc *YoutubeClient) ConvertToArticle(item *gofeed.Item) database.Article { log.Printf("[YouTube] %v", msg) } - var article = database.Article{ - Sourceid: yc.record.ID, - Tags: tags, - Title: item.Title, - Url: item.Link, - Pubdate: *item.PublishedParsed, - Thumbnail: thumb, - Description: item.Description, - Authorname: sql.NullString{String: item.Author.Name}, - Authorimage: sql.NullString{String: yc.avatarUri}, + var article = domain.ArticleEntity{ + SourceID: yc.record.ID, + Tags: tags, + Title: item.Title, + Url: item.Link, + PubDate: *item.PublishedParsed, + Thumbnail: thumb, + Description: item.Description, + AuthorName: item.Author.Name, + AuthorImageUrl: yc.avatarUri, } return article } diff --git a/internal/services/input/youtube_test.go b/internal/services/input/youtube_test.go index 4005f96..7c1a75f 100644 --- a/internal/services/input/youtube_test.go +++ b/internal/services/input/youtube_test.go @@ -3,17 +3,15 @@ package input_test import ( "testing" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "github.com/google/uuid" ) -var YouTubeRecord database.Source = database.Source{ - ID: uuid.New(), - Name: "dadjokes", - Source: "reddit", - Site: "reddit", - Url: "https://youtube.com/gamegrumps", +var YouTubeRecord = domain.SourceEntity{ + ID: 9999, + DisplayName: "dadjokes", + Source: domain.SourceCollectorReddit, + Url: "https://youtube.com/gamegrumps", } func TestGetPageParser(t *testing.T) { -- 2.45.2 From ddf50077b55a766a6ed8b57c356b6555480fe34c Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 1 May 2024 18:26:32 -0700 Subject: [PATCH 03/13] cron is now cleaned up and only running rss for now --- internal/services/cron/collectors.go | 220 ++++++++++++++++ .../cron/{rss_test.go => collectors_test.go} | 4 +- internal/services/cron/rss.go | 59 ----- internal/services/cron/scheduler.go | 248 ++---------------- internal/services/cron/scheduler_test.go | 9 +- 5 files changed, 250 insertions(+), 290 deletions(-) create mode 100644 internal/services/cron/collectors.go rename internal/services/cron/{rss_test.go => collectors_test.go} (91%) delete mode 100644 internal/services/cron/rss.go diff --git a/internal/services/cron/collectors.go b/internal/services/cron/collectors.go new file mode 100644 index 0000000..6e0ed32 --- /dev/null +++ b/internal/services/cron/collectors.go @@ -0,0 +1,220 @@ +package cron + +import ( + "log" + "time" + + "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" + "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" +) + +func (c *Cron) CollectRssPosts() { + log.Println("Starting ") + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss) + if err != nil { + log.Println(err) + } + + for sourceIndex, source := range sources { + if !source.Enabled { + continue + } + + rssClient := input.NewRssClient(source) + articles, err := rssClient.GetArticles() + if err != nil { + log.Println(err) + } + + for _, article := range articles { + _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + if err == nil { + continue + } + + rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + if err != nil { + log.Println(err) + } + if rowsCreated != 1 { + log.Println("Got back the wrong number of rows") + } + } + + if sourceIndex != len(sources) { + time.Sleep(time.Second * 30) + } + } +} + +func (c *Cron) CollectRedditPosts() { + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit) + if err != nil { + log.Printf("[Reddit] No sources found to query - %v\r", err) + } + + for _, source := range sources { + if !source.Enabled { + continue + } + + log.Printf("[Reddit] Checking '%v'...", source.DisplayName) + rc := input.NewRedditClient(source) + raw, err := rc.GetContent() + if err != nil { + log.Println(err) + } + + redditArticles := rc.ConvertToArticles(raw) + for _, article := range redditArticles { + _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + if err == nil { + continue + } + + rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + if err != nil { + log.Printf("Failed to add a new reddit article to the database: %s", err) + } + + if rowsAdded != 1 { + log.Printf("no error came back when data was added to the database but the expected row count is wrong") + } + } + } + log.Print("[Reddit] Done!") +} + +func (c *Cron) CollectYoutubePosts() { + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube) + if err != nil { + log.Printf("[Youtube] No sources found to query - %v\r", err) + } + + for sourceIndex, source := range sources { + if !source.Enabled { + continue + } + + log.Printf("[YouTube] Checking '%v'...", source.DisplayName) + yc := input.NewYoutubeClient(source) + raw, err := yc.GetContent() + if err != nil { + log.Println(err) + } + + for _, article := range raw { + _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + if err == nil { + continue + } + + rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + if err != nil { + log.Printf("Failed to add a new youtube article to the database: %s", err) + } + + if rowsAdded != 1 { + log.Printf("no error came back when data was added to the database but the expected row count is wrong") + } + } + + if sourceIndex != len(sources) { + time.Sleep(time.Second * 30) + } + } + log.Print("[YouTube] Done!") +} + +func (c *Cron) CollectFfxivPosts() { + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv) + if err != nil { + log.Printf("[FFXIV] No sources found to query - %v\r", err) + } + + for sourceIndex, source := range sources { + if !source.Enabled { + continue + } + + fc := input.NewFFXIVClient(source) + items, err := fc.CheckSource() + if err != nil { + log.Println(err) + } + + for _, article := range items { + _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + if err == nil { + continue + } + + rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + if err != nil { + log.Printf("Failed to add a new FFXIV article to the database: %s", err) + } + + if rowsAdded != 1 { + log.Printf("no error came back when data was added to the database but the expected row count is wrong") + } + } + + if sourceIndex != len(sources) { + time.Sleep(time.Second * 30) + } + } + log.Printf("[FFXIV Done!]") +} + +func (c *Cron) CollectTwitchPosts() { + sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch) + if err != nil { + log.Printf("[Twitch] No sources found to query - %v\r", err) + } + + tc, err := input.NewTwitchClient() + if err != nil { + log.Println(err) + return + } + + err = tc.Login() + if err != nil { + log.Println(err) + } + + for sourceIndex, source := range sources { + if !source.Enabled { + continue + } + + log.Printf("[Twitch] Checking '%v'...", source.DisplayName) + tc.ReplaceSourceRecord(source) + items, err := tc.GetContent() + if err != nil { + log.Println(err) + } + + for _, article := range items { + _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + if err == nil { + continue + } + + rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + if err != nil { + log.Printf("Failed to add a new Twitch article to the database: %s", err) + } + + if rowsAdded != 1 { + log.Printf("no error came back when data was added to the database but the expected row count is wrong") + } + } + + if sourceIndex != len(sources) { + time.Sleep(time.Second * 30) + } + } + + log.Print("[Twitch] Done!") +} diff --git a/internal/services/cron/rss_test.go b/internal/services/cron/collectors_test.go similarity index 91% rename from internal/services/cron/rss_test.go rename to internal/services/cron/collectors_test.go index 4ab8da7..090d12d 100644 --- a/internal/services/cron/rss_test.go +++ b/internal/services/cron/collectors_test.go @@ -19,7 +19,7 @@ func TestRssPullsCorrectly(t *testing.T) { ctx := context.Background() db := services.NewRepositoryService(conn) - rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api",true) + rowsCreated, err := db.Sources.Create(ctx, domain.SourceCollectorRss, "Gitea - Newsbot.api", "https://git.jamestombleson.com/jtom38/newsbot-api.rss", "rss,gitea,newsbot.api", true) if err != nil { t.Error(err) t.FailNow() @@ -31,7 +31,7 @@ func TestRssPullsCorrectly(t *testing.T) { } client := cron.NewScheduler(ctx, conn) - client.CheckRssSources() + client.CollectRssPosts() articles, err := db.Articles.ListByPage(ctx, 0, 100) if err != nil { diff --git a/internal/services/cron/rss.go b/internal/services/cron/rss.go deleted file mode 100644 index fce1a0d..0000000 --- a/internal/services/cron/rss.go +++ /dev/null @@ -1,59 +0,0 @@ -package cron - -import ( - "log" - "time" - - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" - "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" -) - -func (c *Cron) CheckRssSources() { - log.Println("Starting ") - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss) - if err != nil { - log.Println(err) - } - - for sourceIndex, source := range sources { - if !source.Enabled { - continue - } - - rssClient := input.NewRssClient(source) - articles, err := rssClient.GetArticles() - if err != nil { - log.Println(err) - } - - for _, article := range articles { - _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) - if err == nil { - continue - } - - rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) - if err != nil { - log.Println(err) - } - if rowsCreated != 1 { - log.Println("Got back the wrong number of rows") - } - } - - if sourceIndex != len(sources) { - time.Sleep(time.Second * 30) - } - } -} - -func (c *Cron) ListAllSourceRecords(sourceType string) ([]domain.SourceEntity, error) { - var records []domain.SourceEntity - - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, sourceType) - if err != nil { - return records, err - } - - return sources, nil -} diff --git a/internal/services/cron/scheduler.go b/internal/services/cron/scheduler.go index f2f978c..594bec4 100644 --- a/internal/services/cron/scheduler.go +++ b/internal/services/cron/scheduler.go @@ -3,102 +3,37 @@ package cron import ( "context" "database/sql" - "fmt" - "log" - "time" - "github.com/google/uuid" _ "github.com/lib/pq" "github.com/robfig/cron/v3" "git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" - "git.jamestombleson.com/jtom38/newsbot-api/internal/services/input" - "git.jamestombleson.com/jtom38/newsbot-api/internal/services/output" ) type Cron struct { Db *database.Queries ctx *context.Context timer *cron.Cron + repo services.RepositoryService } -func openDatabase() (*database.Queries, error) { - _env := services.NewConfig() - connString := _env.GetConfig(services.Sql_Connection_String) - if connString == "" { - panic("Connection String is null!") - } - db, err := sql.Open("postgres", connString) - if err != nil { - panic(err) - } - - queries := database.New(db) - return queries, err -} - -func NewScheduler(ctx context.Context) *Cron { +func NewScheduler(ctx context.Context, conn *sql.DB) *Cron { c := &Cron{ - ctx: &ctx, + ctx: &ctx, + repo: services.NewRepositoryService(conn), } timer := cron.New() - queries, err := openDatabase() - if err != nil { - panic(err) - } - c.Db = queries //timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //features := services.GetEnvConfig() - features := services.NewConfig() - timer.AddFunc("5 * * * *", c.CheckRssSources) - - //if features.RedditEnabled { - // timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) - // log.Print("[Input] Reddit backend was enabled") - // //go c.CheckReddit() - //} - res, _ := features.GetFeature(services.FEATURE_ENABLE_REDDIT_BACKEND) - if res { - timer.AddFunc("5 1-23 * * *", func() { go c.CheckReddit() }) - log.Print("[Input] Reddit backend was enabled") - //go c.CheckReddit() - } - - //if features.YoutubeEnabled { - // timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) - // log.Print("[Input] YouTube backend was enabled") - //} - res, _ = features.GetFeature(services.FEATURE_ENABLE_YOUTUBE_BACKEND) - if res { - timer.AddFunc("10 1-23 * * *", func() { go c.CheckYoutube() }) - log.Print("[Input] YouTube backend was enabled") - } - - //if features.FfxivEnabled { - // timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) - // log.Print("[Input] FFXIV backend was enabled") - //} - res, _ = features.GetFeature(services.FEATURE_ENABLE_FFXIV_BACKEND) - if res { - timer.AddFunc("5 5,10,15,20 * * *", func() { go c.CheckFfxiv() }) - log.Print("[Input] FFXIV backend was enabled") - } - - //if features.TwitchEnabled { - // timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) - // log.Print("[Input] Twitch backend was enabled") - //} - res, _ = features.GetFeature(services.FEATURE_ENABLE_TWITCH_BACKEND) - if res { - timer.AddFunc("15 1-23 * * *", func() { go c.CheckTwitch() }) - log.Print("[Input] Twitch backend was enabled") - } - - timer.AddFunc("*/5 * * * *", func() { go c.CheckDiscordQueue() }) - log.Print("[Output] Discord Output was enabled") + timer.AddFunc("5 * * * *", c.CollectRssPosts) + //timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts) + //timer.AddFunc("10 1-23 * * *", c.CheckYoutube) + //timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv) + //timer.AddFunc("15 1-23 * * *", c.CheckTwitch) + //timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue) c.timer = timer return c @@ -112,105 +47,8 @@ func (c *Cron) Stop() { c.timer.Stop() } -// This is the main entry point to query all the reddit services -func (c *Cron) CheckReddit() { - sources, err := c.Db.ListSourcesBySource(*c.ctx, "reddit") - if err != nil { - log.Printf("[Reddit] No sources found to query - %v\r", err) - } - - for _, source := range sources { - if !source.Enabled { - continue - } - log.Printf("[Reddit] Checking '%v'...", source.Name) - rc := input.NewRedditClient(source) - raw, err := rc.GetContent() - if err != nil { - log.Println(err) - } - redditArticles := rc.ConvertToArticles(raw) - c.checkPosts(redditArticles, "Reddit") - } - log.Print("[Reddit] Done!") -} - -func (c *Cron) CheckYoutube() { - // Add call to the db to request youtube sources. - sources, err := c.Db.ListSourcesBySource(*c.ctx, "youtube") - if err != nil { - log.Printf("[Youtube] No sources found to query - %v\r", err) - } - - for _, source := range sources { - if !source.Enabled { - continue - } - log.Printf("[YouTube] Checking '%v'...", source.Name) - yc := input.NewYoutubeClient(source) - raw, err := yc.GetContent() - if err != nil { - log.Println(err) - } - c.checkPosts(raw, "YouTube") - } - log.Print("[YouTube] Done!") -} - -func (c *Cron) CheckFfxiv() { - sources, err := c.Db.ListSourcesBySource(*c.ctx, "ffxiv") - if err != nil { - log.Printf("[FFXIV] No sources found to query - %v\r", err) - } - - for _, source := range sources { - if !source.Enabled { - continue - } - fc := input.NewFFXIVClient(source) - items, err := fc.CheckSource() - if err != nil { - log.Println(err) - } - c.checkPosts(items, "FFXIV") - } - log.Printf("[FFXIV Done!]") -} - -func (c *Cron) CheckTwitch() error { - sources, err := c.Db.ListSourcesBySource(*c.ctx, "twitch") - if err != nil { - log.Printf("[Twitch] No sources found to query - %v\r", err) - } - - tc, err := input.NewTwitchClient() - if err != nil { - return err - } - - err = tc.Login() - if err != nil { - return err - } - - for _, source := range sources { - if !source.Enabled { - continue - } - log.Printf("[Twitch] Checking '%v'...", source.Name) - tc.ReplaceSourceRecord(source) - items, err := tc.GetContent() - if err != nil { - log.Println(err) - } - c.checkPosts(items, "Twitch") - } - - log.Print("[Twitch] Done!") - return nil -} - -func (c *Cron) CheckDiscordQueue() error { +/* +func (c *Cron) CheckDiscordQueue() { // Get items from the table queueItems, err := c.Db.ListDiscordQueueItems(*c.ctx, 50) if err != nil { @@ -279,55 +117,15 @@ func (c *Cron) CheckDiscordQueue() error { return nil } +*/ -func (c *Cron) checkPosts(posts []database.Article, sourceName string) error { - for _, item := range posts { - _, err := c.Db.GetArticleByUrl(*c.ctx, item.Url) - if err != nil { - id := uuid.New() - - err := c.postArticle(id, item) - if err != nil { - return fmt.Errorf("[%v] Failed to post article - %v - %v.\r", sourceName, item.Url, err) - } - - err = c.addToDiscordQueue(id) - if err != nil { - return err - } - - } - } - time.Sleep(30 * time.Second) - return nil -} - -func (c *Cron) postArticle(id uuid.UUID, item database.Article) error { - err := c.Db.CreateArticle(*c.ctx, database.CreateArticleParams{ - ID: id, - Sourceid: item.Sourceid, - Tags: item.Tags, - Title: item.Title, - Url: item.Url, - Pubdate: item.Pubdate, - Video: item.Video, - Videoheight: item.Videoheight, - Videowidth: item.Videowidth, - Thumbnail: item.Thumbnail, - Description: item.Description, - Authorname: item.Authorname, - Authorimage: item.Authorimage, - }) - return err -} - -func (c *Cron) addToDiscordQueue(Id uuid.UUID) error { - err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{ - ID: uuid.New(), - Articleid: Id, - }) - if err != nil { - return err - } - return nil -} +//func (c *Cron) addToDiscordQueue(Id uuid.UUID) error { +// err := c.Db.CreateDiscordQueue(*c.ctx, database.CreateDiscordQueueParams{ +// ID: uuid.New(), +// Articleid: Id, +// }) +// if err != nil { +// return err +// } +// return nil +//} diff --git a/internal/services/cron/scheduler_test.go b/internal/services/cron/scheduler_test.go index a0dda86..8332a5e 100644 --- a/internal/services/cron/scheduler_test.go +++ b/internal/services/cron/scheduler_test.go @@ -1,12 +1,12 @@ package cron_test import ( - "context" - "testing" + "database/sql" - "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" + "github.com/pressly/goose/v3" ) +/* func TestInvokeTwitch(t *testing.T) { } @@ -15,7 +15,7 @@ func TestInvokeTwitch(t *testing.T) { func TestCheckReddit(t *testing.T) { ctx := context.Background() c := cron.NewScheduler(ctx) - c.CheckReddit() + c.Col() } func TestCheckYouTube(t *testing.T) { @@ -32,6 +32,7 @@ func TestCheckTwitch(t *testing.T) { t.Error(err) } } +*/ func setupInMemoryDb() (*sql.DB, error) { db, err := sql.Open("sqlite", ":memory:") -- 2.45.2 From c44d7f20bc96e5143d603e198564b3999224a774 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 1 May 2024 18:27:03 -0700 Subject: [PATCH 04/13] server passes in the db to cron now --- cmd/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index c028d1d..15ade00 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -11,7 +11,7 @@ import ( "git.jamestombleson.com/jtom38/newsbot-api/docs" "git.jamestombleson.com/jtom38/newsbot-api/internal/database" - "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1" + v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" ) @@ -44,7 +44,7 @@ func main() { queries := database.New(db) - c := cron.NewScheduler(ctx) + c := cron.NewScheduler(ctx, db) c.Start() server := v1.NewServer(ctx, queries, configs, db) -- 2.45.2 From 62fa7224c8512b8b8071316aa7eb9aedcf8d7515 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 1 May 2024 18:27:19 -0700 Subject: [PATCH 05/13] minor changes to make and gitignore --- .gitignore | 1 + makefile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index efcc85d..29a5f2f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ server *.so *.dylib collector +newsbot.db # Test binary, built with `go test -c` *.test diff --git a/makefile b/makefile index c0ccd02..55689f2 100644 --- a/makefile +++ b/makefile @@ -4,8 +4,8 @@ help: ## Shows this help command build: ## builds the application with the current go runtime ~/go/bin/swag f - ~/go/bin/swag i - go build . + ~/go/bin/swag init -g cmd/server.go + go build cmd/server.go docker-build: ## Generates the docker image docker build -t "newsbot.collector.api" . -- 2.45.2 From 9b2cd646b89fa6bd090092f0cce9c120408707dd Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Wed, 1 May 2024 18:27:55 -0700 Subject: [PATCH 06/13] chi has been removed --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index e489115..6dd1414 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.22 require ( github.com/PuerkitoBio/goquery v1.8.0 github.com/glebarez/go-sqlite v1.22.0 - github.com/go-chi/chi/v5 v5.0.7 github.com/go-rod/rod v0.107.1 github.com/google/uuid v1.6.0 github.com/huandu/go-sqlbuilder v1.27.1 diff --git a/go.sum b/go.sum index 2d8d499..1e6558b 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= -github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= -github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -- 2.45.2 From 781dced4efe973354a407688bbbb7a2e99ed79d7 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:36:20 -0700 Subject: [PATCH 07/13] removed the old complex dto package --- internal/services/dto/articles.go | 140 ----------------------- internal/services/dto/discordwebhooks.go | 63 ---------- internal/services/dto/queue.go | 42 ------- internal/services/dto/sources.go | 85 -------------- internal/services/dto/subscriptions.go | 91 --------------- 5 files changed, 421 deletions(-) delete mode 100644 internal/services/dto/articles.go delete mode 100644 internal/services/dto/discordwebhooks.go delete mode 100644 internal/services/dto/queue.go delete mode 100644 internal/services/dto/sources.go delete mode 100644 internal/services/dto/subscriptions.go diff --git a/internal/services/dto/articles.go b/internal/services/dto/articles.go deleted file mode 100644 index d144780..0000000 --- a/internal/services/dto/articles.go +++ /dev/null @@ -1,140 +0,0 @@ -// The converter package lives between the database calls and the API calls. -// This way if any new methods like RPC calls are added later, the API does not need to be reworked as much -package dto - -import ( - "context" - "strings" - - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" - "github.com/google/uuid" -) - -type DtoClient struct { - db *database.Queries -} - -func NewDtoClient(db *database.Queries) *DtoClient { - return &DtoClient{ - db: db, - } -} - -func (c *DtoClient) ListArticles(ctx context.Context, limit, page int) ([]models.ArticleDto, error) { - var res []models.ArticleDto - - a, err := c.db.ListArticles(ctx, database.ListArticlesParams{ - Limit: int32(limit), - Offset: int32(limit * page), - }) - if err != nil { - return res, err - } - - for _, article := range a { - res = append(res, c.convertArticle(article)) - } - return res, nil -} - -func (c *DtoClient) ListArticlesByPage(ctx context.Context, page, limit int32) ([]models.ArticleDto, error) { - var res []models.ArticleDto - - a, err := c.db.ListArticlesByPage(ctx, database.ListArticlesByPageParams{ - Limit: limit, - Offset: page * limit, - }) - if err != nil { - return res, err - } - - for _, article := range a { - res = append(res, c.convertArticle(article)) - } - - return res, nil -} - -func (c *DtoClient) GetArticle(ctx context.Context, ID uuid.UUID) (models.ArticleDto, error) { - a, err := c.db.GetArticleByID(ctx, ID) - if err != nil { - return models.ArticleDto{}, err - } - - return c.convertArticle(a), nil -} - -func (c *DtoClient) GetArticleDetails(ctx context.Context, ID uuid.UUID) (models.ArticleDetailsDto, error) { - a, err := c.db.GetArticleByID(ctx, ID) - if err != nil { - return models.ArticleDetailsDto{}, err - } - - s, err := c.db.GetSourceByID(ctx, a.Sourceid) - if err != nil { - return models.ArticleDetailsDto{}, err - } - - res := c.convertArticleDetails(a, s) - - return res, nil -} - -func (c *DtoClient) ListNewArticlesBySourceId(ctx context.Context, SourceID uuid.UUID, limit, page int) ([]models.ArticleDto, error) { - var res []models.ArticleDto - a, err := c.db.ListNewArticlesBySourceId(ctx, database.ListNewArticlesBySourceIdParams{ - Sourceid: SourceID, - Limit: int32(limit), - Offset: int32(limit * page), - }) - if err != nil { - return res, err - } - - for _, article := range a { - res = append(res, c.convertArticle(article)) - } - - return res, nil -} - -func (c *DtoClient) convertArticle(i database.Article) models.ArticleDto { - return models.ArticleDto{ - ID: i.ID, - Source: i.Sourceid, - Tags: c.SplitTags(i.Tags), - Title: i.Title, - Url: i.Url, - Pubdate: i.Pubdate, - Video: i.Video.String, - Videoheight: i.Videoheight, - Videowidth: i.Videoheight, - Thumbnail: i.Thumbnail, - Description: i.Description, - Authorname: i.Authorname.String, - Authorimage: i.Authorimage.String, - } -} - -func (c *DtoClient) convertArticleDetails(i database.Article, s database.Source) models.ArticleDetailsDto { - return models.ArticleDetailsDto{ - ID: i.ID, - Source: c.ConvertToSource(s), - Tags: c.SplitTags(i.Tags), - Title: i.Title, - Url: i.Url, - Pubdate: i.Pubdate, - Video: i.Video.String, - Videoheight: i.Videoheight, - Videowidth: i.Videoheight, - Thumbnail: i.Thumbnail, - Description: i.Description, - Authorname: i.Authorname.String, - Authorimage: i.Authorimage.String, - } -} - -func (c DtoClient) SplitTags(t string) []string { - return strings.Split(t, ", ") -} diff --git a/internal/services/dto/discordwebhooks.go b/internal/services/dto/discordwebhooks.go deleted file mode 100644 index 881083c..0000000 --- a/internal/services/dto/discordwebhooks.go +++ /dev/null @@ -1,63 +0,0 @@ -package dto - -import ( - "context" - - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" - "github.com/google/uuid" -) - -func (c *DtoClient) ListDiscordWebHooks(ctx context.Context, total int32) ([]models.DiscordWebHooksDto, error) { - var res []models.DiscordWebHooksDto - - items, err := c.db.ListDiscordWebhooks(ctx, total) - if err != nil { - return res, nil - } - - for _, item := range items { - res = append(res, c.ConvertDiscordWebhook(item)) - } - - return res, nil -} - -func (c *DtoClient) GetDiscordWebhook(ctx context.Context, id uuid.UUID) (models.DiscordWebHooksDto, error) { - var res models.DiscordWebHooksDto - - item, err := c.db.GetDiscordWebHooksByID(ctx, id) - if err != nil { - return res, err - } - - return c.ConvertDiscordWebhook(item), nil -} - -func (c *DtoClient) GetDiscordWebHookByServerAndChannel(ctx context.Context, server, channel string) ([]models.DiscordWebHooksDto, error) { - var res []models.DiscordWebHooksDto - - items, err := c.db.GetDiscordWebHooksByServerAndChannel(ctx, database.GetDiscordWebHooksByServerAndChannelParams{ - Server: server, - Channel: channel, - }) - if err != nil { - return res, err - } - - for _, item := range items { - res = append(res, c.ConvertDiscordWebhook(item)) - } - - return res, nil -} - -func (c *DtoClient) ConvertDiscordWebhook(i database.Discordwebhook) models.DiscordWebHooksDto { - return models.DiscordWebHooksDto{ - ID: i.ID, - Url: i.Url, - Server: i.Server, - Channel: i.Channel, - Enabled: i.Enabled, - } -} diff --git a/internal/services/dto/queue.go b/internal/services/dto/queue.go deleted file mode 100644 index ce61fc4..0000000 --- a/internal/services/dto/queue.go +++ /dev/null @@ -1,42 +0,0 @@ -package dto - -import ( - "context" - - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" -) - -func (c *DtoClient) ListDiscordWebhookQueue(ctx context.Context, limit int32) { - -} - -func (c *DtoClient) ListDiscordWebhookQueueDetails(ctx context.Context, limit int32) ([]models.DiscordQueueDetailsDto, error) { - var res []models.DiscordQueueDetailsDto - - items, err := c.db.ListDiscordQueueItems(ctx, limit) - if err != nil { - return res, err - } - - for _, item := range items { - article, err := c.GetArticleDetails(ctx, item.ID) - if err != nil { - return res, err - } - - res = append(res, models.DiscordQueueDetailsDto{ - ID: item.ID, - Article: article, - }) - } - - return res, nil -} - -func (c *DtoClient) ConvertToDiscordQueueDto(i database.Discordqueue) models.DiscordQueueDto { - return models.DiscordQueueDto{ - ID: i.ID, - Articleid: i.Articleid, - } -} diff --git a/internal/services/dto/sources.go b/internal/services/dto/sources.go deleted file mode 100644 index 8bdae27..0000000 --- a/internal/services/dto/sources.go +++ /dev/null @@ -1,85 +0,0 @@ -package dto - -import ( - "context" - "strings" - - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" - "github.com/google/uuid" -) - -func (c *DtoClient) ListSources(ctx context.Context, limit int32) ([]models.SourceDto, error) { - var res []models.SourceDto - - items, err := c.db.ListSources(ctx, limit) - if err != nil { - return res, err - } - - for _, item := range items { - res = append(res, c.ConvertToSource(item)) - } - - return res, nil -} - -func (c *DtoClient) ListSourcesBySource(ctx context.Context, sourceName string) ([]models.SourceDto, error) { - var res []models.SourceDto - - items, err := c.db.ListSourcesBySource(ctx, strings.ToLower(sourceName)) - if err != nil { - return res, err - } - - for _, item := range items { - res = append(res, c.ConvertToSource(item)) - } - - return res, nil -} - -func (c *DtoClient) GetSourceById(ctx context.Context, id uuid.UUID) (models.SourceDto, error) { - var res models.SourceDto - - item, err := c.db.GetSourceByID(ctx, id) - if err != nil { - return res, err - } - - return c.ConvertToSource(item), nil -} - -func (c *DtoClient) GetSourceByNameAndSource(ctx context.Context, name, source string) (models.SourceDto, error) { - var res models.SourceDto - - item, err := c.db.GetSourceByNameAndSource(ctx, database.GetSourceByNameAndSourceParams{ - Name: name, - Source: source, - }) - if err != nil { - return res, err - } - - return c.ConvertToSource(item), nil -} - -func (c *DtoClient) ConvertToSource(i database.Source) models.SourceDto { - var deleted bool - if !i.Deleted.Valid { - deleted = true - } - - return models.SourceDto{ - ID: i.ID, - Site: i.Site, - Name: i.Name, - Source: i.Source, - Type: i.Type, - Value: i.Value.String, - Enabled: i.Enabled, - Url: i.Url, - Tags: c.SplitTags(i.Tags), - Deleted: deleted, - } -} diff --git a/internal/services/dto/subscriptions.go b/internal/services/dto/subscriptions.go deleted file mode 100644 index 6d0f24b..0000000 --- a/internal/services/dto/subscriptions.go +++ /dev/null @@ -1,91 +0,0 @@ -package dto - -import ( - "context" - - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" - "github.com/google/uuid" -) - -func (c *DtoClient) ListSubscriptions(ctx context.Context, limit int32) ([]models.SubscriptionDto, error) { - var res []models.SubscriptionDto - - items, err := c.db.ListSubscriptions(ctx, limit) - if err != nil { - return res, err - } - - for _, item := range items { - res = append(res, c.ConvertSubscription(item)) - } - - return res, nil -} - -func (c *DtoClient) ListSubscriptionDetails(ctx context.Context, limit int32) ([]models.SubscriptionDetailsDto, error) { - var res []models.SubscriptionDetailsDto - - items, err := c.ListSubscriptions(ctx, limit) - if err != nil { - return res, err - } - - for _, item := range items { - dwh, err := c.GetDiscordWebhook(ctx, item.DiscordWebhookId) - if err != nil { - return res, err - } - - source, err := c.GetSourceById(ctx, item.SourceId) - if err != nil { - return res, err - } - - res = append(res, models.SubscriptionDetailsDto{ - ID: item.ID, - Source: source, - DiscordWebHook: dwh, - }) - } - - return res, nil -} - -func (c *DtoClient) ListSubscriptionsByDiscordWebhookId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) { - var res []models.SubscriptionDto - - items, err := c.db.GetSubscriptionsByDiscordWebHookId(ctx, id) - if err != nil { - return res, err - } - - for _, item := range items { - res = append(res, c.ConvertSubscription(item)) - } - - return res, nil -} - -func (c *DtoClient) ListSubscriptionsBySourceId(ctx context.Context, id uuid.UUID) ([]models.SubscriptionDto, error) { - var res []models.SubscriptionDto - - items, err := c.db.GetSubscriptionsBySourceID(ctx, id) - if err != nil { - return res, err - } - - for _, item := range items { - res = append(res, c.ConvertSubscription(item)) - } - - return res, nil -} - -func (c *DtoClient) ConvertSubscription(i database.Subscription) models.SubscriptionDto { - return models.SubscriptionDto{ - ID: i.ID, - DiscordWebhookId: i.Discordwebhookid, - SourceId: i.Sourceid, - } -} -- 2.45.2 From e523cc72d1c3ac7e96c0bd55493415ff2420acb1 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:36:39 -0700 Subject: [PATCH 08/13] updated swagger defs and removed old dto package --- docs/docs.go | 79 ++++++++++++-------------- docs/swagger.json | 79 ++++++++++++-------------- docs/swagger.yaml | 79 ++++++++++++-------------- internal/handler/v1/articles.go | 8 +-- internal/handler/v1/discordwebhooks.go | 32 +++++------ internal/handler/v1/handler.go | 26 +++++---- internal/handler/v1/queue.go | 17 +++--- internal/handler/v1/settings.go | 2 +- internal/handler/v1/sources.go | 22 +++---- internal/handler/v1/subscriptions.go | 73 +++++++++++------------- 10 files changed, 197 insertions(+), 220 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 85bff2f..4dfaa2c 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -16,7 +16,7 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { - "/articles": { + "/v1/articles": { "get": { "produces": [ "application/json" @@ -55,7 +55,7 @@ const docTemplate = `{ } } }, - "/articles/by/sourceid": { + "/v1/articles/by/sourceid": { "get": { "produces": [ "application/json" @@ -101,7 +101,7 @@ const docTemplate = `{ } } }, - "/articles/{ID}": { + "/v1/articles/{ID}": { "get": { "produces": [ "application/json" @@ -141,7 +141,7 @@ const docTemplate = `{ } } }, - "/articles/{ID}/details": { + "/v1/articles/{ID}/details": { "get": { "produces": [ "application/json" @@ -181,14 +181,13 @@ const docTemplate = `{ } } }, - "/discord/webhooks": { + "/v1/discord/webhooks": { "get": { "produces": [ "application/json" ], "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Returns the top 100", "responses": { @@ -213,14 +212,13 @@ const docTemplate = `{ } } }, - "/discord/webhooks/by/serverAndChannel": { + "/v1/discord/webhooks/by/serverAndChannel": { "get": { "produces": [ "application/json" ], "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Returns all the known web hooks based on the Server and Channel given.", "parameters": [ @@ -261,11 +259,10 @@ const docTemplate = `{ } } }, - "/discord/webhooks/new": { + "/v1/discord/webhooks/new": { "post": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Creates a new record for a discord web hook to post data to.", "parameters": [ @@ -313,11 +310,10 @@ const docTemplate = `{ } } }, - "/discord/webhooks/{ID}": { + "/v1/discord/webhooks/{ID}": { "delete": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Deletes a record by ID.", "parameters": [ @@ -351,11 +347,10 @@ const docTemplate = `{ } } }, - "/discord/webhooks/{ID}/disable": { + "/v1/discord/webhooks/{ID}/disable": { "post": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Disables a Webhook from being used.", "parameters": [ @@ -389,11 +384,10 @@ const docTemplate = `{ } } }, - "/discord/webhooks/{ID}/enable": { + "/v1/discord/webhooks/{ID}/enable": { "post": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Enables a source to continue processing.", "parameters": [ @@ -408,14 +402,13 @@ const docTemplate = `{ "responses": {} } }, - "/discord/webhooks/{id}": { + "/v1/discord/webhooks/{id}": { "get": { "produces": [ "application/json" ], "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -449,7 +442,7 @@ const docTemplate = `{ } } }, - "/queue/discord/webhooks": { + "/v1/queue/discord/webhooks": { "get": { "produces": [ "application/json" @@ -468,7 +461,7 @@ const docTemplate = `{ } } }, - "/settings/{key}": { + "/v1/settings/{key}": { "get": { "produces": [ "application/json" @@ -489,7 +482,7 @@ const docTemplate = `{ "responses": {} } }, - "/sources": { + "/v1/sources": { "get": { "produces": [ "application/json" @@ -522,7 +515,7 @@ const docTemplate = `{ } } }, - "/sources/by/source": { + "/v1/sources/by/source": { "get": { "produces": [ "application/json" @@ -568,7 +561,7 @@ const docTemplate = `{ } } }, - "/sources/by/sourceAndName": { + "/v1/sources/by/sourceAndName": { "get": { "produces": [ "application/json" @@ -615,7 +608,7 @@ const docTemplate = `{ } } }, - "/sources/new/reddit": { + "/v1/sources/new/reddit": { "post": { "tags": [ "Source" @@ -659,7 +652,7 @@ const docTemplate = `{ } } }, - "/sources/new/rss": { + "/v1/sources/new/rss": { "post": { "tags": [ "Source" @@ -703,7 +696,7 @@ const docTemplate = `{ } } }, - "/sources/new/twitch": { + "/v1/sources/new/twitch": { "post": { "tags": [ "Source" @@ -721,7 +714,7 @@ const docTemplate = `{ "responses": {} } }, - "/sources/new/youtube": { + "/v1/sources/new/youtube": { "post": { "tags": [ "Source" @@ -746,7 +739,7 @@ const docTemplate = `{ "responses": {} } }, - "/sources/{id}": { + "/v1/sources/{id}": { "get": { "produces": [ "application/json" @@ -802,7 +795,7 @@ const docTemplate = `{ "responses": {} } }, - "/sources/{id}/disable": { + "/v1/sources/{id}/disable": { "post": { "tags": [ "Source" @@ -839,7 +832,7 @@ const docTemplate = `{ } } }, - "/sources/{id}/enable": { + "/v1/sources/{id}/enable": { "post": { "tags": [ "Source" @@ -876,7 +869,7 @@ const docTemplate = `{ } } }, - "/subscriptions": { + "/v1/subscriptions": { "get": { "produces": [ "application/json" @@ -907,7 +900,7 @@ const docTemplate = `{ } } }, - "/subscriptions/by/SourceId": { + "/v1/subscriptions/by/SourceId": { "get": { "produces": [ "application/json" @@ -935,7 +928,7 @@ const docTemplate = `{ } } }, - "/subscriptions/by/discordId": { + "/v1/subscriptions/by/discordId": { "get": { "produces": [ "application/json" @@ -975,7 +968,7 @@ const docTemplate = `{ } } }, - "/subscriptions/details": { + "/v1/subscriptions/details": { "get": { "produces": [ "application/json" @@ -994,7 +987,7 @@ const docTemplate = `{ } } }, - "/subscriptions/discord/webhook/delete": { + "/v1/subscriptions/discord/webhook/delete": { "delete": { "tags": [ "Subscription" @@ -1012,7 +1005,7 @@ const docTemplate = `{ "responses": {} } }, - "/subscriptions/discord/webhook/new": { + "/v1/subscriptions/discord/webhook/new": { "post": { "tags": [ "Subscription" diff --git a/docs/swagger.json b/docs/swagger.json index b4260d7..1457fe6 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -7,7 +7,7 @@ }, "basePath": "/api", "paths": { - "/articles": { + "/v1/articles": { "get": { "produces": [ "application/json" @@ -46,7 +46,7 @@ } } }, - "/articles/by/sourceid": { + "/v1/articles/by/sourceid": { "get": { "produces": [ "application/json" @@ -92,7 +92,7 @@ } } }, - "/articles/{ID}": { + "/v1/articles/{ID}": { "get": { "produces": [ "application/json" @@ -132,7 +132,7 @@ } } }, - "/articles/{ID}/details": { + "/v1/articles/{ID}/details": { "get": { "produces": [ "application/json" @@ -172,14 +172,13 @@ } } }, - "/discord/webhooks": { + "/v1/discord/webhooks": { "get": { "produces": [ "application/json" ], "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Returns the top 100", "responses": { @@ -204,14 +203,13 @@ } } }, - "/discord/webhooks/by/serverAndChannel": { + "/v1/discord/webhooks/by/serverAndChannel": { "get": { "produces": [ "application/json" ], "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Returns all the known web hooks based on the Server and Channel given.", "parameters": [ @@ -252,11 +250,10 @@ } } }, - "/discord/webhooks/new": { + "/v1/discord/webhooks/new": { "post": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Creates a new record for a discord web hook to post data to.", "parameters": [ @@ -304,11 +301,10 @@ } } }, - "/discord/webhooks/{ID}": { + "/v1/discord/webhooks/{ID}": { "delete": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Deletes a record by ID.", "parameters": [ @@ -342,11 +338,10 @@ } } }, - "/discord/webhooks/{ID}/disable": { + "/v1/discord/webhooks/{ID}/disable": { "post": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Disables a Webhook from being used.", "parameters": [ @@ -380,11 +375,10 @@ } } }, - "/discord/webhooks/{ID}/enable": { + "/v1/discord/webhooks/{ID}/enable": { "post": { "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Enables a source to continue processing.", "parameters": [ @@ -399,14 +393,13 @@ "responses": {} } }, - "/discord/webhooks/{id}": { + "/v1/discord/webhooks/{id}": { "get": { "produces": [ "application/json" ], "tags": [ - "Discord", - "Webhook" + "DiscordWebhook" ], "summary": "Returns the top 100 entries from the queue to be processed.", "parameters": [ @@ -440,7 +433,7 @@ } } }, - "/queue/discord/webhooks": { + "/v1/queue/discord/webhooks": { "get": { "produces": [ "application/json" @@ -459,7 +452,7 @@ } } }, - "/settings/{key}": { + "/v1/settings/{key}": { "get": { "produces": [ "application/json" @@ -480,7 +473,7 @@ "responses": {} } }, - "/sources": { + "/v1/sources": { "get": { "produces": [ "application/json" @@ -513,7 +506,7 @@ } } }, - "/sources/by/source": { + "/v1/sources/by/source": { "get": { "produces": [ "application/json" @@ -559,7 +552,7 @@ } } }, - "/sources/by/sourceAndName": { + "/v1/sources/by/sourceAndName": { "get": { "produces": [ "application/json" @@ -606,7 +599,7 @@ } } }, - "/sources/new/reddit": { + "/v1/sources/new/reddit": { "post": { "tags": [ "Source" @@ -650,7 +643,7 @@ } } }, - "/sources/new/rss": { + "/v1/sources/new/rss": { "post": { "tags": [ "Source" @@ -694,7 +687,7 @@ } } }, - "/sources/new/twitch": { + "/v1/sources/new/twitch": { "post": { "tags": [ "Source" @@ -712,7 +705,7 @@ "responses": {} } }, - "/sources/new/youtube": { + "/v1/sources/new/youtube": { "post": { "tags": [ "Source" @@ -737,7 +730,7 @@ "responses": {} } }, - "/sources/{id}": { + "/v1/sources/{id}": { "get": { "produces": [ "application/json" @@ -793,7 +786,7 @@ "responses": {} } }, - "/sources/{id}/disable": { + "/v1/sources/{id}/disable": { "post": { "tags": [ "Source" @@ -830,7 +823,7 @@ } } }, - "/sources/{id}/enable": { + "/v1/sources/{id}/enable": { "post": { "tags": [ "Source" @@ -867,7 +860,7 @@ } } }, - "/subscriptions": { + "/v1/subscriptions": { "get": { "produces": [ "application/json" @@ -898,7 +891,7 @@ } } }, - "/subscriptions/by/SourceId": { + "/v1/subscriptions/by/SourceId": { "get": { "produces": [ "application/json" @@ -926,7 +919,7 @@ } } }, - "/subscriptions/by/discordId": { + "/v1/subscriptions/by/discordId": { "get": { "produces": [ "application/json" @@ -966,7 +959,7 @@ } } }, - "/subscriptions/details": { + "/v1/subscriptions/details": { "get": { "produces": [ "application/json" @@ -985,7 +978,7 @@ } } }, - "/subscriptions/discord/webhook/delete": { + "/v1/subscriptions/discord/webhook/delete": { "delete": { "tags": [ "Subscription" @@ -1003,7 +996,7 @@ "responses": {} } }, - "/subscriptions/discord/webhook/new": { + "/v1/subscriptions/discord/webhook/new": { "post": { "tags": [ "Subscription" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9638e3f..a77fcff 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -241,7 +241,7 @@ info: title: NewsBot collector version: "0.1" paths: - /articles: + /v1/articles: get: parameters: - description: page number @@ -266,7 +266,7 @@ paths: summary: Lists the top 25 records ordering from newest to oldest. tags: - Articles - /articles/{ID}: + /v1/articles/{ID}: get: parameters: - description: int @@ -292,7 +292,7 @@ paths: summary: Returns an article based on defined ID. tags: - Articles - /articles/{ID}/details: + /v1/articles/{ID}/details: get: parameters: - description: int @@ -318,7 +318,7 @@ paths: summary: Returns an article and source based on defined ID. tags: - Articles - /articles/by/sourceid: + /v1/articles/by/sourceid: get: parameters: - description: source id @@ -349,7 +349,7 @@ paths: 25. tags: - Articles - /discord/webhooks: + /v1/discord/webhooks: get: produces: - application/json @@ -368,9 +368,8 @@ paths: $ref: '#/definitions/domain.BaseResponse' summary: Returns the top 100 tags: - - Discord - - Webhook - /discord/webhooks/{ID}: + - DiscordWebhook + /v1/discord/webhooks/{ID}: delete: parameters: - description: id @@ -393,9 +392,8 @@ paths: $ref: '#/definitions/domain.BaseResponse' summary: Deletes a record by ID. tags: - - Discord - - Webhook - /discord/webhooks/{ID}/disable: + - DiscordWebhook + /v1/discord/webhooks/{ID}/disable: post: parameters: - description: id @@ -418,9 +416,8 @@ paths: $ref: '#/definitions/domain.BaseResponse' summary: Disables a Webhook from being used. tags: - - Discord - - Webhook - /discord/webhooks/{ID}/enable: + - DiscordWebhook + /v1/discord/webhooks/{ID}/enable: post: parameters: - description: id @@ -431,9 +428,8 @@ paths: responses: {} summary: Enables a source to continue processing. tags: - - Discord - - Webhook - /discord/webhooks/{id}: + - DiscordWebhook + /v1/discord/webhooks/{id}: get: parameters: - description: id @@ -458,9 +454,8 @@ paths: $ref: '#/definitions/domain.BaseResponse' summary: Returns the top 100 entries from the queue to be processed. tags: - - Discord - - Webhook - /discord/webhooks/by/serverAndChannel: + - DiscordWebhook + /v1/discord/webhooks/by/serverAndChannel: get: parameters: - description: Fancy Server @@ -490,9 +485,8 @@ paths: $ref: '#/definitions/domain.BaseResponse' summary: Returns all the known web hooks based on the Server and Channel given. tags: - - Discord - - Webhook - /discord/webhooks/new: + - DiscordWebhook + /v1/discord/webhooks/new: post: parameters: - description: url @@ -525,9 +519,8 @@ paths: $ref: '#/definitions/domain.BaseResponse' summary: Creates a new record for a discord web hook to post data to. tags: - - Discord - - Webhook - /queue/discord/webhooks: + - DiscordWebhook + /v1/queue/discord/webhooks: get: produces: - application/json @@ -539,7 +532,7 @@ paths: summary: Returns the top 100 entries from the queue to be processed. tags: - Queue - /settings/{key}: + /v1/settings/{key}: get: parameters: - description: Settings Key value @@ -553,7 +546,7 @@ paths: summary: Returns a object based on the Key that was given. tags: - Settings - /sources: + /v1/sources: get: parameters: - description: page number @@ -574,7 +567,7 @@ paths: summary: Lists the top 50 records tags: - Source - /sources/{id}: + /v1/sources/{id}: get: parameters: - description: uuid @@ -611,7 +604,7 @@ paths: summary: Marks a source as deleted based on its ID value. tags: - Source - /sources/{id}/disable: + /v1/sources/{id}/disable: post: parameters: - description: id @@ -635,7 +628,7 @@ paths: summary: Disables a source from processing. tags: - Source - /sources/{id}/enable: + /v1/sources/{id}/enable: post: parameters: - description: id @@ -659,7 +652,7 @@ paths: summary: Enables a source to continue processing. tags: - Source - /sources/by/source: + /v1/sources/by/source: get: parameters: - description: Source Name @@ -689,7 +682,7 @@ paths: summary: 'Lists the top 50 records based on the name given. Example: reddit' tags: - Source - /sources/by/sourceAndName: + /v1/sources/by/sourceAndName: get: parameters: - description: dadjokes @@ -720,7 +713,7 @@ paths: summary: Returns a single entity by ID tags: - Source - /sources/new/reddit: + /v1/sources/new/reddit: post: parameters: - description: name @@ -749,7 +742,7 @@ paths: summary: Creates a new reddit source to monitor. tags: - Source - /sources/new/rss: + /v1/sources/new/rss: post: parameters: - description: Site Name @@ -778,7 +771,7 @@ paths: summary: Creates a new rss source to monitor. tags: - Source - /sources/new/twitch: + /v1/sources/new/twitch: post: parameters: - description: name @@ -790,7 +783,7 @@ paths: summary: Creates a new twitch source to monitor. tags: - Source - /sources/new/youtube: + /v1/sources/new/youtube: post: parameters: - description: name @@ -807,7 +800,7 @@ paths: summary: Creates a new youtube source to monitor. tags: - Source - /subscriptions: + /v1/subscriptions: get: produces: - application/json @@ -827,7 +820,7 @@ paths: summary: Returns the top 100 entries from the queue to be processed. tags: - Subscription - /subscriptions/by/SourceId: + /v1/subscriptions/by/SourceId: get: parameters: - description: id @@ -845,7 +838,7 @@ paths: summary: Returns the top 100 entries from the queue to be processed. tags: - Subscription - /subscriptions/by/discordId: + /v1/subscriptions/by/discordId: get: parameters: - description: id @@ -871,7 +864,7 @@ paths: summary: Returns the top 100 entries from the queue to be processed. tags: - Subscription - /subscriptions/details: + /v1/subscriptions/details: get: produces: - application/json @@ -883,7 +876,7 @@ paths: summary: Returns the top 50 entries with full deatils on the source and output. tags: - Subscription - /subscriptions/discord/webhook/delete: + /v1/subscriptions/discord/webhook/delete: delete: parameters: - description: id @@ -895,7 +888,7 @@ paths: summary: Removes a Discord WebHook Subscription based on the Subscription ID. tags: - Subscription - /subscriptions/discord/webhook/new: + /v1/subscriptions/discord/webhook/new: post: parameters: - description: discordWebHookId diff --git a/internal/handler/v1/articles.go b/internal/handler/v1/articles.go index 02ca2ef..0febc5d 100644 --- a/internal/handler/v1/articles.go +++ b/internal/handler/v1/articles.go @@ -14,7 +14,7 @@ import ( // @Produce application/json // @Param page query string false "page number" // @Tags Articles -// @Router /articles [get] +// @Router /v1/articles [get] // @Success 200 {object} domain.ArticleResponse // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -44,7 +44,7 @@ func (s *Handler) listArticles(c echo.Context) error { // @Param ID path string true "int" // @Produce application/json // @Tags Articles -// @Router /articles/{ID} [get] +// @Router /v1/articles/{ID} [get] // @Success 200 {object} domain.ArticleResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -78,7 +78,7 @@ func (s *Handler) getArticle(c echo.Context) error { // @Param ID path string true "int" // @Produce application/json // @Tags Articles -// @Router /articles/{ID}/details [get] +// @Router /v1/articles/{ID}/details [get] // @Success 200 {object} domain.ArticleDetailedResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -119,7 +119,7 @@ func (s *Handler) getArticleDetails(c echo.Context) error { // @Param page query int false "Page to query" // @Produce application/json // @Tags Articles -// @Router /articles/by/sourceid [get] +// @Router /v1/articles/by/sourceid [get] // @Success 200 {object} domain.ArticleResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse diff --git a/internal/handler/v1/discordwebhooks.go b/internal/handler/v1/discordwebhooks.go index 3e2cb11..328d179 100644 --- a/internal/handler/v1/discordwebhooks.go +++ b/internal/handler/v1/discordwebhooks.go @@ -13,8 +13,8 @@ import ( // ListDiscordWebhooks // @Summary Returns the top 100 // @Produce application/json -// @Tags Discord, Webhook -// @Router /discord/webhooks [get] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks [get] // @Success 200 {object} domain.DiscordWebhookResponse // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -37,8 +37,8 @@ func (s *Handler) ListDiscordWebHooks(c echo.Context) error { // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json // @Param id path int true "id" -// @Tags Discord, Webhook -// @Router /discord/webhooks/{id} [get] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/{id} [get] // @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -69,8 +69,8 @@ func (s *Handler) GetDiscordWebHooksById(c echo.Context) error { // @Produce application/json // @Param server query string true "Fancy Server" // @Param channel query string true "memes" -// @Tags Discord, Webhook -// @Router /discord/webhooks/by/serverAndChannel [get] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/by/serverAndChannel [get] // @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -105,8 +105,8 @@ func (s *Handler) GetDiscordWebHooksByServerAndChannel(c echo.Context) error { // @Param url query string true "url" // @Param server query string true "Server name" // @Param channel query string true "Channel name" -// @Tags Discord, Webhook -// @Router /discord/webhooks/new [post] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/new [post] // @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -164,8 +164,8 @@ func (s *Handler) NewDiscordWebHook(c echo.Context) error { // DisableDiscordWebHooks // @Summary Disables a Webhook from being used. // @Param id path int true "id" -// @Tags Discord, Webhook -// @Router /discord/webhooks/{ID}/disable [post] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/{ID}/disable [post] // @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -212,8 +212,8 @@ func (s *Handler) disableDiscordWebHook(c echo.Context) error { // EnableDiscordWebHook // @Summary Enables a source to continue processing. // @Param id path int true "id" -// @Tags Discord, Webhook -// @Router /discord/webhooks/{ID}/enable [post] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/{ID}/enable [post] func (s *Handler) enableDiscordWebHook(c echo.Context) error { id, err := strconv.Atoi(c.Param("ID")) if err != nil { @@ -253,8 +253,8 @@ func (s *Handler) enableDiscordWebHook(c echo.Context) error { // DeleteDiscordWebHook // @Summary Deletes a record by ID. // @Param id path string true "id" -// @Tags Discord, Webhook -// @Router /discord/webhooks/{ID} [delete] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/{ID} [delete] // @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -298,8 +298,8 @@ func (s *Handler) deleteDiscordWebHook(c echo.Context) error { // UpdateDiscordWebHook // @Summary Updates a valid discord webhook ID based on the body given. // @Param id path string true "id" -// @Tags Discord, Webhook -// @Router /discord/webhooks/{id} [patch] +// @Tags DiscordWebhook +// @Router /v1/discord/webhooks/{id} [patch] // @Success 200 {object} domain.DiscordWebhookResponse "OK" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse diff --git a/internal/handler/v1/handler.go b/internal/handler/v1/handler.go index c510f65..149de1f 100644 --- a/internal/handler/v1/handler.go +++ b/internal/handler/v1/handler.go @@ -5,19 +5,19 @@ import ( "database/sql" "github.com/labstack/echo/v4" - _ "github.com/lib/pq" + "github.com/labstack/echo/v4/middleware" swagger "github.com/swaggo/echo-swagger" + _ "git.jamestombleson.com/jtom38/newsbot-api/docs" "git.jamestombleson.com/jtom38/newsbot-api/internal/database" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" - "git.jamestombleson.com/jtom38/newsbot-api/internal/services/dto" ) type Handler struct { Router *echo.Echo Db *database.Queries - dto *dto.DtoClient + //dto *dto.DtoClient config services.Configs repo services.RepositoryService } @@ -39,23 +39,26 @@ var ( ErrUnableToConvertToJson string = "Unable to convert to json" ) -func NewServer(ctx context.Context, db *database.Queries, configs services.Configs, conn *sql.DB) *Handler { +func NewServer(ctx context.Context, configs services.Configs, conn *sql.DB) *Handler { s := &Handler{ - Db: db, - dto: dto.NewDtoClient(db), + //Db: db, + //dto: dto.NewDtoClient(db), config: configs, repo: services.NewRepositoryService(conn), } router := echo.New() + router.Pre(middleware.RemoveTrailingSlash()) + router.Pre(middleware.Logger()) + router.Pre(middleware.Recover()) router.GET("/swagger/*", swagger.WrapHandler) v1 := router.Group("/api/v1") articles := v1.Group("/articles") - articles.GET("/", s.listArticles) - articles.GET("/:id", s.getArticle) - articles.GET("/:id/details", s.getArticleDetails) - articles.GET("/by/source/:id", s.ListArticlesBySourceId) + articles.GET("", s.listArticles) + articles.GET(":id", s.getArticle) + articles.GET(":id/details", s.getArticleDetails) + articles.GET("by/source/:id", s.ListArticlesBySourceId) //dwh := v1.Group("/discord/webhooks") //dwh.GET("/", s.ListDiscordWebHooks) @@ -73,7 +76,7 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi //settings.GET("/", s.getSettings) sources := v1.Group("/sources") - sources.GET("/", s.listSources) + sources.GET("", s.listSources) sources.GET("/by/source", s.listSourcesBySource) sources.GET("/by/sourceAndName", s.GetSourceBySourceAndName) //sources.POST("/new/reddit", s.newRedditSource) @@ -93,6 +96,7 @@ func NewServer(ctx context.Context, db *database.Queries, configs services.Confi subs.POST("/discord/webhook/new", s.newDiscordWebHookSubscription) subs.DELETE("/discord/webhook/delete", s.DeleteDiscordWebHookSubscription) + s.Router = router return s } diff --git a/internal/handler/v1/queue.go b/internal/handler/v1/queue.go index deadb89..92878ae 100644 --- a/internal/handler/v1/queue.go +++ b/internal/handler/v1/queue.go @@ -3,7 +3,6 @@ package v1 import ( "net/http" - "git.jamestombleson.com/jtom38/newsbot-api/internal/domain" "git.jamestombleson.com/jtom38/newsbot-api/internal/domain/models" "github.com/labstack/echo/v4" ) @@ -17,7 +16,7 @@ type ListDiscordWebHooksQueueResults struct { // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json // @Tags Queue -// @Router /queue/discord/webhooks [get] +// @Router /v1/queue/discord/webhooks [get] // @Success 200 {object} ListDiscordWebHooksQueueResults "ok" func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error { p := ListDiscordWebHooksQueueResults{ @@ -28,13 +27,13 @@ func (s *Handler) ListDiscordWebhookQueue(c echo.Context) error { } // Get the raw resp from sql - res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50) - if err != nil { - return c.JSON(http.StatusInternalServerError, domain.BaseResponse{ - Message: err.Error(), - }) - } + //res, err := s.dto.ListDiscordWebhookQueueDetails(c.Request().Context(), 50) + //if err != nil { + // return c.JSON(http.StatusInternalServerError, domain.BaseResponse{ + // Message: err.Error(), + // }) + //} - p.Payload = res + //p.Payload = res return c.JSON(http.StatusOK, p) } diff --git a/internal/handler/v1/settings.go b/internal/handler/v1/settings.go index 114500d..26367af 100644 --- a/internal/handler/v1/settings.go +++ b/internal/handler/v1/settings.go @@ -14,7 +14,7 @@ import ( // @Param key path string true "Settings Key value" // @Produce application/json // @Tags Settings -// @Router /settings/{key} [get] +// @Router /v1/settings/{key} [get] func (s *Handler) getSettings(c echo.Context) error { id := c.Param("ID") diff --git a/internal/handler/v1/sources.go b/internal/handler/v1/sources.go index 49e4d8f..2297858 100644 --- a/internal/handler/v1/sources.go +++ b/internal/handler/v1/sources.go @@ -31,7 +31,7 @@ type GetSource struct { // @Param page query string false "page number" // @Produce application/json // @Tags Source -// @Router /sources [get] +// @Router /v1/sources [get] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse "Unable to reach SQL or Data problems" func (s *Handler) listSources(c echo.Context) error { @@ -62,7 +62,7 @@ func (s *Handler) listSources(c echo.Context) error { // @Param page query string false "page number" // @Produce application/json // @Tags Source -// @Router /sources/by/source [get] +// @Router /v1/sources/by/source [get] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -100,7 +100,7 @@ func (s *Handler) listSourcesBySource(c echo.Context) error { // @Param id path int true "uuid" // @Produce application/json // @Tags Source -// @Router /sources/{id} [get] +// @Router /v1/sources/{id} [get] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -135,7 +135,7 @@ func (s *Handler) getSource(c echo.Context) error { // @Param source query string true "reddit" // @Produce application/json // @Tags Source -// @Router /sources/by/sourceAndName [get] +// @Router /v1/sources/by/sourceAndName [get] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -170,7 +170,7 @@ func (s *Handler) GetSourceBySourceAndName(c echo.Context) error { // @Param name query string true "name" // @Param url query string true "url" // @Tags Source -// @Router /sources/new/reddit [post] +// @Router /v1/sources/new/reddit [post] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -226,7 +226,7 @@ func (s *Handler) newRedditSource(c echo.Context) error { // @Param name query string true "name" // @Param url query string true "url" // @Tags Source -// @Router /sources/new/youtube [post] +// @Router /v1/sources/new/youtube [post] func (s *Handler) newYoutubeSource(c echo.Context) error { var param domain.NewSourceParamRequest err := c.Bind(¶m) @@ -289,7 +289,7 @@ func (s *Handler) newYoutubeSource(c echo.Context) error { // @Summary Creates a new twitch source to monitor. // @Param name query string true "name" // @Tags Source -// @Router /sources/new/twitch [post] +// @Router /v1/sources/new/twitch [post] func (s *Handler) newTwitchSource(c echo.Context) error { var param domain.NewSourceParamRequest err := c.Bind(¶m) @@ -337,7 +337,7 @@ func (s *Handler) newTwitchSource(c echo.Context) error { // @Param name query string true "Site Name" // @Param url query string true "RSS Url" // @Tags Source -// @Router /sources/new/rss [post] +// @Router /v1/sources/new/rss [post] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -387,7 +387,7 @@ func (s *Handler) newRssSource(c echo.Context) error { // @Summary Marks a source as deleted based on its ID value. // @Param id path string true "id" // @Tags Source -// @Router /sources/{id} [POST] +// @Router /v1/sources/{id} [POST] func (s *Handler) deleteSources(c echo.Context) error { id := c.Param("ID") uuid, err := uuid.Parse(id) @@ -432,7 +432,7 @@ func (s *Handler) deleteSources(c echo.Context) error { // @Summary Disables a source from processing. // @Param id path int true "id" // @Tags Source -// @Router /sources/{id}/disable [post] +// @Router /v1/sources/{id}/disable [post] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse @@ -474,7 +474,7 @@ func (s *Handler) disableSource(c echo.Context) error { // @Summary Enables a source to continue processing. // @Param id path string true "id" // @Tags Source -// @Router /sources/{id}/enable [post] +// @Router /v1/sources/{id}/enable [post] // @Success 200 {object} domain.SourcesResponse "ok" // @Failure 400 {object} domain.BaseResponse // @Failure 500 {object} domain.BaseResponse diff --git a/internal/handler/v1/subscriptions.go b/internal/handler/v1/subscriptions.go index eff2512..dcb7440 100644 --- a/internal/handler/v1/subscriptions.go +++ b/internal/handler/v1/subscriptions.go @@ -31,7 +31,7 @@ type ListSubscriptionDetails struct { // @Summary Returns the top 100 entries from the queue to be processed. // @Produce application/json // @Tags Subscription -// @Router /subscriptions [get] +// @Router /v1/subscriptions [get] // @Success 200 {object} ListSubscriptions "ok" // @Failure 400 {object} ApiError "Unable to reach SQL." // @Failure 500 {object} ApiError "Failed to process data from SQL." @@ -43,12 +43,11 @@ func (s *Handler) ListSubscriptions(c echo.Context) error { }, } - res, err := s.dto.ListSubscriptions(c.Request().Context(), 50) - if err != nil { - return s.WriteError(c, err, http.StatusBadRequest) - } - - payload.Payload = res + //res, err := s.dto.ListSubscriptions(c.Request().Context(), 50) + //if err != nil { + // return s.WriteError(c, err, http.StatusBadRequest) + //} + //payload.Payload = res return c.JSON(http.StatusOK, payload) } @@ -56,7 +55,7 @@ func (s *Handler) ListSubscriptions(c echo.Context) error { // @Summary Returns the top 50 entries with full deatils on the source and output. // @Produce application/json // @Tags Subscription -// @Router /subscriptions/details [get] +// @Router /v1/subscriptions/details [get] // @Success 200 {object} ListSubscriptionDetails "ok" func (s *Handler) ListSubscriptionDetails(c echo.Context) error { payload := ListSubscriptionDetails{ @@ -66,12 +65,11 @@ func (s *Handler) ListSubscriptionDetails(c echo.Context) error { }, } - res, err := s.dto.ListSubscriptionDetails(c.Request().Context(), 50) - if err != nil { - return s.WriteError(c, err, http.StatusInternalServerError) - } - - payload.Payload = res + //res, err := s.dto.ListSubscriptionDetails(c.Request().Context(), 50) + //if err != nil { + // return s.WriteError(c, err, http.StatusInternalServerError) + //} + //payload.Payload = res return c.JSON(http.StatusOK, payload) } @@ -80,7 +78,7 @@ func (s *Handler) ListSubscriptionDetails(c echo.Context) error { // @Produce application/json // @Param id query string true "id" // @Tags Subscription -// @Router /subscriptions/by/discordId [get] +// @Router /v1/subscriptions/by/discordId [get] // @Success 200 {object} ListSubscriptions "ok" // @Failure 400 {object} ApiError "Unable to reach SQL or Data problems" // @Failure 500 {object} ApiError "Data problems" @@ -97,18 +95,16 @@ func (s *Handler) GetSubscriptionsByDiscordId(c echo.Context) error { return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest) } - uuid, err := uuid.Parse(id) - if err != nil { - return s.WriteError(c, errors.New(ErrValueNotUuid), http.StatusBadRequest) + //uuid, err := uuid.Parse(id) + //if err != nil { + // return s.WriteError(c, errors.New(ErrValueNotUuid), http.StatusBadRequest) + //} - } - - res, err := s.dto.ListSubscriptionsByDiscordWebhookId(context.Background(), uuid) - if err != nil { - return s.WriteError(c, err, http.StatusNoContent) - } - - p.Payload = res + //res, err := s.dto.ListSubscriptionsByDiscordWebhookId(context.Background(), uuid) + //if err != nil { + // return s.WriteError(c, err, http.StatusNoContent) + //} + //p.Payload = res return c.JSON(http.StatusOK, p) } @@ -117,7 +113,7 @@ func (s *Handler) GetSubscriptionsByDiscordId(c echo.Context) error { // @Produce application/json // @Param id query string true "id" // @Tags Subscription -// @Router /subscriptions/by/SourceId [get] +// @Router /v1/subscriptions/by/SourceId [get] // @Success 200 {object} ListSubscriptions "ok" func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error { p := ListSubscriptions{ @@ -132,17 +128,16 @@ func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error { return s.WriteError(c, errors.New(ErrIdValueMissing), http.StatusBadRequest) } - uuid, err := uuid.Parse(_id) - if err != nil { - return s.WriteError(c, err, http.StatusBadRequest) - } + //uuid, err := uuid.Parse(_id) + //if err != nil { + // return s.WriteError(c, err, http.StatusBadRequest) + //} - res, err := s.dto.ListSubscriptionsBySourceId(context.Background(), uuid) - if err != nil { - return s.WriteError(c, err, http.StatusNoContent) - } - - p.Payload = res + //res, err := s.dto.ListSubscriptionsBySourceId(context.Background(), uuid) + //if err != nil { + // return s.WriteError(c, err, http.StatusNoContent) + //} + //p.Payload = res return c.JSON(http.StatusOK, p) } @@ -151,7 +146,7 @@ func (s *Handler) GetSubscriptionsBySourceId(c echo.Context) error { // @Param discordWebHookId query string true "discordWebHookId" // @Param sourceId query string true "sourceId" // @Tags Subscription -// @Router /subscriptions/discord/webhook/new [post] +// @Router /v1/subscriptions/discord/webhook/new [post] func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error { // Extract the values given discordWebHookId := c.QueryParam("discordWebHookId") @@ -207,7 +202,7 @@ func (s *Handler) newDiscordWebHookSubscription(c echo.Context) error { // @Summary Removes a Discord WebHook Subscription based on the Subscription ID. // @Param id query string true "id" // @Tags Subscription -// @Router /subscriptions/discord/webhook/delete [delete] +// @Router /v1/subscriptions/discord/webhook/delete [delete] func (s *Handler) DeleteDiscordWebHookSubscription(c echo.Context) error { var ErrMissingSubscriptionID string = "the request was missing a 'Id'" -- 2.45.2 From 852db161c990913174ca73b2e80d059f62028d91 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:36:56 -0700 Subject: [PATCH 09/13] changed how cron uses context --- internal/services/cron/collectors.go | 30 ++++++++++++++-------------- internal/services/cron/scheduler.go | 14 ++++++------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/services/cron/collectors.go b/internal/services/cron/collectors.go index 6e0ed32..4d4a93d 100644 --- a/internal/services/cron/collectors.go +++ b/internal/services/cron/collectors.go @@ -10,7 +10,7 @@ import ( func (c *Cron) CollectRssPosts() { log.Println("Starting ") - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorRss) + sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorRss) if err != nil { log.Println(err) } @@ -27,12 +27,12 @@ func (c *Cron) CollectRssPosts() { } for _, article := range articles { - _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } - rowsCreated, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + rowsCreated, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Println(err) } @@ -48,7 +48,7 @@ func (c *Cron) CollectRssPosts() { } func (c *Cron) CollectRedditPosts() { - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorReddit) + sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorReddit) if err != nil { log.Printf("[Reddit] No sources found to query - %v\r", err) } @@ -67,12 +67,12 @@ func (c *Cron) CollectRedditPosts() { redditArticles := rc.ConvertToArticles(raw) for _, article := range redditArticles { - _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } - rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new reddit article to the database: %s", err) } @@ -86,7 +86,7 @@ func (c *Cron) CollectRedditPosts() { } func (c *Cron) CollectYoutubePosts() { - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorYoutube) + sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorYoutube) if err != nil { log.Printf("[Youtube] No sources found to query - %v\r", err) } @@ -104,12 +104,12 @@ func (c *Cron) CollectYoutubePosts() { } for _, article := range raw { - _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } - rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new youtube article to the database: %s", err) } @@ -127,7 +127,7 @@ func (c *Cron) CollectYoutubePosts() { } func (c *Cron) CollectFfxivPosts() { - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorFfxiv) + sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorFfxiv) if err != nil { log.Printf("[FFXIV] No sources found to query - %v\r", err) } @@ -144,12 +144,12 @@ func (c *Cron) CollectFfxivPosts() { } for _, article := range items { - _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } - rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new FFXIV article to the database: %s", err) } @@ -167,7 +167,7 @@ func (c *Cron) CollectFfxivPosts() { } func (c *Cron) CollectTwitchPosts() { - sources, err := c.repo.Sources.ListBySource(*c.ctx, 0, 1000, domain.SourceCollectorTwitch) + sources, err := c.repo.Sources.ListBySource(c.ctx, 0, 1000, domain.SourceCollectorTwitch) if err != nil { log.Printf("[Twitch] No sources found to query - %v\r", err) } @@ -196,12 +196,12 @@ func (c *Cron) CollectTwitchPosts() { } for _, article := range items { - _, err := c.repo.Articles.GetByUrl(*c.ctx, article.Url) + _, err := c.repo.Articles.GetByUrl(c.ctx, article.Url) if err == nil { continue } - rowsAdded, err := c.repo.Articles.CreateFromEntity(*c.ctx, article) + rowsAdded, err := c.repo.Articles.CreateFromEntity(c.ctx, article) if err != nil { log.Printf("Failed to add a new Twitch article to the database: %s", err) } diff --git a/internal/services/cron/scheduler.go b/internal/services/cron/scheduler.go index 594bec4..41d3e75 100644 --- a/internal/services/cron/scheduler.go +++ b/internal/services/cron/scheduler.go @@ -13,14 +13,14 @@ import ( type Cron struct { Db *database.Queries - ctx *context.Context + ctx context.Context timer *cron.Cron repo services.RepositoryService } func NewScheduler(ctx context.Context, conn *sql.DB) *Cron { c := &Cron{ - ctx: &ctx, + ctx: ctx, repo: services.NewRepositoryService(conn), } timer := cron.New() @@ -28,11 +28,11 @@ func NewScheduler(ctx context.Context, conn *sql.DB) *Cron { //timer.AddFunc("*/5 * * * *", func() { go CheckCache() }) //features := services.GetEnvConfig() - timer.AddFunc("5 * * * *", c.CollectRssPosts) - //timer.AddFunc("5 1-23 * * *", c.CollectRedditPosts) - //timer.AddFunc("10 1-23 * * *", c.CheckYoutube) - //timer.AddFunc("5 5,10,15,20 * * *", c.CheckFfxiv) - //timer.AddFunc("15 1-23 * * *", c.CheckTwitch) + timer.AddFunc("5 * * * *", func() { go c.CollectRssPosts() }) + //timer.AddFunc("10 * * * *", c.CollectRedditPosts) + //timer.AddFunc("15 * * * *", c.CheckYoutube) + //timer.AddFunc("20 * * * *", c.CheckFfxiv) + //timer.AddFunc("25 * * * *", c.CheckTwitch) //timer.AddFunc("*/5 * * * *", c.CheckDiscordQueue) c.timer = timer -- 2.45.2 From f1f2142fdf20542c0e35f8a4af6cd7dfcc83978a Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:37:18 -0700 Subject: [PATCH 10/13] correct rss to null check the author --- internal/services/input/httpClient.go | 4 ++-- internal/services/input/rss.go | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/internal/services/input/httpClient.go b/internal/services/input/httpClient.go index d942158..9091abe 100644 --- a/internal/services/input/httpClient.go +++ b/internal/services/input/httpClient.go @@ -2,7 +2,7 @@ package input import ( "crypto/tls" - "io/ioutil" + "io" "log" "net/http" ) @@ -35,7 +35,7 @@ func getHttpContent(uri string) ([]byte, error) { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/internal/services/input/rss.go b/internal/services/input/rss.go index 1faa40c..e5dbccd 100644 --- a/internal/services/input/rss.go +++ b/internal/services/input/rss.go @@ -39,7 +39,11 @@ func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) { Description: post.Content, Url: post.Link, PubDate: *post.PublishedParsed, - AuthorName: post.Author.Email, + //AuthorName: post.Authors[0].Email, + } + + if len(post.Authors) != 0 { + article.AuthorName = post.Authors[0].Email } var postTags []string @@ -47,6 +51,21 @@ func (rc rssClient) GetArticles() ([]domain.ArticleEntity, error) { postTags = append(postTags, post.Categories...) article.Tags = strings.Join(postTags, ",") + /* + pageContent, err := getHttpContent(article.Url) + if err != nil { + continue + } + + htmlNode, err := html.Parse(bytes.NewReader(pageContent)) + if err != nil { + continue + } + htmlNode. + + fmt.Println(htmlNode) + */ + if post.Image == nil { article.Thumbnail = "" } -- 2.45.2 From 4eff80c1fd2cc112ee7f0467ecb85ead231ef9e8 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:37:35 -0700 Subject: [PATCH 11/13] minor cleanup --- internal/services/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/services/config.go b/internal/services/config.go index 2099d44..b1acbf6 100644 --- a/internal/services/config.go +++ b/internal/services/config.go @@ -11,9 +11,9 @@ import ( ) const ( - ServerAddress = "SERVER_ADDRESS" + ServerAddress = "ServerAddress" - Sql_Connection_String = "SQL_CONNECTION_STRING" + //Sql_Connection_String = "SQL_CONNECTION_STRING" FEATURE_ENABLE_REDDIT_BACKEND = "FEATURE_ENABLE_REDDIT_BACKEND" REDDIT_PULL_TOP = "REDDIT_PULL_TOP" -- 2.45.2 From 20ec963e780cf5e2470fdd5ad9337d5d78f9ba0f Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:38:30 -0700 Subject: [PATCH 12/13] minor updates --- api.http | 6 ++++++ cmd/server.go | 11 ++--------- go.mod | 2 ++ go.sum | 4 ++++ makefile | 4 ++-- 5 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 api.http diff --git a/api.http b/api.http new file mode 100644 index 0000000..c87abcc --- /dev/null +++ b/api.http @@ -0,0 +1,6 @@ +### Select Sources fro mthe top +GET http://localhost:8081/api/v1/sources/ + + +### Select Sources by type +GET http://localhost:8081/api/v1/sources/by/source?source=rss diff --git a/cmd/server.go b/cmd/server.go index 15ade00..48f4826 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -4,13 +4,11 @@ import ( "context" "database/sql" "fmt" - "net/http" _ "github.com/glebarez/go-sqlite" "github.com/pressly/goose/v3" "git.jamestombleson.com/jtom38/newsbot-api/docs" - "git.jamestombleson.com/jtom38/newsbot-api/internal/database" v1 "git.jamestombleson.com/jtom38/newsbot-api/internal/handler/v1" "git.jamestombleson.com/jtom38/newsbot-api/internal/services" "git.jamestombleson.com/jtom38/newsbot-api/internal/services/cron" @@ -42,19 +40,14 @@ func main() { panic(err) } - queries := database.New(db) - c := cron.NewScheduler(ctx, db) c.Start() - server := v1.NewServer(ctx, queries, configs, db) + server := v1.NewServer(ctx, configs, db) fmt.Println("API is online and waiting for requests.") fmt.Printf("API: http://%v:8081/api\r\n", configs.ServerAddress) fmt.Printf("Swagger: http://%v:8081/swagger/index.html\r\n", configs.ServerAddress) - err = http.ListenAndServe(":8081", server.Router) - if err != nil { - panic(err) - } + server.Router.Start(":8081") } diff --git a/go.mod b/go.mod index 6dd1414..f4bf926 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -35,6 +36,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sync v0.7.0 // indirect + golang.org/x/time v0.5.0 // indirect modernc.org/libc v1.41.0 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.7.2 // indirect diff --git a/go.sum b/go.sum index 1e6558b..a92e69f 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-rod/rod v0.107.1 h1:wRxTTAXJ0JUnoSGcyGAOubpdrToWIKPCnLu3av8EDFY= github.com/go-rod/rod v0.107.1/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -161,6 +163,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= diff --git a/makefile b/makefile index 55689f2..b2d1ed4 100644 --- a/makefile +++ b/makefile @@ -12,10 +12,10 @@ docker-build: ## Generates the docker image docker image ls | grep newsbot.collector.api migrate-dev: ## Apply sql migrations to dev db - goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" up + goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db up migrate-dev-down: ## revert sql migrations to dev db - goose -dir "./internal/database/migrations" postgres "user=postgres password=postgres dbname=postgres sslmode=disable" down + goose -dir "./internal/database/migrations" sqlite3 ./cmd/newsbot.db down swag: ## Generates the swagger documentation with the swag tool ~/go/bin/swag f -- 2.45.2 From fb471659d9691307e1b86cbe5bed4a7aa29ebf72 Mon Sep 17 00:00:00 2001 From: James Tombleson Date: Thu, 2 May 2024 17:39:07 -0700 Subject: [PATCH 13/13] minor table touches like related ID's and updated seed --- .../migrations/20240425083756_init.sql | 8 ++--- .../migrations/20240425092459_seed.sql | 32 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/internal/database/migrations/20240425083756_init.sql b/internal/database/migrations/20240425083756_init.sql index 8d83028..96e82ae 100644 --- a/internal/database/migrations/20240425083756_init.sql +++ b/internal/database/migrations/20240425083756_init.sql @@ -12,8 +12,6 @@ CREATE TABLE Articles ( Url TEXT NOT NULL, PubDate DATETIME NOT NULL, IsVideo TEXT NOT NULL, - --VideoHeight int NOT NULL, - --VideoWidth int NOT NULL, ThumbnailUrl TEXT NOT NULL, Description TEXT NOT NULL, AuthorName TEXT NOT NULL, @@ -34,6 +32,7 @@ CREATE Table DiscordWebHooks ( CreatedAt DATETIME NOT NULL, UpdatedAt DATETIME NOT NULL, DeletedAt DATETIME NOT NULL, + UserID INTEGER NOT NULL, --Name TEXT NOT NULL, -- Defines webhook purpose --Key TEXT, Url TEXT NOT NULL, -- Webhook Url @@ -78,8 +77,9 @@ CREATE TABLE Subscriptions ( CreatedAt DATETIME NOT NULL, UpdatedAt DATETIME NOT NULL, DeletedAt DATETIME, - DiscordWebHookId NUMBER NOT NULL, - SourceId NUMBER NOT NULL + DiscordWebHookID NUMBER NOT NULL, + SourceID NUMBER NOT NULL, + UserID NUMBER NOT NULL ); CREATE TABLE Users ( diff --git a/internal/database/migrations/20240425092459_seed.sql b/internal/database/migrations/20240425092459_seed.sql index 8cef6a7..6d71fcb 100644 --- a/internal/database/migrations/20240425092459_seed.sql +++ b/internal/database/migrations/20240425092459_seed.sql @@ -7,33 +7,33 @@ SELECT 'up SQL query'; -- Final Fantasy XIV Entries INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - NA', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - NA', 'ffxiv', TRUE, 'https://na.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, na, lodestone'); INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - JP', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - JP', 'ffxiv', FALSE, 'https://jp.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, jp, lodestone'); INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - EU', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - EU', 'ffxiv', FALSE, 'https://eu.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, eu, lodestone'); INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - FR', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - FR', 'ffxiv', FALSE, 'https://fr.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, fr, lodestone'); INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'ffxiv', 'Final Fantasy XIV - DE', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Final Fantasy XIV - DE', 'ffxiv', FALSE, 'https://de.finalfantasyxiv.com/lodestone/', 'ffxiv, final, fantasy, xiv, de, lodestone'); -- Reddit Entries INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'reddit', 'dadjokes', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'dadjokes', 'reddit', TRUE, 'https://reddit.com/r/dadjokes', 'reddit, dadjokes'); INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'reddit', 'steamdeck', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steamdeck', 'reddit', TRUE, 'https://reddit.com/r/steamdeck', 'reddit, steam deck, steam, deck'); -- Youtube Entries INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'youtube', 'Game Grumps', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Game Grumps', 'youtube', TRUE, 'https://www.youtube.com/user/GameGrumps', 'youtube, game grumps, game, grumps'); -- RSS Entries INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered', 'steam deck', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'steampowered - steam deck', 'rss', TRUE, 'https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english&snr=1_2108_9__2107', 'rss, steampowered, steam, deck, steam deck'); -- Twitch Entries INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabled, Url, Tags) VALUES -("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'twitch', 'Nintendo', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo'); +("2024-04-25 18:37:43.852367", "2024-04-25 18:37:43.852367", "0001-01-01 00:00:00", 'Nintendo', 'twitch', TRUE, 'https://twitch.tv/nintendo', 'twitch, nintendo'); -- +goose StatementEnd @@ -41,10 +41,10 @@ INSERT INTO sources (CreatedAt, UpdatedAt, DeletedAt, DisplayName, Source, Enabl -- +goose StatementBegin --SELECT 'down SQL query'; -DELETE FROM sources where source = 'reddit' and name = 'dadjokes'; -DELETE FROM sources where source = 'reddit' and name = 'steamdeck'; -DELETE FROM sources where source = 'ffxiv'; -DELETE FROM sources WHERE source = 'twitch' and name = 'Nintendo'; -DELETE FROM sources WHERE source = 'youtube' and name = 'Game Grumps'; -DELETE FROM SOURCES WHERE source = 'rss' and name = 'steam deck'; +DELETE FROM sources where Source = 'reddit' and DisplayName = 'dadjokes'; +DELETE FROM sources where Source = 'reddit' and DisplayName = 'steamdeck'; +DELETE FROM sources where Source = 'ffxiv'; +DELETE FROM sources WHERE Source = 'twitch' and DisplayName = 'Nintendo'; +DELETE FROM sources WHERE Source = 'youtube' and DisplayName = 'Game Grumps'; +DELETE FROM SOURCES WHERE Source = 'rss' and DisplayName = 'steampowered - steam deck'; -- +goose StatementEnd -- 2.45.2