From f6836e811e598b856ef9766254e5b124b4e1f824 Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Wed, 20 Sep 2023 07:20:56 -0600 Subject: [PATCH] Auto migrate demos to s3. --- frontend/src/api/demo.ts | 2 + frontend/src/api/media.ts | 11 ++ frontend/src/component/MDEditor.tsx | 2 +- frontend/src/component/STVListVIew.tsx | 2 +- internal/app/app.go | 154 +++++++++++++----- internal/app/app_test.go | 2 +- internal/app/http_api.go | 14 +- internal/store/demo.go | 26 ++- .../migrations/000068_add_s3_fields.up.sql | 3 + internal/store/wiki.go | 4 +- 10 files changed, 159 insertions(+), 61 deletions(-) diff --git a/frontend/src/api/demo.ts b/frontend/src/api/demo.ts index 16ad67c2..934dcb08 100644 --- a/frontend/src/api/demo.ts +++ b/frontend/src/api/demo.ts @@ -1,5 +1,6 @@ import { apiCall } from './common'; import { parseDateTime } from '../util/text'; +import { Asset } from './media'; export interface DemoFile { demo_id: number; @@ -12,6 +13,7 @@ export interface DemoFile { downloads: number; map_name: string; archive: boolean; + asset: Asset; } export interface demoFilters { diff --git a/frontend/src/api/media.ts b/frontend/src/api/media.ts index 7fe129cf..fb766844 100644 --- a/frontend/src/api/media.ts +++ b/frontend/src/api/media.ts @@ -8,6 +8,17 @@ export interface BaseUploadedMedia extends TimeStamped { name: string; contents: Uint8Array; deleted: boolean; + asset: Asset; +} + +export interface Asset { + asset_id: string; + bucket: string; + path: string; + name: string; + mime_type: string; + size: number; + old_id: number; } export interface MediaUploadResponse extends BaseUploadedMedia { diff --git a/frontend/src/component/MDEditor.tsx b/frontend/src/component/MDEditor.tsx index 404c49bc..667c1242 100644 --- a/frontend/src/component/MDEditor.tsx +++ b/frontend/src/component/MDEditor.tsx @@ -70,7 +70,7 @@ export const MDEditor = ({ setOpen(false); const newBody = bodyMD.slice(0, cursorPos) + - `![${resp.name}](media://${resp.media_id})` + + `![${resp.asset.name}](media://${resp.asset.asset_id})` + bodyMD.slice(cursorPos); setBodyMD(newBody); onSuccess && onSuccess(); diff --git a/frontend/src/component/STVListVIew.tsx b/frontend/src/component/STVListVIew.tsx index 7ea218aa..d46627d6 100644 --- a/frontend/src/component/STVListVIew.tsx +++ b/frontend/src/component/STVListVIew.tsx @@ -185,7 +185,7 @@ export const STVListVIew = () => { return ( diff --git a/internal/app/app.go b/internal/app/app.go index 842dd0a6..86f99240 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -201,15 +201,71 @@ func (app *App) Init(ctx context.Context) error { app.log.Info("Loaded filter list", zap.Int("count", len(app.wordFilters.wordFilters))) } - if errMigrateS3 := app.migrateS3(ctx); errMigrateS3 != nil { + if errMigrateS3 := app.migrateS3Media(ctx); errMigrateS3 != nil { + panic(errMigrateS3) + } + + if errMigrateS3 := app.migrateS3Demo(ctx); errMigrateS3 != nil { panic(errMigrateS3) } return nil } -// TODO remove this eventually -func (app *App) migrateS3(ctx context.Context) error { +// TODO remove this eventually. +func (app *App) migrateS3Demo(ctx context.Context) error { + if errBucket := app.assetStore.CreateBucketIfNotExists(ctx, app.conf.S3.BucketDemo); errBucket != nil { + return errBucket + } + + demos, errDemos := app.db.GetDemos(ctx, store.GetDemosOptions{}) + if errDemos != nil { + if errors.Is(errDemos, store.ErrNoResult) { + return nil + } + + return errors.Wrap(errDemos, "Failed to get demos") + } + + for _, demo := range demos { + var full store.DemoFile + if errFull := app.db.GetDemoByID(ctx, demo.DemoID, &full); errFull != nil { + app.log.Error("Failed to get demo", zap.Error(errFull)) + + continue + } + + ass, errAss := store.NewAsset(full.Data, app.conf.S3.BucketDemo, demo.Title) + if errAss != nil { + return errors.Wrap(errAss, "Failed to create asset") + } + + ass.MimeType = "application/octet-stream" + full.AssetID = ass.AssetID + + if errSave := app.db.SaveAsset(ctx, &ass); errSave != nil { + return errors.Wrap(errSave, "Failed to save asset") + } + + if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketDemo, ass.Name, + bytes.NewReader(full.Data), ass.Size, ass.MimeType); errPut != nil { + return errPut + } + + full.Size = int64(len(full.Data)) + + if errSaveMedia := app.db.SaveDemo(ctx, &full); errSaveMedia != nil { + return errors.Wrap(errSaveMedia, "Failed to save media") + } + + app.log.Info("Migrated demo successfully", zap.String("name", full.Title)) + } + + return nil +} + +// TODO remove this eventually. +func (app *App) migrateS3Media(ctx context.Context) error { if errBucket := app.assetStore.CreateBucketIfNotExists(ctx, app.conf.S3.BucketMedia); errBucket != nil { return errBucket } @@ -219,55 +275,60 @@ func (app *App) migrateS3(ctx context.Context) error { Limit: 100000, }, }) - if errReports != nil { - return errReports + if errReports != nil && !errors.Is(errReports, store.ErrNoResult) { + return errors.Wrap(errReports, "Failed to get reports") } findIdsRx := regexp.MustCompile(`!\[(.+?)]\(media://(\d+)\)`) - for _, report := range reports { + for _, reportVal := range reports { + report := reportVal // Update links in message descriptions findIds := findIdsRx.FindAllStringSubmatch(report.Description, -1) - for _, id := range findIds { - v, e := strconv.ParseInt(id[2], 10, 32) - if e != nil { - return e + for _, foundID := range findIds { + idValue, errParse := strconv.ParseInt(foundID[2], 10, 32) + if errParse != nil { + return errors.Wrap(errParse, "Failed to parse id value") } - var m store.Media - if err := app.db.GetMediaByID(ctx, int(v), &m); err != nil { + + var newMedia store.Media + if err := app.db.GetMediaByID(ctx, int(idValue), &newMedia); err != nil { app.log.Error("Failed to get media to migrate") + continue } - ass, errAss := store.NewAsset(m.Contents, app.conf.S3.BucketMedia, "") + ass, errAss := store.NewAsset(newMedia.Contents, app.conf.S3.BucketMedia, "") if errAss != nil { - return errAss + return errors.Wrap(errAss, "Failed to create asset") } - ass.OldID = int64(m.MediaID) + ass.OldID = int64(newMedia.MediaID) if errSave := app.db.SaveAsset(ctx, &ass); errSave != nil { - return errSave + return errors.Wrap(errSave, "Failed to save asset") } - if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(m.Contents), m.Size, m.MimeType); errPut != nil { + if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(newMedia.Contents), newMedia.Size, newMedia.MimeType); errPut != nil { return errPut } - m.Asset = ass + newMedia.Asset = ass - if errSaveMedia := app.db.SaveMedia(ctx, &m); errSaveMedia != nil { - return errSaveMedia + if errSaveMedia := app.db.SaveMedia(ctx, &newMedia); errSaveMedia != nil { + return errors.Wrap(errSaveMedia, "Failed to save media") } report.Description = strings.Replace( report.Description, - fmt.Sprintf("![%s](media://%s)", id[1], id[2]), - fmt.Sprintf("![%s](media://%s)", id[1], ass.AssetID.String()), 1) + fmt.Sprintf("![%s](media://%s)", foundID[1], foundID[2]), + fmt.Sprintf("![%s](media://%s)", foundID[1], ass.AssetID.String()), 1) if errSave := app.db.SaveReport(ctx, &report); errSave != nil { - return errSave + return errors.Wrap(errSave, "Failed to save report") } + + app.log.Info("Migrated report successfully", zap.Int64("id", report.ReportID)) } // Update links in report messages @@ -276,51 +337,58 @@ func (app *App) migrateS3(ctx context.Context) error { continue } - for _, msg := range msgs { + for _, message := range msgs { + msg := message msgIds := findIdsRx.FindAllStringSubmatch(msg.Contents, -1) - for _, id := range msgIds { - v, e := strconv.ParseInt(id[2], 10, 32) - if e != nil { - return e + + for _, msgID := range msgIds { + msgIDValue, errParse := strconv.ParseInt(msgID[2], 10, 32) + if errParse != nil { + app.log.Error("Failed to parse int media id", zap.Error(errParse)) + + continue } - var m store.Media - if err := app.db.GetMediaByID(ctx, int(v), &m); err != nil { + + var media store.Media + if err := app.db.GetMediaByID(ctx, int(msgIDValue), &media); err != nil { app.log.Error("Failed to get media to migrate") + continue } - ass, errAss := store.NewAsset(m.Contents, app.conf.S3.BucketMedia, "") + ass, errAss := store.NewAsset(media.Contents, app.conf.S3.BucketMedia, "") if errAss != nil { - return errAss + return errors.Wrap(errAss, "Failed to create asset") } - ass.OldID = int64(m.MediaID) + ass.OldID = int64(media.MediaID) if errSave := app.db.SaveAsset(ctx, &ass); errSave != nil { - return errSave + return errors.Wrap(errSave, "Failed to save asset") } - if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(m.Contents), m.Size, m.MimeType); errPut != nil { - return errPut + if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(media.Contents), media.Size, media.MimeType); errPut != nil { + return errors.Wrap(errPut, "Failed to store asset") } - m.Asset = ass + media.Asset = ass - if errSaveMedia := app.db.SaveMedia(ctx, &m); errSaveMedia != nil { - return errSaveMedia + if errSaveMedia := app.db.SaveMedia(ctx, &media); errSaveMedia != nil { + return errors.Wrap(errSaveMedia, "Failed to update media") } msg.Contents = strings.Replace( msg.Contents, - fmt.Sprintf("![%s](media://%s)", id[1], id[2]), - fmt.Sprintf("![%s](media://%s)", id[1], ass.AssetID.String()), 1) + fmt.Sprintf("![%s](media://%s)", msgID[1], msgID[2]), + fmt.Sprintf("![%s](media://%s)", msgID[1], ass.AssetID.String()), 1) if errSave := app.db.SaveReportMessage(ctx, &msg); errSave != nil { - return errSave + return errors.Wrap(errSave, "Failed to save report message") } + + app.log.Info("Migrated report message successfully", zap.Int64("id", msg.MessageID)) } } - } return nil diff --git a/internal/app/app_test.go b/internal/app/app_test.go index fc7b0d00..0e5c5f4b 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -63,7 +63,7 @@ func TestApp(t *testing.T) { } }) - app := New(&config, database, nil, zap.NewNop()) + app := New(&config, database, nil, zap.NewNop(), nil) t.Run("match_sum", testMatchSum(&app)) } diff --git a/internal/app/http_api.go b/internal/app/http_api.go index 8c23d5ac..a219a060 100644 --- a/internal/app/http_api.go +++ b/internal/app/http_api.go @@ -2707,6 +2707,14 @@ func onAPISaveMedia(app *App) gin.HandlerFunc { return } + if errSaveAsset := app.db.SaveAsset(ctx, &asset); errSaveAsset != nil { + responseErr(ctx, http.StatusInternalServerError, errors.New("Could not save asset")) + + log.Error("Failed to save user asset to s3 backend", zap.Error(errSaveAsset)) + } + + media.Asset = asset + media.Contents = nil if !fp.Contains(MediaSafeMimeTypesImages, media.MimeType) { @@ -2731,12 +2739,6 @@ func onAPISaveMedia(app *App) gin.HandlerFunc { return } - if app.conf.S3.Enabled { - if errLoad := app.db.GetMediaByAssetID(ctx, media.Asset.AssetID, &media); errLoad != nil { - responseErr(ctx, http.StatusInternalServerError, errors.New("Could not load new media")) - } - } - ctx.JSON(http.StatusCreated, media) } } diff --git a/internal/store/demo.go b/internal/store/demo.go index 1fbac1e1..7351ec73 100644 --- a/internal/store/demo.go +++ b/internal/store/demo.go @@ -3,11 +3,11 @@ package store import ( "context" "fmt" - "github.com/gabriel-vasile/mimetype" - "github.com/gofrs/uuid/v5" "time" sq "github.com/Masterminds/squirrel" + "github.com/gabriel-vasile/mimetype" + "github.com/gofrs/uuid/v5" "github.com/leighmacdonald/gbans/internal/consts" "github.com/leighmacdonald/srcdsup/srcdsup" "github.com/leighmacdonald/steamid/v3/steamid" @@ -28,6 +28,7 @@ type DemoFile struct { MapName string `json:"map_name"` Archive bool `json:"archive"` // When true, will not get auto deleted when flushing old demos Stats map[steamid.SID64]srcdsup.PlayerStats `json:"stats"` + AssetID uuid.UUID `json:"asset_id"` } // func NewDemoFile(serverId int64, title string, rawData []byte) (DemoFile, error) { @@ -106,7 +107,7 @@ func (db *Store) GetDemos(ctx context.Context, opts GetDemosOptions) ([]DemoFile builder := db.sb. Select("d.demo_id", "d.server_id", "d.title", "d.created_on", "d.size", "d.downloads", - "d.map_name", "d.archive", "d.stats", "s.short_name", "s.name"). + "d.map_name", "d.archive", "d.stats", "s.short_name", "s.name", "d.asset_id"). From("demo d"). LeftJoin("server s ON s.server_id = d.server_id"). OrderBy("created_on DESC"). @@ -145,11 +146,19 @@ func (db *Store) GetDemos(ctx context.Context, opts GetDemosOptions) ([]DemoFile defer rows.Close() for rows.Next() { - var demoFile DemoFile + var ( + demoFile DemoFile + uuidScan *uuid.UUID // TODO remove this and make column not-null once migrations are complete + ) + if errScan := rows.Scan(&demoFile.DemoID, &demoFile.ServerID, &demoFile.Title, &demoFile.CreatedOn, &demoFile.Size, &demoFile.Downloads, &demoFile.MapName, &demoFile.Archive, &demoFile.Stats, - &demoFile.ServerNameShort, &demoFile.ServerNameLong); errScan != nil { - return nil, Err(errQuery) + &demoFile.ServerNameShort, &demoFile.ServerNameLong, &uuidScan); errScan != nil { + return nil, Err(errScan) + } + + if uuidScan != nil { + demoFile.AssetID = *uuidScan } demos = append(demos, demoFile) @@ -194,9 +203,9 @@ func (db *Store) SaveDemo(ctx context.Context, demoFile *DemoFile) error { func (db *Store) insertDemo(ctx context.Context, demoFile *DemoFile) error { query, args, errQueryArgs := db.sb. Insert(string(tableDemo)). - Columns("server_id", "title", "raw_data", "created_on", "size", "downloads", "map_name", "archive", "stats"). + Columns("server_id", "title", "raw_data", "created_on", "size", "downloads", "map_name", "archive", "stats", "asset_id"). Values(demoFile.ServerID, demoFile.Title, demoFile.Data, demoFile.CreatedOn, - demoFile.Size, demoFile.Downloads, demoFile.MapName, demoFile.Archive, demoFile.Stats). + demoFile.Size, demoFile.Downloads, demoFile.MapName, demoFile.Archive, demoFile.Stats, demoFile.AssetID). Suffix("RETURNING demo_id"). ToSql() if errQueryArgs != nil { @@ -222,6 +231,7 @@ func (db *Store) updateDemo(ctx context.Context, demoFile *DemoFile) error { Set("map_name", demoFile.MapName). Set("archive", demoFile.Archive). Set("stats", demoFile.Stats). + Set("asset_id", demoFile.AssetID). Where(sq.Eq{"demo_id": demoFile.DemoID}). ToSql() if errQueryArgs != nil { diff --git a/internal/store/migrations/000068_add_s3_fields.up.sql b/internal/store/migrations/000068_add_s3_fields.up.sql index 1cf195aa..914708ea 100644 --- a/internal/store/migrations/000068_add_s3_fields.up.sql +++ b/internal/store/migrations/000068_add_s3_fields.up.sql @@ -45,4 +45,7 @@ ALTER TABLE media -- CHECK ((contents IS NOT NULL OR asset_id IS NOT NULL) AND NOT (contents IS NOT NULL AND asset_id IS NOT NULL)); +ALTER TABLE media + ALTER COLUMN contents drop not null; + COMMIT; diff --git a/internal/store/wiki.go b/internal/store/wiki.go index 4d1ad39a..e3afc1bd 100644 --- a/internal/store/wiki.go +++ b/internal/store/wiki.go @@ -2,12 +2,12 @@ package store import ( "context" - "github.com/gofrs/uuid/v5" "strings" "time" sq "github.com/Masterminds/squirrel" "github.com/gabriel-vasile/mimetype" + "github.com/gofrs/uuid/v5" "github.com/leighmacdonald/gbans/pkg/util" "github.com/leighmacdonald/gbans/pkg/wiki" "github.com/leighmacdonald/steamid/v3/steamid" @@ -117,6 +117,7 @@ func (db *Store) SaveMedia(ctx context.Context, media *Media) error { UPDATE media SET author_id = $2, mime_type = $3, name = $4, contents = $5, size = $6, deleted = $7, updated_on = $8, asset_id = $9 WHERE media_id = $1` + if errQuery := db.Exec(ctx, query, media.MediaID, media.AuthorID, media.MimeType, media.Name, media.Contents, media.Size, media.Deleted, media.UpdatedOn, media.Asset.AssetID); errQuery != nil { return errQuery @@ -152,6 +153,7 @@ func (db *Store) SaveMedia(ctx context.Context, media *Media) error { zap.String("mime", util.SanitizeLog(media.MimeType)), ) } + return nil }