Auto migrate demos to s3.

frontend/src/api/demo.ts

@@ -1,5 +1,6 @@
 import { apiCall } from './common';
 import { parseDateTime } from '../util/text';
+import { Asset } from './media';
 
 export interface DemoFile {
     demo_id: number;
@@ -12,6 +13,7 @@ export interface DemoFile {
     downloads: number;
     map_name: string;
     archive: boolean;
+    asset: Asset;
 }
 
 export interface demoFilters {

frontend/src/api/media.ts

@@ -8,6 +8,17 @@ export interface BaseUploadedMedia extends TimeStamped {
     name: string;
     contents: Uint8Array;
     deleted: boolean;
+    asset: Asset;
+}
+
+export interface Asset {
+    asset_id: string;
+    bucket: string;
+    path: string;
+    name: string;
+    mime_type: string;
+    size: number;
+    old_id: number;
 }
 
 export interface MediaUploadResponse extends BaseUploadedMedia {

frontend/src/component/MDEditor.tsx

@@ -70,7 +70,7 @@ export const MDEditor = ({
                         setOpen(false);
                         const newBody =
                             bodyMD.slice(0, cursorPos) +
-                            `![${resp.name}](media://${resp.media_id})` +
+                            `![${resp.asset.name}](media://${resp.asset.asset_id})` +
                             bodyMD.slice(cursorPos);
                         setBodyMD(newBody);
                         onSuccess && onSuccess();

frontend/src/component/STVListVIew.tsx

@@ -185,7 +185,7 @@ export const STVListVIew = () => {
                                 return (
                                     <IconButton
                                         component={Link}
-                                        href={`/demos/${row.demo_id}`}
+                                        href={`${window.gbans.asset_url}/${window.gbans.bucket_demo}/${row.title}`}
                                         color={'primary'}
                                     >
                                         <FileDownloadIcon />

internal/app/app.go

@@ -201,15 +201,71 @@ func (app *App) Init(ctx context.Context) error {
 		app.log.Info("Loaded filter list", zap.Int("count", len(app.wordFilters.wordFilters)))
 	}
 
-	if errMigrateS3 := app.migrateS3(ctx); errMigrateS3 != nil {
+	if errMigrateS3 := app.migrateS3Media(ctx); errMigrateS3 != nil {
+		panic(errMigrateS3)
+	}
+
+	if errMigrateS3 := app.migrateS3Demo(ctx); errMigrateS3 != nil {
 		panic(errMigrateS3)
 	}
 
 	return nil
 }
 
-// TODO remove this eventually
-func (app *App) migrateS3(ctx context.Context) error {
+// TODO remove this eventually.
+func (app *App) migrateS3Demo(ctx context.Context) error {
+	if errBucket := app.assetStore.CreateBucketIfNotExists(ctx, app.conf.S3.BucketDemo); errBucket != nil {
+		return errBucket
+	}
+
+	demos, errDemos := app.db.GetDemos(ctx, store.GetDemosOptions{})
+	if errDemos != nil {
+		if errors.Is(errDemos, store.ErrNoResult) {
+			return nil
+		}
+
+		return errors.Wrap(errDemos, "Failed to get demos")
+	}
+
+	for _, demo := range demos {
+		var full store.DemoFile
+		if errFull := app.db.GetDemoByID(ctx, demo.DemoID, &full); errFull != nil {
+			app.log.Error("Failed to get demo", zap.Error(errFull))
+
+			continue
+		}
+
+		ass, errAss := store.NewAsset(full.Data, app.conf.S3.BucketDemo, demo.Title)
+		if errAss != nil {
+			return errors.Wrap(errAss, "Failed to create asset")
+		}
+
+		ass.MimeType = "application/octet-stream"
+		full.AssetID = ass.AssetID
+
+		if errSave := app.db.SaveAsset(ctx, &ass); errSave != nil {
+			return errors.Wrap(errSave, "Failed to save asset")
+		}
+
+		if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketDemo, ass.Name,
+			bytes.NewReader(full.Data), ass.Size, ass.MimeType); errPut != nil {
+			return errPut
+		}
+
+		full.Size = int64(len(full.Data))
+
+		if errSaveMedia := app.db.SaveDemo(ctx, &full); errSaveMedia != nil {
+			return errors.Wrap(errSaveMedia, "Failed to save media")
+		}
+
+		app.log.Info("Migrated demo successfully", zap.String("name", full.Title))
+	}
+
+	return nil
+}
+
+// TODO remove this eventually.
+func (app *App) migrateS3Media(ctx context.Context) error {
 	if errBucket := app.assetStore.CreateBucketIfNotExists(ctx, app.conf.S3.BucketMedia); errBucket != nil {
 		return errBucket
 	}
@@ -219,55 +275,60 @@ func (app *App) migrateS3(ctx context.Context) error {
 			Limit: 100000,
 		},
 	})
-	if errReports != nil {
-		return errReports
+	if errReports != nil && !errors.Is(errReports, store.ErrNoResult) {
+		return errors.Wrap(errReports, "Failed to get reports")
 	}
 
 	findIdsRx := regexp.MustCompile(`!\[(.+?)]\(media://(\d+)\)`)
 
-	for _, report := range reports {
+	for _, reportVal := range reports {
+		report := reportVal
 		// Update links in message descriptions
 		findIds := findIdsRx.FindAllStringSubmatch(report.Description, -1)
-		for _, id := range findIds {
-			v, e := strconv.ParseInt(id[2], 10, 32)
-			if e != nil {
-				return e
+		for _, foundID := range findIds {
+			idValue, errParse := strconv.ParseInt(foundID[2], 10, 32)
+			if errParse != nil {
+				return errors.Wrap(errParse, "Failed to parse id value")
 			}
-			var m store.Media
-			if err := app.db.GetMediaByID(ctx, int(v), &m); err != nil {
+
+			var newMedia store.Media
+			if err := app.db.GetMediaByID(ctx, int(idValue), &newMedia); err != nil {
 				app.log.Error("Failed to get media to migrate")
+
 				continue
 			}
 
-			ass, errAss := store.NewAsset(m.Contents, app.conf.S3.BucketMedia, "")
+			ass, errAss := store.NewAsset(newMedia.Contents, app.conf.S3.BucketMedia, "")
 			if errAss != nil {
-				return errAss
+				return errors.Wrap(errAss, "Failed to create asset")
 			}
 
-			ass.OldID = int64(m.MediaID)
+			ass.OldID = int64(newMedia.MediaID)
 
 			if errSave := app.db.SaveAsset(ctx, &ass); errSave != nil {
-				return errSave
+				return errors.Wrap(errSave, "Failed to save asset")
 			}
 
-			if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(m.Contents), m.Size, m.MimeType); errPut != nil {
+			if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(newMedia.Contents), newMedia.Size, newMedia.MimeType); errPut != nil {
 				return errPut
 			}
 
-			m.Asset = ass
+			newMedia.Asset = ass
 
-			if errSaveMedia := app.db.SaveMedia(ctx, &m); errSaveMedia != nil {
-				return errSaveMedia
+			if errSaveMedia := app.db.SaveMedia(ctx, &newMedia); errSaveMedia != nil {
+				return errors.Wrap(errSaveMedia, "Failed to save media")
 			}
 
 			report.Description = strings.Replace(
 				report.Description,
-				fmt.Sprintf("![%s](media://%s)", id[1], id[2]),
-				fmt.Sprintf("![%s](media://%s)", id[1], ass.AssetID.String()), 1)
+				fmt.Sprintf("![%s](media://%s)", foundID[1], foundID[2]),
+				fmt.Sprintf("![%s](media://%s)", foundID[1], ass.AssetID.String()), 1)
 
 			if errSave := app.db.SaveReport(ctx, &report); errSave != nil {
-				return errSave
+				return errors.Wrap(errSave, "Failed to save report")
 			}
+
+			app.log.Info("Migrated report successfully", zap.Int64("id", report.ReportID))
 		}
 
 		// Update links in report messages
@@ -276,51 +337,58 @@ func (app *App) migrateS3(ctx context.Context) error {
 			continue
 		}
 
-		for _, msg := range msgs {
+		for _, message := range msgs {
+			msg := message
 			msgIds := findIdsRx.FindAllStringSubmatch(msg.Contents, -1)
-			for _, id := range msgIds {
-				v, e := strconv.ParseInt(id[2], 10, 32)
-				if e != nil {
-					return e
+
+			for _, msgID := range msgIds {
+				msgIDValue, errParse := strconv.ParseInt(msgID[2], 10, 32)
+				if errParse != nil {
+					app.log.Error("Failed to parse int media id", zap.Error(errParse))
+
+					continue
 				}
-				var m store.Media
-				if err := app.db.GetMediaByID(ctx, int(v), &m); err != nil {
+
+				var media store.Media
+				if err := app.db.GetMediaByID(ctx, int(msgIDValue), &media); err != nil {
 					app.log.Error("Failed to get media to migrate")
+
 					continue
 				}
 
-				ass, errAss := store.NewAsset(m.Contents, app.conf.S3.BucketMedia, "")
+				ass, errAss := store.NewAsset(media.Contents, app.conf.S3.BucketMedia, "")
 				if errAss != nil {
-					return errAss
+					return errors.Wrap(errAss, "Failed to create asset")
 				}
 
-				ass.OldID = int64(m.MediaID)
+				ass.OldID = int64(media.MediaID)
 
 				if errSave := app.db.SaveAsset(ctx, &ass); errSave != nil {
-					return errSave
+					return errors.Wrap(errSave, "Failed to save asset")
 				}
 
-				if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(m.Contents), m.Size, m.MimeType); errPut != nil {
-					return errPut
+				if errPut := app.assetStore.Put(ctx, app.conf.S3.BucketMedia, ass.Name, bytes.NewReader(media.Contents), media.Size, media.MimeType); errPut != nil {
+					return errors.Wrap(errPut, "Failed to store asset")
 				}
 
-				m.Asset = ass
+				media.Asset = ass
 
-				if errSaveMedia := app.db.SaveMedia(ctx, &m); errSaveMedia != nil {
-					return errSaveMedia
+				if errSaveMedia := app.db.SaveMedia(ctx, &media); errSaveMedia != nil {
+					return errors.Wrap(errSaveMedia, "Failed to update media")
 				}
 
 				msg.Contents = strings.Replace(
 					msg.Contents,
-					fmt.Sprintf("![%s](media://%s)", id[1], id[2]),
-					fmt.Sprintf("![%s](media://%s)", id[1], ass.AssetID.String()), 1)
+					fmt.Sprintf("![%s](media://%s)", msgID[1], msgID[2]),
+					fmt.Sprintf("![%s](media://%s)", msgID[1], ass.AssetID.String()), 1)
 
 				if errSave := app.db.SaveReportMessage(ctx, &msg); errSave != nil {
-					return errSave
+					return errors.Wrap(errSave, "Failed to save report message")
 				}
+
+				app.log.Info("Migrated report message successfully", zap.Int64("id", msg.MessageID))
 			}
 		}
-
 	}
 
 	return nil

internal/app/app_test.go

@@ -63,7 +63,7 @@ func TestApp(t *testing.T) {
 		}
 	})
 
-	app := New(&config, database, nil, zap.NewNop())
+	app := New(&config, database, nil, zap.NewNop(), nil)
 
 	t.Run("match_sum", testMatchSum(&app))
 }

internal/app/http_api.go

@@ -2707,6 +2707,14 @@ func onAPISaveMedia(app *App) gin.HandlerFunc {
 			return
 		}
 
+		if errSaveAsset := app.db.SaveAsset(ctx, &asset); errSaveAsset != nil {
+			responseErr(ctx, http.StatusInternalServerError, errors.New("Could not save asset"))
+
+			log.Error("Failed to save user asset to s3 backend", zap.Error(errSaveAsset))
+		}
+
+		media.Asset = asset
+
 		media.Contents = nil
 
 		if !fp.Contains(MediaSafeMimeTypesImages, media.MimeType) {
@@ -2731,12 +2739,6 @@ func onAPISaveMedia(app *App) gin.HandlerFunc {
 			return
 		}
 
-		if app.conf.S3.Enabled {
-			if errLoad := app.db.GetMediaByAssetID(ctx, media.Asset.AssetID, &media); errLoad != nil {
-				responseErr(ctx, http.StatusInternalServerError, errors.New("Could not load new media"))
-			}
-		}
-
 		ctx.JSON(http.StatusCreated, media)
 	}
 }

internal/store/demo.go

@@ -3,11 +3,11 @@ package store
 import (
 	"context"
 	"fmt"
-	"github.com/gabriel-vasile/mimetype"
-	"github.com/gofrs/uuid/v5"
 	"time"
 
 	sq "github.com/Masterminds/squirrel"
+	"github.com/gabriel-vasile/mimetype"
+	"github.com/gofrs/uuid/v5"
 	"github.com/leighmacdonald/gbans/internal/consts"
 	"github.com/leighmacdonald/srcdsup/srcdsup"
 	"github.com/leighmacdonald/steamid/v3/steamid"
@@ -28,6 +28,7 @@ type DemoFile struct {
 	MapName         string                                `json:"map_name"`
 	Archive         bool                                  `json:"archive"` // When true, will not get auto deleted when flushing old demos
 	Stats           map[steamid.SID64]srcdsup.PlayerStats `json:"stats"`
+	AssetID         uuid.UUID                             `json:"asset_id"`
 }
 
 // func NewDemoFile(serverId int64, title string, rawData []byte) (DemoFile, error) {
@@ -106,7 +107,7 @@ func (db *Store) GetDemos(ctx context.Context, opts GetDemosOptions) ([]DemoFile
 
 	builder := db.sb.
 		Select("d.demo_id", "d.server_id", "d.title", "d.created_on", "d.size", "d.downloads",
-			"d.map_name", "d.archive", "d.stats", "s.short_name", "s.name").
+			"d.map_name", "d.archive", "d.stats", "s.short_name", "s.name", "d.asset_id").
 		From("demo d").
 		LeftJoin("server s ON s.server_id = d.server_id").
 		OrderBy("created_on DESC").
@@ -145,11 +146,19 @@ func (db *Store) GetDemos(ctx context.Context, opts GetDemosOptions) ([]DemoFile
 	defer rows.Close()
 
 	for rows.Next() {
-		var demoFile DemoFile
+		var (
+			demoFile DemoFile
+			uuidScan *uuid.UUID // TODO remove this and make column not-null once migrations are complete
+		)
+
 		if errScan := rows.Scan(&demoFile.DemoID, &demoFile.ServerID, &demoFile.Title, &demoFile.CreatedOn,
 			&demoFile.Size, &demoFile.Downloads, &demoFile.MapName, &demoFile.Archive, &demoFile.Stats,
-			&demoFile.ServerNameShort, &demoFile.ServerNameLong); errScan != nil {
-			return nil, Err(errQuery)
+			&demoFile.ServerNameShort, &demoFile.ServerNameLong, &uuidScan); errScan != nil {
+			return nil, Err(errScan)
+		}
+
+		if uuidScan != nil {
+			demoFile.AssetID = *uuidScan
 		}
 
 		demos = append(demos, demoFile)
@@ -194,9 +203,9 @@ func (db *Store) SaveDemo(ctx context.Context, demoFile *DemoFile) error {
 func (db *Store) insertDemo(ctx context.Context, demoFile *DemoFile) error {
 	query, args, errQueryArgs := db.sb.
 		Insert(string(tableDemo)).
-		Columns("server_id", "title", "raw_data", "created_on", "size", "downloads", "map_name", "archive", "stats").
+		Columns("server_id", "title", "raw_data", "created_on", "size", "downloads", "map_name", "archive", "stats", "asset_id").
 		Values(demoFile.ServerID, demoFile.Title, demoFile.Data, demoFile.CreatedOn,
-			demoFile.Size, demoFile.Downloads, demoFile.MapName, demoFile.Archive, demoFile.Stats).
+			demoFile.Size, demoFile.Downloads, demoFile.MapName, demoFile.Archive, demoFile.Stats, demoFile.AssetID).
 		Suffix("RETURNING demo_id").
 		ToSql()
 	if errQueryArgs != nil {
@@ -222,6 +231,7 @@ func (db *Store) updateDemo(ctx context.Context, demoFile *DemoFile) error {
 		Set("map_name", demoFile.MapName).
 		Set("archive", demoFile.Archive).
 		Set("stats", demoFile.Stats).
+		Set("asset_id", demoFile.AssetID).
 		Where(sq.Eq{"demo_id": demoFile.DemoID}).
 		ToSql()
 	if errQueryArgs != nil {