From 6d9b592729c36d1e3c1137b82b8de4cf47d611e5 Mon Sep 17 00:00:00 2001 From: lberrymage Date: Thu, 25 Jul 2024 15:06:48 -0700 Subject: [PATCH] Use db as source of truth for repository metadata The current behavior of S3PublishService when publishing an edit is to fetch the old repository metadata from the S3 bucket, patch it with new data, then republish the patch metadata over the old metadata. The problem with this approach is that it requires unnecessary data fetches and doesn't maintain a well-defined source of truth for the repository. Instead, store generated repository metadata in the database and patch _that_ when publishing edits so that we can consistently source the repository from a single location. The data is relatively small, so storing it in the database shouldn't have a significant performance impact. If we really need to in the future, we can store the metadata as files in a separate storage bucket. --- .../accrescent/parcelo/console/data/App.kt | 2 + .../console/publish/S3PublishService.kt | 42 ++++++------ .../V10__Save_repodata_to_database.kt | 65 +++++++++++++++++++ ...V11__Make_repository_metadata_not_null.sql | 17 +++++ ...__Add_repository_metadata_field_to_app.sql | 5 ++ 5 files changed, 110 insertions(+), 21 deletions(-) create mode 100644 console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt create mode 100644 console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql create mode 100644 console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt index f5cb0ed1..474fd143 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt @@ -20,6 +20,7 @@ object Apps : IdTable("apps") { val reviewIssueGroupId = reference("review_issue_group_id", ReviewIssueGroups, ReferenceOption.NO_ACTION).nullable() val updating = bool("updating").default(false) + val repositoryMetadata = blob("repository_metadata") override val primaryKey = PrimaryKey(id) } @@ -31,6 +32,7 @@ class App(id: EntityID) : Entity(id), ToSerializable - val newRepoData = s3Client.getObject(getOldDataReq) { resp -> - val oldRepoData = - resp.body?.toInputStream()?.use { Json.decodeFromStream(it) } - ?: throw FileNotFoundException() - RepoData( - version = oldRepoData.version, - versionCode = oldRepoData.versionCode, - abiSplits = oldRepoData.abiSplits, - densitySplits = oldRepoData.densitySplits, - langSplits = oldRepoData.langSplits, - shortDescription = shortDescription ?: oldRepoData.shortDescription - ) - } + // Fetch the old app metadata from the database + val app = transaction { App.findById(appId) } ?: throw Exception("app not found") + val oldRepoData = app.repositoryMetadata.inputStream + .use { Json.decodeFromStream(it) } + // Modify the old app metadata to produce the new app metadata + val newRepoData = RepoData( + version = oldRepoData.version, + versionCode = oldRepoData.versionCode, + abiSplits = oldRepoData.abiSplits, + densitySplits = oldRepoData.densitySplits, + langSplits = oldRepoData.langSplits, + shortDescription = shortDescription ?: oldRepoData.shortDescription + ).let { Json.encodeToString(it) }.toByteArray() + + // Publish the new app metadata val updateDataReq = PutObjectRequest { bucket = s3Bucket key = "apps/$appId/repodata.json" - body = ByteStream.fromString(Json.encodeToString(newRepoData)) + body = ByteStream.fromBytes(newRepoData) } s3Client.putObject(updateDataReq) + + // Save the new app metadata to the database + transaction { app.repositoryMetadata = ExposedBlob(newRepoData) } } } diff --git a/console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt b/console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt new file mode 100644 index 00000000..f61105f7 --- /dev/null +++ b/console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt @@ -0,0 +1,65 @@ +// Copyright 2024 Logan Magee +// +// SPDX-License-Identifier: AGPL-3.0-only + +package db.migration + +import app.accrescent.parcelo.console.Config +import app.accrescent.parcelo.console.data.App +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.GetObjectRequest +import aws.smithy.kotlin.runtime.content.toInputStream +import aws.smithy.kotlin.runtime.net.url.Url +import kotlinx.coroutines.runBlocking +import org.flywaydb.core.api.migration.BaseJavaMigration +import org.flywaydb.core.api.migration.Context +import org.jetbrains.exposed.sql.statements.api.ExposedBlob +import org.jetbrains.exposed.sql.transactions.transaction +import org.koin.java.KoinJavaComponent.inject +import java.io.FileNotFoundException + +/** + * A versioned migration which saves published repository to the database. + */ +class V10__Save_repodata_to_database : BaseJavaMigration() { + private val config: Config by inject(Config::class.java) + + /** + * Downloads all published repository metadata, saving it to the database alongside its + * associated app. + */ + override fun migrate(context: Context) { + val oldRepoDataReqs = transaction { App.all().map { it.id.value } } + .map { appId -> + val req = GetObjectRequest { + bucket = config.s3.bucket + key = "apps/$appId/repodata.json" + } + Pair(appId, req) + } + S3Client { + endpointUrl = Url.parse(config.s3.endpointUrl) + region = config.s3.region + credentialsProvider = StaticCredentialsProvider { + accessKeyId = config.s3.accessKeyId + secretAccessKey = config.s3.secretAccessKey + } + }.use { s3Client -> + oldRepoDataReqs.forEach { (appId, req) -> + runBlocking { + s3Client.getObject(req) { resp -> + val data = resp.body + ?.toInputStream() + ?.use { it.readBytes() } + ?: throw FileNotFoundException() + + transaction { + App.findById(appId)?.repositoryMetadata = ExposedBlob(data) + } + } + } + } + } + } +} diff --git a/console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql b/console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql new file mode 100644 index 00000000..c2a90453 --- /dev/null +++ b/console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql @@ -0,0 +1,17 @@ +-- Copyright 2024 Logan Magee +-- +-- SPDX-License-Identifier: AGPL-3.0-only + +-- Add a NOT NULL constraint to the repository_metadata column +PRAGMA foreign_keys = OFF; + +-- The previous apps table with the repository_metadata column made NOT NULL +CREATE TABLE apps2 (id TEXT NOT NULL PRIMARY KEY, version_code INT NOT NULL, version_name TEXT NOT NULL, file_id INT NOT NULL, review_issue_group_id INT NULL, updating BOOLEAN DEFAULT 0 NOT NULL, repository_metadata NOT NULL, CONSTRAINT fk_apps_file_id__id FOREIGN KEY (file_id) REFERENCES files(id) ON UPDATE RESTRICT, CONSTRAINT fk_apps_review_issue_group_id__id FOREIGN KEY (review_issue_group_id) REFERENCES review_issue_groups(id) ON UPDATE RESTRICT); + +INSERT INTO apps2 SELECT * FROM apps; +DROP TABLE apps; +ALTER TABLE apps2 RENAME TO apps; + +PRAGMA foreign_key_check; + +PRAGMA foreign_keys = ON; diff --git a/console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql b/console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql new file mode 100644 index 00000000..959ad47e --- /dev/null +++ b/console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql @@ -0,0 +1,5 @@ +-- Copyright 2024 Logan Magee +-- +-- SPDX-License-Identifier: AGPL-3.0-only + +ALTER TABLE apps ADD COLUMN repository_metadata BLOB;