From 10c424e44fedadf5bf4d4d1790699de2b96b7f0b Mon Sep 17 00:00:00 2001 From: Jonas Herzig Date: Tue, 10 Oct 2023 14:31:50 +0200 Subject: [PATCH] Migrations: Add Linear: EM-2038 GitHub: #82 --- api/Vigilance.api | 5 + build.gradle.kts | 4 + .../kotlin/gg/essential/vigilance/Vigilant.kt | 34 ++++ .../gg/essential/vigilance/data/Migration.kt | 8 + .../gg/essential/vigilance/impl/migration.kt | 108 +++++++++++++ .../essential/vigilance/impl/MigrationTest.kt | 149 ++++++++++++++++++ 6 files changed, 308 insertions(+) create mode 100644 src/main/kotlin/gg/essential/vigilance/data/Migration.kt create mode 100644 src/main/kotlin/gg/essential/vigilance/impl/migration.kt create mode 100644 src/test/kotlin/gg/essential/vigilance/impl/MigrationTest.kt diff --git a/api/Vigilance.api b/api/Vigilance.api index 55c8032b..8248ee95 100644 --- a/api/Vigilance.api +++ b/api/Vigilance.api @@ -26,6 +26,7 @@ public abstract class gg/essential/vigilance/Vigilant { public final fun getCategories ()Ljava/util/List; public final fun getCategoryFromSearch (Ljava/lang/String;)Lgg/essential/vigilance/data/Category; public final fun getGuiTitle ()Ljava/lang/String; + protected fun getMigrations ()Ljava/util/List; public final fun getSortingBehavior ()Lgg/essential/vigilance/data/SortingBehavior; public final fun gui ()Lgg/essential/vigilance/gui/SettingsGui; public final fun hiddenIf (Lkotlin/reflect/KProperty;Lkotlin/jvm/functions/Function0;)V @@ -153,6 +154,10 @@ public final class gg/essential/vigilance/data/MethodBackedPropertyValue : gg/es public fun invoke (Lgg/essential/vigilance/Vigilant;)V } +public abstract interface class gg/essential/vigilance/data/Migration { + public abstract fun apply (Ljava/util/Map;)V +} + public abstract interface annotation class gg/essential/vigilance/data/Property : java/lang/annotation/Annotation { public abstract fun allowAlpha ()Z public abstract fun category ()Ljava/lang/String; diff --git a/build.gradle.kts b/build.gradle.kts index dcc61a19..554130a2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,3 +57,7 @@ apiValidation { ignoredPackages.add("gg.essential.vigilance.example") nonPublicMarkers.add("org.jetbrains.annotations.ApiStatus\$Internal") } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/gg/essential/vigilance/Vigilant.kt b/src/main/kotlin/gg/essential/vigilance/Vigilant.kt index dae4c38a..9f49564b 100644 --- a/src/main/kotlin/gg/essential/vigilance/Vigilant.kt +++ b/src/main/kotlin/gg/essential/vigilance/Vigilant.kt @@ -4,6 +4,7 @@ import gg.essential.universal.UChat import gg.essential.vigilance.data.* import gg.essential.vigilance.gui.SettingsGui import gg.essential.vigilance.impl.I18n +import gg.essential.vigilance.impl.migrate import gg.essential.vigilance.impl.nightconfig.core.file.FileConfig import java.awt.Color import java.io.File @@ -38,7 +39,38 @@ abstract class Vigilant @JvmOverloads constructor( c.initEmptyFile(f) false }.build() + private val categoryDescription = mutableMapOf() + + /** + * List of migrations to apply to the config file when it is loaded. + * + * Each entry in the list is a "migration" which should be a pure function that transforms a given old config + * to a newer format. + * The config is passed to the migration as a [MutableMap] of paths to arbitrary values. + * Each path consist of one or more parts joint by `.` characters. + * To get the path for a given property, join its category, (optionally) subcategory, and name with a `.`, lowercase + * everything and replace all spaces with `_` (other special characters are unaffected). + * E.g. `@Property(name = "My Fancy Setting", category = "General", subcategory = "Fancy Stuff")` + * becomes `general.fancy_stuff.my_fancy_setting`. + * See also [gg.essential.vigilance.data.fullPropertyPath]. + * + * The config file keeps track of how many migrations have already been applied to it, so a migration will only be + * applied if it has not yet been applied. + * Note that for this to work properly, the list must effectively be treated as append-only. Removing or re-ordering + * migrations in the list will change their index and may cause them to be re-applied / other migrations to not be + * applied at all. + * + * The config file also keeps track of what changes each migration made and stores this information in the file, + * such that, if the mod is downgraded, it can roll back those changes. + * Note that this will only roll back things which have actually changed. If a new version of your mod adds a new + * option to a selector, and the user manually selects that option and then downgrades the mod, the old version + * will see that new index and likely error. If you wish to prevent this via the migrations/rollback system, you + * must artificially modify that setting in a migration (e.g. increase its value by 1) and then change it back to + * its actual value in a second migration (e.g. decrease its value again by 1). + */ + protected open val migrations: List = emptyList() + private var dirty = false private var hasError = false @@ -265,6 +297,8 @@ abstract class Vigilant @JvmOverloads constructor( private fun readData() { fileConfig.load() + migrate(fileConfig, migrations) + propertyCollector.getProperties().filter { it.value.writeDataToFile }.forEach { val fullPath = it.attributesExt.fullPropertyPath() diff --git a/src/main/kotlin/gg/essential/vigilance/data/Migration.kt b/src/main/kotlin/gg/essential/vigilance/data/Migration.kt new file mode 100644 index 00000000..07463437 --- /dev/null +++ b/src/main/kotlin/gg/essential/vigilance/data/Migration.kt @@ -0,0 +1,8 @@ +package gg.essential.vigilance.data + +import gg.essential.vigilance.Vigilant + +/** See [Vigilant.migrations]. */ +fun interface Migration { + fun apply(config: MutableMap) +} diff --git a/src/main/kotlin/gg/essential/vigilance/impl/migration.kt b/src/main/kotlin/gg/essential/vigilance/impl/migration.kt new file mode 100644 index 00000000..96c503cb --- /dev/null +++ b/src/main/kotlin/gg/essential/vigilance/impl/migration.kt @@ -0,0 +1,108 @@ +package gg.essential.vigilance.impl + +import gg.essential.vigilance.data.Migration +import gg.essential.vigilance.impl.nightconfig.core.Config + +private const val META_KEY = "__meta" +private val VERSION_KEY = listOf(META_KEY, "version") +private fun migrationLogKey(migration: Int): List = listOf(META_KEY, "migration_log", "$migration") + +internal fun migrate(root: Config, migrations: List) { + val fileVersion = root[VERSION_KEY] ?: 0 + if (fileVersion < migrations.size) { + var oldMap = root.toMap() + for (migration in fileVersion until migrations.size) { + val newMap = oldMap.toMutableMap() + + migrations[migration].apply(newMap) + + applyMigration(root, migration, oldMap, newMap) + + root.update(VERSION_KEY, migration + 1) + + oldMap = newMap + assert(oldMap == root.toMap()) + } + } else if (fileVersion > migrations.size) { + for (migration in fileVersion - 1 downTo migrations.size) { + rollbackMigration(root, migration) + root.update(VERSION_KEY, migration) + } + } +} + +private fun applyMigration(root: Config, migration: Int, oldMap: Map, newMap: Map) { + val migrationLog = root.createSubConfig() + + for ((key, oldValue) in oldMap) { + if (key !in newMap) { + migrationLog.update(listOf("changed", key), oldValue) + root.purge(key.split(".")) + } else { + val newValue = newMap[key] + if (newValue != oldValue) { + migrationLog.update(listOf("changed", key), oldValue) + root.update(key, newValue) + } + } + } + + val added = mutableListOf() + for ((key, newValue) in newMap) { + if (key !in oldMap) { + added.add(key) + root.update(key, newValue) + } + } + if (added.isNotEmpty()) { + migrationLog.update(listOf("added"), added) + } + + if (!migrationLog.isEmpty) { + root.update(migrationLogKey(migration), migrationLog) + } +} + +private fun rollbackMigration(root: Config, migration: Int) { + val migrationLog = root.purge(migrationLogKey(migration)) ?: return + val added = migrationLog.get>(listOf("added")) ?: emptyList() + val changed = migrationLog.get(listOf("changed"))?.valueMap() ?: emptyMap() + + for (key in added) { + root.purge(key.split(".")) + } + for ((key, oldValue) in changed) { + root.update(key, oldValue) + } +} + +/** Removes the value at the given key as well as any now-empty intermediate nodes. */ +private fun Config.purge(keys: List): T? { + return if (keys.size > 1) { + val child = get(listOf(keys[0])) as? Config ?: return null + val removed = child.purge(keys.subList(1, keys.size)) + if (child.isEmpty) { + remove(listOf(keys[0])) + } + removed + } else { + remove(keys) + } +} + +/** Converts the nested [Config] into a flat [Map]. Does not include any [META_KEY] entries. */ +private fun Config.toMap(): Map { + val result = mutableMapOf() + fun visit(config: Config, prefix: String) { + for ((key, value) in config.valueMap()) { + if (key == META_KEY) continue + if (value is Config) { + visit(value, "$prefix$key.") + } else { + result[prefix + key] = value + } + } + } + visit(this, "") + return result +} diff --git a/src/test/kotlin/gg/essential/vigilance/impl/MigrationTest.kt b/src/test/kotlin/gg/essential/vigilance/impl/MigrationTest.kt new file mode 100644 index 00000000..1eccc983 --- /dev/null +++ b/src/test/kotlin/gg/essential/vigilance/impl/MigrationTest.kt @@ -0,0 +1,149 @@ +package gg.essential.vigilance.impl + +import gg.essential.vigilance.data.Migration +import gg.essential.vigilance.impl.nightconfig.core.Config +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class MigrationTest { + @Test + fun testUpToDate() { + val input = config("__meta" to mapOf("version" to 2), "test" to 42) + migrate(input, listOf(Migration {}, Migration {})) + assertEquals(config("__meta" to mapOf("version" to 2), "test" to 42), input) + } + + @Test + fun testNoOpMigration() { + val input = config("__meta" to mapOf("version" to 1), "test" to 42) + migrate(input, listOf(Migration {}, Migration {})) + assertEquals(config("__meta" to mapOf("version" to 2), "test" to 42), input) + } + + @Test + fun testAddMigration() { + val input = config("__meta" to mapOf("version" to 1), "test" to 42) + migrate(input, listOf(Migration {}, Migration { it["new"] = 43 })) + assertEquals(config( + "__meta" to mapOf("version" to 2, "migration_log" to mapOf("1" to mapOf("added" to listOf("new")))), + "test" to 42, + "new" to 43, + ), input) + } + + @Test + fun testChangeMigration() { + val input = config("__meta" to mapOf("version" to 1), "test" to 42) + migrate(input, listOf(Migration {}, Migration { it["test"] = it["test"] as Int + 1 })) + assertEquals(config( + "__meta" to mapOf("version" to 2, "migration_log" to mapOf("1" to mapOf("changed" to mapOf("test" to 42)))), + "test" to 43, + ), input) + } + + @Test + fun testRemoveMigration() { + val input = config("__meta" to mapOf("version" to 1), "test" to 42) + migrate(input, listOf(Migration {}, Migration { it.remove("test") })) + assertEquals(config( + "__meta" to mapOf("version" to 2, "migration_log" to mapOf("1" to mapOf("changed" to mapOf("test" to 42)))), + ), input) + } + + @Test + fun testMultipleMigrations() { + val input = config("__meta" to mapOf("version" to 1), "test" to 42) + migrate(input, listOf(Migration {}, Migration { it["test"] = 1 }, Migration { it["test"] = it["test"] as Int + 1 })) + assertEquals(config( + "__meta" to mapOf("version" to 3, "migration_log" to mapOf( + "1" to mapOf("changed" to mapOf("test" to 42)), + "2" to mapOf("changed" to mapOf("test" to 1)), + )), + "test" to 2, + ), input) + } + + @Test + fun testAddRollback() { + val input = config( + "__meta" to mapOf("version" to 2, "migration_log" to mapOf("1" to mapOf("added" to listOf("new")))), + "test" to 42, + "new" to 43, + ) + migrate(input, listOf(Migration {})) + assertEquals(config("__meta" to mapOf("version" to 1), "test" to 42), input) + } + + @Test + fun testChangeRollback() { + val input = config( + "__meta" to mapOf("version" to 2, "migration_log" to mapOf("1" to mapOf("changed" to mapOf("test" to 42)))), + "test" to 43, + ) + migrate(input, listOf(Migration {})) + assertEquals(config("__meta" to mapOf("version" to 1), "test" to 42), input) + } + + @Test + fun testRemoveRollback() { + val input = config( + "__meta" to mapOf("version" to 2, "migration_log" to mapOf("1" to mapOf("changed" to mapOf("test" to 42)))), + ) + migrate(input, listOf(Migration {})) + assertEquals(config("__meta" to mapOf("version" to 1), "test" to 42), input) + } + + @Test + fun testMultipleRollback() { + val input = config( + "__meta" to mapOf("version" to 3, "migration_log" to mapOf( + "1" to mapOf("changed" to mapOf("test" to 42)), + "2" to mapOf("changed" to mapOf("test" to 1)), + )), + "test" to 2, + ) + migrate(input, listOf(Migration {})) + assertEquals(config("__meta" to mapOf("version" to 1), "test" to 42), input) + } + + @Test + fun testNoMigrations() { + val input = config("test" to 42) + migrate(input, listOf()) + assertEquals(config("test" to 42), input) + } + + @Test + fun testInitialMigration() { + val input = config("test" to 42) + migrate(input, listOf(Migration { config -> + assert(config == mapOf("test" to 42)) + config["test"] = 43 + })) + assertEquals(config( + "__meta" to mapOf("version" to 1, "migration_log" to mapOf("0" to mapOf("changed" to mapOf("test" to 42)))), + "test" to 43, + ), input) + } + + private fun config(vararg entries: Pair): Config = + config(entries.toMap()) + + private fun config(map: Map): Config { + fun visit(map: Map, config: Config) { + for ((key, value) in map) { + if (value is Map<*, *>) { + val inner = config.createSubConfig() + @Suppress("UNCHECKED_CAST") + visit(value as Map, inner) + config.update(listOf(key), inner) + } else { + config.update(listOf(key), value) + } + } + } + val config = Config.inMemory() + visit(map, config) + return config + } +}