Change backend database from SQLite to PostgreSQL

As part of our containerization efforts, we need to move to a networked
database so that persistent state needn't be stored directly alongside
our application. PostgreSQL will also give us a lot less grief about
database locking, provide stronger data integrity guarantees, enable us
to easily implement atomic database backups, and give us access to much
more expressive data types than SQLite.

Because this is a major breaking change, it uses the opportunity to
establish a new baseline for future database migrations (instead of
"migrating" the existing database migrations to the PostgreSQL flavor of
SQL).

Transferring existing data from SQLite to PostgreSQL is left to the
administrator to be performed manually, i.e., without any scripts
provided by Accrescent.

.run/console.run.xml

@@ -8,9 +8,10 @@ SPDX-License-Identifier: AGPL-3.0-only
   <configuration default="false" name="console" type="JetRunConfigurationType">
     <envs>
       <env name="BASE_URL" value="http://localhost:8080" />
-      <env name="CONSOLE_DATABASE_PATH" value="appdata/console.db" />
       <env name="DEBUG_USER_EMAIL" value="example@example.com" />
       <env name="DEBUG_USER_REVIEWER_EMAIL" value="example@example.com" />
+      <env name="POSTGRESQL_PASSWORD" value="password" />
+      <env name="POSTGRESQL_SSL" value="false" />
       <env name="GITHUB_OAUTH2_REDIRECT_URL" value="http://localhost:4200/auth/github/callback" />
       <env name="HOST" value="0.0.0.0" />
       <env name="PORT" value="8080" />

console/build.gradle.kts

@@ -37,6 +37,7 @@ dependencies {
     implementation(libs.exposed.dao)
     implementation(libs.exposed.jdbc)
     implementation(libs.flyway)
+    implementation(libs.flyway.postgresql)
     implementation(libs.fourkoma)
     implementation(libs.github)
     implementation(libs.jobrunr)
@@ -50,7 +51,7 @@ dependencies {
     implementation(libs.ktor.server.netty)
     implementation(libs.ktor.server.resources)
     implementation(libs.logback)
-    implementation(libs.sqlite)
+    implementation(libs.postgresql)
     testImplementation(libs.ktor.server.tests)
     testImplementation(libs.kotlin.test)
 }

console/src/main/kotlin/app/accrescent/parcelo/console/Application.kt

@@ -31,6 +31,11 @@ import org.koin.logger.slf4jLogger
 import java.nio.file.Path
 
 private const val DEFAULT_CONFIG_PATH = "/etc/pconsole/config.toml"
+private const val POSTGRESQL_DEFAULT_SERVER_NAME = "localhost"
+private const val POSTGRESQL_DEFAULT_DATABASE_NAME = "postgres"
+private const val POSTGRESQL_DEFAULT_PORT = 5432
+private const val POSTGRESQL_DEFAULT_USER = "postgres"
+private const val POSTGRESQL_DEFAULT_SSL = true
 
 fun main(args: Array<String>) = EngineMain.main(args)
 
@@ -42,7 +47,17 @@ fun Application.module() {
         Config(
             application = Config.Application(
                 baseUrl = System.getenv("BASE_URL"),
-                databasePath = System.getenv("CONSOLE_DATABASE_PATH"),
+            ),
+            postgresql = Config.Postgresql(
+                serverName = System.getenv("POSTGRESQL_SERVER_NAME")
+                    ?: POSTGRESQL_DEFAULT_SERVER_NAME,
+                databaseName = System.getenv("POSTGRESQL_DATABASE_NAME")
+                    ?: POSTGRESQL_DEFAULT_DATABASE_NAME,
+                portNumber = System.getenv("POSTGRESQL_PORT_NUMBER")?.toInt()
+                    ?: POSTGRESQL_DEFAULT_PORT,
+                user = System.getenv("POSTGRESQL_USER") ?: POSTGRESQL_DEFAULT_USER,
+                password = System.getenv("POSTGRESQL_PASSWORD"),
+                ssl = System.getenv("POSTGRESQL_SSL")?.toBooleanStrict() ?: POSTGRESQL_DEFAULT_SSL,
             ),
             privateStorage = Config.S3(
                 endpointUrl = System.getenv("PRIVATE_STORAGE_ENDPOINT_URL"),

console/src/main/kotlin/app/accrescent/parcelo/console/Config.kt

@@ -6,13 +6,20 @@ package app.accrescent.parcelo.console
 
 data class Config(
     val application: Application,
+    val postgresql: Postgresql,
     val privateStorage: S3,
     val s3: S3,
     val github: GitHub,
 ) {
-    data class Application(
-        val baseUrl: String,
-        val databasePath: String,
+    data class Application(val baseUrl: String)
+
+    data class Postgresql(
+        val serverName: String,
+        val databaseName: String,
+        val portNumber: Int,
+        val user: String,
+        val password: String,
+        val ssl: Boolean,
     )
 
     data class S3(

console/src/main/kotlin/app/accrescent/parcelo/console/data/Database.kt

@@ -8,9 +8,12 @@ import app.accrescent.parcelo.console.Config
 import app.accrescent.parcelo.console.data.baseline.BaselineAccessControlLists
 import app.accrescent.parcelo.console.data.baseline.BaselineApps
 import app.accrescent.parcelo.console.data.baseline.BaselineDrafts
+import app.accrescent.parcelo.console.data.baseline.BaselineEdits
 import app.accrescent.parcelo.console.data.baseline.BaselineFiles
 import app.accrescent.parcelo.console.data.baseline.BaselineIcons
+import app.accrescent.parcelo.console.data.baseline.BaselineListings
 import app.accrescent.parcelo.console.data.baseline.BaselineRejectionReasons
+import app.accrescent.parcelo.console.data.baseline.BaselineReviewIssueGroups
 import app.accrescent.parcelo.console.data.baseline.BaselineReviewIssues
 import app.accrescent.parcelo.console.data.baseline.BaselineReviewers
 import app.accrescent.parcelo.console.data.baseline.BaselineReviews
@@ -26,32 +29,36 @@ import org.jetbrains.exposed.sql.insertIgnore
 import org.jetbrains.exposed.sql.insertIgnoreAndGetId
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.koin.ktor.ext.inject
-import org.sqlite.SQLiteDataSource
+import org.postgresql.ds.PGSimpleDataSource
 import javax.sql.DataSource
 
 fun Application.configureDatabase(): DataSource {
     val config: Config by inject()
 
-    val dataSource = SQLiteDataSource().apply {
-        url = "jdbc:sqlite:${config.application.databasePath}?journal_mode=wal"
-
-        setEnforceForeignKeys(true)
+    val dataSource = PGSimpleDataSource().apply {
+        serverNames = arrayOf(config.postgresql.serverName)
+        databaseName = config.postgresql.databaseName
+        portNumbers = intArrayOf(config.postgresql.portNumber)
+        user = config.postgresql.user
+        password = config.postgresql.password
+        ssl = config.postgresql.ssl
     }
-    Database.connect(dataSource, setupConnection = {
-        it.createStatement().executeUpdate("PRAGMA trusted_schema = OFF")
-    })
+    Database.connect(dataSource)
 
     transaction {
         SchemaUtils.create(
             BaselineAccessControlLists,
             BaselineApps,
             BaselineDrafts,
+            BaselineEdits,
             BaselineFiles,
             BaselineIcons,
+            BaselineListings,
             BaselineRejectionReasons,
             BaselineReviewers,
             BaselineReviewIssues,
             BaselineReviews,
+            BaselineReviewIssueGroups,
             BaselineSessions,
             BaselineUpdates,
             BaselineUsers,

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineAccessControlList.kt

@@ -13,6 +13,7 @@ object BaselineAccessControlLists : IntIdTable("access_control_lists") {
     val userId = reference("user_id", BaselineUsers, ReferenceOption.CASCADE)
     val appId = reference("app_id", BaselineApps, ReferenceOption.CASCADE)
     val update = bool("update").default(false)
+    val editMetadata = bool("edit_metadata").default(false)
 
     init {
         uniqueIndex(userId, appId)

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineApp.kt

@@ -11,17 +11,16 @@ import org.jetbrains.exposed.sql.ReferenceOption
 
 object BaselineApps : IdTable<String>("apps") {
     override val id = text("id").entityId()
-    val label = text("label")
     val versionCode = integer("version_code")
     val versionName = text("version_name")
     val fileId = reference("file_id", BaselineFiles, ReferenceOption.NO_ACTION)
-    val iconId = reference("icon_id", BaselineIcons, ReferenceOption.NO_ACTION)
     val reviewIssueGroupId =
         reference(
             "review_issue_group_id",
             BaselineReviewIssueGroups,
             ReferenceOption.NO_ACTION
         ).nullable()
     val updating = bool("updating").default(false)
+    val repositoryMetadata = blob("repository_metadata")
     override val primaryKey = PrimaryKey(id)
 }

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineDraft.kt

@@ -18,6 +18,7 @@ object BaselineDrafts : UUIDTable("drafts") {
     val label = text("label")
     val versionCode = integer("version_code")
     val versionName = text("version_name")
+    val shortDescription = text("short_description").default("")
     val creatorId = reference("creator_id", BaselineUsers, ReferenceOption.CASCADE)
     val creationTime = long("creation_time").clientDefault { System.currentTimeMillis() / 1000 }
     val fileId = reference("file_id", BaselineFiles, ReferenceOption.NO_ACTION)

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineEdit.kt

@@ -0,0 +1,27 @@
+// Copyright 2024 Logan Magee
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// DO NOT MODIFY - DATABASE BASELINE
+
+package app.accrescent.parcelo.console.data.baseline
+
+import org.jetbrains.exposed.dao.id.UUIDTable
+import org.jetbrains.exposed.sql.ReferenceOption
+
+object BaselineEdits : UUIDTable("edits") {
+    val appId = reference("app_id", BaselineApps, ReferenceOption.CASCADE)
+    val shortDescription = text("short_description").nullable()
+    val creationTime = long("creation_time").clientDefault { System.currentTimeMillis() / 1000 }
+    val reviewerId =
+        reference("reviewer_id", BaselineReviewers, ReferenceOption.NO_ACTION).nullable()
+    val reviewId = reference("review_id", BaselineReviews, ReferenceOption.NO_ACTION).nullable()
+    val published = bool("published").default(false)
+
+    init {
+        check {
+            // At least one metadata field must be non-null
+            shortDescription.isNotNull()
+        }
+    }
+}

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineFile.kt

@@ -9,6 +9,6 @@ package app.accrescent.parcelo.console.data.baseline
 import org.jetbrains.exposed.dao.id.IntIdTable
 
 object BaselineFiles : IntIdTable("files") {
-    val localPath = text("local_path")
     val deleted = bool("deleted").default(false)
+    val s3ObjectKey = text("s3_object_key").nullable()
 }

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineIcon.kt

@@ -10,6 +10,5 @@ import org.jetbrains.exposed.dao.id.IntIdTable
 import org.jetbrains.exposed.sql.ReferenceOption
 
 object BaselineIcons : IntIdTable("icons") {
-    val hash = text("hash")
     val fileId = reference("file_id", BaselineFiles, ReferenceOption.NO_ACTION)
 }

console/src/main/kotlin/app/accrescent/parcelo/console/data/baseline/BaselineListing.kt

@@ -0,0 +1,22 @@
+// Copyright 2024 Logan Magee
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// DO NOT MODIFY - DATABASE BASELINE
+
+package app.accrescent.parcelo.console.data.baseline
+
+import org.jetbrains.exposed.dao.id.IntIdTable
+import org.jetbrains.exposed.sql.ReferenceOption
+
+object BaselineListings : IntIdTable("listings") {
+    val appId = reference("app_id", BaselineApps, ReferenceOption.CASCADE)
+    val locale = text("locale")
+    val iconId = reference("icon_id", BaselineIcons, ReferenceOption.NO_ACTION)
+    val label = text("label")
+    val shortDescription = text("short_description")
+
+    init {
+        uniqueIndex(appId, locale)
+    }
+}

console/src/main/kotlin/app/accrescent/parcelo/console/jobs/JobRunr.kt

@@ -6,7 +6,7 @@ package app.accrescent.parcelo.console.jobs
 
 import org.jobrunr.configuration.JobRunr
 import org.jobrunr.scheduling.BackgroundJob
-import org.jobrunr.storage.sql.sqlite.SqLiteStorageProvider
+import org.jobrunr.storage.sql.postgres.PostgresStorageProvider
 import java.time.Duration
 import javax.sql.DataSource
 
@@ -19,7 +19,7 @@ private val FILE_CLEANING_PERIOD = Duration.ofHours(6)
 fun configureJobRunr(dataSource: DataSource) {
     JobRunr
         .configure()
-        .useStorageProvider(SqLiteStorageProvider(dataSource))
+        .useStorageProvider(PostgresStorageProvider(dataSource))
         .useBackgroundJobServer()
         .initialize()