[EMBR-1911] DB Migrations (#211)

* migration prototype
MigrationService - runs list of Migration objects that are passed in
Migration - protocol to define metadata about Migration and `perform` method

* Updates tests to check for migration that duplicates table definition

* Adds test to check migrations run multiple times

* Adds migration to create SpanRecord table

* Adds 'AddSpanRecordMigration' to list of defined migrations
Removes `defineTable` method from SpanRecord.

* Adds AddSessionRecordMigrationTests

Renames test file to include date

* Adds `AddMetadataRecordMigration`. Updates filenames to account for multiple migrations in the same day

* Adds AddLogRecordMigration

* Removes `defineTable` methods from LogRecord, MetadataRecord, SessionRecord

These are now performed by migrations

* Adds tests for Migration identifiers and Migrations+Current definition

* Updates EmbraceStorage+Options to use `fileURL` computed property

* Updates EmbraceStorage to reset database if migrations fail

* Fixes typo in tests

* Re-adds missing `databaseTableName` override to each record

* Adds explict check for databaseTableName in each record test

* Adds changelog entry

---------

Co-authored-by: Fernando Draghi <fdraghi@FDraghi-Embrace-MBP.local>
Co-authored-by: Austin Emmons <austin.emmons@embrace.io>

CHANGELOG.md

@@ -1,5 +1,14 @@
 
 
+**Features**
+
+**Updates**
+
+* Adds `MigrationService` in `EmbraceStorage` target to structure DB migrations that occur. Will perform migrations 
+during SDK setup if any are outstanding. Converted existing DB schema to be initialized using migrations.
+
+
+
 ## 6.0.0
 
 April 22nd, 2024

Sources/EmbraceCore/Internal/Embrace+Setup.swift

@@ -17,7 +17,9 @@ extension Embrace {
         ) {
             do {
                 let storageOptions = EmbraceStorage.Options(baseUrl: storageUrl, fileName: "db.sqlite")
-                return try EmbraceStorage(options: storageOptions)
+                let storage = try EmbraceStorage(options: storageOptions)
+                try storage.performMigration()
+                return storage
             } catch {
                 throw EmbraceSetupError.failedStorageCreation("Failed to create EmbraceStorage")
             }

Sources/EmbraceStorage/EmbraceStorage+Options.swift

@@ -66,10 +66,10 @@ extension EmbraceStorage.Options {
         return nil
     }
 
-    /// Full path to the storage file
-    public var filePath: String? {
+    /// URL to the storage file
+    public var fileURL: URL? {
         if case let .onDisk(url, filename) = storageMechanism {
-            return url.appendingPathComponent(filename).path
+            return url.appendingPathComponent(filename)
         }
         return nil
     }

Sources/EmbraceStorage/EmbraceStorage.swift

@@ -18,73 +18,39 @@ public class EmbraceStorage: Storage {
     /// - Parameters:
     ///   - baseUrl: URL containing the path when the database will be stored.
     public init(options: Options) throws {
-
         self.options = options
-
-        if case let .inMemory(name) = options.storageMechanism {
-            dbQueue = try DatabaseQueue(named: name)
-        } else if case let .onDisk(baseURL, fileName) = options.storageMechanism {
-            // create base directory if necessary
-            try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
-
-            // create sqlite file
-            let filepath = baseURL.appendingPathComponent(fileName)
-
-            dbQueue = try EmbraceStorage.getDBQueueIfPossible(at: filepath)
-        } else {
-            fatalError("Unsupported storage mechansim added")
-        }
-
-        // define tables
-        try dbQueue.write { db in
-            try SessionRecord.defineTable(db: db)
-            try SpanRecord.defineTable(db: db)
-            try LogRecord.defineTable(db: db)
-            try MetadataRecord.defineTable(db: db)
-        }
+        dbQueue = try Self.createDBQueue(options: options)
     }
 
-    /// Will attempt to create or open the DB File. If first attempt fails due to GRDB error, it'll assume the existing DB is corruped and try again after deleting the existing DB file.
-    private static func getDBQueueIfPossible(at fileURL: URL) throws -> DatabaseQueue {
+    /// Performs any DB migrations
+    /// - Parameters:
+    ///   - resetIfError: If true and the migrations fail the DB will be reset entirely.
+    public func performMigration(
+        resetIfError: Bool = true,
+        service: MigrationServiceProtocol = MigrationService()
+    ) throws {
         do {
-            return try DatabaseQueue(path: fileURL.path)
-        } catch {
-            if let dbError = error as? DatabaseError {
-                ConsoleLog.error("""
-GRDB Failed to initialize EmbraceStorage.
-Will attempt to remove existing file and create a new DB.
-Message: \(dbError.message ?? "[empty message]"),
-Result Code: \(dbError.resultCode),
-SQLite Extended Code: \(dbError.extendedResultCode)
-""")
+            try service.perform(dbQueue, migrations: .current)
+        } catch let error {
+            if resetIfError {
+                ConsoleLog.error("Error performing migrations, resetting EmbraceStorage: \(error)")
+                try reset(migrationService: service)
             } else {
-                ConsoleLog.error("""
-Unknown error while trying to initialize EmbraceStorage: \(error)
-Will attempt to recover by deleting existing DB.
-""")
+                ConsoleLog.error("Error performing migrations. Reset not enabled: \(error)")
+                throw error // re-throw error if auto-recover is not enabled
             }
         }
-
-        try EmbraceStorage.deleteDBFile(at: fileURL)
-
-        return try DatabaseQueue(path: fileURL.path)
     }
 
-    /// Will attempt to delete the provided file.
-    private static func deleteDBFile(at fileURL: URL) throws {
-        do {
-            let fileURL = URL(fileURLWithPath: fileURL.path)
+    /// Deletes the database and recreates it from scratch
+    func reset(migrationService: MigrationServiceProtocol = MigrationService()) throws {
+        if let fileURL = options.fileURL {
             try FileManager.default.removeItem(at: fileURL)
-        } catch let error {
-            ConsoleLog.error("""
-EmbraceStorage failed to remove DB file.
-Domain: \(error._domain)
-Code: \(error._code)
-Filepath: \(fileURL)
-""")
         }
-    }
 
+        dbQueue = try Self.createDBQueue(options: options)
+        try performMigration(resetIfError: false, service: migrationService) // Do not perpetuate loop
+    }
 }
 
 // MARK: - Sync operations
@@ -172,3 +138,64 @@ extension EmbraceStorage {
         }, completion: completion)
     }
 }
+
+extension EmbraceStorage {
+
+    private static func createDBQueue(options: EmbraceStorage.Options) throws -> DatabaseQueue {
+        if case let .inMemory(name) = options.storageMechanism {
+            return try DatabaseQueue(named: name)
+        } else if case let .onDisk(baseURL, _) = options.storageMechanism, let fileURL = options.fileURL {
+            // create base directory if necessary
+            try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
+            return try EmbraceStorage.getDBQueueIfPossible(at: fileURL)
+        } else {
+            fatalError("Unsupported storage mechansim added")
+        }
+    }
+
+    /// Will attempt to create or open the DB File. If first attempt fails due to GRDB error, it'll assume the existing DB is corruped and try again after deleting the existing DB file.
+    private static func getDBQueueIfPossible(at fileURL: URL) throws -> DatabaseQueue {
+        do {
+            return try DatabaseQueue(path: fileURL.path)
+        } catch {
+            if let dbError = error as? DatabaseError {
+                ConsoleLog.error(
+                    """
+                    GRDB Failed to initialize EmbraceStorage.
+                    Will attempt to remove existing file and create a new DB.
+                    Message: \(dbError.message ?? "[empty message]"),
+                    Result Code: \(dbError.resultCode),
+                    SQLite Extended Code: \(dbError.extendedResultCode)
+                    """
+                )
+            } else {
+                ConsoleLog.error(
+                    """
+                    Unknown error while trying to initialize EmbraceStorage: \(error)
+                    Will attempt to recover by deleting existing DB.
+                    """
+                )
+            }
+        }
+
+        try EmbraceStorage.deleteDBFile(at: fileURL)
+
+        return try DatabaseQueue(path: fileURL.path)
+    }
+
+    /// Will attempt to delete the provided file.
+    private static func deleteDBFile(at fileURL: URL) throws {
+        do {
+            let fileURL = URL(fileURLWithPath: fileURL.path)
+            try FileManager.default.removeItem(at: fileURL)
+        } catch let error {
+            ConsoleLog.error(
+                """
+                EmbraceStorage failed to remove DB file.
+                Error: \(error.localizedDescription)
+                Filepath: \(fileURL)
+                """
+            )
+        }
+    }
+}

Sources/EmbraceStorage/Migration/Migration.swift

@@ -0,0 +1,27 @@
+//
+//  Copyright © 2023 Embrace Mobile, Inc. All rights reserved.
+//
+
+import Foundation
+import GRDB
+
+public protocol Migration {
+    /// The identifier to register this migration under. Must be unique
+    static var identifier: StringLiteralType { get }
+
+    /// Controls how this migration handles foreign key constraints. Defaults to `immediate`.
+    static var foreignKeyChecks: DatabaseMigrator.ForeignKeyChecks { get }
+
+    /// Operation that performs migration.
+    /// See [GRDB Reference](https://swiftpackageindex.com/groue/grdb.swift/master/documentation/grdb/migrations).
+    func perform(_ db: Database) throws
+}
+
+extension Migration {
+    var identifier: StringLiteralType { Self.identifier }
+    var foreignKeyChecks: DatabaseMigrator.ForeignKeyChecks { Self.foreignKeyChecks }
+}
+
+extension Migration {
+    static var foreignKeyChecks: DatabaseMigrator.ForeignKeyChecks { .immediate }
+}

Sources/EmbraceStorage/Migration/MigrationService.swift

@@ -0,0 +1,39 @@
+//
+//  Copyright © 2023 Embrace Mobile, Inc. All rights reserved.
+//
+
+import GRDB
+import EmbraceCommon
+
+public protocol MigrationServiceProtocol {
+    func perform(_ dbQueue: DatabaseWriter, migrations: [Migration]) throws
+}
+
+final public class MigrationService: MigrationServiceProtocol {
+    public init() { }
+
+    public func perform(_ dbQueue: DatabaseWriter, migrations: [Migration]) throws {
+        guard migrations.count > 0 else {
+            ConsoleLog.debug("No migrations to perform")
+            return
+        }
+
+        var migrator = DatabaseMigrator()
+        migrations.forEach { migration in
+            migrator.registerMigration(migration.identifier,
+                                       foreignKeyChecks: migration.foreignKeyChecks,
+                                       migrate: migration.perform(_:))
+        }
+
+        try dbQueue.read { db in
+            if try migrator.hasCompletedMigrations(db) {
+                ConsoleLog.debug("DB is up to date")
+                return
+            } else {
+                ConsoleLog.debug("Running up to \(migrations.count) migrations")
+            }
+        }
+
+        try migrator.migrate(dbQueue)
+    }
+}

Sources/EmbraceStorage/Migration/Migrations/20240509_00_AddSpanRecordMigration.swift

@@ -0,0 +1,36 @@
+//
+//  Copyright © 2023 Embrace Mobile, Inc. All rights reserved.
+//
+
+import GRDB
+
+struct AddSpanRecordMigration: Migration {
+    static let identifier = "CreateSpanRecordTable" // DEV: Must not change
+
+    func perform(_ db: GRDB.Database) throws {
+        try db.create(table: SpanRecord.databaseTableName, options: .ifNotExists) { t in
+            t.column(SpanRecord.Schema.id.name, .text).notNull()
+            t.column(SpanRecord.Schema.name.name, .text).notNull()
+            t.column(SpanRecord.Schema.traceId.name, .text).notNull()
+            t.primaryKey([SpanRecord.Schema.traceId.name, SpanRecord.Schema.id.name])
+
+            t.column(SpanRecord.Schema.type.name, .text).notNull()
+            t.column(SpanRecord.Schema.startTime.name, .datetime).notNull()
+            t.column(SpanRecord.Schema.endTime.name, .datetime)
+
+            t.column(SpanRecord.Schema.data.name, .blob).notNull()
+        }
+
+        let preventClosedSpanModification = """
+        CREATE TRIGGER IF NOT EXISTS prevent_closed_span_modification
+        BEFORE UPDATE ON \(SpanRecord.databaseTableName)
+        WHEN OLD.end_time IS NOT NULL
+        BEGIN
+            SELECT RAISE(ABORT,'Attempted to modify an already closed span.');
+        END;
+        """
+
+        try db.execute(sql: preventClosedSpanModification)
+    }
+
+}

Sources/EmbraceStorage/Migration/Migrations/20240510_00_AddSessionRecordMigration.swift

@@ -0,0 +1,39 @@
+//
+//  Copyright © 2023 Embrace Mobile, Inc. All rights reserved.
+//
+
+import GRDB
+
+struct AddSessionRecordMigration: Migration {
+    static var identifier = "CreateSessionRecordTable" // DEV: Must not change
+
+    func perform(_ db: GRDB.Database) throws {
+        try db.create(table: SessionRecord.databaseTableName, options: .ifNotExists) { t in
+
+            t.primaryKey(SessionRecord.Schema.id.name, .text).notNull()
+
+            t.column(SessionRecord.Schema.state.name, .text).notNull()
+            t.column(SessionRecord.Schema.processId.name, .text).notNull()
+            t.column(SessionRecord.Schema.traceId.name, .text).notNull()
+            t.column(SessionRecord.Schema.spanId.name, .text).notNull()
+
+            t.column(SessionRecord.Schema.startTime.name, .datetime).notNull()
+            t.column(SessionRecord.Schema.endTime.name, .datetime)
+            t.column(SessionRecord.Schema.lastHeartbeatTime.name, .datetime).notNull()
+
+            t.column(SessionRecord.Schema.coldStart.name, .boolean)
+                .notNull()
+                .defaults(to: false)
+
+            t.column(SessionRecord.Schema.cleanExit.name, .boolean)
+                .notNull()
+                .defaults(to: false)
+
+            t.column(SessionRecord.Schema.appTerminated.name, .boolean)
+                .notNull()
+                .defaults(to: false)
+
+            t.column(SessionRecord.Schema.crashReportId.name, .text)
+        }
+    }
+}

Sources/EmbraceStorage/Migration/Migrations/20240510_01_AddMetadataRecordMigration.swift

@@ -0,0 +1,29 @@
+//
+//  Copyright © 2023 Embrace Mobile, Inc. All rights reserved.
+//
+
+import GRDB
+
+struct AddMetadataRecordMigration: Migration {
+
+    static var identifier = "CreateMetadataRecordTable" // DEV: Must not change
+
+    func perform(_ db: Database) throws {
+        try db.create(table: MetadataRecord.databaseTableName, options: .ifNotExists) { t in
+
+            t.column(MetadataRecord.Schema.key.name, .text).notNull()
+            t.column(MetadataRecord.Schema.value.name, .text).notNull()
+            t.column(MetadataRecord.Schema.type.name, .text).notNull()
+            t.column(MetadataRecord.Schema.lifespan.name, .text).notNull()
+            t.column(MetadataRecord.Schema.lifespanId.name, .text).notNull()
+            t.column(MetadataRecord.Schema.collectedAt.name, .datetime).notNull()
+
+            t.primaryKey([
+                MetadataRecord.Schema.key.name,
+                MetadataRecord.Schema.type.name,
+                MetadataRecord.Schema.lifespan.name,
+                MetadataRecord.Schema.lifespanId.name
+            ])
+        }
+    }
+}

Sources/EmbraceStorage/Migration/Migrations/20240510_02_AddLogRecordMigration.swift

@@ -0,0 +1,21 @@
+//
+//  Copyright © 2023 Embrace Mobile, Inc. All rights reserved.
+//
+
+import GRDB
+
+struct AddLogRecordMigration: Migration {
+
+    static var identifier = "CreateLogRecordTable" // DEV: Must not change
+
+    func perform(_ db: Database) throws {
+        try db.create(table: LogRecord.databaseTableName, options: .ifNotExists) { t in
+            t.primaryKey(LogRecord.Schema.identifier.name, .text).notNull()
+            t.column(LogRecord.Schema.processIdentifier.name, .integer).notNull()
+            t.column(LogRecord.Schema.severity.name, .integer).notNull()
+            t.column(LogRecord.Schema.body.name, .text).notNull()
+            t.column(LogRecord.Schema.timestamp.name, .datetime).notNull()
+            t.column(LogRecord.Schema.attributes.name, .text).notNull()
+        }
+    }
+}