Skip to content

Commit

Permalink
CoreData PoC with EmbraceUploadCache
Browse files Browse the repository at this point in the history
  • Loading branch information
NachoEmbrace committed Dec 16, 2024
1 parent 949d7ee commit 4f06b26
Show file tree
Hide file tree
Showing 9 changed files with 425 additions and 388 deletions.
291 changes: 152 additions & 139 deletions Sources/EmbraceUploadInternal/Cache/EmbraceUploadCache.swift
Original file line number Diff line number Diff line change
Expand Up @@ -5,55 +5,102 @@
import Foundation
import EmbraceOTelInternal
import EmbraceCommonInternal
import GRDB
import CoreData

/// Class that handles all the cached upload data generated by the Embrace SDK.
class EmbraceUploadCache {

private(set) var options: EmbraceUpload.CacheOptions
private(set) var dbQueue: DatabaseQueue
let container: NSPersistentContainer
let context: NSManagedObjectContext
let logger: InternalLogger

init(options: EmbraceUpload.CacheOptions, logger: InternalLogger) throws {
self.options = options
self.logger = logger

// create sqlite file
dbQueue = try Self.createDBQueue(options: options, logger: logger)
// create core data stack
let model = NSManagedObjectModel()
model.entities = [UploadDataRecord.entityDescription]

// define tables
try dbQueue.write { db in
try UploadDataRecord.defineTable(db: db)
self.container = NSPersistentContainer(name: "EmbraceUploadCache", managedObjectModel: model)

switch options.storageMechanism {
case .inMemory:
let description = NSPersistentStoreDescription()
description.type = NSInMemoryStoreType
self.container.persistentStoreDescriptions = [description]

case let .onDisk(baseURL, _):
try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
let description = NSPersistentStoreDescription()
description.url = options.fileURL
self.container.persistentStoreDescriptions = [description]
}

container.loadPersistentStores { _, error in
if let error {
logger.error("Error initializing EmbraceUpload cache!: \(error.localizedDescription)")
}
}

try clearStaleDataIfNeeded()
self.context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType)
self.context.persistentStoreCoordinator = self.container.persistentStoreCoordinator
}

// Saves all changes on the current context to disk
func save() {
context.perform { [weak self] in
do {
try self?.context.save()
} catch {
self?.logger.warning("Erro saving EmbraceUploadCache: \(error.localizedDescription)")
}
}
}

/// Fetches the cached upload data for the given identifier.
/// - Parameters:
/// - id: Identifier of the data
/// - type: Type of the data
/// - Returns: The cached `UploadDataRecord`, if any
public func fetchUploadData(id: String, type: EmbraceUploadType) throws -> UploadDataRecord? {
try dbQueue.read { db in
return try UploadDataRecord.fetchOne(db, key: ["id": id, "type": type.rawValue])
public func fetchUploadData(id: String, type: EmbraceUploadType) -> UploadDataRecord? {

var result: UploadDataRecord?
context.performAndWait {
do {
let request = NSFetchRequest<UploadDataRecord>(entityName: UploadDataRecord.entityName)
request.predicate = NSPredicate(format: "id == %@ AND type == %i", id, type.rawValue)

result = try context.fetch(request).first
} catch { }
}
return result
}

/// Fetches all the cached upload data.
/// - Returns: An array containing all the cached `UploadDataRecords`
public func fetchAllUploadData() throws -> [UploadDataRecord] {
try dbQueue.read { db in
return try UploadDataRecord
.order(Column("date").asc)
.fetchAll(db)

var result: [UploadDataRecord] = []
context.performAndWait {
do {
let request = NSFetchRequest<UploadDataRecord>(entityName: UploadDataRecord.entityName)
result = try context.fetch(request)
} catch { }
}
return result
}

/// Removes stale data based on size or date, if they're limited in options.
@discardableResult public func clearStaleDataIfNeeded() throws -> UInt {
let limitDays = options.cacheDaysLimit
let recordsToDelete = limitDays > 0 ? try fetchRecordsToDeleteBasedOnDate(maxDays: limitDays) : []
guard options.cacheDaysLimit > 0 else {
return 0
}

let now = Date().timeIntervalSince1970
let lastValidTime = now - TimeInterval(options.cacheDaysLimit * 86400) // (60 * 60 * 24) = 86400 seconds per day
let recordsToDelete = fetchRecordsToDelete(dateLimit: Date(timeIntervalSince1970: lastValidTime))
let deleteCount = recordsToDelete.count

if deleteCount > 0 {
Expand All @@ -63,9 +110,9 @@ class EmbraceUploadCache {
attributes: ["removed": "\(deleteCount)"])
.markAsPrivate()
span.setStartTime(time: Date())

let startedSpan = span.startSpan()
try deleteRecords(recordIDs: recordsToDelete)
try dbQueue.vacuum()
deleteRecords(recordsToDelete)
startedSpan.end()

return UInt(deleteCount)
Expand All @@ -79,64 +126,94 @@ class EmbraceUploadCache {
/// - id: Identifier of the data
/// - type: Type of the data
/// - data: Data to cache
/// - Returns: The newly cached `UploadDataRecord`
@discardableResult func saveUploadData(id: String, type: EmbraceUploadType, data: Data) throws -> UploadDataRecord {
let record = UploadDataRecord(id: id, type: type.rawValue, data: data, attemptCount: 0, date: Date())
try saveUploadData(record)
/// - Returns: Boolean indicating if the operation was successful
@discardableResult func saveUploadData(id: String, type: EmbraceUploadType, data: Data) -> Bool {

// update if it already exists
if let record = fetchUploadData(id: id, type: type) {
context.perform { [weak self] in
record.data = data
self?.save()
}

return true
}

// check limit and delete if necessary
checkCountLimit()

// insert new
var result = true

context.performAndWait {
let record = UploadDataRecord.create(
context: context,
id: id,
type: type.rawValue,
data: data,
attemptCount: 0,
date: Date()
)

do {
try context.save()
} catch {
context.delete(record)
result = false
}
}

return record
return result
}

/// Saves the given `UploadDataRecord` to the cache.
/// - Parameter record: `UploadDataRecord` instance to save
func saveUploadData(_ record: UploadDataRecord) throws {
try dbQueue.write { [weak self] db in
// Checks the amount of records stored and deletes the oldest ones if the total amount
// surpasses the limit.
func checkCountLimit() {
guard options.cacheLimit > 0 else {
return
}

// update if its already stored
if try record.exists(db) {
try record.update(db)
context.perform { [weak self] in
guard let strongSelf = self else {
return
}

// check limit and delete if necessary
if let limit = self?.options.cacheLimit, limit > 0 {
let count = try UploadDataRecord.fetchCount(db)
do {
let request = NSFetchRequest<UploadDataRecord>(entityName: UploadDataRecord.entityName)
let count = try strongSelf.context.count(for: request)

if count >= limit {
let recordsToDelete = try UploadDataRecord
.order(Column("date").asc)
.limit(Int(limit))
.fetchAll(db)
if count >= strongSelf.options.cacheLimit {
request.sortDescriptors = [NSSortDescriptor(key: "date", ascending: true)]
request.fetchLimit = max(0, count - Int(strongSelf.options.cacheLimit) + 10)

for recordToDelete in recordsToDelete {
try recordToDelete.delete(db)
let result = try strongSelf.context.fetch(request)
for uploadData in result {
strongSelf.context.delete(uploadData)
}
}
}

try record.insert(db)
strongSelf.save()
}
} catch { }
}
}

/// Deletes the cached data for the given identifier.
/// - Parameters:
/// - id: Identifier of the data
/// - type: Type of the data
/// - Returns: Boolean indicating if the data was successfully deleted
@discardableResult func deleteUploadData(id: String, type: EmbraceUploadType) throws -> Bool {
guard let uploadData = try fetchUploadData(id: id, type: type) else {
return false
func deleteUploadData(id: String, type: EmbraceUploadType) {
guard let uploadData = fetchUploadData(id: id, type: type) else {
return
}

return try deleteUploadData(uploadData)
deleteUploadData(uploadData)
}

/// Deletes the cached `UploadDataRecord`.
/// - Parameter uploadData: `UploadDataRecord` to delete
/// - Returns: Boolean indicating if the data was successfully deleted
func deleteUploadData(_ uploadData: UploadDataRecord) throws -> Bool {
try dbQueue.write { db in
return try uploadData.delete(db)
func deleteUploadData(_ uploadData: UploadDataRecord) {
context.perform { [weak self] in
self?.context.delete(uploadData)
}
}

Expand All @@ -150,104 +227,40 @@ class EmbraceUploadCache {
id: String,
type: EmbraceUploadType,
attemptCount: Int
) throws {
try dbQueue.write { db in
let filter = UploadDataRecord.Schema.id == id && UploadDataRecord.Schema.type == type
try UploadDataRecord.filter(filter)
.updateAll(db, UploadDataRecord.Schema.attemptCount.set(to: attemptCount))
) {
guard let uploadData = fetchUploadData(id: id, type: type) else {
return
}
}

/// Fetches all records that should be deleted based on them being older than __maxDays__ days
/// - Parameter db: The database where to pull the data from, assumes the records to be UploadDataRecord.
/// - Parameter maxDays: The maximum allowed days old a record is allowed to be cached.
/// - Returns: An array of IDs from records that should be deleted.
func fetchRecordsToDeleteBasedOnDate(maxDays: UInt) throws -> [String] {
let sqlQuery = """
SELECT id, date FROM uploads WHERE date <= DATE(DATE(), '-\(maxDays) day')
"""

var result: [String] = []

try dbQueue.read { db in
result = try String.fetchAll(db, sql: sqlQuery)
context.perform { [weak self] in
uploadData.attemptCount = attemptCount
self?.save()
}

return result
}

/// Deletes requested records from the database based on their IDs
/// Assumes the records to be of type __UploadDataRecord__
/// - Parameter recordIDs: The IDs array to delete
func deleteRecords(recordIDs: [String]) throws {
let questionMarks = "\(databaseQuestionMarks(count: recordIDs.count))"
let sqlQuery = "DELETE FROM uploads WHERE id IN (\(questionMarks))"
try dbQueue.write { db in
try db.execute(sql: sqlQuery, arguments: .init(recordIDs))
}
}
}
/// Fetches all records that should be deleted based on them being older than the passed date
func fetchRecordsToDelete(dateLimit: Date) -> [UploadDataRecord] {

extension EmbraceUploadCache {
var result: [UploadDataRecord] = []
context.performAndWait {
do {
let request = NSFetchRequest<UploadDataRecord>(entityName: UploadDataRecord.entityName)
request.predicate = NSPredicate(format: "date < %@", dateLimit as NSDate)

private static func createDBQueue(
options: EmbraceUpload.CacheOptions,
logger: InternalLogger
) throws -> DatabaseQueue {
if case let .inMemory(name) = options.storageMechanism {
return try DatabaseQueue(named: name)
} else if case let .onDisk(baseURL, _) = options.storageMechanism, let fileURL = options.fileURL {
// create base directory if necessary
try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
return try EmbraceUploadCache.getDBQueueIfPossible(at: fileURL, logger: logger)
} else {
fatalError("Unsupported storage mechansim added")
result = try context.fetch(request)
} catch { }
}
return result
}

/// Will attempt to create or open the DB File. If first attempt fails due to GRDB error, it'll assume the existing DB is corruped and try again after deleting the existing DB file.
private static func getDBQueueIfPossible(at fileURL: URL, logger: InternalLogger) throws -> DatabaseQueue {
do {
return try DatabaseQueue(path: fileURL.path)
} catch {
if let dbError = error as? DatabaseError {
logger.error(
"""
GRDB Failed to initialize EmbraceUploadCache.
Will attempt to remove existing file and create a new DB.
Message: \(dbError.message ?? "[empty message]"),
Result Code: \(dbError.resultCode),
SQLite Extended Code: \(dbError.extendedResultCode)
"""
)
} else {
logger.error(
"""
Unknown error while trying to initialize EmbraceUploadCache: \(error)
Will attempt to recover by deleting existing DB.
"""
)
/// Deletes requested records from the database
func deleteRecords(_ records: [UploadDataRecord]) {
context.perform { [weak self] in
for record in records {
self?.context.delete(record)
}
}

try EmbraceUploadCache.deleteDBFile(at: fileURL, logger: logger)

return try DatabaseQueue(path: fileURL.path)
}

/// Will attempt to delete the provided file.
private static func deleteDBFile(at fileURL: URL, logger: InternalLogger) throws {
do {
let fileURL = URL(fileURLWithPath: fileURL.path)
try FileManager.default.removeItem(at: fileURL)
} catch let error {
logger.error(
"""
EmbraceUploadCache failed to remove DB file.
Error: \(error.localizedDescription)
Filepath: \(fileURL)
"""
)
self?.save()
}
}
}
Loading

0 comments on commit 4f06b26

Please sign in to comment.