Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove migrations which are more than a year old. #5093

Draft
wants to merge 1 commit into
base: trunk
Choose a base branch
from
Draft

Remove migrations which are more than a year old. #5093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions parser-typechecker/src/Unison/Codebase/SqliteCodebase.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,10 @@ sqliteCodebase debugName root localOrRemote lockOption migrationStrategy action
DontMigrate -> pure $ Left (OpenCodebaseRequiresMigration fromSv toSv)
MigrateAfterPrompt backupStrategy vacuumStrategy -> do
let shouldPrompt = True
Migrations.ensureCodebaseIsUpToDate localOrRemote root getDeclType termBuffer declBuffer shouldPrompt backupStrategy vacuumStrategy conn
Migrations.ensureCodebaseIsUpToDate localOrRemote root shouldPrompt backupStrategy vacuumStrategy conn
MigrateAutomatically backupStrategy vacuumStrategy -> do
let shouldPrompt = False
Migrations.ensureCodebaseIsUpToDate localOrRemote root getDeclType termBuffer declBuffer shouldPrompt backupStrategy vacuumStrategy conn
Migrations.ensureCodebaseIsUpToDate localOrRemote root shouldPrompt backupStrategy vacuumStrategy conn

case result of
Left err -> pure $ Left err
Expand Down
110 changes: 14 additions & 96 deletions parser-typechecker/src/Unison/Codebase/SqliteCodebase/Migrations.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,77 +3,31 @@
module Unison.Codebase.SqliteCodebase.Migrations where

import Control.Concurrent.MVar
import Control.Concurrent.STM (TVar)
import Control.Monad.Reader
import Data.Map qualified as Map
import Data.Text qualified as Text
import Data.Time.Clock.POSIX (getPOSIXTime)
import System.Console.Regions qualified as Region
import System.FilePath ((</>))
import Text.Printf (printf)
import U.Codebase.Reference qualified as C.Reference
import U.Codebase.Sqlite.DbId (HashVersion (..), SchemaVersion (..))
import U.Codebase.Sqlite.DbId (SchemaVersion (..))
import U.Codebase.Sqlite.Queries qualified as Q
import Unison.Codebase (CodebasePath)
import Unison.Codebase.Init (BackupStrategy (..), VacuumStrategy (..))
import Unison.Codebase.Init.OpenCodebaseError (OpenCodebaseError (OpenCodebaseUnknownSchemaVersion))
import Unison.Codebase.Init.OpenCodebaseError qualified as Codebase
import Unison.Codebase.IntegrityCheck (IntegrityResult (..), integrityCheckAllBranches, integrityCheckAllCausals, prettyPrintIntegrityErrors)
import Unison.Codebase.SqliteCodebase.Migrations.Helpers (abortMigration)
import Unison.Codebase.SqliteCodebase.Migrations.MigrateSchema11To12 (migrateSchema11To12)
import Unison.Codebase.SqliteCodebase.Migrations.MigrateSchema1To2 (migrateSchema1To2)
import Unison.Codebase.SqliteCodebase.Migrations.MigrateSchema3To4 (migrateSchema3To4)
import Unison.Codebase.SqliteCodebase.Migrations.MigrateSchema5To6 (migrateSchema5To6)
import Unison.Codebase.SqliteCodebase.Migrations.MigrateSchema6To7 (migrateSchema6To7)
import Unison.Codebase.SqliteCodebase.Migrations.MigrateSchema7To8 (migrateSchema7To8)
import Unison.Codebase.SqliteCodebase.Operations qualified as Ops2
import Unison.Codebase.SqliteCodebase.Paths (backupCodebasePath)
import Unison.Codebase.Type (LocalOrRemote (..))
import Unison.ConstructorType qualified as CT
import Unison.Hash (Hash)
import Unison.Prelude
import Unison.Sqlite qualified as Sqlite
import Unison.Sqlite.Connection qualified as Sqlite.Connection
import Unison.Util.Monoid (foldMapM)
import Unison.Util.Monoid qualified as Monoid
import Unison.Util.Pretty qualified as Pretty
import UnliftIO qualified

-- | Mapping from schema version to the migration required to get there.
-- E.g. The migration at index 2 must be run on a codebase at version 1.
migrations ::
-- | A 'getDeclType'-like lookup, possibly backed by a cache.
(C.Reference.Reference -> Sqlite.Transaction CT.ConstructorType) ->
TVar (Map Hash Ops2.TermBufferEntry) ->
TVar (Map Hash Ops2.DeclBufferEntry) ->
CodebasePath ->
Map SchemaVersion (Sqlite.Transaction ())
migrations getDeclType termBuffer declBuffer rootCodebasePath =
migrations :: Map SchemaVersion (Sqlite.Transaction ())
migrations =
Map.fromList
[ (2, migrateSchema1To2 getDeclType termBuffer declBuffer),
-- The 1 to 2 migration kept around hash objects of hash version 1, unfortunately this
-- caused an issue:
--
-- The migration would detect causals whose value hash did not have a corresponding branch
-- object, this was caused by a race-condition in sync which could end up in a partial sync.
-- When a branch object was determined to be missing, the migration would replace it with the
-- empty branch. This worked well, but led to a situation where related parent or successors
-- of that causal would have their hash objects mapped to the new v2 object which contained
-- the empty branch in place of missing branches. This is fine, but, if a different codebase
-- migrated the same branch and wasn't missing the branch in question it would migrate
-- successfully and each database now have the same v1 hash object mapped to two distinct v2
-- objects, which rightfully causes a crash when syncing.
--
-- This migration drops all the v1 hash objects to avoid this issue, since these hash objects
-- weren't being used for anything anyways.
sqlMigration 3 (Q.removeHashObjectsByHashingVersion (HashVersion 1)),
(4, migrateSchema3To4),
-- The 4 to 5 migration adds initial support for out-of-order sync i.e. Unison Share
sqlMigration 5 Q.addTempEntityTables,
(6, migrateSchema5To6 rootCodebasePath),
(7, migrateSchema6To7),
(8, migrateSchema7To8),
-- Recreates the name lookup tables because the primary key was missing the root hash id.
[ -- Recreates the name lookup tables because the primary key was missing the root hash id.
sqlMigration 9 Q.fixScopedNameLookupTables,
sqlMigration 10 Q.addProjectTables,
sqlMigration 11 Q.addMostRecentBranchTable,
Expand Down Expand Up @@ -120,16 +74,12 @@ ensureCodebaseIsUpToDate ::
(MonadIO m) =>
LocalOrRemote ->
CodebasePath ->
-- | A 'getDeclType'-like lookup, possibly backed by a cache.
(C.Reference.Reference -> Sqlite.Transaction CT.ConstructorType) ->
TVar (Map Hash Ops2.TermBufferEntry) ->
TVar (Map Hash Ops2.DeclBufferEntry) ->
Bool ->
BackupStrategy ->
VacuumStrategy ->
Sqlite.Connection ->
m (Either Codebase.OpenCodebaseError ())
ensureCodebaseIsUpToDate localOrRemote root getDeclType termBuffer declBuffer shouldPrompt backupStrategy vacuumStrategy conn =
ensureCodebaseIsUpToDate localOrRemote root shouldPrompt backupStrategy vacuumStrategy conn =
(liftIO . UnliftIO.try) do
regionVar <- newEmptyMVar
let finalizeRegion :: IO ()
Expand All @@ -140,7 +90,7 @@ ensureCodebaseIsUpToDate localOrRemote root getDeclType termBuffer declBuffer sh

Region.displayConsoleRegions do
(`UnliftIO.finally` finalizeRegion) do
let migs = migrations getDeclType termBuffer declBuffer root
let migs = migrations
-- The highest schema that this ucm knows how to migrate to.
let highestKnownSchemaVersion = fst . head $ Map.toDescList migs
currentSchemaVersion <- Sqlite.runTransaction conn Q.schemaVersion
Expand All @@ -153,51 +103,19 @@ ensureCodebaseIsUpToDate localOrRemote root getDeclType termBuffer declBuffer sh
Sqlite.runWriteTransaction conn \run -> do
-- Get the schema version again now that we're in a transaction.
currentSchemaVersion <- run Q.schemaVersion
case Map.minViewWithKey migrations of
Nothing -> error "No migrations found"
Just ((minMigrationVersion, _), _) -> do
when (currentSchemaVersion < (minMigrationVersion - 1)) do
putStrLn $ "🚨 Your codebase is at schema version " <> show currentSchemaVersion <> " but this UCM binary only knows how to migrate from codebases of at least version " <> show (minMigrationVersion - 1)
putStrLn $ "🚨 You may need to find a slightly older version of the UCM binary to bridge the gap. Please file an issue or contact the Unison team for help."
error $ "Migration aborted."

let migrationsToRun = Map.filterWithKey (\v _ -> v > currentSchemaVersion) migs
-- This is a bit of a hack, hopefully we can remove this when we have a more
-- reliable way to freeze old migration code in time.
-- The problem is that 'saveObject' has been changed to flush temp entity tables,
-- but old schema versions still use 'saveObject', but don't have the tables!
-- We can create the tables no matter what, there won't be anything to flush, so
-- everything still works as expected.
--
-- Hopefully we can remove this once we've got better methods of freezing migration
-- code in time.
when (currentSchemaVersion < 5) $ run Q.addTempEntityTables
when (currentSchemaVersion < 6) $ run Q.addNamespaceStatsTables
for_ (Map.toAscList migrationsToRun) $ \(SchemaVersion v, migration) -> do
putStrLn $ "🔨 Migrating codebase to version " <> show v <> "..."
run migration
let ranMigrations = not (null migrationsToRun)
when ranMigrations do
region <-
UnliftIO.mask_ do
region <- Region.openConsoleRegion Region.Linear
putMVar regionVar region
pure region
result <- do
-- Ideally we'd check everything here, but certain codebases are known to have objects
-- with missing Hash Objects, we'll want to clean that up in a future migration.
-- integrityCheckAllHashObjects,
let checks =
Monoid.whenM
(currentSchemaVersion < 7) -- Only certain migrations actually make changes which reasonably need to be checked
[ integrityCheckAllBranches,
integrityCheckAllCausals
]

zip [(1 :: Int) ..] checks & foldMapM \(i, check) -> do
Region.setConsoleRegion
region
(Text.pack (printf "🕵️ Checking codebase integrity (step %d of %d)..." i (length checks)))
run check
case result of
NoIntegrityErrors -> pure ()
IntegrityErrorDetected errs -> do
let msg = prettyPrintIntegrityErrors errs
let rendered = Pretty.toPlain 80 (Pretty.border 2 msg)
Region.setConsoleRegion region (Text.pack rendered)
run (abortMigration "Codebase integrity error detected.")
pure ranMigrations
when ranMigrations do
region <- readMVar regionVar
Expand Down
Loading
Loading