update tests for txout variants and running txout migrations

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Plutus.hs

@@ -29,10 +29,7 @@ module Test.Cardano.Db.Mock.Unit.Alonzo.Plutus (
 ) where
 
 import qualified Cardano.Crypto.Hash as Crypto
-import Cardano.Db (TxOutTableType (..))
 import qualified Cardano.Db as DB
-import qualified Cardano.Db.Schema.Core.TxOut as C
-import qualified Cardano.Db.Schema.Variant.TxOut as V
 import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress)
 import Cardano.Ledger.Coin
 import Cardano.Ledger.Mary.Value (MaryValue (..), MultiAsset (..), PolicyID (..))
@@ -76,6 +73,9 @@ import Test.Cardano.Db.Mock.Validate (
   assertScriptCert,
  )
 import Test.Tasty.HUnit (Assertion)
+import Cardano.Db (TxOutTableType(..))
+import qualified Cardano.Db.Schema.Core.TxOut as C
+import qualified Cardano.Db.Schema.Variant.TxOut as V
 
 ----------------------------------------------------------------------------------------------------------
 -- Plutus Spend Scripts
@@ -110,7 +110,7 @@ simpleScript =
           , V.txOutValue txout
           , V.txOutDataHash txout
           )
-        Nothing -> error "AlonzosimpleScript: expected an address"
+        Nothing -> error "AlonzoSimpleScript: expected an address"
     expectedFields =
       ( renderAddress alwaysSucceedsScriptAddr
       , True

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -168,6 +168,7 @@ initPruneConsumeMigration :: Bool -> Bool -> Bool -> Bool -> DB.PruneConsumeMigr
 initPruneConsumeMigration consumed pruneTxOut bootstrap forceTxIn' =
   DB.PruneConsumeMigration
     { DB.pcmPruneTxOut = pruneTxOut || bootstrap
+    , DB.pcmConsumedTxOut = consumed
     , DB.pcmConsumeOrPruneTxOut = consumed || pruneTxOut || bootstrap
     , DB.pcmSkipTxIn = not forceTxIn' && (consumed || pruneTxOut || bootstrap)
     }

cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

@@ -470,6 +470,7 @@ optionsToList SyncInsertOptions {..} =
     , toJsonIfSet "offchain_pool_data" sioOffchainPoolData
     , toJsonIfSet "pool_stats" sioPoolStats
     , toJsonIfSet "json_type" sioJsonType
+    , toJsonIfSet "remove_jsonb_from_schema" sioRemoveJsonbFromSchema
     ]
 
 toJsonIfSet :: ToJSON a => Text -> a -> Maybe Pair
@@ -562,8 +563,8 @@ instance ToJSON TxOutConfig where
 instance FromJSON TxOutConfig where
   parseJSON = Aeson.withObject "tx_out" $ \obj -> do
     val <- obj .: "value"
-    useAddress' <- obj .: "use_address_table" .!= UseTxOutAddress False
     forceTxIn' <- obj .:? "force_tx_in" .!= ForceTxIn False
+    useAddress' <- obj .:? "use_address_table" .!= UseTxOutAddress False
 
     case val :: Text of
       "enable" -> pure (TxOutEnable useAddress')

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Query.hs

@@ -12,7 +12,7 @@ module Cardano.DbSync.Era.Shelley.Query (
 ) where
 
 import Cardano.Db
-import qualified Cardano.DbSync.Api as Db
+import Cardano.DbSync.Api (getTxOutTableType)
 import Cardano.DbSync.Api.Types (SyncEnv)
 import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
 import Cardano.DbSync.Util
@@ -28,16 +28,16 @@ resolveStakeAddress addr = queryStakeAddress addr renderByteArray
 
 resolveInputTxOutId :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, TxOutIdW))
 resolveInputTxOutId syncEnv txIn =
-  queryTxOutId (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutId (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
 
 resolveInputValue :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, DbLovelace))
 resolveInputValue syncEnv txIn =
-  queryTxOutValue (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutValue (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
 
 resolveInputTxOutIdValue :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, TxOutIdW, DbLovelace))
 resolveInputTxOutIdValue syncEnv txIn =
-  queryTxOutIdValue (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutIdValue (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
 
 queryResolveInputCredentials :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (Maybe ByteString, Bool))
 queryResolveInputCredentials syncEnv txIn = do
-  queryTxOutCredentials (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutCredentials (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))

cardano-db-sync/src/Cardano/DbSync/Fix/ConsumedBy.hs

@@ -7,6 +7,8 @@ import qualified Cardano.Chain.Block as Byron hiding (blockHash)
 import qualified Cardano.Chain.UTxO as Byron
 import qualified Cardano.Crypto as Crypto (serializeCborHash)
 import qualified Cardano.Db as DB
+import Cardano.DbSync.Api (getTrace, getTxOutTableType)
+import Cardano.DbSync.Api.Types (SyncEnv)
 import Cardano.DbSync.Era.Byron.Insert
 import Cardano.DbSync.Era.Byron.Util (blockPayload, unTxHash)
 import Cardano.DbSync.Era.Util
@@ -16,8 +18,6 @@ import Cardano.Prelude hiding (length, (.))
 import Database.Persist.SqlBackend.Internal
 import Ouroboros.Consensus.Byron.Ledger (ByronBlock (..))
 import Ouroboros.Consensus.Cardano.Block (HardForkBlock (..))
-import Cardano.DbSync.Api.Types (SyncEnv)
-import Cardano.DbSync.Api (getTxOutTableType, getTrace)
 
 type FixEntry = (DB.TxOutIdW, DB.TxId)
 

cardano-db-sync/test/Cardano/DbSync/Config/TypesTest.hs

@@ -32,7 +32,6 @@ tests =
 prop_syncInsertConfigFromJSON :: Property
 prop_syncInsertConfigFromJSON = property $ do
   json <- forAll genDefaultJson
-
   Aeson.fromJSON json === Aeson.Success (def :: SyncInsertConfig)
 
 prop_syncInsertConfigRoundtrip :: Property
@@ -118,7 +117,9 @@ genDefaultJson =
     [ [aesonQQ|
         {
           "tx_out": {
-            "value": "enable"
+            "value": "enable",
+            "use_address_table": false,
+            "force_tx_in": false
           },
           "ledger": "enable",
           "shelley": {
@@ -148,7 +149,8 @@ genDefaultJson =
     , [aesonQQ|
         {
           "tx_out": {
-            "value": "enable"
+            "value": "enable",
+            "use_address_table": false
           },
           "ledger": "enable",
           "shelley": {

cardano-db-tool/app/cardano-db-tool.hs

@@ -64,7 +64,7 @@ runCommand cmd =
         void $
           runMigrations pgConfig False mdir mldir Fix
     CmdTxOutMigration txOutTableType -> do
-      runWithConnectionNoLogging PGPassDefaultEnv $ migrateTxOut Nothing txOutTableType
+      runWithConnectionNoLogging PGPassDefaultEnv $ migrateTxOutTests txOutTableType
     CmdUtxoSetAtBlock blkid txOutAddressType -> utxoSetAtSlot txOutAddressType blkid
     CmdPrepareSnapshot pargs -> runPrepareSnapshot pargs
     CmdValidateDb txOutAddressType -> runDbValidation txOutAddressType

cardano-db/app/gen-schema-docs.hs

@@ -60,7 +60,7 @@ docHeader branchName =
             , "** which may not accurately reflect the version number)"
             ]
     , "\n"
-    , "**Note:** This file is auto-generated from the documentation in cardano-db/src/Cardano/Db/Schema.hs\
+    , "**Note:** This file is auto-generated from the documentation in cardano-db/src/Cardano/Db/Schema/BaseSchema.hs\
       \ by the command `cabal run -- gen-schema-docs doc/schema.md`. This document should only be updated\
       \ during the release process and updated on the release branch."
     , "\n"

cardano-db/src/Cardano/Db.hs

@@ -6,12 +6,6 @@ module Cardano.Db (
   Tx (..),
   TxIn (..),
   gitRev,
-  -- CTX.migrateTxOut,
-  -- CTX.runExtraMigrations,
-  -- CTX.queryTxConsumedColumnExists,
-  -- CTX.queryTxOutConsumedNullCount,
-  -- CTX.queryTxOutConsumedCount,
-  -- CTX.querySetNullTxOut,
 ) where
 
 import Cardano.Db.Error as X
@@ -25,11 +19,7 @@ import Cardano.Db.Operations.Core.MinId as X
 import Cardano.Db.Operations.Core.Query as X
 import Cardano.Db.Operations.Core.QueryHelper as X
 import Cardano.Db.Operations.Types as X
-
--- import qualified Cardano.Db.Operations.Variant.ConsumedTxOut as CTX
 import Cardano.Db.Operations.Variant.ConsumedTxOut as X
-
--- (migrateTxOut, queryTxConsumedColumnExists, queryTxOutConsumedCount, queryTxOutConsumedNullCount, runExtraMigrations, querySetNullTxOut)
 import Cardano.Db.Operations.Variant.JsonbQuery as X
 import Cardano.Db.Operations.Variant.TxOutDelete as X
 import Cardano.Db.Operations.Variant.TxOutInsert as X

cardano-db/src/Cardano/Db/Migration.hs

@@ -32,6 +32,7 @@ import Cardano.Db.Operations.Core.Query
 import Cardano.Db.PGConfig
 import Cardano.Db.Run
 import Cardano.Db.Schema.BaseSchema
+import Cardano.Db.Schema.Core.TxOut (migrateCoreTxOutCardanoDb)
 import Cardano.Prelude (Typeable, textShow)
 import Control.Exception (Exception, SomeException, handle)
 import Control.Monad.Extra
@@ -238,7 +239,9 @@ createMigration source (MigrationDir migdir) = do
     create :: ReaderT SqlBackend (NoLoggingT IO) (Maybe (MigrationVersion, Text))
     create = do
       ver <- getSchemaVersion
-      statements <- getMigration migrateBaseCardanoDb
+      statementsBase <- getMigration migrateBaseCardanoDb
+      statementsTxOut <- getMigration migrateCoreTxOutCardanoDb
+      let statements = statementsBase <> statementsTxOut
       if null statements
         then pure Nothing
         else do