adding more whitelists

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -269,7 +269,6 @@ mkSyncNodeParams staticDir mutableDir CommandLineArgs {..} = do
       , enpHasMultiAssets = claHasMultiAssets
       , enpHasMetadata = claHasMetadata
       , enpWhitelistMetadataNames = []
-      , enpWhitelistMAPolicies = []
       , enpHasPlutusExtra = True
       , enpHasGov = True
       , enpHasOffChainPoolData = True

cardano-db-sync/app/cardano-db-sync.hs

@@ -89,7 +89,6 @@ pRunDbSyncNode =
     <*> pHasMultiAssets
     <*> pHasMetadata
     <*> pWhiteListTxMetadata
-    <*> pWhiteListMAPolicies
     <*> pHasPlutusExtra
     <*> pHasGov
     <*> pHasOffChainPoolData
@@ -240,14 +239,6 @@ pWhiteListTxMetadata =
         <> Opt.help "Insert a specific set of tx metadata, based on the tx metadata key names"
     )
 
-pWhiteListMAPolicies :: Parser [Word64]
-pWhiteListMAPolicies =
-  Opt.option
-    (parseCommaSeparated <$> Opt.str)
-    ( Opt.long "whitelist-multi-asset-policy"
-        <> Opt.help "Only insert a specific sellected list of multi-assets, based on the multi-asset's policy name"
-    )
-
 parseCommaSeparated :: String -> [Word64]
 parseCommaSeparated str =
   case traverse readMaybe (splitOn "," str) of

cardano-db-sync/src/Cardano/DbSync.hs

@@ -35,7 +35,10 @@ import Cardano.DbSync.Config.Types (
   ConfigFile (..),
   GenesisFile (..),
   LedgerStateDir (..),
+  MetadataConfig (..),
+  MultiAssetConfig (..),
   NetworkName (..),
+  PlutusConfig (..),
   SocketPath (..),
   SyncCommand (..),
   SyncNodeConfig (..),
@@ -56,7 +59,6 @@ import Cardano.Prelude hiding (Nat, (%))
 import Cardano.Slotting.Slot (EpochNo (..))
 import Control.Concurrent.Async
 import Control.Monad.Extra (whenJust)
-import qualified Data.Strict.Maybe as Strict
 import qualified Data.Text as Text
 import Data.Version (showVersion)
 import Database.Persist.Postgresql (ConnectionString, withPostgresqlConn)
@@ -233,14 +235,6 @@ extractSyncOptions snp aop =
     , snapshotEveryLagging = enpSnEveryLagging snp
     }
   where
-    maybeWhitelistMDNames = whitelistToMaybe (enpWhitelistMetadataNames snp)
-    maybeWhitelistMAPolicies = whitelistToMaybe (enpWhitelistMAPolicies snp)
-
-    whitelistToMaybe wList =
-      if null wList
-        then Strict.Nothing
-        else Strict.Just wList
-
     iopts
       | enpOnlyGov snp = onlyGovInsertOptions useLedger
       | enpOnlyUTxO snp = onlyUTxOInsertOptions
@@ -252,11 +246,10 @@ extractSyncOptions snp aop =
             , ioUseLedger = useLedger
             , ioShelley = enpHasShelley snp
             , ioRewards = True
-            , ioMultiAssets = enpHasMultiAssets snp
-            , ioMetadata = enpHasMetadata snp
-            , ioWhitelistMetadataNames = maybeWhitelistMDNames
-            , ioWhitelistMAPolicies = maybeWhitelistMAPolicies
-            , ioPlutusExtra = enpHasPlutusExtra snp
+            , -- TODO: cmdv: this is where we plug configs
+              ioMultiAssets = MultiAssetDisable
+            , ioMetadata = MetadataDisable
+            , ioPlutusExtra = PlutusDisable
             , ioOffChainPoolData = enpHasOffChainPoolData snp
             , ioGov = enpHasGov snp
             }

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -206,11 +206,9 @@ fullInsertOptions useLedger =
     , ioUseLedger = useLedger
     , ioShelley = True
     , ioRewards = True
-    , ioMultiAssets = True
-    , ioMetadata = True
-    , ioWhitelistMetadataNames = Strict.Nothing
-    , ioWhitelistMAPolicies = Strict.Nothing
-    , ioPlutusExtra = True
+    , ioMultiAssets = MultiAssetEnable
+    , ioMetadata = MetadataEnable
+    , ioPlutusExtra = PlutusEnable
     , ioOffChainPoolData = True
     , ioGov = True
     }
@@ -222,11 +220,9 @@ onlyUTxOInsertOptions =
     , ioUseLedger = False
     , ioShelley = False
     , ioRewards = False
-    , ioMultiAssets = True
-    , ioMetadata = False
-    , ioWhitelistMetadataNames = Strict.Nothing
-    , ioWhitelistMAPolicies = Strict.Nothing
-    , ioPlutusExtra = False
+    , ioMultiAssets = MultiAssetEnable
+    , ioMetadata = MetadataDisable
+    , ioPlutusExtra = PlutusDisable
     , ioOffChainPoolData = False
     , ioGov = False
     }
@@ -241,11 +237,9 @@ disableAllInsertOptions useLedger =
     , ioUseLedger = useLedger
     , ioShelley = False
     , ioRewards = False
-    , ioMultiAssets = False
-    , ioMetadata = False
-    , ioWhitelistMetadataNames = Strict.Nothing
-    , ioWhitelistMAPolicies = Strict.Nothing
-    , ioPlutusExtra = False
+    , ioMultiAssets = MultiAssetEnable
+    , ioMetadata = MetadataDisable
+    , ioPlutusExtra = PlutusDisable
     , ioOffChainPoolData = False
     , ioGov = False
     }

cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs

@@ -142,7 +142,8 @@ storePage ::
   ExceptT SyncNodeError (ReaderT SqlBackend m) ()
 storePage syncEnv cache percQuantum (n, ls) = do
   when (n `mod` 10 == 0) $ liftIO $ logInfo trce $ "Bootstrap in progress " <> prc <> "%"
-  txOuts <- mapM (prepareTxOut syncEnv cache) ls
+  txOuts <- do
+    mapM (prepareTxOut syncEnv cache) ls
   txOutIds <- lift . DB.insertManyTxOutPlex True False $ etoTxOut . fst <$> txOuts
   let maTxOuts = concatMap mkmaTxOuts $ zip txOutIds (snd <$> txOuts)
   void . lift $ DB.insertManyMaTxOut maTxOuts
@@ -167,7 +168,7 @@ prepareTxOut syncEnv txCache (TxIn txHash (TxIx index), txOut) = do
   let txHashByteString = Generic.safeHashToByteString $ unTxId txHash
   let genTxOut = fromTxOut index txOut
   txId <- queryTxIdWithCache txCache txHashByteString
-  Insert.prepareTxOut syncEnv trce cache iopts (txId, txHashByteString) genTxOut
+  Insert.prepareTxOut trce iopts cache (txId, txHashByteString) genTxOut
   where
     trce = getTrace syncEnv
     cache = envCache syncEnv

cardano-db-sync/src/Cardano/DbSync/Api/Types.hs

@@ -15,6 +15,7 @@ module Cardano.DbSync.Api.Types (
 
 import qualified Cardano.Db as DB
 import Cardano.DbSync.Cache.Types (Cache)
+import Cardano.DbSync.Config.Types (MetadataConfig, MultiAssetConfig, PlutusConfig)
 import Cardano.DbSync.Ledger.Types (HasLedgerEnv)
 import Cardano.DbSync.LocalStateQuery (NoLedgerEnv)
 import Cardano.DbSync.Types (
@@ -23,7 +24,7 @@ import Cardano.DbSync.Types (
   OffChainVoteResult,
   OffChainVoteWorkQueue,
  )
-import Cardano.Prelude (Bool, Eq, IO, Show, Word64)
+import Cardano.Prelude (Bool (..), Eq, IO, Show, Word64)
 import Cardano.Slotting.Slot (EpochNo (..))
 import Control.Concurrent.Class.MonadSTM.Strict (
   StrictTVar,
@@ -76,11 +77,9 @@ data InsertOptions = InsertOptions
   , ioUseLedger :: !Bool
   , ioShelley :: !Bool
   , ioRewards :: !Bool
-  , ioMultiAssets :: !Bool
-  , ioMetadata :: !Bool
-  , ioWhitelistMetadataNames :: Strict.Maybe [Word64]
-  , ioWhitelistMAPolicies :: Strict.Maybe [Word64]
-  , ioPlutusExtra :: !Bool
+  , ioMultiAssets :: !MultiAssetConfig
+  , ioMetadata :: !MetadataConfig
+  , ioPlutusExtra :: !PlutusConfig
   , ioOffChainPoolData :: !Bool
   , ioGov :: !Bool
   }

cardano-db-sync/src/Cardano/DbSync/Config.hs

@@ -20,16 +20,19 @@ module Cardano.DbSync.Config (
   readCardanoGenesisConfig,
   readSyncNodeConfig,
   configureLogging,
+  plutusWhitelistCheckTxOut,
 ) where
 
 import qualified Cardano.BM.Configuration.Model as Logging
 import qualified Cardano.BM.Setup as Logging
 import Cardano.BM.Trace (Trace)
 import qualified Cardano.BM.Trace as Logging
+import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv, SyncOptions (..), envOptions)
 import Cardano.DbSync.Config.Cardano
 import Cardano.DbSync.Config.Node (NodeConfig (..), parseNodeConfig, parseSyncPreConfig, readByteStringFromFile)
 import Cardano.DbSync.Config.Shelley
 import Cardano.DbSync.Config.Types
+import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
 import Cardano.Prelude
 import System.FilePath (takeDirectory, (</>))
 
@@ -91,3 +94,28 @@ coalesceConfig pcfg ncfg adjustGenesisPath = do
 
 mkAdjustPath :: SyncPreConfig -> (FilePath -> FilePath)
 mkAdjustPath cfg fp = takeDirectory (pcNodeConfigFilePath cfg) </> fp
+
+-- do a whitelist check against a list of TxOut and if one matches we keep them all
+plutusWhitelistCheckTxOut :: SyncEnv -> [Generic.TxOut] -> Bool
+plutusWhitelistCheckTxOut syncEnv txOuts = do
+  let iopts = soptInsertOptions $ envOptions syncEnv
+  case ioPlutusExtra iopts of
+    PlutusEnable -> True
+    PlutusDisable -> False
+    PlutusWhitelistScripts whitelist -> do
+      -- we map over our txOuts and check if txOutAddress OR txOutScript are in the whitelist
+      let whitelistCheck =
+            ( \txOut ->
+                case (Generic.txOutScript txOut, Generic.maybePaymentCred $ Generic.txOutAddress txOut) of
+                  (Just script, _) ->
+                    if Generic.txScriptHash script `elem` whitelist
+                      then Just txOut
+                      else Nothing
+                  (_, Just address) ->
+                    if address `elem` whitelist
+                      then Just txOut
+                      else Nothing
+                  (Nothing, Nothing) -> Nothing
+            )
+              <$> txOuts
+      any isJust whitelistCheck

cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

@@ -2,6 +2,7 @@
 {-# LANGUAGE DerivingStrategies #-}
 {-# LANGUAGE FlexibleInstances #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE NoImplicitPrelude #-}
@@ -18,11 +19,17 @@ module Cardano.DbSync.Config.Types (
   GenesisHashConway (..),
   SyncNodeConfig (..),
   SyncPreConfig (..),
+  MetadataConfig (..),
+  MultiAssetConfig (..),
+  PlutusConfig (..),
   LedgerStateDir (..),
   LogFileDir (..),
   NetworkName (..),
   NodeConfigFile (..),
   SocketPath (..),
+  isMetadataEnableOrWhiteList,
+  isMultiAssetEnableOrWhitelist,
+  isPlutusEnableOrWhitelist,
   adjustGenesisFilePath,
   adjustNodeConfigFilePath,
   pcNodeConfigFilePath,
@@ -73,7 +80,6 @@ data SyncNodeParams = SyncNodeParams
   , enpHasMultiAssets :: !Bool
   , enpHasMetadata :: !Bool
   , enpWhitelistMetadataNames :: ![Word64]
-  , enpWhitelistMAPolicies :: ![Word64]
   , enpHasPlutusExtra :: !Bool
   , enpHasGov :: !Bool
   , enpHasOffChainPoolData :: !Bool
@@ -132,6 +138,42 @@ data SyncPreConfig = SyncPreConfig
   , pcPrometheusPort :: !Int
   }
 
+data MetadataConfig
+  = MetadataEnable
+  | MetadataDisable
+  | MetadataWhitelistKeys (NonEmpty ByteString)
+  deriving (Eq, Show)
+
+isMetadataEnableOrWhiteList :: MetadataConfig -> Bool
+isMetadataEnableOrWhiteList = \case
+  MetadataEnable -> True
+  MetadataDisable -> False
+  MetadataWhitelistKeys _ -> True
+
+data MultiAssetConfig
+  = MultiAssetEnable
+  | MultiAssetDisable
+  | MultiAssetWhitelistPolicies (NonEmpty ByteString)
+  deriving (Eq, Show)
+
+isMultiAssetEnableOrWhitelist :: MultiAssetConfig -> Bool
+isMultiAssetEnableOrWhitelist = \case
+  MultiAssetEnable -> True
+  MultiAssetDisable -> False
+  MultiAssetWhitelistPolicies _ -> True
+
+data PlutusConfig
+  = PlutusEnable
+  | PlutusDisable
+  | PlutusWhitelistScripts (NonEmpty ByteString)
+  deriving (Eq, Show)
+
+isPlutusEnableOrWhitelist :: PlutusConfig -> Bool
+isPlutusEnableOrWhitelist = \case
+  PlutusEnable -> True
+  PlutusDisable -> False
+  PlutusWhitelistScripts _ -> True
+
 newtype GenesisFile = GenesisFile
   { unGenesisFile :: FilePath
   }

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs

@@ -25,6 +25,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Block (
 
 import qualified Cardano.Crypto.Hash as Crypto
 import qualified Cardano.Crypto.KES.Class as KES
+import Cardano.DbSync.Config.Types (PlutusConfig)
 import Cardano.DbSync.Era.Shelley.Generic.Tx
 import Cardano.DbSync.Types
 import Cardano.DbSync.Util.Bech32 (serialiseVerKeyVrfToBech32)
@@ -120,7 +121,7 @@ fromMaryBlock blk =
     , blkTxs = map fromMaryTx (getTxs blk)
     }
 
-fromAlonzoBlock :: Bool -> Maybe Prices -> ShelleyBlock TPraosStandard StandardAlonzo -> Block
+fromAlonzoBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock TPraosStandard StandardAlonzo -> Block
 fromAlonzoBlock iope mprices blk =
   Block
     { blkEra = Alonzo
@@ -137,7 +138,7 @@ fromAlonzoBlock iope mprices blk =
     , blkTxs = map (fromAlonzoTx iope mprices) (getTxs blk)
     }
 
-fromBabbageBlock :: Bool -> Maybe Prices -> ShelleyBlock PraosStandard StandardBabbage -> Block
+fromBabbageBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock PraosStandard StandardBabbage -> Block
 fromBabbageBlock iope mprices blk =
   Block
     { blkEra = Babbage
@@ -154,7 +155,7 @@ fromBabbageBlock iope mprices blk =
     , blkTxs = map (fromBabbageTx iope mprices) (getTxs blk)
     }
 
-fromConwayBlock :: Bool -> Maybe Prices -> ShelleyBlock PraosStandard StandardConway -> Block
+fromConwayBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock PraosStandard StandardConway -> Block
 fromConwayBlock iope mprices blk =
   Block
     { blkEra = Conway

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Alonzo.hs

@@ -26,6 +26,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Tx.Alonzo (
 
 import qualified Cardano.Crypto.Hash as Crypto
 import Cardano.Db (ScriptType (..))
+import Cardano.DbSync.Config.Types (PlutusConfig, isPlutusEnableOrWhitelist)
 import Cardano.DbSync.Era.Shelley.Generic.Metadata
 import Cardano.DbSync.Era.Shelley.Generic.Script (fromTimelock)
 import Cardano.DbSync.Era.Shelley.Generic.ScriptData (ScriptData (..))
@@ -65,7 +66,7 @@ import qualified Data.Set as Set
 import Lens.Micro
 import Ouroboros.Consensus.Cardano.Block (EraCrypto, StandardAlonzo, StandardCrypto)
 
-fromAlonzoTx :: Bool -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardAlonzo) -> Tx
+fromAlonzoTx :: PlutusConfig -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardAlonzo) -> Tx
 fromAlonzoTx ioExtraPlutus mprices (blkIndex, tx) =
   Tx
     { txHash = txHashId tx
@@ -178,13 +179,13 @@ resolveRedeemers ::
   , Core.EraTx era
   , Alonzo.MaryEraTxBody era
   ) =>
-  Bool ->
+  PlutusConfig ->
   Maybe Alonzo.Prices ->
   Core.Tx era ->
   (TxCert era -> Cert) ->
   (RedeemerMaps, [(Word64, TxRedeemer)])
 resolveRedeemers ioExtraPlutus mprices tx toCert =
-  if not ioExtraPlutus
+  if not $ isPlutusEnableOrWhitelist ioExtraPlutus
     then (initRedeemersMaps, [])
     else
       mkRdmrAndUpdateRec (initRedeemersMaps, []) $

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Babbage.hs

@@ -13,6 +13,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Tx.Babbage (
   fromTxOut,
 ) where
 
+import Cardano.DbSync.Config.Types (PlutusConfig)
 import Cardano.DbSync.Era.Shelley.Generic.Metadata
 import Cardano.DbSync.Era.Shelley.Generic.Tx.Allegra (getInterval)
 import Cardano.DbSync.Era.Shelley.Generic.Tx.Alonzo
@@ -36,7 +37,7 @@ import qualified Data.Map.Strict as Map
 import Lens.Micro
 import Ouroboros.Consensus.Shelley.Eras (StandardBabbage, StandardCrypto)
 
-fromBabbageTx :: Bool -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardBabbage) -> Tx
+fromBabbageTx :: PlutusConfig -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardBabbage) -> Tx
 fromBabbageTx ioExtraPlutus mprices (blkIndex, tx) =
   Tx
     { txHash = txHashId tx