From 4e51ef963823116ad3248230d87775b7604d7ecc Mon Sep 17 00:00:00 2001 From: Cmdv Date: Thu, 10 Oct 2024 14:45:19 +0100 Subject: [PATCH 1/2] add logging to Rollbacks --- cardano-db-sync/app/cardano-db-sync.hs | 6 +- cardano-db-sync/src/Cardano/DbSync/Api.hs | 1 + .../src/Cardano/DbSync/Rollback.hs | 20 +- .../src/Cardano/Db/Operations/Delete.hs | 427 ++++++++++++------ .../src/Cardano/Db/Operations/Insert.hs | 16 +- .../Db/Operations/Other/ConsumedTxOut.hs | 49 +- cardano-db/src/Cardano/Db/Operations/Query.hs | 59 ++- cardano-db/test/Test/IO/Cardano/Db/Insert.hs | 2 +- cardano-db/test/Test/IO/Cardano/Db/Util.hs | 2 +- 9 files changed, 363 insertions(+), 219 deletions(-) diff --git a/cardano-db-sync/app/cardano-db-sync.hs b/cardano-db-sync/app/cardano-db-sync.hs index 7190cdad5..7e6e0162a 100644 --- a/cardano-db-sync/app/cardano-db-sync.hs +++ b/cardano-db-sync/app/cardano-db-sync.hs @@ -110,7 +110,7 @@ pRunDbSyncNode = do <*> pHasInOut <*> pure 500 <*> pure 10000 - <*> optional pSlotNo + <*> optional pRollbackSlotNo pConfigFile :: Parser ConfigFile pConfigFile = @@ -210,8 +210,8 @@ pSocketPath = <> Opt.metavar "FILEPATH" ) -pSlotNo :: Parser SlotNo -pSlotNo = +pRollbackSlotNo :: Parser SlotNo +pRollbackSlotNo = SlotNo <$> Opt.option Opt.auto diff --git a/cardano-db-sync/src/Cardano/DbSync/Api.hs b/cardano-db-sync/src/Cardano/DbSync/Api.hs index cbf52c111..02f0b9745 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Api.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Api.hs @@ -30,6 +30,7 @@ module Cardano.DbSync.Api ( whenConsumeOrPruneTxOut, whenPruneTxOut, getTxOutTableType, + getPruneConsume, getHasConsumedOrPruneTxOut, getSkipTxIn, getPrunes, diff --git a/cardano-db-sync/src/Cardano/DbSync/Rollback.hs b/cardano-db-sync/src/Cardano/DbSync/Rollback.hs index dcb09d60b..1ba1d13f8 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Rollback.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Rollback.hs @@ -9,7 +9,7 @@ module Cardano.DbSync.Rollback ( unsafeRollback, ) where -import Cardano.BM.Trace (Trace, logInfo) +import Cardano.BM.Trace (Trace, logInfo, logWarning) import qualified Cardano.Db as DB import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (SyncEnv (..)) @@ -48,17 +48,7 @@ rollbackFromBlockNo syncEnv blkNo = do , textShow blkNo ] lift $ do - (mTxId, deletedBlockCount) <- DB.deleteBlocksBlockId trce txOutTable blockId - whenConsumeOrPruneTxOut syncEnv $ - DB.querySetNullTxOut trce txOutTable mTxId - DB.deleteEpochRows epochNo - DB.deleteDrepDistr epochNo - DB.deleteRewardRest epochNo - DB.deletePoolStat epochNo - DB.setNullEnacted epochNo - DB.setNullRatified epochNo - DB.setNullDropped epochNo - DB.setNullExpired epochNo + deletedBlockCount <- DB.deleteBlocksBlockId trce txOutTableType blockId epochNo (Just (DB.pcmConsumedTxOut $ getPruneConsume syncEnv)) when (deletedBlockCount > 0) $ do -- We use custom constraints to improve input speeds when syncing. -- If they don't already exists we add them here as once a rollback has happened @@ -71,7 +61,7 @@ rollbackFromBlockNo syncEnv blkNo = do where trce = getTrace syncEnv cache = envCache syncEnv - txOutTable = getTxOutTableType syncEnv + txOutTableType = getTxOutTableType syncEnv prepareRollback :: SyncEnv -> CardanoPoint -> Tip CardanoBlock -> IO (Either SyncNodeError Bool) prepareRollback syncEnv point serverTip = @@ -120,5 +110,5 @@ prepareRollback syncEnv point serverTip = -- For testing and debugging. unsafeRollback :: Trace IO Text -> DB.TxOutTableType -> DB.PGConfig -> SlotNo -> IO (Either SyncNodeError ()) unsafeRollback trce txOutTableType config slotNo = do - logInfo trce $ "Forced rollback to slot " <> textShow (unSlotNo slotNo) - Right <$> DB.runDbNoLogging (DB.PGPassCached config) (void $ DB.deleteBlocksSlotNo trce txOutTableType slotNo) + logWarning trce $ "Starting a forced rollback to slot: " <> textShow (unSlotNo slotNo) + Right <$> DB.runDbNoLogging (DB.PGPassCached config) (void $ DB.deleteBlocksSlotNo trce txOutTableType slotNo Nothing) diff --git a/cardano-db/src/Cardano/Db/Operations/Delete.hs b/cardano-db/src/Cardano/Db/Operations/Delete.hs index e9e7e4b10..69b27d127 100644 --- a/cardano-db/src/Cardano/Db/Operations/Delete.hs +++ b/cardano-db/src/Cardano/Db/Operations/Delete.hs @@ -13,18 +13,19 @@ module Cardano.Db.Operations.Delete ( deleteBlocksSlotNoNoTrace, deleteDelistedPool, deleteBlocksBlockId, - deleteBlocksBlockIdNotrace, + deleteBlocksForTests, deleteBlock, - deleteEpochRows, - deleteDrepDistr, - deleteRewardRest, - deletePoolStat, - deleteAdaPots, - -- for testing - queryFirstAndDeleteAfter, + queryDelete, ) where -import Cardano.BM.Trace (Trace, logWarning, nullTracer) +import Cardano.BM.Trace (Trace, logInfo, logWarning, nullTracer) +import Cardano.Db.Operations.Insert ( + setNullDropped, + setNullEnacted, + setNullExpired, + setNullRatified, + ) +import Cardano.Db.Operations.Other.ConsumedTxOut (querySetNullTxOut) import Cardano.Db.Operations.Other.MinId (MinIds (..), MinIdsWrapper (..), completeMinId, textToMinIds) import Cardano.Db.Operations.Query import Cardano.Db.Operations.Types (TxOutTableType (..)) @@ -34,53 +35,77 @@ import qualified Cardano.Db.Schema.Variant.TxOut as V import Cardano.Prelude (Int64) import Cardano.Slotting.Slot (SlotNo (..)) import Control.Monad (void) -import Control.Monad.Extra (whenJust) import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Trans.Reader (ReaderT) import Data.ByteString (ByteString) +import Data.List (partition) import Data.Maybe (isJust) -import Data.Text (Text) +import Data.Text (Text, intercalate, pack) import Data.Word (Word64) -import Database.Esqueleto.Experimental (PersistEntity, PersistField, persistIdField) -import Database.Persist.Class.PersistQuery (deleteWhere) -import Database.Persist.Sql ( +import Database.Esqueleto.Experimental (persistIdField) +import Database.Persist ( + PersistEntity, PersistEntityBackend, - SqlBackend, - delete, - deleteWhereCount, - selectKeysList, + PersistField, (!=.), (==.), (>.), (>=.), ) +import Database.Persist.Sql (Filter, SqlBackend, delete, deleteWhere, deleteWhereCount, selectKeysList) deleteBlocksSlotNoNoTrace :: MonadIO m => TxOutTableType -> SlotNo -> ReaderT SqlBackend m Bool -deleteBlocksSlotNoNoTrace = deleteBlocksSlotNo nullTracer +deleteBlocksSlotNoNoTrace txOutTableType slotNo = deleteBlocksSlotNo nullTracer txOutTableType slotNo Nothing -- | Delete a block if it exists. Returns 'True' if it did exist and has been -- deleted and 'False' if it did not exist. -deleteBlocksSlotNo :: MonadIO m => Trace IO Text -> TxOutTableType -> SlotNo -> ReaderT SqlBackend m Bool -deleteBlocksSlotNo trce txOutTableType (SlotNo slotNo) = do - mBlockId <- queryBlockSlotNo slotNo +deleteBlocksSlotNo :: + MonadIO m => + Trace IO Text -> + TxOutTableType -> + SlotNo -> + Maybe Bool -> + ReaderT SqlBackend m Bool +deleteBlocksSlotNo trce txOutTableType (SlotNo slotNo) mIsConsumedTxOut = do + mBlockId <- queryNearestBlockSlotNo slotNo case mBlockId of - Nothing -> pure False - Just blockId -> do - void $ deleteBlocksBlockId trce txOutTableType blockId + Nothing -> do + liftIO $ logWarning trce $ "deleteBlocksSlotNo: No block contains the the slot: " <> pack (show slotNo) + pure False + Just (blockId, epochN) -> do + void $ deleteBlocksBlockId trce txOutTableType blockId epochN mIsConsumedTxOut pure True -deleteBlocksBlockIdNotrace :: MonadIO m => TxOutTableType -> BlockId -> ReaderT SqlBackend m () -deleteBlocksBlockIdNotrace txOutTableType = void . deleteBlocksBlockId nullTracer txOutTableType +deleteBlocksForTests :: MonadIO m => TxOutTableType -> BlockId -> Word64 -> ReaderT SqlBackend m () +deleteBlocksForTests txOutTableType blockId epochN = do + void $ deleteBlocksBlockId nullTracer txOutTableType blockId epochN Nothing -- | Delete starting from a 'BlockId'. -deleteBlocksBlockId :: MonadIO m => Trace IO Text -> TxOutTableType -> BlockId -> ReaderT SqlBackend m (Maybe TxId, Int64) -deleteBlocksBlockId trce txOutTableType blockId = do +deleteBlocksBlockId :: + MonadIO m => + Trace IO Text -> + TxOutTableType -> + BlockId -> + -- | The 'EpochNo' of the block to delete. + Word64 -> + -- | Is ConsumeTxout + Maybe Bool -> + ReaderT SqlBackend m Int64 +deleteBlocksBlockId trce txOutTableType blockId epochN mIsConsumedTxOut = do mMinIds <- fmap (textToMinIds txOutTableType =<<) <$> queryReverseIndexBlockId blockId (cminIds, completed) <- findMinIdsRec mMinIds mempty mTxId <- queryMinRefId TxBlockId blockId minIds <- if completed then pure cminIds else completeMinId mTxId cminIds - blockCountInt <- deleteTablesAfterBlockId txOutTableType blockId mTxId minIds - pure (mTxId, blockCountInt) + deleteEpochLogs <- deleteUsingEpochNo epochN + (deleteBlockCount, blockDeleteLogs) <- deleteTablesAfterBlockId txOutTableType blockId mTxId minIds + setNullLogs <- + maybe + (pure ("ConsumedTxOut is not active so no Nulls set", 0)) + (\_ -> querySetNullTxOut txOutTableType mTxId) + mIsConsumedTxOut + -- log all the deleted rows in the rollback + liftIO $ logInfo trce $ mkRollbackSummary (deleteEpochLogs <> blockDeleteLogs) setNullLogs + pure deleteBlockCount where findMinIdsRec :: MonadIO m => [Maybe MinIdsWrapper] -> MinIdsWrapper -> ReaderT SqlBackend m (MinIdsWrapper, Bool) findMinIdsRec [] minIds = pure (minIds, True) @@ -102,106 +127,227 @@ deleteBlocksBlockId trce txOutTableType blockId = do CMinIdsWrapper (MinIds m1 m2 m3) -> isJust m1 && isJust m2 && isJust m3 VMinIdsWrapper (MinIds m1 m2 m3) -> isJust m1 && isJust m2 && isJust m3 -deleteTablesAfterBlockId :: MonadIO m => TxOutTableType -> BlockId -> Maybe TxId -> MinIdsWrapper -> ReaderT SqlBackend m Int64 -deleteTablesAfterBlockId txOutTableType blkId mtxId minIdsW = do - deleteWhere [AdaPotsBlockId >=. blkId] - deleteWhere [ReverseIndexBlockId >=. blkId] - deleteWhere [EpochParamBlockId >=. blkId] - mvaId <- queryMinRefId VotingAnchorBlockId blkId - whenJust mvaId $ \vaId -> do - mocvdId <- queryMinRefId OffChainVoteDataVotingAnchorId vaId - whenJust mocvdId $ \ocvdId -> do - queryFirstAndDeleteAfter OffChainVoteGovActionDataOffChainVoteDataId ocvdId - queryFirstAndDeleteAfter OffChainVoteDrepDataOffChainVoteDataId ocvdId - queryFirstAndDeleteAfter OffChainVoteAuthorOffChainVoteDataId ocvdId - queryFirstAndDeleteAfter OffChainVoteReferenceOffChainVoteDataId ocvdId - queryFirstAndDeleteAfter OffChainVoteExternalUpdateOffChainVoteDataId ocvdId - deleteWhere [OffChainVoteDataId >=. ocvdId] - queryFirstAndDeleteAfter OffChainVoteDataVotingAnchorId vaId - queryFirstAndDeleteAfter OffChainVoteFetchErrorVotingAnchorId vaId - deleteWhere [VotingAnchorId >=. vaId] - deleteTablesAfterTxId txOutTableType mtxId minIdsW - deleteWhereCount [BlockId >=. blkId] - -deleteTablesAfterTxId :: (MonadIO m) => TxOutTableType -> Maybe TxId -> MinIdsWrapper -> ReaderT SqlBackend m () -deleteTablesAfterTxId txOutTableType mtxId minIdsW = do - case minIdsW of - CMinIdsWrapper (MinIds mtxInId mtxOutId mmaTxOutId) -> do - whenJust mtxInId $ \txInId -> deleteWhere [TxInId >=. txInId] - whenJust mtxOutId $ \txOutId -> deleteWhere [C.TxOutId >=. txOutId] - whenJust mmaTxOutId $ \maTxOutId -> deleteWhere [C.MaTxOutId >=. maTxOutId] - VMinIdsWrapper (MinIds mtxInId mtxOutId mmaTxOutId) -> do - whenJust mtxInId $ \txInId -> deleteWhere [TxInId >=. txInId] - whenJust mtxOutId $ \txOutId -> deleteWhere [V.TxOutId >=. txOutId] - whenJust mmaTxOutId $ \maTxOutId -> deleteWhere [V.MaTxOutId >=. maTxOutId] - - whenJust mtxId $ \txId -> do - case txOutTableType of - TxOutCore -> queryFirstAndDeleteAfter C.CollateralTxOutTxId txId - TxOutVariantAddress -> queryFirstAndDeleteAfter V.CollateralTxOutTxId txId - queryFirstAndDeleteAfter CollateralTxInTxInId txId - queryFirstAndDeleteAfter ReferenceTxInTxInId txId - queryFirstAndDeleteAfter PoolRetireAnnouncedTxId txId - queryFirstAndDeleteAfter StakeRegistrationTxId txId - queryFirstAndDeleteAfter StakeDeregistrationTxId txId - queryFirstAndDeleteAfter DelegationTxId txId - queryFirstAndDeleteAfter TxMetadataTxId txId - queryFirstAndDeleteAfter WithdrawalTxId txId - queryFirstAndDeleteAfter TreasuryTxId txId - queryFirstAndDeleteAfter ReserveTxId txId - queryFirstAndDeleteAfter PotTransferTxId txId - queryFirstAndDeleteAfter MaTxMintTxId txId - queryFirstAndDeleteAfter RedeemerTxId txId - queryFirstAndDeleteAfter ScriptTxId txId - queryFirstAndDeleteAfter DatumTxId txId - queryFirstAndDeleteAfter RedeemerDataTxId txId - queryFirstAndDeleteAfter ExtraKeyWitnessTxId txId - queryFirstAndDeleteAfter TxCborTxId txId - queryFirstAndDeleteAfter ParamProposalRegisteredTxId txId - queryFirstAndDeleteAfter DelegationVoteTxId txId - queryFirstAndDeleteAfter CommitteeRegistrationTxId txId - queryFirstAndDeleteAfter CommitteeDeRegistrationTxId txId - queryFirstAndDeleteAfter DrepRegistrationTxId txId - queryFirstAndDeleteAfter VotingProcedureTxId txId - mgaId <- queryMinRefId GovActionProposalTxId txId - whenJust mgaId $ \gaId -> do - queryFirstAndDeleteAfter TreasuryWithdrawalGovActionProposalId gaId - queryFirstAndDeleteAfter' CommitteeGovActionProposalId gaId - queryFirstAndDeleteAfter' ConstitutionGovActionProposalId gaId - deleteWhere [GovActionProposalId >=. gaId] - minPmr <- queryMinRefId PoolMetadataRefRegisteredTxId txId - whenJust minPmr $ \pmrId -> do - queryFirstAndDeleteAfter OffChainPoolDataPmrId pmrId - queryFirstAndDeleteAfter OffChainPoolFetchErrorPmrId pmrId - deleteWhere [PoolMetadataRefId >=. pmrId] - minPoolUpdate <- queryMinRefId PoolUpdateRegisteredTxId txId - whenJust minPoolUpdate $ \puid -> do - queryFirstAndDeleteAfter PoolOwnerPoolUpdateId puid - queryFirstAndDeleteAfter PoolRelayUpdateId puid - deleteWhere [PoolUpdateId >=. puid] - deleteWhere [TxId >=. txId] - -queryFirstAndDeleteAfter :: +deleteUsingEpochNo :: MonadIO m => Word64 -> ReaderT SqlBackend m [(Text, Int64)] +deleteUsingEpochNo epochN = do + countLogs <- + concat + <$> sequence + [ onlyDelete "Epoch" [EpochNo ==. epochN] + , onlyDelete "DrepDistr" [DrepDistrEpochNo >. epochN] + , onlyDelete "RewardRest" [RewardRestSpendableEpoch >. epochN] + , onlyDelete "PoolStat" [PoolStatEpochNo >. epochN] + ] + nullLogs <- do + a <- setNullEnacted epochN + b <- setNullRatified epochN + c <- setNullDropped epochN + e <- setNullExpired epochN + pure [("GovActionProposal Nulled", a + b + c + e)] + pure $ countLogs <> nullLogs + +queryDelete :: forall m record field. (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => EntityField record field -> field -> ReaderT SqlBackend m () -queryFirstAndDeleteAfter txIdField txId = do - mRecordId <- queryMinRefId txIdField txId - whenJust mRecordId $ \recordId -> - deleteWhere [persistIdField @record >=. recordId] +queryDelete fieldIdField fieldId = do + mRecordId <- queryMinRefId fieldIdField fieldId + case mRecordId of + Nothing -> pure () + Just recordId -> deleteWhere [persistIdField @record >=. recordId] -queryFirstAndDeleteAfter' :: +queryDeleteAndLog :: forall m record field. (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => + Text -> + EntityField record field -> + field -> + ReaderT SqlBackend m [(Text, Int64)] +queryDeleteAndLog tableName txIdField fieldId = do + mRecordId <- queryMinRefId txIdField fieldId + case mRecordId of + Nothing -> pure [(tableName, 0)] + Just recordId -> do + count <- deleteWhereCount [persistIdField @record >=. recordId] + pure [(tableName, count)] + +onlyDelete :: + forall m record. + (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) => + Text -> + [Filter record] -> + ReaderT SqlBackend m [(Text, Int64)] +onlyDelete tableName filters = do + count <- deleteWhereCount filters + pure [(tableName, count)] + +deleteTablesAfterBlockId :: + MonadIO m => + TxOutTableType -> + BlockId -> + Maybe TxId -> + MinIdsWrapper -> + ReaderT SqlBackend m (Int64, [(Text, Int64)]) +deleteTablesAfterBlockId txOutTableType blkId mtxId minIdsW = do + initialLogs <- + concat + <$> sequence + [ onlyDelete "AdaPots" [AdaPotsBlockId >=. blkId] + , onlyDelete "ReverseIndex" [ReverseIndexBlockId >=. blkId] + , onlyDelete "EpochParam" [EpochParamBlockId >=. blkId] + ] + + -- Handle off-chain related deletions + mvaId <- queryMinRefId VotingAnchorBlockId blkId + offChainLogs <- case mvaId of + Nothing -> pure [] + Just vaId -> do + mocvdId <- queryMinRefId OffChainVoteDataVotingAnchorId vaId + logsVoting <- case mocvdId of + Nothing -> pure [] + Just ocvdId -> + concat + <$> sequence + [ queryDeleteAndLog "OffChainVoteGovActionData" OffChainVoteGovActionDataOffChainVoteDataId ocvdId + , queryDeleteAndLog "OffChainVoteDrepData" OffChainVoteDrepDataOffChainVoteDataId ocvdId + , queryDeleteAndLog "OffChainVoteAuthor" OffChainVoteAuthorOffChainVoteDataId ocvdId + , queryDeleteAndLog "OffChainVoteReference" OffChainVoteReferenceOffChainVoteDataId ocvdId + , queryDeleteAndLog "OffChainVoteExternalUpdate" OffChainVoteExternalUpdateOffChainVoteDataId ocvdId + ] + + offChain <- + concat + <$> sequence + [ queryDeleteAndLog "OffChainVoteData" OffChainVoteDataVotingAnchorId vaId + , queryDeleteAndLog "OffChainVoteFetchError" OffChainVoteFetchErrorVotingAnchorId vaId + , onlyDelete "VotingAnchor" [VotingAnchorId >=. vaId] + ] + pure $ logsVoting <> offChain + -- Additional deletions based on TxId and minimum IDs + afterTxIdLogs <- deleteTablesAfterTxId txOutTableType mtxId minIdsW + -- Final block deletions + blockLogs <- onlyDelete "Block" [BlockId >=. blkId] + -- Aggregate and return all logs + pure (sum $ map snd blockLogs, initialLogs <> blockLogs <> offChainLogs <> afterTxIdLogs) + +deleteTablesAfterTxId :: + (MonadIO m) => + TxOutTableType -> + Maybe TxId -> + MinIdsWrapper -> + ReaderT SqlBackend m [(Text, Int64)] +deleteTablesAfterTxId txOutTableType mtxId minIdsW = do + -- Handle deletions and log accumulation from MinIdsWrapper + minIdsLogs <- case minIdsW of + CMinIdsWrapper (MinIds mtxInId mtxOutId mmaTxOutId) -> + concat + <$> sequence + [ maybe (pure []) (\txInId -> onlyDelete "TxIn" [TxInId >=. txInId]) mtxInId + , maybe (pure []) (\txOutId -> onlyDelete "TxOut" [C.TxOutId >=. txOutId]) mtxOutId + , maybe (pure []) (\maTxOutId -> onlyDelete "MaTxOut" [C.MaTxOutId >=. maTxOutId]) mmaTxOutId + ] + VMinIdsWrapper (MinIds mtxInId mtxOutId mmaTxOutId) -> + concat + <$> sequence + [ maybe (pure []) (\txInId -> onlyDelete "TxIn" [TxInId >=. txInId]) mtxInId + , maybe (pure []) (\txOutId -> onlyDelete "TxOut" [V.TxOutId >=. txOutId]) mtxOutId + , maybe (pure []) (\maTxOutId -> onlyDelete "MaTxOut" [V.MaTxOutId >=. maTxOutId]) mmaTxOutId + ] + -- Handle deletions and log accumulation using the specified TxId + txIdLogs <- case mtxId of + Nothing -> pure [] -- If no TxId is provided, skip further deletions + Just txId -> do + result <- + -- Sequentially delete records with associated transaction ID + concat + <$> sequence + [ case txOutTableType of + TxOutCore -> queryDeleteAndLog "CollateralTxOut" C.CollateralTxOutTxId txId + TxOutVariantAddress -> queryDeleteAndLog "CollateralTxOut" V.CollateralTxOutTxId txId + , queryDeleteAndLog "CollateralTxIn" CollateralTxInTxInId txId + , queryDeleteAndLog "ReferenceTxIn" ReferenceTxInTxInId txId + , queryDeleteAndLog "PoolRetire" PoolRetireAnnouncedTxId txId + , queryDeleteAndLog "StakeRegistration" StakeRegistrationTxId txId + , queryDeleteAndLog "StakeDeregistration" StakeDeregistrationTxId txId + , queryDeleteAndLog "Delegation" DelegationTxId txId + , queryDeleteAndLog "TxMetadata" TxMetadataTxId txId + , queryDeleteAndLog "Withdrawal" WithdrawalTxId txId + , queryDeleteAndLog "Treasury" TreasuryTxId txId + , queryDeleteAndLog "Reserve" ReserveTxId txId + , queryDeleteAndLog "PotTransfer" PotTransferTxId txId + , queryDeleteAndLog "MaTxMint" MaTxMintTxId txId + , queryDeleteAndLog "Redeemer" RedeemerTxId txId + , queryDeleteAndLog "Script" ScriptTxId txId + , queryDeleteAndLog "Datum" DatumTxId txId + , queryDeleteAndLog "RedeemerData" RedeemerDataTxId txId + , queryDeleteAndLog "ExtraKeyWitness" ExtraKeyWitnessTxId txId + , queryDeleteAndLog "TxCbor" TxCborTxId txId + , queryDeleteAndLog "ParamProposal" ParamProposalRegisteredTxId txId + , queryDeleteAndLog "DelegationVote" DelegationVoteTxId txId + , queryDeleteAndLog "CommitteeRegistration" CommitteeRegistrationTxId txId + , queryDeleteAndLog "CommitteeDeRegistration" CommitteeDeRegistrationTxId txId + , queryDeleteAndLog "DrepRegistration" DrepRegistrationTxId txId + , queryDeleteAndLog "VotingProcedure" VotingProcedureTxId txId + ] + -- Handle GovActionProposal related deletions if present + mgaId <- queryMinRefId GovActionProposalTxId txId + gaLogs <- case mgaId of + Nothing -> pure [] -- No GovActionProposal ID found, skip this step + Just gaId -> + -- Delete records related to the GovActionProposal ID + concat + <$> sequence + [ queryDeleteAndLog "TreasuryWithdrawal" TreasuryWithdrawalGovActionProposalId gaId + , queryThenNull "Committee" CommitteeGovActionProposalId gaId + , queryThenNull "Constitution" ConstitutionGovActionProposalId gaId + , onlyDelete "GovActionProposal" [GovActionProposalId >=. gaId] + ] + -- Handle PoolMetadataRef related deletions if present + minPmr <- queryMinRefId PoolMetadataRefRegisteredTxId txId + pmrLogs <- case minPmr of + Nothing -> pure [] -- No PoolMetadataRef ID found, skip this step + Just pmrId -> + -- Delete records related to PoolMetadataRef + concat + <$> sequence + [ queryDeleteAndLog "OffChainPoolData" OffChainPoolDataPmrId pmrId + , queryDeleteAndLog "OffChainPoolFetchError" OffChainPoolFetchErrorPmrId pmrId + , onlyDelete "PoolMetadataRef" [PoolMetadataRefId >=. pmrId] + ] + -- Handle PoolUpdate related deletions if present + minPoolUpdate <- queryMinRefId PoolUpdateRegisteredTxId txId + poolUpdateLogs <- case minPoolUpdate of + Nothing -> pure [] -- No PoolUpdate ID found, skip this step + Just puid -> do + -- Delete records related to PoolUpdate + concat + <$> sequence + [ queryDeleteAndLog "PoolOwner" PoolOwnerPoolUpdateId puid + , queryDeleteAndLog "PoolRelay" PoolRelayUpdateId puid + , onlyDelete "PoolUpdate" [PoolUpdateId >=. puid] + ] + -- Final deletions for the given TxId + txLogs <- onlyDelete "Tx" [TxId >=. txId] + -- Combine all logs from the operations above + pure $ result <> gaLogs <> pmrLogs <> poolUpdateLogs <> txLogs + -- Return the combined logs of all operations + pure $ minIdsLogs <> txIdLogs + +queryThenNull :: + forall m record field. + (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => + Text -> EntityField record (Maybe field) -> field -> - ReaderT SqlBackend m () -queryFirstAndDeleteAfter' txIdField txId = do + ReaderT SqlBackend m [(Text, Int64)] +queryThenNull tableName txIdField txId = do mRecordId <- queryMinRefIdNullable txIdField txId - whenJust mRecordId $ \recordId -> - deleteWhere [persistIdField @record >=. recordId, txIdField !=. Nothing] + case mRecordId of + Nothing -> pure [(tableName, 0)] + Just recordId -> do + count <- deleteWhereCount [persistIdField @record >=. recordId, txIdField !=. Nothing] + pure [(tableName, count)] -- | Delete a delisted pool if it exists. Returns 'True' if it did exist and has been -- deleted and 'False' if it did not exist. @@ -215,29 +361,34 @@ deleteDelistedPool poolHash = do -- deleted and 'False' if it did not exist. deleteBlock :: MonadIO m => TxOutTableType -> Block -> ReaderT SqlBackend m Bool deleteBlock txOutTableType block = do - mBlockId <- listToMaybe <$> selectKeysList [BlockHash ==. blockHash block] [] + mBlockId <- queryBlockHash block case mBlockId of Nothing -> pure False - Just blockId -> do - void $ deleteBlocksBlockId nullTracer txOutTableType blockId + Just (blockId, epochN) -> do + void $ deleteBlocksBlockId nullTracer txOutTableType blockId epochN Nothing pure True -deleteEpochRows :: MonadIO m => Word64 -> ReaderT SqlBackend m () -deleteEpochRows epochNum = - deleteWhere [EpochNo >=. epochNum] +mkRollbackSummary :: [(Text, Int64)] -> (Text, Int64) -> Text +mkRollbackSummary logs setNullLogs = + "\n----------------------- Rollback Summary: ----------------------- \n" + <> formattedLog + <> zeroDeletedEntry + <> formatSetNullLog setNullLogs + <> "\n" + where + (zeroDeletes, nonZeroDeletes) = partition ((== 0) . snd) logs -deleteDrepDistr :: MonadIO m => Word64 -> ReaderT SqlBackend m () -deleteDrepDistr epochNum = - deleteWhere [DrepDistrEpochNo >. epochNum] + formattedLog = intercalate "\n" (map formatEntry nonZeroDeletes) -deleteRewardRest :: MonadIO m => Word64 -> ReaderT SqlBackend m () -deleteRewardRest epochNum = - deleteWhere [RewardRestSpendableEpoch >. epochNum] + zeroDeletedEntry + | null zeroDeletes = "" + | otherwise = "\n\nNo Deletes in tables: " <> intercalate ", " (map fst zeroDeletes) -deletePoolStat :: MonadIO m => Word64 -> ReaderT SqlBackend m () -deletePoolStat epochNum = do - deleteWhere [PoolStatEpochNo >. epochNum] + formatEntry (tableName, rowCount) = + "Table: " <> tableName <> " - Count: " <> pack (show rowCount) -deleteAdaPots :: MonadIO m => BlockId -> ReaderT SqlBackend m () -deleteAdaPots blkId = do - deleteWhere [AdaPotsBlockId ==. blkId] + formatSetNullLog (nullMessage, nullCount) = + "\n\nSet Null: " + <> if nullCount == 0 + then nullMessage + else "\n\nSet Null: " <> nullMessage <> " - Count: " <> pack (show nullCount) diff --git a/cardano-db/src/Cardano/Db/Operations/Insert.hs b/cardano-db/src/Cardano/Db/Operations/Insert.hs index f1a12947c..f498ae285 100644 --- a/cardano-db/src/Cardano/Db/Operations/Insert.hs +++ b/cardano-db/src/Cardano/Db/Operations/Insert.hs @@ -391,21 +391,21 @@ updateGovActionExpired :: MonadIO m => GovActionProposalId -> Word64 -> ReaderT updateGovActionExpired gaid eNo = updateWhere [GovActionProposalId ==. gaid, GovActionProposalExpiredEpoch ==. Nothing] [GovActionProposalExpiredEpoch =. Just eNo] -setNullEnacted :: MonadIO m => Word64 -> ReaderT SqlBackend m () +setNullEnacted :: MonadIO m => Word64 -> ReaderT SqlBackend m Int64 setNullEnacted eNo = - updateWhere [GovActionProposalEnactedEpoch !=. Nothing, GovActionProposalEnactedEpoch >. Just eNo] [GovActionProposalEnactedEpoch =. Nothing] + updateWhereCount [GovActionProposalEnactedEpoch !=. Nothing, GovActionProposalEnactedEpoch >. Just eNo] [GovActionProposalEnactedEpoch =. Nothing] -setNullRatified :: MonadIO m => Word64 -> ReaderT SqlBackend m () +setNullRatified :: MonadIO m => Word64 -> ReaderT SqlBackend m Int64 setNullRatified eNo = - updateWhere [GovActionProposalRatifiedEpoch !=. Nothing, GovActionProposalRatifiedEpoch >. Just eNo] [GovActionProposalRatifiedEpoch =. Nothing] + updateWhereCount [GovActionProposalRatifiedEpoch !=. Nothing, GovActionProposalRatifiedEpoch >. Just eNo] [GovActionProposalRatifiedEpoch =. Nothing] -setNullExpired :: MonadIO m => Word64 -> ReaderT SqlBackend m () +setNullExpired :: MonadIO m => Word64 -> ReaderT SqlBackend m Int64 setNullExpired eNo = - updateWhere [GovActionProposalExpiredEpoch !=. Nothing, GovActionProposalExpiredEpoch >. Just eNo] [GovActionProposalExpiredEpoch =. Nothing] + updateWhereCount [GovActionProposalExpiredEpoch !=. Nothing, GovActionProposalExpiredEpoch >. Just eNo] [GovActionProposalExpiredEpoch =. Nothing] -setNullDropped :: MonadIO m => Word64 -> ReaderT SqlBackend m () +setNullDropped :: MonadIO m => Word64 -> ReaderT SqlBackend m Int64 setNullDropped eNo = - updateWhere [GovActionProposalDroppedEpoch !=. Nothing, GovActionProposalDroppedEpoch >. Just eNo] [GovActionProposalDroppedEpoch =. Nothing] + updateWhereCount [GovActionProposalDroppedEpoch !=. Nothing, GovActionProposalDroppedEpoch >. Just eNo] [GovActionProposalDroppedEpoch =. Nothing] replaceAdaPots :: (MonadBaseControl IO m, MonadIO m) => BlockId -> AdaPots -> ReaderT SqlBackend m Bool replaceAdaPots blockId adapots = do diff --git a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs index 3a663a670..6c6c79f32 100644 --- a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs +++ b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs @@ -30,6 +30,7 @@ import Control.Monad.Extra (unless, when, whenJust) import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Trans.Control (MonadBaseControl) import Control.Monad.Trans.Reader (ReaderT) +import Data.Int (Int64) import Data.Text (Text) import qualified Data.Text as Text import Data.Word (Word64) @@ -52,13 +53,20 @@ data ConsumedTriplet = ConsumedTriplet -------------------------------------------------------------------------------------------------- -- Queries -------------------------------------------------------------------------------------------------- -querySetNullTxOut :: MonadIO m => Trace IO Text -> TxOutTableType -> Maybe TxId -> ReaderT SqlBackend m () -querySetNullTxOut trce txOutTableType mMinTxId = do - whenJust mMinTxId $ \txId -> do - txOutIds <- getTxOutConsumedAfter txId - mapM_ setNullTxOutConsumedAfter txOutIds - let updatedEntries = length txOutIds - liftIO $ logInfo trce $ "Set to null " <> textShow updatedEntries <> " tx_out.consumed_by_tx_id" +querySetNullTxOut :: + MonadIO m => + TxOutTableType -> + Maybe TxId -> + ReaderT SqlBackend m (Text, Int64) +querySetNullTxOut txOutTableType mMinTxId = do + case mMinTxId of + Nothing -> do + pure ("No tx_out to set to null", 0) + Just txId -> do + txOutIds <- getTxOutConsumedAfter txId + mapM_ setNullTxOutConsumedAfter txOutIds + let updatedEntriesCount = length txOutIds + pure ("tx_out.consumed_by_tx_id", fromIntegral updatedEntriesCount) where -- \| This requires an index at TxOutConsumedByTxId. getTxOutConsumedAfter :: MonadIO m => TxId -> ReaderT SqlBackend m [TxOutIdW] @@ -114,7 +122,7 @@ runExtraMigrations trce txOutTableType blockNoDiff pcm = do DBExtraMigration "runExtraMigrations: The configuration option 'tx_out.use_address_table' was previously set and the database updated. Unfortunately reverting this isn't possible." -- Has the user given txout address config && the migration wasn't previously set when (isTxOutVariant && not isTxOutAddressSet) $ do - updateTxOutAndCreateAddress trce + updateTxOutAndCreateAddress insertExtraMigration TxOutAddressPreviouslySet -- first check if pruneTxOut flag is missing and it has previously been used when (isPruneTxOutPreviouslySet migrationValues && not (pcmPruneTxOut pcm)) $ @@ -407,29 +415,18 @@ createPruneConstraintTxOut = do exceptHandler e = liftIO $ throwIO (DBPruneConsumed $ show e) --- Be very mindfull that these queries can fail silently and make tests fail making it hard to know why. --- To help mitigate this, logs are printed after each query is ran, so one can know where it stopped. updateTxOutAndCreateAddress :: forall m. ( MonadBaseControl IO m , MonadIO m ) => - Trace IO Text -> ReaderT SqlBackend m () -updateTxOutAndCreateAddress trc = do +updateTxOutAndCreateAddress = do handle exceptHandler $ rawExecute dropViewsQuery [] - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Dropped views" handle exceptHandler $ rawExecute alterTxOutQuery [] - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Altered tx_out" - handle exceptHandler $ rawExecute alterCollateralTxOutQuery [] - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Altered collateral_tx_out" handle exceptHandler $ rawExecute createAddressTableQuery [] - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Created address table" handle exceptHandler $ rawExecute createIndexPaymentCredQuery [] - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Created index payment_cred" handle exceptHandler $ rawExecute createIndexRawQuery [] - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Created index raw" - liftIO $ logInfo trc "updateTxOutAndCreateAddress: Completed" where dropViewsQuery = Text.unlines @@ -443,16 +440,8 @@ updateTxOutAndCreateAddress trc = do , " ADD COLUMN \"address_id\" INT8 NOT NULL," , " DROP COLUMN \"address\"," , " DROP COLUMN \"address_has_script\"," - , " DROP COLUMN \"payment_cred\"" - ] - - alterCollateralTxOutQuery = - Text.unlines - [ "ALTER TABLE \"collateral_tx_out\"" - , " ADD COLUMN \"address_id\" INT8 NOT NULL," - , " DROP COLUMN \"address\"," - , " DROP COLUMN \"address_has_script\"," - , " DROP COLUMN \"payment_cred\"" + , " DROP COLUMN \"payment_cred\"," + , " DROP COLUMN \"stake_address_id\"" ] createAddressTableQuery = diff --git a/cardano-db/src/Cardano/Db/Operations/Query.hs b/cardano-db/src/Cardano/Db/Operations/Query.hs index 7b8934455..904ed1646 100644 --- a/cardano-db/src/Cardano/Db/Operations/Query.hs +++ b/cardano-db/src/Cardano/Db/Operations/Query.hs @@ -12,7 +12,8 @@ module Cardano.Db.Operations.Query ( queryBlockHashBlockNo, queryBlockNo, queryBlockNoAndEpoch, - queryBlockSlotNo, + queryNearestBlockSlotNo, + queryBlockHash, queryReverseIndexBlockId, queryMinIdsAfterReverseIndex, queryBlockTxCount, @@ -158,6 +159,7 @@ import Database.Esqueleto.Experimental ( (>=.), (?.), (^.), + (||.), type (:&) ((:&)), ) import Database.Persist.Class.PersistQuery (selectList) @@ -218,22 +220,42 @@ queryBlockNoAndEpoch blkNo = do blk <- from $ table @Block where_ (blk ^. BlockBlockNo ==. just (val blkNo)) pure (blk ^. BlockId, blk ^. BlockEpochNo) - pure $ convert (listToMaybe res) - where - convert :: Maybe (Value (Key Block), Value (Maybe Word64)) -> Maybe (BlockId, Word64) - convert mr = - case mr of - Nothing -> Nothing - Just (_, Value Nothing) -> Nothing -- Should not ever happen. - Just (Value blkid, Value (Just epoch)) -> Just (blkid, epoch) + pure $ convertBlockQuery (listToMaybe res) -queryBlockSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m (Maybe BlockId) -queryBlockSlotNo slotNo = do +-- | Retrieves the nearest block with a slot number equal to or greater than the given slot number. +queryNearestBlockSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m (Maybe (BlockId, Word64)) +queryNearestBlockSlotNo slotNo = do res <- select $ do blk <- from $ table @Block - where_ (blk ^. BlockSlotNo ==. just (val slotNo)) - pure (blk ^. BlockId) - pure $ fmap unValue (listToMaybe res) + where_ (isNothing (blk ^. BlockSlotNo) ||. blk ^. BlockSlotNo >=. just (val slotNo)) + orderBy [asc (blk ^. BlockSlotNo)] + limit 1 + pure (blk ^. BlockId, blk ^. BlockBlockNo) + pure $ convertBlockQuery (listToMaybe res) + +queryBlockHash :: MonadIO m => Block -> ReaderT SqlBackend m (Maybe (BlockId, Word64)) +queryBlockHash hash = do + res <- select $ do + blk <- from $ table @Block + where_ (blk ^. BlockHash ==. val (blockHash hash)) + pure (blk ^. BlockId, blk ^. BlockEpochNo) + pure $ convertBlockQuery (listToMaybe res) + +queryMinBlock :: MonadIO m => ReaderT SqlBackend m (Maybe (BlockId, Word64)) +queryMinBlock = do + res <- select $ do + blk <- from $ table @Block + orderBy [asc (blk ^. BlockId)] + limit 1 + pure (blk ^. BlockId, blk ^. BlockBlockNo) + pure $ convertBlockQuery (listToMaybe res) + +convertBlockQuery :: Maybe (Value (Key Block), Value (Maybe Word64)) -> Maybe (BlockId, Word64) +convertBlockQuery mr = + case mr of + Nothing -> Nothing + Just (_, Value Nothing) -> Nothing -- Should never happen. + Just (Value blkid, Value (Just epoch)) -> Just (blkid, epoch) queryReverseIndexBlockId :: MonadIO m => BlockId -> ReaderT SqlBackend m [Maybe Text] queryReverseIndexBlockId blockId = do @@ -1187,12 +1209,3 @@ queryPreviousSlotNo slotNo = do where_ (blk ^. BlockSlotNo ==. just (val slotNo)) pure $ pblk ^. BlockSlotNo pure $ unValue =<< listToMaybe res - -queryMinBlock :: MonadIO m => ReaderT SqlBackend m (Maybe BlockId) -queryMinBlock = do - res <- select $ do - blk <- from $ table @Block - orderBy [asc (blk ^. BlockId)] - limit 1 - pure $ blk ^. BlockId - pure $ unValue <$> listToMaybe res diff --git a/cardano-db/test/Test/IO/Cardano/Db/Insert.hs b/cardano-db/test/Test/IO/Cardano/Db/Insert.hs index 616a585cb..233a4400c 100644 --- a/cardano-db/test/Test/IO/Cardano/Db/Insert.hs +++ b/cardano-db/test/Test/IO/Cardano/Db/Insert.hs @@ -84,7 +84,7 @@ insertForeignKeyMissing = do assertBool (show count0 ++ "/= 1") (count0 == 1) -- Delete all OffChainFetchErrorTypeCount after pmrid - queryFirstAndDeleteAfter OffChainPoolFetchErrorPmrId pmrid + queryDelete OffChainPoolFetchErrorPmrId pmrid deleteWhere [PoolMetadataRefId >=. pmrid] count1 <- offChainPoolFetchErrorCount assertBool (show count1 ++ "/= 0") (count1 == 0) diff --git a/cardano-db/test/Test/IO/Cardano/Db/Util.hs b/cardano-db/test/Test/IO/Cardano/Db/Util.hs index edb05dea2..fdee97c4c 100644 --- a/cardano-db/test/Test/IO/Cardano/Db/Util.hs +++ b/cardano-db/test/Test/IO/Cardano/Db/Util.hs @@ -36,7 +36,7 @@ assertBool msg bool = deleteAllBlocks :: MonadIO m => ReaderT SqlBackend m () deleteAllBlocks = do mblkId <- queryMinBlock - whenJust mblkId $ deleteBlocksBlockIdNotrace TxOutCore + whenJust mblkId $ \(blkId, epochN) -> deleteBlocksForTests TxOutCore blkId epochN dummyUTCTime :: UTCTime dummyUTCTime = UTCTime (ModifiedJulianDay 0) 0 From 32f6a5f87512daf3d9f07a9447c0d74807ade5e4 Mon Sep 17 00:00:00 2001 From: Cmdv Date: Tue, 22 Oct 2024 10:52:16 +0100 Subject: [PATCH 2/2] add cmd line docs and update rollback queries --- .../src/Cardano/Db/Operations/Delete.hs | 74 +++++++++---------- .../Db/Operations/Other/ConsumedTxOut.hs | 27 ++++++- cardano-db/test/Test/IO/Cardano/Db/Util.hs | 2 +- doc/command-line-options | 36 +++++++++ 4 files changed, 97 insertions(+), 42 deletions(-) create mode 100644 doc/command-line-options diff --git a/cardano-db/src/Cardano/Db/Operations/Delete.hs b/cardano-db/src/Cardano/Db/Operations/Delete.hs index 69b27d127..ebcf84966 100644 --- a/cardano-db/src/Cardano/Db/Operations/Delete.hs +++ b/cardano-db/src/Cardano/Db/Operations/Delete.hs @@ -145,43 +145,6 @@ deleteUsingEpochNo epochN = do pure [("GovActionProposal Nulled", a + b + c + e)] pure $ countLogs <> nullLogs -queryDelete :: - forall m record field. - (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => - EntityField record field -> - field -> - ReaderT SqlBackend m () -queryDelete fieldIdField fieldId = do - mRecordId <- queryMinRefId fieldIdField fieldId - case mRecordId of - Nothing -> pure () - Just recordId -> deleteWhere [persistIdField @record >=. recordId] - -queryDeleteAndLog :: - forall m record field. - (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => - Text -> - EntityField record field -> - field -> - ReaderT SqlBackend m [(Text, Int64)] -queryDeleteAndLog tableName txIdField fieldId = do - mRecordId <- queryMinRefId txIdField fieldId - case mRecordId of - Nothing -> pure [(tableName, 0)] - Just recordId -> do - count <- deleteWhereCount [persistIdField @record >=. recordId] - pure [(tableName, count)] - -onlyDelete :: - forall m record. - (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) => - Text -> - [Filter record] -> - ReaderT SqlBackend m [(Text, Int64)] -onlyDelete tableName filters = do - count <- deleteWhereCount filters - pure [(tableName, count)] - deleteTablesAfterBlockId :: MonadIO m => TxOutTableType -> @@ -334,6 +297,43 @@ deleteTablesAfterTxId txOutTableType mtxId minIdsW = do -- Return the combined logs of all operations pure $ minIdsLogs <> txIdLogs +queryDelete :: + forall m record field. + (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => + EntityField record field -> + field -> + ReaderT SqlBackend m () +queryDelete fieldIdField fieldId = do + mRecordId <- queryMinRefId fieldIdField fieldId + case mRecordId of + Nothing -> pure () + Just recordId -> deleteWhere [persistIdField @record >=. recordId] + +queryDeleteAndLog :: + forall m record field. + (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => + Text -> + EntityField record field -> + field -> + ReaderT SqlBackend m [(Text, Int64)] +queryDeleteAndLog tableName txIdField fieldId = do + mRecordId <- queryMinRefId txIdField fieldId + case mRecordId of + Nothing -> pure [(tableName, 0)] + Just recordId -> do + count <- deleteWhereCount [persistIdField @record >=. recordId] + pure [(tableName, count)] + +onlyDelete :: + forall m record. + (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) => + Text -> + [Filter record] -> + ReaderT SqlBackend m [(Text, Int64)] +onlyDelete tableName filters = do + count <- deleteWhereCount filters + pure [(tableName, count)] + queryThenNull :: forall m record field. (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => diff --git a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs index 6c6c79f32..47f68e513 100644 --- a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs +++ b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs @@ -122,7 +122,7 @@ runExtraMigrations trce txOutTableType blockNoDiff pcm = do DBExtraMigration "runExtraMigrations: The configuration option 'tx_out.use_address_table' was previously set and the database updated. Unfortunately reverting this isn't possible." -- Has the user given txout address config && the migration wasn't previously set when (isTxOutVariant && not isTxOutAddressSet) $ do - updateTxOutAndCreateAddress + updateTxOutAndCreateAddress trce insertExtraMigration TxOutAddressPreviouslySet -- first check if pruneTxOut flag is missing and it has previously been used when (isPruneTxOutPreviouslySet migrationValues && not (pcmPruneTxOut pcm)) $ @@ -415,18 +415,29 @@ createPruneConstraintTxOut = do exceptHandler e = liftIO $ throwIO (DBPruneConsumed $ show e) +-- Be very mindfull that these queries can fail silently and make tests fail making it hard to know why. +-- To help mitigate this, logs are printed after each query is ran, so one can know where it stopped. updateTxOutAndCreateAddress :: forall m. ( MonadBaseControl IO m , MonadIO m ) => + Trace IO Text -> ReaderT SqlBackend m () -updateTxOutAndCreateAddress = do +updateTxOutAndCreateAddress trc = do handle exceptHandler $ rawExecute dropViewsQuery [] + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Dropped views" handle exceptHandler $ rawExecute alterTxOutQuery [] + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Altered tx_out" + handle exceptHandler $ rawExecute alterCollateralTxOutQuery [] + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Altered collateral_tx_out" handle exceptHandler $ rawExecute createAddressTableQuery [] + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Created address table" handle exceptHandler $ rawExecute createIndexPaymentCredQuery [] + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Created index payment_cred" handle exceptHandler $ rawExecute createIndexRawQuery [] + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Created index raw" + liftIO $ logInfo trc "updateTxOutAndCreateAddress: Completed" where dropViewsQuery = Text.unlines @@ -440,8 +451,16 @@ updateTxOutAndCreateAddress = do , " ADD COLUMN \"address_id\" INT8 NOT NULL," , " DROP COLUMN \"address\"," , " DROP COLUMN \"address_has_script\"," - , " DROP COLUMN \"payment_cred\"," - , " DROP COLUMN \"stake_address_id\"" + , " DROP COLUMN \"payment_cred\"" + ] + + alterCollateralTxOutQuery = + Text.unlines + [ "ALTER TABLE \"collateral_tx_out\"" + , " ADD COLUMN \"address_id\" INT8 NOT NULL," + , " DROP COLUMN \"address\"," + , " DROP COLUMN \"address_has_script\"," + , " DROP COLUMN \"payment_cred\"" ] createAddressTableQuery = diff --git a/cardano-db/test/Test/IO/Cardano/Db/Util.hs b/cardano-db/test/Test/IO/Cardano/Db/Util.hs index fdee97c4c..1bf6cece7 100644 --- a/cardano-db/test/Test/IO/Cardano/Db/Util.hs +++ b/cardano-db/test/Test/IO/Cardano/Db/Util.hs @@ -36,7 +36,7 @@ assertBool msg bool = deleteAllBlocks :: MonadIO m => ReaderT SqlBackend m () deleteAllBlocks = do mblkId <- queryMinBlock - whenJust mblkId $ \(blkId, epochN) -> deleteBlocksForTests TxOutCore blkId epochN + whenJust mblkId $ uncurry (deleteBlocksForTests TxOutCore) dummyUTCTime :: UTCTime dummyUTCTime = UTCTime (ModifiedJulianDay 0) 0 diff --git a/doc/command-line-options b/doc/command-line-options new file mode 100644 index 000000000..7c7bdf299 --- /dev/null +++ b/doc/command-line-options @@ -0,0 +1,36 @@ +### Command Line Options + +- **General Options:** + - `--config FILEPATH`: Path to the db-sync node config file + - `--socket-path FILEPATH`: Path to a cardano-node socket + - `--state-dir FILEPATH`: The directory for persisting ledger state. + - `--schema-dir FILEPATH`: The directory containing the migrations. + - `--pg-pass-env ENV`: Alternative env variable to use, defaults to `PGPASSFILE` env variable. + - `--disable-epoch`: Makes epoch table remain empty. + - `--skip-fix`: Disables the db-sync fix procedure for the wrong datum and redeemer_data bytes. + - `--force-indexes`: Forces the Index creation at the start of db-sync. Normally they're created later. + - `--fix-only`: Runs only the db-sync fix procedure for the wrong datum, redeemer_data, and plutus script bytes and exits. + - `--disable-cache`: Disables the db-sync caches. Reduces memory usage but it takes longer to sync. + - `--rollback-to-slot SLOTNO`: Force a rollback to the specified slot, if the given slot doesn't exist it will use the next greater slot. + - `--disable-in-out`: Disables the `tx_in` and `tx_out` table. + +- **Deprecated Options (for historical reference only):** + - For more details, refer to [Configuration Documentation](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md) + - `--disable-offline-data` + - `--disable-ledger` + - `--dont-use-ledger` + - `--keep-tx-metadata` + - `--disable-shelley` + - `--disable-multiassets` + - `--disable-metadata` + - `--disable-plutus-extra` + - `--disable-gov` + - `--disable-offchain-pool-data` + - `--force-tx-in` + - `--disable-all` + - `--full` + - `--only-utxo` + - `--only-gov` + - `--consumed-tx-out` + - `--prune-tx-out` + - `--bootstrap-tx-out`