From 8cab25a56ec9a867fec83b7c72f2e0969c227db8 Mon Sep 17 00:00:00 2001 From: Cmdv Date: Tue, 22 Oct 2024 10:52:16 +0100 Subject: [PATCH] add cmd line docs and update rollback queries --- .../src/Cardano/Db/Operations/Delete.hs | 119 ++++++++++-------- .../Db/Operations/Other/ConsumedTxOut.hs | 22 ++-- cardano-db/src/Cardano/Db/Operations/Query.hs | 12 +- doc/command-line-options | 36 ++++++ 4 files changed, 129 insertions(+), 60 deletions(-) create mode 100644 doc/command-line-options diff --git a/cardano-db/src/Cardano/Db/Operations/Delete.hs b/cardano-db/src/Cardano/Db/Operations/Delete.hs index a780cedbf..fafab39ae 100644 --- a/cardano-db/src/Cardano/Db/Operations/Delete.hs +++ b/cardano-db/src/Cardano/Db/Operations/Delete.hs @@ -42,10 +42,10 @@ import qualified Cardano.Db.Schema.Variant.TxOut as V import Cardano.Prelude (Int64) import Cardano.Slotting.Slot (SlotNo (..)) import Control.Monad (void) -import Control.Monad.Extra (whenJust) import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Trans.Reader (ReaderT) import Data.ByteString (ByteString) +import Data.List (partition) import Data.Maybe (isJust) import Data.Text (Text, intercalate, pack) import Data.Word (Word64) @@ -74,7 +74,7 @@ deleteBlocksSlotNo :: Maybe Bool -> ReaderT SqlBackend m Bool deleteBlocksSlotNo trce txOutTableType (SlotNo slotNo) mIsConsumedTxOut = do - mBlockId <- queryBlockSlotNo slotNo + mBlockId <- queryNearestBlockSlotNo slotNo case mBlockId of Nothing -> do liftIO $ logWarning trce $ "deleteBlocksSlotNo: No block contains the the slot: " <> pack (show slotNo) @@ -105,10 +105,13 @@ deleteBlocksBlockId trce txOutTableType blockId epochN mIsConsumedTxOut = do minIds <- if completed then pure cminIds else completeMinId mTxId cminIds deleteEpochLogs <- deleteUsingEpochNo epochN (deleteBlockCount, blockDeleteLogs) <- deleteTablesAfterBlockId blockId mTxId minIds - whenJust mIsConsumedTxOut $ \_ -> do - querySetNullTxOut trce txOutTableType mTxId + setNullLogs <- + maybe + (pure ("ConsumedTxOut is not active so no Nulls set", 0)) + (\_ -> querySetNullTxOut txOutTableType mTxId) + mIsConsumedTxOut -- log all the deleted rows in the rollback - liftIO $ logInfo trce $ mkRollbackSummary (deleteEpochLogs <> blockDeleteLogs) + liftIO $ logInfo trce $ mkRollbackSummary (deleteEpochLogs <> blockDeleteLogs) setNullLogs pure deleteBlockCount where findMinIdsRec :: MonadIO m => [Maybe MinIdsWrapper] -> MinIdsWrapper -> ReaderT SqlBackend m (MinIdsWrapper, Bool) @@ -131,13 +134,6 @@ deleteBlocksBlockId trce txOutTableType blockId epochN mIsConsumedTxOut = do CMinIdsWrapper (MinIds m1 m2 m3) -> isJust m1 && isJust m2 && isJust m3 VMinIdsWrapper (MinIds m1 m2 m3) -> isJust m1 && isJust m2 && isJust m3 -mkRollbackSummary :: [(Text, Int64)] -> Text -mkRollbackSummary logs = "\n ----------------------- Rollback Summary: ----------------------- \n" <> formattedLog - where - formattedLog = intercalate "\n" (map formatEntry logs) - formatEntry (tableName, rowCount) = - "Table: " <> tableName <> " - Count: " <> pack (show rowCount) - deleteUsingEpochNo :: MonadIO m => Word64 -> ReaderT SqlBackend m [(Text, Int64)] deleteUsingEpochNo epochN = do countLogs <- @@ -156,43 +152,6 @@ deleteUsingEpochNo epochN = do pure [("GovActionProposal Nulled", a + b + c + e)] pure $ countLogs <> nullLogs -queryDelete :: - forall m record field. - (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => - EntityField record field -> - field -> - ReaderT SqlBackend m () -queryDelete fieldIdField fieldId = do - mRecordId <- queryMinRefId fieldIdField fieldId - case mRecordId of - Nothing -> pure () - Just recordId -> deleteWhere [persistIdField @record >=. recordId] - -queryDeleteAndLog :: - forall m record field. - (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => - Text -> - EntityField record field -> - field -> - ReaderT SqlBackend m [(Text, Int64)] -queryDeleteAndLog tableName txIdField fieldId = do - mRecordId <- queryMinRefId txIdField fieldId - case mRecordId of - Nothing -> pure [(tableName, 0)] - Just recordId -> do - count <- deleteWhereCount [persistIdField @record >=. recordId] - pure [(tableName, count)] - -onlyDelete :: - forall m record. - (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) => - Text -> - [Filter record] -> - ReaderT SqlBackend m [(Text, Int64)] -onlyDelete tableName filters = do - count <- deleteWhereCount filters - pure [(tableName, count)] - deleteTablesAfterBlockId :: MonadIO m => BlockId -> @@ -341,6 +300,43 @@ deleteTablesAfterTxId mtxId minIdsW = do -- Return the combined logs of all operations pure $ minIdsLogs <> txIdLogs +queryDelete :: + forall m record field. + (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => + EntityField record field -> + field -> + ReaderT SqlBackend m () +queryDelete fieldIdField fieldId = do + mRecordId <- queryMinRefId fieldIdField fieldId + case mRecordId of + Nothing -> pure () + Just recordId -> deleteWhere [persistIdField @record >=. recordId] + +queryDeleteAndLog :: + forall m record field. + (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => + Text -> + EntityField record field -> + field -> + ReaderT SqlBackend m [(Text, Int64)] +queryDeleteAndLog tableName txIdField fieldId = do + mRecordId <- queryMinRefId txIdField fieldId + case mRecordId of + Nothing -> pure [(tableName, 0)] + Just recordId -> do + count <- deleteWhereCount [persistIdField @record >=. recordId] + pure [(tableName, count)] + +onlyDelete :: + forall m record. + (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) => + Text -> + [Filter record] -> + ReaderT SqlBackend m [(Text, Int64)] +onlyDelete tableName filters = do + count <- deleteWhereCount filters + pure [(tableName, count)] + queryThenNull :: forall m record field. (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) => @@ -374,3 +370,28 @@ deleteBlock txOutTableType block = do Just (blockId, epochN) -> do void $ deleteBlocksBlockId nullTracer txOutTableType blockId epochN Nothing pure True + +mkRollbackSummary :: [(Text, Int64)] -> (Text, Int64) -> Text +mkRollbackSummary logs setNullLogs = + "\n----------------------- Rollback Summary: ----------------------- \n" + <> formattedLog + <> zeroDeletedEntry + <> formatSetNullLog setNullLogs + <> "\n" + where + (zeroDeletes, nonZeroDeletes) = partition ((== 0) . snd) logs + + formattedLog = intercalate "\n" (map formatEntry nonZeroDeletes) + + zeroDeletedEntry + | null zeroDeletes = "" + | otherwise = "\n\nNo Deletes in tables: " <> intercalate ", " (map fst zeroDeletes) + + formatEntry (tableName, rowCount) = + "Table: " <> tableName <> " - Count: " <> pack (show rowCount) + + formatSetNullLog (nullMessage, nullCount) = + "\n\nSet Null: " + <> if nullCount == 0 + then nullMessage + else "\n\nSet Null: " <> nullMessage <> " - Count: " <> pack (show nullCount) diff --git a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs index 2a4ad3238..536d7b637 100644 --- a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs +++ b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs @@ -30,6 +30,7 @@ import Control.Monad.Extra (unless, when, whenJust) import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Trans.Control (MonadBaseControl) import Control.Monad.Trans.Reader (ReaderT) +import Data.Int (Int64) import Data.Text (Text) import qualified Data.Text as Text import Data.Word (Word64) @@ -52,13 +53,20 @@ data ConsumedTriplet = ConsumedTriplet -------------------------------------------------------------------------------------------------- -- Queries -------------------------------------------------------------------------------------------------- -querySetNullTxOut :: MonadIO m => Trace IO Text -> TxOutTableType -> Maybe TxId -> ReaderT SqlBackend m () -querySetNullTxOut trce txOutTableType mMinTxId = do - whenJust mMinTxId $ \txId -> do - txOutIds <- getTxOutConsumedAfter txId - mapM_ setNullTxOutConsumedAfter txOutIds - let updatedEntries = length txOutIds - liftIO $ logInfo trce $ "querySetNullTxOut: Set to null " <> textShow updatedEntries <> " tx_out.consumed_by_tx_id" +querySetNullTxOut :: + MonadIO m => + TxOutTableType -> + Maybe TxId -> + ReaderT SqlBackend m (Text, Int64) +querySetNullTxOut txOutTableType mMinTxId = do + case mMinTxId of + Nothing -> do + pure ("querySetNullTxOut: No tx_out to set to null:", 0) + Just txId -> do + txOutIds <- getTxOutConsumedAfter txId + mapM_ setNullTxOutConsumedAfter txOutIds + let updatedEntriesCount = length txOutIds + pure ("querySetNullTxOut: Set to null tx_out.consumed_by_tx_id", fromIntegral updatedEntriesCount) where -- \| This requires an index at TxOutConsumedByTxId. getTxOutConsumedAfter :: MonadIO m => TxId -> ReaderT SqlBackend m [TxOutIdW] diff --git a/cardano-db/src/Cardano/Db/Operations/Query.hs b/cardano-db/src/Cardano/Db/Operations/Query.hs index 4f7c87717..904ed1646 100644 --- a/cardano-db/src/Cardano/Db/Operations/Query.hs +++ b/cardano-db/src/Cardano/Db/Operations/Query.hs @@ -12,7 +12,7 @@ module Cardano.Db.Operations.Query ( queryBlockHashBlockNo, queryBlockNo, queryBlockNoAndEpoch, - queryBlockSlotNo, + queryNearestBlockSlotNo, queryBlockHash, queryReverseIndexBlockId, queryMinIdsAfterReverseIndex, @@ -159,6 +159,7 @@ import Database.Esqueleto.Experimental ( (>=.), (?.), (^.), + (||.), type (:&) ((:&)), ) import Database.Persist.Class.PersistQuery (selectList) @@ -221,11 +222,14 @@ queryBlockNoAndEpoch blkNo = do pure (blk ^. BlockId, blk ^. BlockEpochNo) pure $ convertBlockQuery (listToMaybe res) -queryBlockSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m (Maybe (BlockId, Word64)) -queryBlockSlotNo slotNo = do +-- | Retrieves the nearest block with a slot number equal to or greater than the given slot number. +queryNearestBlockSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m (Maybe (BlockId, Word64)) +queryNearestBlockSlotNo slotNo = do res <- select $ do blk <- from $ table @Block - where_ (blk ^. BlockSlotNo ==. just (val slotNo)) + where_ (isNothing (blk ^. BlockSlotNo) ||. blk ^. BlockSlotNo >=. just (val slotNo)) + orderBy [asc (blk ^. BlockSlotNo)] + limit 1 pure (blk ^. BlockId, blk ^. BlockBlockNo) pure $ convertBlockQuery (listToMaybe res) diff --git a/doc/command-line-options b/doc/command-line-options new file mode 100644 index 000000000..7c7bdf299 --- /dev/null +++ b/doc/command-line-options @@ -0,0 +1,36 @@ +### Command Line Options + +- **General Options:** + - `--config FILEPATH`: Path to the db-sync node config file + - `--socket-path FILEPATH`: Path to a cardano-node socket + - `--state-dir FILEPATH`: The directory for persisting ledger state. + - `--schema-dir FILEPATH`: The directory containing the migrations. + - `--pg-pass-env ENV`: Alternative env variable to use, defaults to `PGPASSFILE` env variable. + - `--disable-epoch`: Makes epoch table remain empty. + - `--skip-fix`: Disables the db-sync fix procedure for the wrong datum and redeemer_data bytes. + - `--force-indexes`: Forces the Index creation at the start of db-sync. Normally they're created later. + - `--fix-only`: Runs only the db-sync fix procedure for the wrong datum, redeemer_data, and plutus script bytes and exits. + - `--disable-cache`: Disables the db-sync caches. Reduces memory usage but it takes longer to sync. + - `--rollback-to-slot SLOTNO`: Force a rollback to the specified slot, if the given slot doesn't exist it will use the next greater slot. + - `--disable-in-out`: Disables the `tx_in` and `tx_out` table. + +- **Deprecated Options (for historical reference only):** + - For more details, refer to [Configuration Documentation](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md) + - `--disable-offline-data` + - `--disable-ledger` + - `--dont-use-ledger` + - `--keep-tx-metadata` + - `--disable-shelley` + - `--disable-multiassets` + - `--disable-metadata` + - `--disable-plutus-extra` + - `--disable-gov` + - `--disable-offchain-pool-data` + - `--force-tx-in` + - `--disable-all` + - `--full` + - `--only-utxo` + - `--only-gov` + - `--consumed-tx-out` + - `--prune-tx-out` + - `--bootstrap-tx-out`