add cmd line docs and update rollback queries

cardano-db/src/Cardano/Db/Operations/Delete.hs

@@ -46,6 +46,7 @@ import Control.Monad.Extra (whenJust)
 import Control.Monad.IO.Class (MonadIO, liftIO)
 import Control.Monad.Trans.Reader (ReaderT)
 import Data.ByteString (ByteString)
+import Data.List (partition)
 import Data.Maybe (isJust)
 import Data.Text (Text, intercalate, pack)
 import Data.Word (Word64)
@@ -74,7 +75,7 @@ deleteBlocksSlotNo ::
   Maybe Bool ->
   ReaderT SqlBackend m Bool
 deleteBlocksSlotNo trce txOutTableType (SlotNo slotNo) mIsConsumedTxOut = do
-  mBlockId <- queryBlockSlotNo slotNo
+  mBlockId <- queryNearestBlockSlotNo slotNo
   case mBlockId of
     Nothing -> do
       liftIO $ logWarning trce $ "deleteBlocksSlotNo: No block contains the the slot: " <> pack (show slotNo)
@@ -105,10 +106,13 @@ deleteBlocksBlockId trce txOutTableType blockId epochN mIsConsumedTxOut = do
   minIds <- if completed then pure cminIds else completeMinId mTxId cminIds
   deleteEpochLogs <- deleteUsingEpochNo epochN
   (deleteBlockCount, blockDeleteLogs) <- deleteTablesAfterBlockId blockId mTxId minIds
-  whenJust mIsConsumedTxOut $ \_ -> do
-    querySetNullTxOut trce txOutTableType mTxId
+  setNullLogs <-
+    maybe
+      (pure ("ConsumedTxOut is not active so no Nulls set", 0))
+      (\_ -> querySetNullTxOut txOutTableType mTxId)
+      mIsConsumedTxOut
   -- log all the deleted rows in the rollback
-  liftIO $ logInfo trce $ mkRollbackSummary (deleteEpochLogs <> blockDeleteLogs)
+  liftIO $ logInfo trce $ mkRollbackSummary (deleteEpochLogs <> blockDeleteLogs) setNullLogs
   pure deleteBlockCount
   where
     findMinIdsRec :: MonadIO m => [Maybe MinIdsWrapper] -> MinIdsWrapper -> ReaderT SqlBackend m (MinIdsWrapper, Bool)
@@ -131,13 +135,6 @@ deleteBlocksBlockId trce txOutTableType blockId epochN mIsConsumedTxOut = do
       CMinIdsWrapper (MinIds m1 m2 m3) -> isJust m1 && isJust m2 && isJust m3
       VMinIdsWrapper (MinIds m1 m2 m3) -> isJust m1 && isJust m2 && isJust m3
 
-mkRollbackSummary :: [(Text, Int64)] -> Text
-mkRollbackSummary logs = "\n ----------------------- Rollback Summary: ----------------------- \n" <> formattedLog
-  where
-    formattedLog = intercalate "\n" (map formatEntry logs)
-    formatEntry (tableName, rowCount) =
-      "Table: " <> tableName <> " - Count: " <> pack (show rowCount)
-
 deleteUsingEpochNo :: MonadIO m => Word64 -> ReaderT SqlBackend m [(Text, Int64)]
 deleteUsingEpochNo epochN = do
   countLogs <-
@@ -156,43 +153,6 @@ deleteUsingEpochNo epochN = do
     pure [("GovActionProposal Nulled", a + b + c + e)]
   pure $ countLogs <> nullLogs
 
-queryDelete ::
-  forall m record field.
-  (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) =>
-  EntityField record field ->
-  field ->
-  ReaderT SqlBackend m ()
-queryDelete fieldIdField fieldId = do
-  mRecordId <- queryMinRefId fieldIdField fieldId
-  case mRecordId of
-    Nothing -> pure ()
-    Just recordId -> deleteWhere [persistIdField @record >=. recordId]
-
-queryDeleteAndLog ::
-  forall m record field.
-  (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) =>
-  Text ->
-  EntityField record field ->
-  field ->
-  ReaderT SqlBackend m [(Text, Int64)]
-queryDeleteAndLog tableName txIdField fieldId = do
-  mRecordId <- queryMinRefId txIdField fieldId
-  case mRecordId of
-    Nothing -> pure [(tableName, 0)]
-    Just recordId -> do
-      count <- deleteWhereCount [persistIdField @record >=. recordId]
-      pure [(tableName, count)]
-
-onlyDelete ::
-  forall m record.
-  (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) =>
-  Text ->
-  [Filter record] ->
-  ReaderT SqlBackend m [(Text, Int64)]
-onlyDelete tableName filters = do
-  count <- deleteWhereCount filters
-  pure [(tableName, count)]
-
 deleteTablesAfterBlockId ::
   MonadIO m =>
   BlockId ->
@@ -341,6 +301,43 @@ deleteTablesAfterTxId mtxId minIdsW = do
   -- Return the combined logs of all operations
   pure $ minIdsLogs <> txIdLogs
 
+queryDelete ::
+  forall m record field.
+  (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) =>
+  EntityField record field ->
+  field ->
+  ReaderT SqlBackend m ()
+queryDelete fieldIdField fieldId = do
+  mRecordId <- queryMinRefId fieldIdField fieldId
+  case mRecordId of
+    Nothing -> pure ()
+    Just recordId -> deleteWhere [persistIdField @record >=. recordId]
+
+queryDeleteAndLog ::
+  forall m record field.
+  (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) =>
+  Text ->
+  EntityField record field ->
+  field ->
+  ReaderT SqlBackend m [(Text, Int64)]
+queryDeleteAndLog tableName txIdField fieldId = do
+  mRecordId <- queryMinRefId txIdField fieldId
+  case mRecordId of
+    Nothing -> pure [(tableName, 0)]
+    Just recordId -> do
+      count <- deleteWhereCount [persistIdField @record >=. recordId]
+      pure [(tableName, count)]
+
+onlyDelete ::
+  forall m record.
+  (MonadIO m, PersistEntity record, PersistEntityBackend record ~ SqlBackend) =>
+  Text ->
+  [Filter record] ->
+  ReaderT SqlBackend m [(Text, Int64)]
+onlyDelete tableName filters = do
+  count <- deleteWhereCount filters
+  pure [(tableName, count)]
+
 queryThenNull ::
   forall m record field.
   (MonadIO m, PersistEntity record, PersistField field, PersistEntityBackend record ~ SqlBackend) =>
@@ -374,3 +371,26 @@ deleteBlock txOutTableType block = do
     Just (blockId, epochN) -> do
       void $ deleteBlocksBlockId nullTracer txOutTableType blockId epochN Nothing
       pure True
+
+mkRollbackSummary :: [(Text, Int64)] -> (Text, Int64) -> Text
+mkRollbackSummary logs setNullLogs =
+  "\n ----------------------- Rollback Summary: ----------------------- \n"
+    <> formattedLog
+    <> formatSetNullLog setNullLogs
+  where
+    (zeroDeletes, nonZeroDeletes) = partition ((== 0) . snd) logs
+
+    formattedLog = intercalate "\n" (zeroDeletedEntry : map formatEntry nonZeroDeletes)
+
+    zeroDeletedEntry
+      | null zeroDeletes = ""
+      | otherwise = "No Deletes: " <> intercalate ", " (map fst zeroDeletes)
+
+    formatEntry (tableName, rowCount) =
+      "Table: " <> tableName <> " - Count: " <> pack (show rowCount)
+
+    formatSetNullLog (nullMessage, nullCount) =
+      "\n Set Null:"
+        <> if nullCount == 0
+          then nullMessage
+          else "Set Null: " <> nullMessage <> " - Count: " <> pack (show nullCount)

cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs

@@ -30,6 +30,7 @@ import Control.Monad.Extra (unless, when, whenJust)
 import Control.Monad.IO.Class (MonadIO, liftIO)
 import Control.Monad.Trans.Control (MonadBaseControl)
 import Control.Monad.Trans.Reader (ReaderT)
+import Data.Int (Int64)
 import Data.Text (Text)
 import qualified Data.Text as Text
 import Data.Word (Word64)
@@ -52,13 +53,20 @@ data ConsumedTriplet = ConsumedTriplet
 --------------------------------------------------------------------------------------------------
 -- Queries
 --------------------------------------------------------------------------------------------------
-querySetNullTxOut :: MonadIO m => Trace IO Text -> TxOutTableType -> Maybe TxId -> ReaderT SqlBackend m ()
-querySetNullTxOut trce txOutTableType mMinTxId = do
-  whenJust mMinTxId $ \txId -> do
-    txOutIds <- getTxOutConsumedAfter txId
-    mapM_ setNullTxOutConsumedAfter txOutIds
-    let updatedEntries = length txOutIds
-    liftIO $ logInfo trce $ "querySetNullTxOut: Set to null " <> textShow updatedEntries <> " tx_out.consumed_by_tx_id"
+querySetNullTxOut ::
+  MonadIO m =>
+  TxOutTableType ->
+  Maybe TxId ->
+  ReaderT SqlBackend m (Text, Int64)
+querySetNullTxOut txOutTableType mMinTxId = do
+  case mMinTxId of
+    Nothing -> do
+      pure ("querySetNullTxOut: No tx_out to set to null:", 0)
+    Just txId -> do
+      txOutIds <- getTxOutConsumedAfter txId
+      mapM_ setNullTxOutConsumedAfter txOutIds
+      let updatedEntriesCount = length txOutIds
+      pure ("querySetNullTxOut: Set to null tx_out.consumed_by_tx_id", fromIntegral updatedEntriesCount)
   where
     -- \| This requires an index at TxOutConsumedByTxId.
     getTxOutConsumedAfter :: MonadIO m => TxId -> ReaderT SqlBackend m [TxOutIdW]

cardano-db/src/Cardano/Db/Operations/Query.hs

@@ -12,7 +12,7 @@ module Cardano.Db.Operations.Query (
   queryBlockHashBlockNo,
   queryBlockNo,
   queryBlockNoAndEpoch,
-  queryBlockSlotNo,
+  queryNearestBlockSlotNo,
   queryBlockHash,
   queryReverseIndexBlockId,
   queryMinIdsAfterReverseIndex,
@@ -159,6 +159,7 @@ import Database.Esqueleto.Experimental (
   (>=.),
   (?.),
   (^.),
+  (||.),
   type (:&) ((:&)),
  )
 import Database.Persist.Class.PersistQuery (selectList)
@@ -221,11 +222,14 @@ queryBlockNoAndEpoch blkNo = do
     pure (blk ^. BlockId, blk ^. BlockEpochNo)
   pure $ convertBlockQuery (listToMaybe res)
 
-queryBlockSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m (Maybe (BlockId, Word64))
-queryBlockSlotNo slotNo = do
+-- | Retrieves the nearest block with a slot number equal to or greater than the given slot number.
+queryNearestBlockSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m (Maybe (BlockId, Word64))
+queryNearestBlockSlotNo slotNo = do
   res <- select $ do
     blk <- from $ table @Block
-    where_ (blk ^. BlockSlotNo ==. just (val slotNo))
+    where_ (isNothing (blk ^. BlockSlotNo) ||. blk ^. BlockSlotNo >=. just (val slotNo))
+    orderBy [asc (blk ^. BlockSlotNo)]
+    limit 1
     pure (blk ^. BlockId, blk ^. BlockBlockNo)
   pure $ convertBlockQuery (listToMaybe res)
 

doc/command-line-options

@@ -0,0 +1,36 @@
+### Command Line Options
+
+- **General Options:**
+  - `--config FILEPATH`: Path to the db-sync node config file
+  - `--socket-path FILEPATH`: Path to a cardano-node socket
+  - `--state-dir FILEPATH`: The directory for persisting ledger state.
+  - `--schema-dir FILEPATH`: The directory containing the migrations.
+  - `--pg-pass-env ENV`: Alternative env variable to use, defaults to `PGPASSFILE` env variable.
+  - `--disable-epoch`: Makes epoch table remain empty.
+  - `--skip-fix`: Disables the db-sync fix procedure for the wrong datum and redeemer_data bytes.
+  - `--force-indexes`: Forces the Index creation at the start of db-sync. Normally they're created later.
+  - `--fix-only`: Runs only the db-sync fix procedure for the wrong datum, redeemer_data, and plutus script bytes and exits.
+  - `--disable-cache`: Disables the db-sync caches. Reduces memory usage but it takes longer to sync.
+  - `--rollback-to-slot SLOTNO`: Force a rollback to the specified slot, if the given slot doesn't exist it will use the next greater slot.
+  - `--disable-in-out`: Disables the `tx_in` and `tx_out` table.
+
+- **Deprecated Options (for historical reference only):**
+  - For more details, refer to [Configuration Documentation](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md)
+  - `--disable-offline-data`
+  - `--disable-ledger`
+  - `--dont-use-ledger`
+  - `--keep-tx-metadata`
+  - `--disable-shelley`
+  - `--disable-multiassets`
+  - `--disable-metadata`
+  - `--disable-plutus-extra`
+  - `--disable-gov`
+  - `--disable-offchain-pool-data`
+  - `--force-tx-in`
+  - `--disable-all`
+  - `--full`
+  - `--only-utxo`
+  - `--only-gov`
+  - `--consumed-tx-out`
+  - `--prune-tx-out`
+  - `--bootstrap-tx-out`