Merge pull request #219 from kcalvinalvin/2024-11-14-fix-relay-utreex…

…otx-bug

blockchain, main, wire, indexers: change MsgUtreexoTx serialization

blockchain/indexers/flatutreexoproofindex.go

@@ -657,10 +657,13 @@ func (idx *FlatUtreexoProofIndex) GenerateUDataPartial(dels []wire.LeafData, pos
 	ud := new(wire.UData)
 	ud.LeafDatas = dels
 
-	// Get the positions of the targets of delHashes.
-	delHashes, err := wire.HashesFromLeafDatas(ud.LeafDatas)
-	if err != nil {
-		return nil, err
+	delHashes := make([]utreexo.Hash, 0, len(dels))
+	for _, del := range dels {
+		// We can't calculate the correct hash if the leaf data is in
+		// the compact state.
+		if !del.IsUnconfirmed() {
+			delHashes = append(delHashes, del.LeafHash())
+		}
 	}
 
 	hashes := make([]utreexo.Hash, len(positions))

blockchain/indexers/utreexoproofindex.go

@@ -451,10 +451,13 @@ func (idx *UtreexoProofIndex) GenerateUDataPartial(dels []wire.LeafData, positio
 	ud := new(wire.UData)
 	ud.LeafDatas = dels
 
-	// Get the positions of the targets of delHashes.
-	delHashes, err := wire.HashesFromLeafDatas(ud.LeafDatas)
-	if err != nil {
-		return nil, err
+	delHashes := make([]utreexo.Hash, 0, len(dels))
+	for _, del := range dels {
+		// We can't calculate the correct hash if the leaf data is in
+		// the compact state.
+		if !del.IsUnconfirmed() {
+			delHashes = append(delHashes, del.LeafHash())
+		}
 	}
 
 	hashes := make([]utreexo.Hash, len(positions))

blockchain/utreexoviewpoint.go

@@ -377,47 +377,11 @@ func reconstructUData(ud *wire.UData, block *btcutil.Block, chainView *chainView
 			}
 
 			ld := &ud.LeafDatas[ldIdx]
-
-			// Get BlockHash.
-			blockNode := chainView.NodeByHeight(ld.Height)
-			if blockNode == nil {
-				return nil, fmt.Errorf("Couldn't find blockNode for height %d",
-					ld.Height)
-			}
-			ld.BlockHash = blockNode.hash
-
-			// Get OutPoint.
-			op := wire.OutPoint{
-				Hash:  txIn.PreviousOutPoint.Hash,
-				Index: txIn.PreviousOutPoint.Index,
-			}
-			ld.OutPoint = op
-
-			if ld.ReconstructablePkType != wire.OtherTy &&
-				ld.PkScript == nil {
-
-				var class txscript.ScriptClass
-
-				switch ld.ReconstructablePkType {
-				case wire.PubKeyHashTy:
-					class = txscript.PubKeyHashTy
-				case wire.ScriptHashTy:
-					class = txscript.ScriptHashTy
-				case wire.WitnessV0PubKeyHashTy:
-					class = txscript.WitnessV0PubKeyHashTy
-				case wire.WitnessV0ScriptHashTy:
-					class = txscript.WitnessV0ScriptHashTy
-				}
-
-				scriptToUse, err := txscript.ReconstructScript(
-					txIn.SignatureScript, txIn.Witness, class)
-				if err != nil {
-					return nil, err
-				}
-
-				ld.PkScript = scriptToUse
+			var err error
+			ld, err = reconstructLeafData(ld, txIn, chainView)
+			if err != nil {
+				return nil, err
 			}
-
 			delHashes = append(delHashes, ld.LeafHash())
 
 			blockInIdx++
@@ -428,6 +392,76 @@ func reconstructUData(ud *wire.UData, block *btcutil.Block, chainView *chainView
 	return delHashes, nil
 }
 
+// ReconstructLeafDatas reconstruct the passed in leaf datas with the given txIns.
+//
+// NOTE: the length of the leafdatas MUST match the TxIns. Otherwise it'll return an error.
+func (b *BlockChain) ReconstructLeafDatas(lds []wire.LeafData, txIns []*wire.TxIn) ([]wire.LeafData, error) {
+	if len(lds) == 0 {
+		return lds, nil
+	}
+
+	if len(lds) != len(txIns) {
+		err := fmt.Errorf("Can't reconstruct leaf datas.  Have %d txins but %d leaf datas",
+			len(txIns), len(lds))
+		return nil, err
+	}
+
+	for i, txIn := range txIns {
+		if lds[i].IsUnconfirmed() {
+			continue
+		}
+
+		ld, err := reconstructLeafData(&lds[i], txIn, b.bestChain)
+		if err != nil {
+			return nil, err
+		}
+		lds[i] = *ld
+	}
+
+	return lds, nil
+}
+
+// reconstructLeafData reconstructs a single leafdata given the associated txIn and the chainview.
+func reconstructLeafData(ld *wire.LeafData, txIn *wire.TxIn, chainView *chainView) (*wire.LeafData, error) {
+	// Get BlockHash.
+	blockNode := chainView.NodeByHeight(ld.Height)
+	if blockNode == nil {
+		return nil, fmt.Errorf("Couldn't find blockNode for height %d",
+			ld.Height)
+	}
+	ld.BlockHash = blockNode.hash
+
+	// Get OutPoint.
+	ld.OutPoint = txIn.PreviousOutPoint
+
+	if ld.ReconstructablePkType != wire.OtherTy &&
+		ld.PkScript == nil {
+
+		var class txscript.ScriptClass
+
+		switch ld.ReconstructablePkType {
+		case wire.PubKeyHashTy:
+			class = txscript.PubKeyHashTy
+		case wire.ScriptHashTy:
+			class = txscript.ScriptHashTy
+		case wire.WitnessV0PubKeyHashTy:
+			class = txscript.WitnessV0PubKeyHashTy
+		case wire.WitnessV0ScriptHashTy:
+			class = txscript.WitnessV0ScriptHashTy
+		}
+
+		scriptToUse, err := txscript.ReconstructScript(
+			txIn.SignatureScript, txIn.Witness, class)
+		if err != nil {
+			return nil, err
+		}
+
+		ld.PkScript = scriptToUse
+	}
+
+	return ld, nil
+}
+
 // IsUnspendable determines whether a tx is spendable or not.
 // returns true if spendable, false if unspendable.
 func IsUnspendable(o *wire.TxOut) bool {
@@ -874,69 +908,19 @@ func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn, remember bo
 			"Cannot validate utreexo accumulator proof")
 	}
 
-	// Check that there are equal amount of LeafDatas for txIns.
-	if len(txIns) != len(ud.LeafDatas) {
-		str := fmt.Sprintf("VerifyUData(): length of txIns and LeafDatas differ. "+
-			"%d txIns, but %d LeafDatas. TxIns PreviousOutPoints are:\n",
-			len(txIns), len(ud.LeafDatas))
-		for _, txIn := range txIns {
-			str += fmt.Sprintf("%s\n", txIn.PreviousOutPoint.String())
-		}
-
-		return fmt.Errorf("%v", str)
+	var err error
+	ud.LeafDatas, err = b.ReconstructLeafDatas(ud.LeafDatas, txIns)
+	if err != nil {
+		return err
 	}
 
-	// Make a slice of hashes from LeafDatas. These are the hash commitments
-	// to be proven.
 	delHashes := make([]utreexo.Hash, 0, len(ud.LeafDatas))
-	for i, txIn := range txIns {
-		ld := &ud.LeafDatas[i]
-
-		// Get OutPoint.
-		op := wire.OutPoint{
-			Hash:  txIn.PreviousOutPoint.Hash,
-			Index: txIn.PreviousOutPoint.Index,
+	for _, ld := range ud.LeafDatas {
+		if ld.IsCompact() || ld.IsUnconfirmed() {
+			continue
 		}
-		ld.OutPoint = op
-
-		// Only append and try to fetch blockHash for confirmed txs.  Skip
-		// all unconfirmed txs.
-		if !ld.IsUnconfirmed() {
-			// Get BlockHash.
-			blockNode := b.bestChain.NodeByHeight(ld.Height)
-			if blockNode == nil {
-				return fmt.Errorf("Couldn't find blockNode for height %d for outpoint %s",
-					ld.Height, txIn.PreviousOutPoint.String())
-			}
-			ld.BlockHash = blockNode.hash
-
-			if ld.ReconstructablePkType != wire.OtherTy &&
-				ld.PkScript == nil {
-
-				var class txscript.ScriptClass
-
-				switch ld.ReconstructablePkType {
-				case wire.PubKeyHashTy:
-					class = txscript.PubKeyHashTy
-				case wire.ScriptHashTy:
-					class = txscript.ScriptHashTy
-				case wire.WitnessV0PubKeyHashTy:
-					class = txscript.WitnessV0PubKeyHashTy
-				case wire.WitnessV0ScriptHashTy:
-					class = txscript.WitnessV0ScriptHashTy
-				}
-
-				scriptToUse, err := txscript.ReconstructScript(
-					txIn.SignatureScript, txIn.Witness, class)
-				if err != nil {
-					return err
-				}
-
-				ld.PkScript = scriptToUse
-			}
 
-			delHashes = append(delHashes, ld.LeafHash())
-		}
+		delHashes = append(delHashes, ld.LeafHash())
 	}
 
 	// Acquire read lock before accessing the accumulator state.
@@ -945,7 +929,7 @@ func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn, remember bo
 
 	// VerifyBatchProof checks that the utreexo proofs are valid without
 	// mutating the accumulator.
-	err := b.utreexoView.accumulator.VerifyPartialProof(ud.AccProof.Targets, delHashes, ud.AccProof.Proof, remember)
+	err = b.utreexoView.accumulator.VerifyPartialProof(ud.AccProof.Targets, delHashes, ud.AccProof.Proof, remember)
 	if err != nil {
 		str := "Verify fail. All txIns-leaf datas:\n"
 		for i, txIn := range txIns {

mempool/mempool.go

@@ -195,6 +195,7 @@ type TxPool struct {
 	pool          map[chainhash.Hash]*TxDesc
 	poolLeaves    map[chainhash.Hash][]wire.LeafData
 	orphans       map[chainhash.Hash]*orphanTx
+	orphanUData   map[chainhash.Hash]*wire.UData
 	orphansByPrev map[wire.OutPoint]map[chainhash.Hash]*btcutil.Tx
 	outpoints     map[wire.OutPoint]*btcutil.Tx
 	pennyTotal    float64 // exponentially decaying total for penny spends.
@@ -225,6 +226,21 @@ func (mp *TxPool) removeOrphan(tx *btcutil.Tx, removeRedeemers bool) {
 		return
 	}
 
+	if mp.cfg.IsUtreexoViewActive() {
+		ud, found := mp.orphanUData[*tx.Hash()]
+		if found {
+			// Remove related utreexo data.
+			//
+			// TODO uncache from the accumulator.
+			delete(mp.orphanUData, *tx.Hash())
+			err := mp.cfg.PruneFromAccumulator(ud.LeafDatas)
+			if err != nil {
+				log.Debugf("error while pruning proof for orphan tx %v"+
+					"from the accumulator.", tx.Hash())
+			}
+		}
+	}
+
 	// Remove the reference from the previous orphan index.
 	for _, txIn := range otx.tx.MsgTx().TxIn {
 		orphans, exists := mp.orphansByPrev[txIn.PreviousOutPoint]
@@ -337,7 +353,7 @@ func (mp *TxPool) limitNumOrphans() error {
 // addOrphan adds an orphan transaction to the orphan pool.
 //
 // This function MUST be called with the mempool lock held (for writes).
-func (mp *TxPool) addOrphan(tx *btcutil.Tx, tag Tag) {
+func (mp *TxPool) addOrphan(tx *btcutil.Tx, utreexoData *wire.UData, tag Tag) {
 	// Nothing to do if no orphans are allowed.
 	if mp.cfg.Policy.MaxOrphanTxs <= 0 {
 		return
@@ -360,6 +376,12 @@ func (mp *TxPool) addOrphan(tx *btcutil.Tx, tag Tag) {
 		}
 		mp.orphansByPrev[txIn.PreviousOutPoint][*tx.Hash()] = tx
 	}
+	if mp.cfg.IsUtreexoViewActive() {
+		// Ingest the proof. Shouldn't error out with the proof being invalid
+		// here since we've already verified it above.
+		mp.cfg.VerifyUData(utreexoData, tx.MsgTx().TxIn, true)
+		mp.orphanUData[*tx.Hash()] = utreexoData
+	}
 
 	log.Debugf("Stored orphan transaction %v (total: %d)", tx.Hash(),
 		len(mp.orphans))
@@ -368,7 +390,7 @@ func (mp *TxPool) addOrphan(tx *btcutil.Tx, tag Tag) {
 // maybeAddOrphan potentially adds an orphan to the orphan pool.
 //
 // This function MUST be called with the mempool lock held (for writes).
-func (mp *TxPool) maybeAddOrphan(tx *btcutil.Tx, tag Tag) error {
+func (mp *TxPool) maybeAddOrphan(tx *btcutil.Tx, utreexoData *wire.UData, tag Tag) error {
 	// Ignore orphan transactions that are too large.  This helps avoid
 	// a memory exhaustion attack based on sending a lot of really large
 	// orphans.  In the case there is a valid transaction larger than this,
@@ -388,7 +410,7 @@ func (mp *TxPool) maybeAddOrphan(tx *btcutil.Tx, tag Tag) error {
 	}
 
 	// Add the orphan if the none of the above disqualified it.
-	mp.addOrphan(tx, tag)
+	mp.addOrphan(tx, utreexoData, tag)
 
 	return nil
 }
@@ -1127,8 +1149,18 @@ func (mp *TxPool) processOrphans(acceptedTx *btcutil.Tx) []*TxDesc {
 
 			// Potentially accept an orphan into the tx pool.
 			for _, tx := range orphans {
+				var uData *wire.UData
+				if mp.cfg.IsUtreexoViewActive() {
+					var found bool
+					uData, found = mp.orphanUData[*tx.Hash()]
+					if !found {
+						log.Debugf("is a utreexo node but is missing udata for"+
+							"orphan tx %v", tx.Hash())
+						continue
+					}
+				}
 				missing, txD, err := mp.maybeAcceptTransaction(
-					tx, nil, true, true, false)
+					tx, uData, true, true, false)
 				if err != nil {
 					// The orphan is now invalid, so there
 					// is no way any other orphans which
@@ -1252,7 +1284,7 @@ func (mp *TxPool) ProcessTransaction(tx *btcutil.Tx, utreexoData *wire.UData, al
 	}
 
 	// Potentially add the orphan transaction to the orphan pool.
-	err = mp.maybeAddOrphan(tx, tag)
+	err = mp.maybeAddOrphan(tx, utreexoData, tag)
 	return nil, err
 }
 
@@ -1882,6 +1914,7 @@ func New(cfg *Config) *TxPool {
 		pool:           make(map[chainhash.Hash]*TxDesc),
 		poolLeaves:     make(map[chainhash.Hash][]wire.LeafData),
 		orphans:        make(map[chainhash.Hash]*orphanTx),
+		orphanUData:    make(map[chainhash.Hash]*wire.UData),
 		orphansByPrev:  make(map[wire.OutPoint]map[chainhash.Hash]*btcutil.Tx),
 		nextExpireScan: time.Now().Add(orphanExpireScanInterval),
 		outpoints:      make(map[wire.OutPoint]*btcutil.Tx),

mempool/mempool_test.go

@@ -325,13 +325,14 @@ func newPoolHarness(chainParams *chaincfg.Params) (*poolHarness, []spendableOutp
 				MinRelayTxFee:        1000, // 1 Satoshi per byte
 				MaxTxVersion:         1,
 			},
-			ChainParams:      chainParams,
-			FetchUtxoView:    chain.FetchUtxoView,
-			BestHeight:       chain.BestHeight,
-			MedianTimePast:   chain.MedianTimePast,
-			CalcSequenceLock: chain.CalcSequenceLock,
-			SigCache:         nil,
-			AddrIndex:        nil,
+			ChainParams:         chainParams,
+			FetchUtxoView:       chain.FetchUtxoView,
+			IsUtreexoViewActive: func() bool { return false },
+			BestHeight:          chain.BestHeight,
+			MedianTimePast:      chain.MedianTimePast,
+			CalcSequenceLock:    chain.CalcSequenceLock,
+			SigCache:            nil,
+			AddrIndex:           nil,
 		}),
 	}