Start using nox and pre-commit, linted code

.gitignore

@@ -142,4 +142,4 @@ cython_debug/
 benchmarks
 
 errors
-tmp
+tmp

.pre-commit-config.yaml

@@ -0,0 +1,104 @@
+# Skip execution of one or more hooks using the SKIP environment variable:
+# $ SKIP=pylint git commit -m "foo"
+# $ SKIP=mypy,pylint pre-commit run --all-files
+#
+# If want to disable all hooks while committing, use the --no-verify/-n option:
+# $ git commit -n -m "foo"
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: check-case-conflict
+      - id: check-yaml
+      - id: check-merge-conflict
+      - id: check-vcs-permalinks
+      - id: detect-private-key
+      - id: debug-statements
+      - id: check-docstring-first
+      - id: check-toml
+  - repo: https://github.com/DavidAnson/markdownlint-cli2
+    rev: v0.8.1
+    hooks:
+      - id: markdownlint-cli2-fix
+        name: markdownlint-cli2-fix (in place fixes)
+  - repo: https://github.com/leoll2/copyright_notice_precommit
+    rev: 0.1.1
+    hooks:
+      - id: copyright-notice
+        args: [--notice=COPYRIGHT]
+        files: python
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.10.1
+    hooks:
+      - id: pyupgrade
+        args: [--py38-plus]
+        # exclude: *fixtures
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place
+          - --expand-star-imports
+          - --remove-all-unused-imports
+          - --ignore-init-module-imports
+          - --remove-duplicate-keys
+          - --remove-unused-variables
+          - --remove-rhs-for-unused-variables
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (black profile, in place fixes)
+        args: ["--profile", "black", "--filter-files"]
+  - repo: https://github.com/DanielNoord/pydocstringformatter
+    rev: v0.7.3
+    hooks:
+      - id: pydocstringformatter
+  - repo: https://github.com/asottile/yesqa
+    rev: v1.5.0
+    hooks:
+      - id: yesqa
+  - repo: https://github.com/psf/black
+    rev: 23.9.1
+    hooks:
+      - id: black
+        name: black (in place fixes)
+        # args: [--diff, --check]
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        # language_version: python3.11
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        additional_dependencies: [Flake8-pyproject]
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.5
+    hooks:
+      - id: bandit
+        name: bandit (checksig)
+        exclude: ".*test.*"
+        language: python
+        types: [python]
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.5
+    hooks:
+      - id: bandit
+        name: bandit (tests)
+        # for the test folder disable
+        # B101, Test for use of assert
+        args: ["--skip", "B101,B311"]
+        exclude: checksig
+        language: python
+        types: [python]
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.5.1
+    hooks:
+      - id: mypy

TODO.md

@@ -1,3 +1,5 @@
+# TODO
+
 - Add bloom filter messages
 - Add address store system. stop using dns for finding peers
 - Improve time spent while connecting when starting to download blocks

btclib_node/__init__.py

@@ -64,7 +64,6 @@ def stop_handler(signal, frame):
         self.rpc_manager = RpcManager(self, self.rpc_port)
 
     def run(self):
-
         self.logger.info("Starting main loop")
 
         if self.p2p_port:

btclib_node/block_db/__init__.py

@@ -83,7 +83,6 @@ def serialize(self):
 
 class BlockDB:
     def __init__(self, data_dir, logger):
-
         self.logger = logger
 
         self.data_dir = data_dir / "blocks"
@@ -127,7 +126,7 @@ def __find_block_file(self):
         else:
             filename = f"{self.file_index:06d}.blk"
             file_metadata = self.files[filename]
-            if file_metadata.size > 128 * 1000 ** 2:  # 128MB
+            if file_metadata.size > 128 * 1000**2:  # 128MB
                 new_file = True
         if new_file:
             self.file_index += 1

btclib_node/chains.py

@@ -15,7 +15,7 @@ def create_genesis(time, nonce, difficulty, version, reward):
         [
             "FFFF001D",
             b"\x04",
-            "The Times 03/Jan/2009 Chancellor on brink of second bailout for banks".encode(),
+            b"The Times 03/Jan/2009 Chancellor on brink of second bailout for banks",
         ]
     )
     script_pub_key = script.serialize(
@@ -80,7 +80,7 @@ def __init__(self):
             "seed.bitcoin.wiz.biz",
         ]
         self.genesis = create_genesis(
-            1231006505, 2083236893, 0x1D00FFFF, 1, 50 * 10 ** 8
+            1231006505, 2083236893, 0x1D00FFFF, 1, 50 * 10**8
         )
         self.flags = [
             (170061, "P2SH"),
@@ -106,7 +106,7 @@ def __init__(self):
             "testnet-seed.bluematt.me",
         ]
         self.genesis = create_genesis(
-            1296688602, 414098458, 0x1D00FFFF, 1, 50 * 10 ** 8
+            1296688602, 414098458, 0x1D00FFFF, 1, 50 * 10**8
         )
         self.flags = [
             (395, "P2SH"),
@@ -126,7 +126,7 @@ def __init__(self):
         self.port = 38333
         self.magic = "0a03cf40"  # default signet
         self.addresses = ["178.128.221.177"]
-        self.genesis = create_genesis(1598918400, 52613770, 0x1E0377AE, 1, 50 * 10 ** 8)
+        self.genesis = create_genesis(1598918400, 52613770, 0x1E0377AE, 1, 50 * 10**8)
         self.flags = [
             (0, "P2SH"),
             (0, "DERSIG"),
@@ -145,7 +145,7 @@ def __init__(self):
         self.port = 18444
         self.magic = "fabfb5da"
         self.addresses = []
-        self.genesis = create_genesis(1296688602, 2, 0x207FFFFF, 1, 50 * 10 ** 8)
+        self.genesis = create_genesis(1296688602, 2, 0x207FFFFF, 1, 50 * 10**8)
         self.flags = [
             (0, "P2SH"),
             (0, "DERSIG"),

btclib_node/chainstate/__init__.py

@@ -1,8 +1,4 @@
 import plyvel
-from btclib.tx.tx_in import OutPoint
-from btclib.tx.tx_out import TxOut
-
-from btclib_node.block_db import RevBlock
 
 from .block_index import BlockIndex
 from .utxo_index import UtxoIndex

btclib_node/chainstate/block_index.py

@@ -2,7 +2,6 @@
 from collections import deque
 from dataclasses import dataclass
 
-import plyvel
 from btclib import var_int
 from btclib.block import BlockHeader
 from btclib.utils import bytesio_from_binarydata
@@ -12,7 +11,7 @@
 def calculate_work(header):
     target = int.from_bytes(header.bits[-3:], "big")
     exp = pow(256, (header.bits[0] - 3))
-    return int(256 ** 32 / target / exp)
+    return int(256**32 / target / exp)
 
 
 class BlockStatus(enum.IntEnum):
@@ -49,7 +48,6 @@ def serialize(self):
 
 class BlockIndex:
     def __init__(self, parent_db, chain, logger):
-
         self.logger = logger
 
         self.db = parent_db
@@ -188,7 +186,6 @@ def add_headers(self, headers):
         added = False  # flag that signals if there is a new header in this message
         current_work = self.get_block_info(self.active_chain[-1]).chainwork
         for header in headers:
-
             # TODO: validate timestamp and difficulty
 
             header_hash = header.hash

btclib_node/chainstate/utxo_index.py

@@ -1,4 +1,3 @@
-import plyvel
 from btclib.tx.tx_in import OutPoint
 from btclib.tx.tx_out import TxOut
 
@@ -7,7 +6,6 @@
 
 class UtxoIndex:
     def __init__(self, parent_db, logger):
-
         self.db = parent_db
 
         self.removed_utxos = set()
@@ -16,7 +14,6 @@ def __init__(self, parent_db, logger):
         self.logger = logger
 
     def add_block(self, block):
-
         removed = []
         added = []
         complete_transactions = []
@@ -27,13 +24,11 @@ def add_block(self, block):
             added.append(out_point)
 
         for tx in block.transactions[1:]:
-
             tx_id = tx.id
 
             prev_outputs = []
 
             for tx_in in tx.vin:
-
                 prevout_bytes = tx_in.prev_out.serialize(check_validity=False)
 
                 if prevout_bytes in self.removed_utxos:
@@ -68,7 +63,6 @@ def add_block(self, block):
 
     def apply_rev_block(self, rev_block):
         for out_point in rev_block.to_remove:
-
             out_point_bytes = out_point.serialize(check_validity=False)
 
             if out_point_bytes in self.removed_utxos:

btclib_node/exceptions.py

@@ -1,5 +1,5 @@
-"""Expception classes"""
+"""Expception classes."""
 
 
 class MissingPrevoutError(ValueError):
-    pass
+    pass

btclib_node/interpreter.py

@@ -1,8 +1,8 @@
 import traceback
+from copy import deepcopy
 from itertools import chain
 from pathlib import Path
 from typing import Tuple
-from copy import deepcopy
 
 from btclib.script.engine import verify_input, verify_transaction
 
@@ -13,12 +13,12 @@ def get_flags(config, index) -> Tuple[str]:
 
 def f(prevouts, tx, i, flags):
     try:
-        # no need to deepcopy the values as 
+        # no need to deepcopy the values as
         # they are not reused
         # TODO: are we really sure this is safe?
         # To check and fix upstream
         verify_input(prevouts, tx, i, flags)
-    except Exception as e:
+    except Exception:
         err_dir = Path("errors", tx.id.hex(), str(i))
         err_dir.mkdir(parents=True, exist_ok=True)
         with open(err_dir / "flags", "w") as f:
@@ -33,7 +33,6 @@ def f(prevouts, tx, i, flags):
 
 
 def check_transactions(transaction_data, index, node):
-
     if not transaction_data:
         return
     if any(len(x[0]) != len(x[1].vin) for x in transaction_data):

btclib_node/main.py

@@ -4,7 +4,6 @@
 from btclib_node.constants import NodeStatus
 from btclib_node.exceptions import MissingPrevoutError
 from btclib_node.interpreter import check_transaction, check_transactions
-from btclib_node.p2p.messages.filters import Filterclear
 
 
 def update_block_status(index, hash, status, wb):
@@ -28,7 +27,6 @@ def finish_sync(node):
 
 # TODO: support for failed updates
 def update_chain(node):
-
     if node.status < NodeStatus.HeaderSynced:
         return
 

btclib_node/mempool.py

@@ -1,13 +1,10 @@
-from dataclasses import dataclass, field
 from typing import Dict
 
 from btclib.tx.tx import Tx
 
 
 class Mempool:
-
     def __init__(self, logger):
-
         self.logger = logger
 
         self.transactions: Dict[bytes, Tx] = {}

btclib_node/p2p/address.py

@@ -1,10 +1,9 @@
 import asyncio
 import enum
-import random
+import secrets
 import socket
 import time
 from dataclasses import dataclass
-from ipaddress import AddressValueError, IPv6Address
 
 from btclib import var_bytes, var_int
 from btclib.utils import bytesio_from_binarydata
@@ -121,7 +120,7 @@ async def connect(self):
                     try:
                         client.getpeername()
                         return client
-                    except socket.error:
+                    except OSError:
                         pass
                 client.close()
         else:
@@ -172,7 +171,7 @@ def is_empty(self):
 
     def random_address(self):
         while True:
-            address = random.choice(list(self.addresses))
+            address = secrets.choice(list(self.addresses))
             if address.can_connect:
                 return address
 

btclib_node/p2p/callbacks.py

@@ -1,7 +1,5 @@
 import time
 
-from btclib.exceptions import BTClibValueError
-
 from btclib_node.constants import NodeStatus, P2pConnStatus, ProtocolVersion, Services
 from btclib_node.exceptions import MissingPrevoutError
 from btclib_node.main import verify_mempool_acceptance
@@ -217,7 +215,9 @@ def not_found(node, msg, conn):
 
 def reject(node, msg, conn):
     reject = Reject.deserialize(msg)
-    err_msg = f"Reject received: {reject.code.name}, {reject.reason}, {reject.data.hex()}"
+    err_msg = (
+        f"Reject received: {reject.code.name}, {reject.reason}, {reject.data.hex()}"
+    )
     node.logger.warning(err_msg)