Merge pull request #3035 from CounterpartyXCP/catchup

Optimize catch up

Author: ouziel-slama

.github/workflows/integrations_test.yml

@@ -24,7 +24,8 @@ jobs:
           - name: Mainnet API Load
             test-path: integrations/load_test.py
             os: Linux-Large-1
-            only_on_develop: true
+            only_on_develop: false
+            no_cov_report: true
 
           # bootstrap, reparse 1000 blocks,
           # rollback 3 checkpoints and catchup
@@ -46,7 +47,7 @@ jobs:
           # bootstrap, reparse 1000 blocks,
           # rollback 3 checkpoints and catchup
           - name: Testnet4 Bootstrap And Catchup
-            test-path: integrations/testnet4_test.py 
+            test-path: integrations/testnet4_test.py
 
           - name: Testnet4 Start and Shutdown
             test-path: integrations/shutdown_test.py
@@ -61,11 +62,16 @@ jobs:
           - name: Compare Hashes
             test-path: integrations/comparehashes_test.py
 
+          - name: RSFetcher Test
+            test-path: integrations/rsfetcher_test.py
+
     # run pytest_action.yml for the matrix
     uses: ./.github/workflows/pytest_action.yml
     with:
       name: ${{ matrix.name }}
       test-path: ${{ matrix.test-path }}
       install_bitcoin: ${{ matrix.install_bitcoin || false }}
       only_on_develop:  ${{ matrix.only_on_develop || false }}
+      os: ${{ matrix.os || 'ubuntu-22.04' }}
+      no_cov_report: ${{ matrix.no_cov_report || false }}
     secrets: inherit

.github/workflows/pytest_action.yml

@@ -18,12 +18,12 @@ on:
       install_bitcoin:
         type: boolean
         default: false
-      run_coveralls:
-        type: boolean
-        default: true
       only_on_develop:
         type: boolean
         default: false
+      no_cov_report:
+        type: boolean
+        default: false
 
 jobs:
   pytest:
@@ -67,9 +67,11 @@ jobs:
       - name: Run tests
         run: |
           cd counterparty-core
-          hatch run pytest counterpartycore/test/${{ inputs.test-path }} -s -vv -x \
-            --cov=counterpartycore/lib --cov-report=term-missing --cov-report=
-          mv .coverage ../
+          hatch run pytest counterpartycore/test/${{ inputs.test-path }} \
+            ${{ !inputs.no_cov_report && '-s -vv -x --cov=counterpartycore/lib --cov-report=term-missing --cov-report=' || '' }}
+          if [ "${{ !inputs.no_cov_report }}" = "true" ]; then
+            mv .coverage ../
+          fi
 
       - name: Upload coverage
         uses: codecov/codecov-action@v5

.github/workflows/ruff_scanner.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: chartboost/ruff-action@v1
         with:
           args: "format --check"
-          version: 0.8.2
+          version: 0.9.7
       - uses: chartboost/ruff-action@v1
         with:
-          version: 0.8.2
+          version: 0.9.7

.github/workflows/rust_test.yml

@@ -0,0 +1,27 @@
+name: Rust Test
+
+on:
+  push:
+    branches: "**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Cargo:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          default: true
+
+      - name: Run tests
+        run: |
+          cd counterparty-rs
+          cargo test

.github/workflows/scanners.yml

@@ -5,7 +5,7 @@ FROM ubuntu:22.04
 
 RUN apt update
 # install dependencies
-RUN apt install -y python3 python3-dev python3-pip libleveldb-dev curl gnupg libclang-dev
+RUN apt install -y python3 python3-dev python3-pip libleveldb-dev curl gnupg libclang-dev pkg-config libssl-dev
 
 # install rust
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y

Dockerfile

@@ -282,7 +282,15 @@ def run(self, server_ready_value, shared_backend_height):
         CurrentState().set_backend_height_value(shared_backend_height)
         self.current_state_thread = NodeStatusCheckerThread(shared_backend_height)
         self.current_state_thread.start()
-        self.server.run()
+        try:
+            self.server.run()
+        except OSError as e:
+            if e.errno == 9:
+                logger.debug(
+                    "Ignoring OSError [Errno 9] Bad file descriptor during waitress server shutdown."
+                )
+            else:
+                raise
 
     def stop(self):
         self.current_state_thread.stop()

counterparty-core/counterpartycore/lib/api/wsgi.py

@@ -539,6 +539,21 @@ def list_unspent(source, allow_unconfirmed_inputs):
     return []
 
 
+def get_vin_info(vin, no_retry=False):
+    vin_info = vin.get("info")
+    if vin_info is None:
+        logger.error(f"vin_info not found for vin {vin}")
+        try:
+            vin_ctx = get_decoded_transaction(vin["hash"], no_retry=no_retry)
+            is_segwit = vin_ctx["segwit"]
+            vout = vin_ctx["vout"][vin["n"]]
+            return vout["value"], vout["script_pub_key"], is_segwit
+        except exceptions.BitcoindRPCError as e:
+            raise exceptions.DecodeError("vin not found") from e
+    else:
+        return vin_info["value"], vin_info["script_pub_key"], vin_info["is_segwit"]
+
+
 def get_vins_info(vins, no_retry=False):
     hashes = [vin["hash"] for vin in vins]
     inputs_txs = getrawtransaction_batch(hashes, verbose=False, return_dict=True, no_retry=no_retry)
@@ -554,6 +569,25 @@ def get_vins_info(vins, no_retry=False):
     return vins_info
 
 
+def complete_vins_info(decoded_tx, no_retry=False):
+    missing_vins = []
+    for vin in decoded_tx["vin"]:
+        if "info" not in vin or vin["info"] is None:
+            missing_vins.append(vin)
+
+    if len(missing_vins) > 0:
+        missing_vins_info = get_vins_info(missing_vins, no_retry=no_retry)
+        for i, vin in enumerate(missing_vins):
+            vin_info = missing_vins_info[i]
+            vin["info"] = {
+                "value": vin_info[0],
+                "script_pub_key": vin_info[1],
+                "is_segwit": vin_info[2],
+            }
+
+    return decoded_tx
+
+
 def get_transaction(tx_hash: str, format: str = "json"):
     """
     Get a transaction from the blockchain

counterparty-core/counterpartycore/lib/backend/bitcoind.py

@@ -102,7 +102,6 @@ def start(self, start_height=0):
         self.prefetch_queue_initialized = False
 
     def get_block(self, retry=0):
-        logger.trace("Fetching block with Rust backend.")
         block = self.get_prefetched_block()
 
         if block is None:

counterparty-core/counterpartycore/lib/backend/rsfetcher.py

@@ -393,6 +393,14 @@ def float_range_checker(arg):
             "help": "Catch up mode (default: normal)",
         },
     ],
+    [
+        ("--profile",),
+        {
+            "action": "store_true",
+            "default": False,
+            "help": "Enable cProfile profiling for catchup; dumps output to a file in the cache dir",
+        },
+    ],
 ]