ef_tests/cases/

blockchain_test.rs

//! Test runners for `BlockchainTests` in <https://github.com/ethereum/tests>

use crate::{
    models::{BlockchainTest, ForkSpec},
    Case, Error, Suite,
};
use alloy_rlp::{Decodable, Encodable};
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_chainspec::ChainSpec;
use reth_consensus::{Consensus, HeaderValidator};
use reth_db_common::init::{insert_genesis_hashes, insert_genesis_history, insert_genesis_state};
use reth_ethereum_consensus::{validate_block_post_execution, EthBeaconConsensus};
use reth_ethereum_primitives::{Block, TransactionSigned};
use reth_evm::{execute::Executor, ConfigureEvm};
use reth_evm_ethereum::EthEvmConfig;
use reth_primitives_traits::{Block as BlockTrait, RecoveredBlock, SealedBlock};
use reth_provider::{
    test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, DatabaseProviderFactory,
    ExecutionOutcome, HeaderProvider, HistoryWriter, OriginalValuesKnown, StateProofProvider,
    StateWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter,
};
use reth_revm::{database::StateProviderDatabase, witness::ExecutionWitnessRecord, State};
use reth_stateless::{
    trie::StatelessSparseTrie, validation::stateless_validation_with_trie, ExecutionWitness,
    UncompressedPublicKey,
};
use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
use reth_trie_db::DatabaseStateRoot;
use std::{
    collections::BTreeMap,
    fs,
    path::{Path, PathBuf},
    sync::Arc,
};

/// A handler for the blockchain test suite.
#[derive(Debug)]
pub struct BlockchainTests {
    suite_path: PathBuf,
}

impl BlockchainTests {
    /// Create a new suite for tests with blockchain tests format.
    pub const fn new(suite_path: PathBuf) -> Self {
        Self { suite_path }
    }
}

impl Suite for BlockchainTests {
    type Case = BlockchainTestCase;

    fn suite_path(&self) -> &Path {
        &self.suite_path
    }
}

/// An Ethereum blockchain test.
#[derive(Debug, PartialEq, Eq)]
pub struct BlockchainTestCase {
    /// The tests within this test case.
    pub tests: BTreeMap<String, BlockchainTest>,
    /// Whether to skip this test case.
    pub skip: bool,
}

impl BlockchainTestCase {
    /// Returns `true` if the fork is not supported.
    const fn excluded_fork(network: ForkSpec) -> bool {
        matches!(
            network,
            ForkSpec::ByzantiumToConstantinopleAt5 |
                ForkSpec::Constantinople |
                ForkSpec::ConstantinopleFix |
                ForkSpec::MergeEOF |
                ForkSpec::MergeMeterInitCode |
                ForkSpec::MergePush0
        )
    }

    /// Checks if the test case is a particular test called `UncleFromSideChain`
    ///
    /// This fixture fails as expected, however it fails at the wrong block number.
    /// Given we no longer have uncle blocks, this test case was pulled out such
    /// that we ensure it still fails as expected, however we do not check the block number.
    #[inline]
    fn is_uncle_sidechain_case(name: &str) -> bool {
        name.contains("UncleFromSideChain")
    }

    /// If the test expects an exception, return the block number
    /// at which it must occur together with the original message.
    ///
    /// Note: There is a +1 here because the genesis block is not included
    /// in the set of blocks, so the first block is actually block number 1
    /// and not block number 0.
    #[inline]
    fn expected_failure(case: &BlockchainTest) -> Option<(u64, String)> {
        case.blocks.iter().enumerate().find_map(|(idx, blk)| {
            blk.expect_exception.as_ref().map(|msg| ((idx + 1) as u64, msg.clone()))
        })
    }

    /// Execute a single `BlockchainTest`, validating the outcome against the
    /// expectations encoded in the JSON file. Returns the list of executed blocks
    /// with their execution witnesses.
    pub fn run_single_case(
        name: &str,
        case: &BlockchainTest,
    ) -> Result<Vec<(RecoveredBlock<Block>, ExecutionWitness)>, Error> {
        let expectation = Self::expected_failure(case);
        match run_case(case) {
            // All blocks executed successfully.
            Ok(program_inputs) => {
                // Check if the test case specifies that it should have failed
                if let Some((block, msg)) = expectation {
                    Err(Error::Assertion(format!(
                        "Test case: {name}\nExpected failure at block {block} - {msg}, but all blocks succeeded",
                    )))
                } else {
                    Ok(program_inputs)
                }
            }

            // A block processing failure occurred.
            Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }) => {
                match expectation {
                    // It happened on exactly the block we were told to fail on
                    Some((expected, _)) if block_number == expected => Ok(partial_program_inputs),

                    // Uncle side‑chain edge case, we accept as long as it failed.
                    // But we don't check the exact block number.
                    _ if Self::is_uncle_sidechain_case(name) => Ok(partial_program_inputs),

                    // Expected failure, but block number does not match
                    Some((expected, _)) => Err(Error::Assertion(format!(
                        "Test case: {name}\nExpected failure at block {expected}\nGot failure at block {block_number}",
                    ))),

                    // No failure expected at all - bubble up original error.
                    None => Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }),
                }
            }

            // Non‑processing error – forward as‑is.
            //
            // This should only happen if we get an unexpected error from processing the block.
            // Since it is unexpected, we treat it as a test failure.
            //
            // One reason for this happening is when one forgets to wrap the error from `run_case`
            // so that it produces an `Error::BlockProcessingFailed`
            Err(other) => Err(other),
        }
    }
}

impl Case for BlockchainTestCase {
    fn load(path: &Path) -> Result<Self, Error> {
        Ok(Self {
            tests: {
                let s = fs::read_to_string(path)
                    .map_err(|error| Error::Io { path: path.into(), error })?;
                serde_json::from_str(&s)
                    .map_err(|error| Error::CouldNotDeserialize { path: path.into(), error })?
            },
            skip: should_skip(path),
        })
    }

    /// Runs the test cases for the Ethereum Forks test suite.
    ///
    /// # Errors
    /// Returns an error if the test is flagged for skipping or encounters issues during execution.
    fn run(&self) -> Result<(), Error> {
        // If the test is marked for skipping, return a Skipped error immediately.
        if self.skip {
            return Err(Error::Skipped);
        }

        // Iterate through test cases, filtering by the network type to exclude specific forks.
        self.tests
            .iter()
            .filter(|(_, case)| !Self::excluded_fork(case.network))
            .par_bridge()
            .try_for_each(|(name, case)| Self::run_single_case(name, case).map(|_| ()))?;

        Ok(())
    }
}

/// Executes a single `BlockchainTest` returning an error as soon as any block has a consensus
/// validation failure.
///
/// A `BlockchainTest` represents a self-contained scenario:
/// - It initializes a fresh blockchain state.
/// - It sequentially decodes, executes, and inserts a predefined set of blocks.
/// - It then verifies that the resulting blockchain state (post-state) matches the expected
///   outcome.
///
/// Returns:
/// - `Ok(_)` if all blocks execute successfully, returning recovered blocks and full block
///   execution witness.
/// - `Err(Error)` if any block fails to execute correctly, returning a partial block execution
///   witness if the error is of variant `BlockProcessingFailed`.
fn run_case(
    case: &BlockchainTest,
) -> Result<Vec<(RecoveredBlock<Block>, ExecutionWitness)>, Error> {
    // Create a new test database and initialize a provider for the test case.
    let chain_spec: Arc<ChainSpec> = Arc::new(case.network.into());
    let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
    let provider = factory.database_provider_rw().unwrap();

    // Insert initial test state into the provider.
    let genesis_block = SealedBlock::<Block>::from_sealed_parts(
        case.genesis_block_header.clone().into(),
        Default::default(),
    )
    .try_recover()
    .unwrap();

    provider
        .insert_block(genesis_block.clone())
        .map_err(|err| Error::block_failed(0, Default::default(), err))?;

    // Increment block number for receipts static file
    provider
        .static_file_provider()
        .latest_writer(StaticFileSegment::Receipts)
        .and_then(|mut writer| writer.increment_block(0))
        .map_err(|err| Error::block_failed(0, Default::default(), err))?;

    let genesis_state = case.pre.clone().into_genesis_state();
    insert_genesis_state(&provider, genesis_state.iter())
        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
    insert_genesis_hashes(&provider, genesis_state.iter())
        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
    insert_genesis_history(&provider, genesis_state.iter())
        .map_err(|err| Error::block_failed(0, Default::default(), err))?;

    // Decode blocks
    let blocks = decode_blocks(&case.blocks)?;

    let executor_provider = EthEvmConfig::ethereum(chain_spec.clone());
    let mut parent = genesis_block;
    let mut program_inputs = Vec::new();

    for (block_index, block) in blocks.iter().enumerate() {
        // Note: same as the comment on `decode_blocks` as to why we cannot use block.number
        let block_number = (block_index + 1) as u64;

        // Insert the block into the database
        provider
            .insert_block(block.clone())
            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
        // Commit static files, so we can query the headers for stateless execution below
        provider
            .static_file_provider()
            .commit()
            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;

        // Consensus checks before block execution
        pre_execution_checks(chain_spec.clone(), &parent, block).map_err(|err| {
            program_inputs.push((block.clone(), execution_witness_with_parent(&parent)));
            Error::block_failed(block_number, program_inputs.clone(), err)
        })?;

        let mut witness_record = ExecutionWitnessRecord::default();

        // Execute the block
        let state_provider = provider.latest();
        let state_db = StateProviderDatabase(&state_provider);
        let executor = executor_provider.batch_executor(state_db);

        let output = executor
            .execute_with_state_closure_always(&(*block).clone(), |statedb: &State<_>| {
                witness_record.record_executed_state(statedb);
            })
            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;

        // Consensus checks after block execution
        validate_block_post_execution(block, &chain_spec, &output.receipts, &output.requests)
            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;

        // Generate the stateless witness
        // TODO: Most of this code is copy-pasted from debug_executionWitness
        let ExecutionWitnessRecord { hashed_state, codes, keys, lowest_block_number } =
            witness_record;
        let state = state_provider.witness(Default::default(), hashed_state)?;
        let mut exec_witness = ExecutionWitness { state, codes, keys, headers: Default::default() };

        let smallest = lowest_block_number.unwrap_or_else(|| {
            // Return only the parent header, if there were no calls to the
            // BLOCKHASH opcode.
            block_number.saturating_sub(1)
        });

        let range = smallest..block_number;

        exec_witness.headers = provider
            .headers_range(range)?
            .into_iter()
            .map(|header| {
                let mut serialized_header = Vec::new();
                header.encode(&mut serialized_header);
                serialized_header.into()
            })
            .collect();

        program_inputs.push((block.clone(), exec_witness));

        // Compute and check the post state root
        let hashed_state =
            HashedPostState::from_bundle_state::<KeccakKeyHasher>(output.state.state());
        let (computed_state_root, _) =
            StateRoot::overlay_root_with_updates(provider.tx_ref(), hashed_state.clone())
                .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
        if computed_state_root != block.state_root {
            return Err(Error::block_failed(
                block_number,
                program_inputs.clone(),
                Error::Assertion("state root mismatch".to_string()),
            ));
        }

        // Commit the post state/state diff to the database
        provider
            .write_state(&ExecutionOutcome::single(block.number, output), OriginalValuesKnown::Yes)
            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;

        provider
            .write_hashed_state(&hashed_state.into_sorted())
            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
        provider
            .update_history_indices(block.number..=block.number)
            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;

        // Since there were no errors, update the parent block
        parent = block.clone()
    }

    match &case.post_state {
        Some(expected_post_state) => {
            // Validate the post-state for the test case.
            //
            // If we get here then it means that the post-state root checks
            // made after we execute each block was successful.
            //
            // If an error occurs here, then it is:
            // - Either an issue with the test setup
            // - Possibly an error in the test case where the post-state root in the last block does
            //   not match the post-state values.
            for (address, account) in expected_post_state {
                account.assert_db(*address, provider.tx_ref())?;
            }
        }
        None => {
            // Some tests may not have post-state (e.g., state-heavy benchmark tests).
            // In this case, we can skip the post-state validation.
        }
    }

    // Now validate using the stateless client if everything else passes
    for (recovered_block, execution_witness) in &program_inputs {
        let block = recovered_block.clone().into_block();

        // Recover the actual public keys from the transaction signatures
        let public_keys = recover_signers(block.body().transactions())
            .expect("Failed to recover public keys from transaction signatures");

        stateless_validation_with_trie::<StatelessSparseTrie, _, _>(
            block,
            public_keys,
            execution_witness.clone(),
            chain_spec.clone(),
            EthEvmConfig::new(chain_spec.clone()),
        )
        .expect("stateless validation failed");
    }

    Ok(program_inputs)
}

fn decode_blocks(
    test_case_blocks: &[crate::models::Block],
) -> Result<Vec<RecoveredBlock<Block>>, Error> {
    let mut blocks = Vec::with_capacity(test_case_blocks.len());
    for (block_index, block) in test_case_blocks.iter().enumerate() {
        // The blocks do not include the genesis block which is why we have the plus one.
        // We also cannot use block.number because for invalid blocks, this may be incorrect.
        let block_number = (block_index + 1) as u64;

        let decoded = SealedBlock::<Block>::decode(&mut block.rlp.as_ref())
            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;

        let recovered_block = decoded
            .clone()
            .try_recover()
            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;

        blocks.push(recovered_block);
    }

    Ok(blocks)
}

fn pre_execution_checks(
    chain_spec: Arc<ChainSpec>,
    parent: &RecoveredBlock<Block>,
    block: &RecoveredBlock<Block>,
) -> Result<(), Error> {
    let consensus: EthBeaconConsensus<ChainSpec> = EthBeaconConsensus::new(chain_spec);

    let sealed_header = block.sealed_header();

    <EthBeaconConsensus<ChainSpec> as Consensus<Block>>::validate_body_against_header(
        &consensus,
        block.body(),
        sealed_header,
    )?;
    consensus.validate_header_against_parent(sealed_header, parent.sealed_header())?;
    consensus.validate_header(sealed_header)?;
    consensus.validate_block_pre_execution(block)?;

    Ok(())
}

/// Recover public keys from transaction signatures.
fn recover_signers<'a, I>(txs: I) -> Result<Vec<UncompressedPublicKey>, Box<dyn std::error::Error>>
where
    I: IntoIterator<Item = &'a TransactionSigned>,
{
    txs.into_iter()
        .enumerate()
        .map(|(i, tx)| {
            tx.signature()
                .recover_from_prehash(&tx.signature_hash())
                .map(|keys| {
                    UncompressedPublicKey(
                        keys.to_encoded_point(false).as_bytes().try_into().unwrap(),
                    )
                })
                .map_err(|e| format!("failed to recover signature for tx #{i}: {e}").into())
        })
        .collect::<Result<Vec<UncompressedPublicKey>, _>>()
}

/// Returns whether the test at the given path should be skipped.
///
/// Some tests are edge cases that cannot happen on mainnet, while others are skipped for
/// convenience (e.g. they take a long time to run) or are temporarily disabled.
///
/// The reason should be documented in a comment above the file name(s).
pub fn should_skip(path: &Path) -> bool {
    let path_str = path.to_str().expect("Path is not valid UTF-8");
    let name = path.file_name().unwrap().to_str().unwrap();
    matches!(
        name,
        // funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require
        // custom json parser. https://github.com/ethereum/tests/issues/971
        | "ValueOverflow.json"
        | "ValueOverflowParis.json"

        // txbyte is of type 02 and we don't parse tx bytes for this test to fail.
        | "typeTwoBerlin.json"

        // Test checks if nonce overflows. We are handling this correctly but we are not parsing
        // exception in testsuite. There are more nonce overflow tests that are internal
        // call/create, and those tests are passing and are enabled.
        | "CreateTransactionHighNonce.json"

        // Test check if gas price overflows, we handle this correctly but does not match tests specific
        // exception.
        | "HighGasPrice.json"
        | "HighGasPriceParis.json"

        // Skip test where basefee/accesslist/difficulty is present but it shouldn't be supported in
        // London/Berlin/TheMerge. https://github.com/ethereum/tests/blob/5b7e1ab3ffaf026d99d20b17bb30f533a2c80c8b/GeneralStateTests/stExample/eip1559.json#L130
        // It is expected to not execute these tests.
        | "accessListExample.json"
        | "basefeeExample.json"
        | "eip1559.json"
        | "mergeTest.json"

        // These tests are passing, but they take a lot of time to execute so we are going to skip them.
        | "loopExp.json"
        | "Call50000_sha256.json"
        | "static_Call50000_sha256.json"
        | "loopMul.json"
        | "CALLBlake2f_MaxRounds.json"
        | "shiftCombinations.json"

        // Skipped by revm as well: <https://github.com/bluealloy/revm/blob/be92e1db21f1c47b34c5a58cfbf019f6b97d7e4b/bins/revme/src/cmd/statetest/runner.rs#L115-L125>
        | "RevertInCreateInInit_Paris.json"
        | "RevertInCreateInInit.json"
        | "dynamicAccountOverwriteEmpty.json"
        | "dynamicAccountOverwriteEmpty_Paris.json"
        | "RevertInCreateInInitCreate2Paris.json"
        | "create2collisionStorage.json"
        | "RevertInCreateInInitCreate2.json"
        | "create2collisionStorageParis.json"
        | "InitCollision.json"
        | "InitCollisionParis.json"
    )
    // Ignore outdated EOF tests that haven't been updated for Cancun yet.
    || path_contains(path_str, &["EIPTests", "stEOF"])
}

/// `str::contains` but for a path. Takes into account the OS path separator (`/` or `\`).
fn path_contains(path_str: &str, rhs: &[&str]) -> bool {
    let rhs = rhs.join(std::path::MAIN_SEPARATOR_STR);
    path_str.contains(&rhs)
}

fn execution_witness_with_parent(parent: &RecoveredBlock<Block>) -> ExecutionWitness {
    let mut serialized_header = Vec::new();
    parent.header().encode(&mut serialized_header);
    ExecutionWitness { headers: vec![serialized_header.into()], ..Default::default() }
}