ef_tests/cases/
blockchain_test.rs

1//! Test runners for `BlockchainTests` in <https://github.com/ethereum/tests>
2
3use crate::{
4    models::{BlockchainTest, ForkSpec},
5    Case, Error, Suite,
6};
7use alloy_rlp::{Decodable, Encodable};
8use rayon::iter::{ParallelBridge, ParallelIterator};
9use reth_chainspec::ChainSpec;
10use reth_consensus::{Consensus, HeaderValidator};
11use reth_db_common::init::{insert_genesis_hashes, insert_genesis_history, insert_genesis_state};
12use reth_ethereum_consensus::{validate_block_post_execution, EthBeaconConsensus};
13use reth_ethereum_primitives::Block;
14use reth_evm::{execute::Executor, ConfigureEvm};
15use reth_evm_ethereum::EthEvmConfig;
16use reth_primitives_traits::{RecoveredBlock, SealedBlock};
17use reth_provider::{
18    test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, DatabaseProviderFactory,
19    ExecutionOutcome, HeaderProvider, HistoryWriter, OriginalValuesKnown, StateProofProvider,
20    StateWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter,
21};
22use reth_revm::{database::StateProviderDatabase, witness::ExecutionWitnessRecord, State};
23use reth_stateless::{validation::stateless_validation, ExecutionWitness};
24use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
25use reth_trie_db::DatabaseStateRoot;
26use std::{
27    collections::BTreeMap,
28    fs,
29    path::{Path, PathBuf},
30    sync::Arc,
31};
32
33/// A handler for the blockchain test suite.
34#[derive(Debug)]
35pub struct BlockchainTests {
36    suite_path: PathBuf,
37}
38
39impl BlockchainTests {
40    /// Create a new suite for tests with blockchain tests format.
41    pub const fn new(suite_path: PathBuf) -> Self {
42        Self { suite_path }
43    }
44}
45
46impl Suite for BlockchainTests {
47    type Case = BlockchainTestCase;
48
49    fn suite_path(&self) -> &Path {
50        &self.suite_path
51    }
52}
53
54/// An Ethereum blockchain test.
55#[derive(Debug, PartialEq, Eq)]
56pub struct BlockchainTestCase {
57    /// The tests within this test case.
58    pub tests: BTreeMap<String, BlockchainTest>,
59    /// Whether to skip this test case.
60    pub skip: bool,
61}
62
63impl BlockchainTestCase {
64    /// Returns `true` if the fork is not supported.
65    const fn excluded_fork(network: ForkSpec) -> bool {
66        matches!(
67            network,
68            ForkSpec::ByzantiumToConstantinopleAt5 |
69                ForkSpec::Constantinople |
70                ForkSpec::ConstantinopleFix |
71                ForkSpec::MergeEOF |
72                ForkSpec::MergeMeterInitCode |
73                ForkSpec::MergePush0
74        )
75    }
76
77    /// Checks if the test case is a particular test called `UncleFromSideChain`
78    ///
79    /// This fixture fails as expected, however it fails at the wrong block number.
80    /// Given we no longer have uncle blocks, this test case was pulled out such
81    /// that we ensure it still fails as expected, however we do not check the block number.
82    #[inline]
83    fn is_uncle_sidechain_case(name: &str) -> bool {
84        name.contains("UncleFromSideChain")
85    }
86
87    /// If the test expects an exception, return the block number
88    /// at which it must occur together with the original message.
89    ///
90    /// Note: There is a +1 here because the genesis block is not included
91    /// in the set of blocks, so the first block is actually block number 1
92    /// and not block number 0.
93    #[inline]
94    fn expected_failure(case: &BlockchainTest) -> Option<(u64, String)> {
95        case.blocks.iter().enumerate().find_map(|(idx, blk)| {
96            blk.expect_exception.as_ref().map(|msg| ((idx + 1) as u64, msg.clone()))
97        })
98    }
99
100    /// Execute a single `BlockchainTest`, validating the outcome against the
101    /// expectations encoded in the JSON file. Returns the list of executed blocks
102    /// with their execution witnesses.
103    pub fn run_single_case(
104        name: &str,
105        case: &BlockchainTest,
106    ) -> Result<Vec<(RecoveredBlock<Block>, ExecutionWitness)>, Error> {
107        let expectation = Self::expected_failure(case);
108        match run_case(case) {
109            // All blocks executed successfully.
110            Ok(program_inputs) => {
111                // Check if the test case specifies that it should have failed
112                if let Some((block, msg)) = expectation {
113                    Err(Error::Assertion(format!(
114                        "Test case: {name}\nExpected failure at block {block} - {msg}, but all blocks succeeded",
115                    )))
116                } else {
117                    Ok(program_inputs)
118                }
119            }
120
121            // A block processing failure occurred.
122            Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }) => {
123                match expectation {
124                    // It happened on exactly the block we were told to fail on
125                    Some((expected, _)) if block_number == expected => Ok(partial_program_inputs),
126
127                    // Uncle side‑chain edge case, we accept as long as it failed.
128                    // But we don't check the exact block number.
129                    _ if Self::is_uncle_sidechain_case(name) => Ok(partial_program_inputs),
130
131                    // Expected failure, but block number does not match
132                    Some((expected, _)) => Err(Error::Assertion(format!(
133                        "Test case: {name}\nExpected failure at block {expected}\nGot failure at block {block_number}",
134                    ))),
135
136                    // No failure expected at all - bubble up original error.
137                    None => Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }),
138                }
139            }
140
141            // Non‑processing error – forward as‑is.
142            //
143            // This should only happen if we get an unexpected error from processing the block.
144            // Since it is unexpected, we treat it as a test failure.
145            //
146            // One reason for this happening is when one forgets to wrap the error from `run_case`
147            // so that it produces an `Error::BlockProcessingFailed`
148            Err(other) => Err(other),
149        }
150    }
151}
152
153impl Case for BlockchainTestCase {
154    fn load(path: &Path) -> Result<Self, Error> {
155        Ok(Self {
156            tests: {
157                let s = fs::read_to_string(path)
158                    .map_err(|error| Error::Io { path: path.into(), error })?;
159                serde_json::from_str(&s)
160                    .map_err(|error| Error::CouldNotDeserialize { path: path.into(), error })?
161            },
162            skip: should_skip(path),
163        })
164    }
165
166    /// Runs the test cases for the Ethereum Forks test suite.
167    ///
168    /// # Errors
169    /// Returns an error if the test is flagged for skipping or encounters issues during execution.
170    fn run(&self) -> Result<(), Error> {
171        // If the test is marked for skipping, return a Skipped error immediately.
172        if self.skip {
173            return Err(Error::Skipped);
174        }
175
176        // Iterate through test cases, filtering by the network type to exclude specific forks.
177        self.tests
178            .iter()
179            .filter(|(_, case)| !Self::excluded_fork(case.network))
180            .par_bridge()
181            .try_for_each(|(name, case)| Self::run_single_case(name, case).map(|_| ()))?;
182
183        Ok(())
184    }
185}
186
187/// Executes a single `BlockchainTest` returning an error as soon as any block has a consensus
188/// validation failure.
189///
190/// A `BlockchainTest` represents a self-contained scenario:
191/// - It initializes a fresh blockchain state.
192/// - It sequentially decodes, executes, and inserts a predefined set of blocks.
193/// - It then verifies that the resulting blockchain state (post-state) matches the expected
194///   outcome.
195///
196/// Returns:
197/// - `Ok(_)` if all blocks execute successfully, returning recovered blocks and full block
198///   execution witness.
199/// - `Err(Error)` if any block fails to execute correctly, returning a partial block execution
200///   witness if the error is of variant `BlockProcessingFailed`.
201fn run_case(
202    case: &BlockchainTest,
203) -> Result<Vec<(RecoveredBlock<Block>, ExecutionWitness)>, Error> {
204    // Create a new test database and initialize a provider for the test case.
205    let chain_spec: Arc<ChainSpec> = Arc::new(case.network.into());
206    let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
207    let provider = factory.database_provider_rw().unwrap();
208
209    // Insert initial test state into the provider.
210    let genesis_block = SealedBlock::<Block>::from_sealed_parts(
211        case.genesis_block_header.clone().into(),
212        Default::default(),
213    )
214    .try_recover()
215    .unwrap();
216
217    provider
218        .insert_block(genesis_block.clone())
219        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
220
221    // Increment block number for receipts static file
222    provider
223        .static_file_provider()
224        .latest_writer(StaticFileSegment::Receipts)
225        .and_then(|mut writer| writer.increment_block(0))
226        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
227
228    let genesis_state = case.pre.clone().into_genesis_state();
229    insert_genesis_state(&provider, genesis_state.iter())
230        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
231    insert_genesis_hashes(&provider, genesis_state.iter())
232        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
233    insert_genesis_history(&provider, genesis_state.iter())
234        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
235
236    // Decode blocks
237    let blocks = decode_blocks(&case.blocks)?;
238
239    let executor_provider = EthEvmConfig::ethereum(chain_spec.clone());
240    let mut parent = genesis_block;
241    let mut program_inputs = Vec::new();
242
243    for (block_index, block) in blocks.iter().enumerate() {
244        // Note: same as the comment on `decode_blocks` as to why we cannot use block.number
245        let block_number = (block_index + 1) as u64;
246
247        // Insert the block into the database
248        provider
249            .insert_block(block.clone())
250            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
251        // Commit static files, so we can query the headers for stateless execution below
252        provider
253            .static_file_provider()
254            .commit()
255            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
256
257        // Consensus checks before block execution
258        pre_execution_checks(chain_spec.clone(), &parent, block).map_err(|err| {
259            program_inputs.push((block.clone(), execution_witness_with_parent(&parent)));
260            Error::block_failed(block_number, program_inputs.clone(), err)
261        })?;
262
263        let mut witness_record = ExecutionWitnessRecord::default();
264
265        // Execute the block
266        let state_provider = provider.latest();
267        let state_db = StateProviderDatabase(&state_provider);
268        let executor = executor_provider.batch_executor(state_db);
269
270        let output = executor
271            .execute_with_state_closure_always(&(*block).clone(), |statedb: &State<_>| {
272                witness_record.record_executed_state(statedb);
273            })
274            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
275
276        // Consensus checks after block execution
277        validate_block_post_execution(block, &chain_spec, &output.receipts, &output.requests)
278            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
279
280        // Generate the stateless witness
281        // TODO: Most of this code is copy-pasted from debug_executionWitness
282        let ExecutionWitnessRecord { hashed_state, codes, keys, lowest_block_number } =
283            witness_record;
284        let state = state_provider.witness(Default::default(), hashed_state)?;
285        let mut exec_witness = ExecutionWitness { state, codes, keys, headers: Default::default() };
286
287        let smallest = lowest_block_number.unwrap_or_else(|| {
288            // Return only the parent header, if there were no calls to the
289            // BLOCKHASH opcode.
290            block_number.saturating_sub(1)
291        });
292
293        let range = smallest..block_number;
294
295        exec_witness.headers = provider
296            .headers_range(range)?
297            .into_iter()
298            .map(|header| {
299                let mut serialized_header = Vec::new();
300                header.encode(&mut serialized_header);
301                serialized_header.into()
302            })
303            .collect();
304
305        program_inputs.push((block.clone(), exec_witness));
306
307        // Compute and check the post state root
308        let hashed_state =
309            HashedPostState::from_bundle_state::<KeccakKeyHasher>(output.state.state());
310        let (computed_state_root, _) =
311            StateRoot::overlay_root_with_updates(provider.tx_ref(), hashed_state.clone())
312                .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
313        if computed_state_root != block.state_root {
314            return Err(Error::block_failed(
315                block_number,
316                program_inputs.clone(),
317                Error::Assertion("state root mismatch".to_string()),
318            ));
319        }
320
321        // Commit the post state/state diff to the database
322        provider
323            .write_state(&ExecutionOutcome::single(block.number, output), OriginalValuesKnown::Yes)
324            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
325
326        provider
327            .write_hashed_state(&hashed_state.into_sorted())
328            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
329        provider
330            .update_history_indices(block.number..=block.number)
331            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
332
333        // Since there were no errors, update the parent block
334        parent = block.clone()
335    }
336
337    match &case.post_state {
338        Some(expected_post_state) => {
339            // Validate the post-state for the test case.
340            //
341            // If we get here then it means that the post-state root checks
342            // made after we execute each block was successful.
343            //
344            // If an error occurs here, then it is:
345            // - Either an issue with the test setup
346            // - Possibly an error in the test case where the post-state root in the last block does
347            //   not match the post-state values.
348            for (address, account) in expected_post_state {
349                account.assert_db(*address, provider.tx_ref())?;
350            }
351        }
352        None => {
353            // Some tests may not have post-state (e.g., state-heavy benchmark tests).
354            // In this case, we can skip the post-state validation.
355        }
356    }
357
358    // Now validate using the stateless client if everything else passes
359    for (block, execution_witness) in &program_inputs {
360        stateless_validation(
361            block.clone(),
362            execution_witness.clone(),
363            chain_spec.clone(),
364            EthEvmConfig::new(chain_spec.clone()),
365        )
366        .expect("stateless validation failed");
367    }
368
369    Ok(program_inputs)
370}
371
372fn decode_blocks(
373    test_case_blocks: &[crate::models::Block],
374) -> Result<Vec<RecoveredBlock<Block>>, Error> {
375    let mut blocks = Vec::with_capacity(test_case_blocks.len());
376    for (block_index, block) in test_case_blocks.iter().enumerate() {
377        // The blocks do not include the genesis block which is why we have the plus one.
378        // We also cannot use block.number because for invalid blocks, this may be incorrect.
379        let block_number = (block_index + 1) as u64;
380
381        let decoded = SealedBlock::<Block>::decode(&mut block.rlp.as_ref())
382            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
383
384        let recovered_block = decoded
385            .clone()
386            .try_recover()
387            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
388
389        blocks.push(recovered_block);
390    }
391
392    Ok(blocks)
393}
394
395fn pre_execution_checks(
396    chain_spec: Arc<ChainSpec>,
397    parent: &RecoveredBlock<Block>,
398    block: &RecoveredBlock<Block>,
399) -> Result<(), Error> {
400    let consensus: EthBeaconConsensus<ChainSpec> = EthBeaconConsensus::new(chain_spec);
401
402    let sealed_header = block.sealed_header();
403
404    <EthBeaconConsensus<ChainSpec> as Consensus<Block>>::validate_body_against_header(
405        &consensus,
406        block.body(),
407        sealed_header,
408    )?;
409    consensus.validate_header_against_parent(sealed_header, parent.sealed_header())?;
410    consensus.validate_header(sealed_header)?;
411    consensus.validate_block_pre_execution(block)?;
412
413    Ok(())
414}
415
416/// Returns whether the test at the given path should be skipped.
417///
418/// Some tests are edge cases that cannot happen on mainnet, while others are skipped for
419/// convenience (e.g. they take a long time to run) or are temporarily disabled.
420///
421/// The reason should be documented in a comment above the file name(s).
422pub fn should_skip(path: &Path) -> bool {
423    let path_str = path.to_str().expect("Path is not valid UTF-8");
424    let name = path.file_name().unwrap().to_str().unwrap();
425    matches!(
426        name,
427        // funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require
428        // custom json parser. https://github.com/ethereum/tests/issues/971
429        | "ValueOverflow.json"
430        | "ValueOverflowParis.json"
431
432        // txbyte is of type 02 and we don't parse tx bytes for this test to fail.
433        | "typeTwoBerlin.json"
434
435        // Test checks if nonce overflows. We are handling this correctly but we are not parsing
436        // exception in testsuite. There are more nonce overflow tests that are internal
437        // call/create, and those tests are passing and are enabled.
438        | "CreateTransactionHighNonce.json"
439
440        // Test check if gas price overflows, we handle this correctly but does not match tests specific
441        // exception.
442        | "HighGasPrice.json"
443        | "HighGasPriceParis.json"
444
445        // Skip test where basefee/accesslist/difficulty is present but it shouldn't be supported in
446        // London/Berlin/TheMerge. https://github.com/ethereum/tests/blob/5b7e1ab3ffaf026d99d20b17bb30f533a2c80c8b/GeneralStateTests/stExample/eip1559.json#L130
447        // It is expected to not execute these tests.
448        | "accessListExample.json"
449        | "basefeeExample.json"
450        | "eip1559.json"
451        | "mergeTest.json"
452
453        // These tests are passing, but they take a lot of time to execute so we are going to skip them.
454        | "loopExp.json"
455        | "Call50000_sha256.json"
456        | "static_Call50000_sha256.json"
457        | "loopMul.json"
458        | "CALLBlake2f_MaxRounds.json"
459        | "shiftCombinations.json"
460
461        // Skipped by revm as well: <https://github.com/bluealloy/revm/blob/be92e1db21f1c47b34c5a58cfbf019f6b97d7e4b/bins/revme/src/cmd/statetest/runner.rs#L115-L125>
462        | "RevertInCreateInInit_Paris.json"
463        | "RevertInCreateInInit.json"
464        | "dynamicAccountOverwriteEmpty.json"
465        | "dynamicAccountOverwriteEmpty_Paris.json"
466        | "RevertInCreateInInitCreate2Paris.json"
467        | "create2collisionStorage.json"
468        | "RevertInCreateInInitCreate2.json"
469        | "create2collisionStorageParis.json"
470        | "InitCollision.json"
471        | "InitCollisionParis.json"
472    )
473    // Ignore outdated EOF tests that haven't been updated for Cancun yet.
474    || path_contains(path_str, &["EIPTests", "stEOF"])
475}
476
477/// `str::contains` but for a path. Takes into account the OS path separator (`/` or `\`).
478fn path_contains(path_str: &str, rhs: &[&str]) -> bool {
479    let rhs = rhs.join(std::path::MAIN_SEPARATOR_STR);
480    path_str.contains(&rhs)
481}
482
483fn execution_witness_with_parent(parent: &RecoveredBlock<Block>) -> ExecutionWitness {
484    let mut serialized_header = Vec::new();
485    parent.header().encode(&mut serialized_header);
486    ExecutionWitness { headers: vec![serialized_header.into()], ..Default::default() }
487}
ef_tests/cases/blockchain_test.rs

ef_tests/cases/
blockchain_test.rs
