ef_tests/
result.rs

1//! Test results and errors
2
3use crate::Case;
4use reth_db::DatabaseError;
5use reth_provider::ProviderError;
6use std::path::{Path, PathBuf};
7use thiserror::Error;
8
9/// Test errors
10///
11/// # Note
12///
13/// `Error::Skipped` should not be treated as a test failure.
14#[derive(Debug, Error)]
15#[non_exhaustive]
16pub enum Error {
17    /// The test was skipped
18    #[error("test was skipped")]
19    Skipped,
20    /// No post state found in test
21    #[error("no post state found for validation")]
22    MissingPostState,
23    /// Block processing failed
24    /// Note: This includes but is not limited to execution.
25    /// For example, the header number could be incorrect.
26    #[error("block {block_number} failed to process: {err}")]
27    BlockProcessingFailed {
28        /// The block number for the block that failed
29        block_number: u64,
30        /// The specific error
31        #[source]
32        err: Box<dyn std::error::Error + Send + Sync>,
33    },
34    /// An IO error occurred
35    #[error("an error occurred interacting with the file system at {path}: {error}")]
36    Io {
37        /// The path to the file or directory
38        path: PathBuf,
39        /// The specific error
40        #[source]
41        error: std::io::Error,
42    },
43    /// A deserialization error occurred
44    #[error("an error occurred deserializing the test at {path}: {error}")]
45    CouldNotDeserialize {
46        /// The path to the file we wanted to deserialize
47        path: PathBuf,
48        /// The specific error
49        #[source]
50        error: serde_json::Error,
51    },
52    /// A database error occurred.
53    #[error(transparent)]
54    Database(#[from] DatabaseError),
55    /// A test assertion failed.
56    #[error("test failed: {0}")]
57    Assertion(String),
58    /// An error internally in reth occurred.
59    #[error("test failed: {0}")]
60    Provider(#[from] ProviderError),
61    /// An error occurred while decoding RLP.
62    #[error("an error occurred deserializing RLP: {0}")]
63    RlpDecodeError(#[from] alloy_rlp::Error),
64    /// A consensus error occurred.
65    #[error("an error occurred during consensus checks: {0}")]
66    ConsensusError(#[from] reth_consensus::ConsensusError),
67}
68
69impl Error {
70    /// Create a new [`Error::BlockProcessingFailed`] error.
71    pub fn block_failed(
72        block_number: u64,
73        err: impl std::error::Error + Send + Sync + 'static,
74    ) -> Self {
75        Self::BlockProcessingFailed { block_number, err: Box::new(err) }
76    }
77}
78
79/// The result of running a test.
80#[derive(Debug)]
81pub struct CaseResult {
82    /// A description of the test.
83    pub desc: String,
84    /// The full path to the test.
85    pub path: PathBuf,
86    /// The result of the test.
87    pub result: Result<(), Error>,
88}
89
90impl CaseResult {
91    /// Create a new test result.
92    pub fn new(path: &Path, case: &impl Case, result: Result<(), Error>) -> Self {
93        Self { desc: case.description(), path: path.into(), result }
94    }
95}
96
97/// Assert that all the given tests passed and print the results to stdout.
98pub(crate) fn assert_tests_pass(suite_name: &str, path: &Path, results: &[CaseResult]) {
99    let (passed, failed, skipped) = categorize_results(results);
100
101    print_results(suite_name, path, &passed, &failed, &skipped);
102
103    assert!(failed.is_empty(), "Some tests failed (see above)");
104}
105
106/// Categorize test results into `(passed, failed, skipped)`.
107pub(crate) fn categorize_results(
108    results: &[CaseResult],
109) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) {
110    let mut passed = Vec::new();
111    let mut failed = Vec::new();
112    let mut skipped = Vec::new();
113
114    for case in results {
115        match case.result.as_ref().err() {
116            Some(Error::Skipped) => skipped.push(case),
117            Some(_) => failed.push(case),
118            None => passed.push(case),
119        }
120    }
121
122    (passed, failed, skipped)
123}
124
125/// Display the given test results to stdout.
126pub(crate) fn print_results(
127    suite_name: &str,
128    path: &Path,
129    passed: &[&CaseResult],
130    failed: &[&CaseResult],
131    skipped: &[&CaseResult],
132) {
133    println!("Suite: {suite_name} (at {})", path.display());
134    println!(
135        "Ran {} tests ({} passed, {} failed, {} skipped)",
136        passed.len() + failed.len() + skipped.len(),
137        passed.len(),
138        failed.len(),
139        skipped.len()
140    );
141
142    for case in skipped {
143        println!("[S] Case {} skipped", case.path.display());
144    }
145
146    for case in failed {
147        let error = case.result.as_ref().unwrap_err();
148        println!("[!] Case {} failed (description: {}): {}", case.path.display(), case.desc, error);
149    }
150}
ef_tests/result.rs

ef_tests/
result.rs
