reth_cli_commands/test_vectors/
compact.rs

1use alloy_eips::eip4895::Withdrawals;
2use alloy_primitives::{hex, Signature, TxKind, B256};
3use arbitrary::Arbitrary;
4use eyre::{Context, Result};
5use proptest::{
6    prelude::{ProptestConfig, RngCore},
7    test_runner::{TestRng, TestRunner},
8};
9use reth_codecs::alloy::{
10    authorization_list::Authorization,
11    genesis_account::GenesisAccount,
12    header::{Header, HeaderExt},
13    transaction::{
14        eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702,
15        legacy::TxLegacy,
16    },
17    withdrawal::Withdrawal,
18};
19use reth_db::{
20    models::{
21        AccountBeforeTx, StaticFileBlockWithdrawals, StoredBlockBodyIndices, StoredBlockOmmers,
22        StoredBlockWithdrawals,
23    },
24    ClientVersion,
25};
26use reth_ethereum_primitives::{Receipt, Transaction, TransactionSigned, TxType};
27use reth_fs_util as fs;
28use reth_primitives_traits::{Account, Log, LogData, StorageEntry};
29use reth_prune_types::{PruneCheckpoint, PruneMode};
30use reth_stages_types::{
31    AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint,
32    HeadersCheckpoint, IndexHistoryCheckpoint, StageCheckpoint, StageUnitCheckpoint,
33    StorageHashingCheckpoint,
34};
35use reth_trie::{hash_builder::HashBuilderValue, TrieMask};
36use reth_trie_common::{hash_builder::HashBuilderState, StoredNibbles, StoredNibblesSubKey};
37use std::{fs::File, io::BufReader};
38
39pub const VECTORS_FOLDER: &str = "testdata/micro/compact";
40pub const VECTOR_SIZE: usize = 100;
41
42#[macro_export]
43macro_rules! compact_types {
44    (regular: [$($regular_ty:ident),*], identifier: [$($id_ty:ident),*]) => {
45        pub const GENERATE_VECTORS: &[fn(&mut TestRunner) -> eyre::Result<()>] = &[
46            $(
47                generate_vector::<$regular_ty> as fn(&mut TestRunner) -> eyre::Result<()>,
48            )*
49            $(
50                generate_vector::<$id_ty> as fn(&mut TestRunner) -> eyre::Result<()>,
51            )*
52        ];
53
54        pub const READ_VECTORS: &[fn() -> eyre::Result<()>] = &[
55            $(
56                read_vector::<$regular_ty> as fn() -> eyre::Result<()>,
57            )*
58            $(
59                read_vector::<$id_ty> as fn() -> eyre::Result<()>,
60            )*
61        ];
62
63        pub static IDENTIFIER_TYPE: std::sync::LazyLock<std::collections::HashSet<String>> = std::sync::LazyLock::new(|| {
64            let mut map = std::collections::HashSet::new();
65            $(
66                 map.insert(type_name::<$id_ty>());
67            )*
68            map
69        });
70    };
71}
72
73// The type that **actually** implements `Compact` should go here. If it's an alloy type, import the
74// auxiliary type from reth_codecs::alloy instead.
75compact_types!(
76    regular: [
77        // reth-primitives
78        Account,
79        Receipt,
80        // reth_codecs::alloy
81        Authorization,
82        GenesisAccount,
83        Header,
84        HeaderExt,
85        Withdrawal,
86        Withdrawals,
87        TxEip2930,
88        TxEip1559,
89        TxEip4844,
90        TxEip7702,
91        TxLegacy,
92        HashBuilderValue,
93        LogData,
94        Log,
95        // BranchNodeCompact, // todo requires arbitrary
96        TrieMask,
97        // reth_prune_types
98        PruneCheckpoint,
99        PruneMode,
100        // reth_stages_types
101        AccountHashingCheckpoint,
102        StorageHashingCheckpoint,
103        ExecutionCheckpoint,
104        HeadersCheckpoint,
105        IndexHistoryCheckpoint,
106        EntitiesCheckpoint,
107        CheckpointBlockRange,
108        StageCheckpoint,
109        StageUnitCheckpoint,
110        // reth_db_api
111        StoredBlockOmmers,
112        StoredBlockBodyIndices,
113        StoredBlockWithdrawals,
114        StaticFileBlockWithdrawals,
115        // Manual implementations
116        TransactionSigned,
117        // Bytecode, // todo revm arbitrary
118        StorageEntry,
119        // MerkleCheckpoint, // todo storedsubnode -> branchnodecompact arbitrary
120        AccountBeforeTx,
121        ClientVersion,
122        StoredNibbles,
123        StoredNibblesSubKey,
124        // StorageTrieEntry, // todo branchnodecompact arbitrary
125        // StoredSubNode, // todo branchnodecompact arbitrary
126        HashBuilderState
127    ],
128    // These types require an extra identifier which is usually stored elsewhere (eg. parent type).
129    identifier: [
130        Signature,
131        Transaction,
132        TxType,
133        TxKind
134    ]
135);
136
137/// Generates a vector of type `T` to a file.
138pub fn generate_vectors() -> Result<()> {
139    generate_vectors_with(GENERATE_VECTORS)
140}
141
142pub fn read_vectors() -> Result<()> {
143    read_vectors_with(READ_VECTORS)
144}
145
146/// Generates a vector of type `T` to a file.
147pub fn generate_vectors_with(generator: &[fn(&mut TestRunner) -> eyre::Result<()>]) -> Result<()> {
148    // Prepare random seed for test (same method as used by proptest)
149    let seed = B256::random();
150    println!("Seed for compact test vectors: {:?}", hex::encode_prefixed(seed));
151
152    // Start the runner with the seed
153    let config = ProptestConfig::default();
154    let rng = TestRng::from_seed(config.rng_algorithm, &seed.0);
155    let mut runner = TestRunner::new_with_rng(config, rng);
156
157    fs::create_dir_all(VECTORS_FOLDER)?;
158
159    for generate_fn in generator {
160        generate_fn(&mut runner)?;
161    }
162
163    Ok(())
164}
165
166/// Reads multiple vectors of different types ensuring their correctness by decoding and
167/// re-encoding.
168pub fn read_vectors_with(read: &[fn() -> eyre::Result<()>]) -> Result<()> {
169    fs::create_dir_all(VECTORS_FOLDER)?;
170    let mut errors = None;
171
172    for read_fn in read {
173        if let Err(err) = read_fn() {
174            errors.get_or_insert_with(Vec::new).push(err);
175        }
176    }
177
178    if let Some(err_list) = errors {
179        for error in err_list {
180            eprintln!("{error:?}");
181        }
182        return Err(eyre::eyre!(
183            "If there are missing types, make sure to run `reth test-vectors compact --write` first.\n
184             If it happened during CI, ignore IF it's a new proposed type that `main` branch does not have."
185        ));
186    }
187
188    Ok(())
189}
190
191/// Generates test vectors for a specific type `T`.
192pub fn generate_vector<T>(runner: &mut TestRunner) -> Result<()>
193where
194    T: for<'a> Arbitrary<'a> + reth_codecs::Compact,
195{
196    let type_name = type_name::<T>();
197    print!("{}", &type_name);
198
199    let mut bytes = std::iter::repeat_n(0u8, 256).collect::<Vec<u8>>();
200    let mut compact_buffer = vec![];
201
202    let mut values = Vec::with_capacity(VECTOR_SIZE);
203    for _ in 0..VECTOR_SIZE {
204        runner.rng().fill_bytes(&mut bytes);
205        compact_buffer.clear();
206
207        // Sometimes type T, might require extra arbitrary data, so we retry it a few times.
208        let mut tries = 0;
209        let obj = loop {
210            match T::arbitrary(&mut arbitrary::Unstructured::new(&bytes)) {
211                Ok(obj) => break obj,
212                Err(err) => {
213                    if tries < 5 && matches!(err, arbitrary::Error::NotEnoughData) {
214                        tries += 1;
215                        bytes.extend(std::iter::repeat_n(0u8, 256));
216                    } else {
217                        return Err(err)?
218                    }
219                }
220            }
221        };
222        let res = obj.to_compact(&mut compact_buffer);
223
224        if IDENTIFIER_TYPE.contains(&type_name) {
225            compact_buffer.push(res as u8);
226        }
227
228        values.push(hex::encode(&compact_buffer));
229    }
230
231    serde_json::to_writer(
232        std::io::BufWriter::new(
233            std::fs::File::create(format!("{VECTORS_FOLDER}/{}.json", &type_name)).unwrap(),
234        ),
235        &values,
236    )?;
237
238    println!(" ✅");
239
240    Ok(())
241}
242
243/// Reads a vector of type `T` from a file and compares each item with its reconstructed version
244/// using `T::from_compact`.
245pub fn read_vector<T>() -> Result<()>
246where
247    T: reth_codecs::Compact,
248{
249    let type_name = type_name::<T>();
250    print!("{}", &type_name);
251
252    // Read the file where the vectors are stored
253    let file_path = format!("{VECTORS_FOLDER}/{}.json", &type_name);
254    let file =
255        File::open(&file_path).wrap_err_with(|| format!("Failed to open vector {type_name}."))?;
256    let reader = BufReader::new(file);
257
258    let stored_values: Vec<String> = serde_json::from_reader(reader)?;
259    let mut buffer = vec![];
260
261    for hex_str in stored_values {
262        let mut compact_bytes = hex::decode(hex_str)?;
263        let mut identifier = None;
264        buffer.clear();
265
266        if IDENTIFIER_TYPE.contains(&type_name) {
267            identifier = compact_bytes.pop().map(|b| b as usize);
268        }
269        let len_or_identifier = identifier.unwrap_or(compact_bytes.len());
270
271        let (reconstructed, _) = T::from_compact(&compact_bytes, len_or_identifier);
272        reconstructed.to_compact(&mut buffer);
273        assert_eq!(buffer, compact_bytes, "mismatch {type_name}");
274    }
275
276    println!(" ✅");
277
278    Ok(())
279}
280
281/// Returns the type name for the given type.
282pub fn type_name<T>() -> String {
283    // With alloy type transition <https://github.com/paradigmxyz/reth/pull/15768> the types are renamed, we map them here to the original name so that test vector files remain consistent
284    let name = std::any::type_name::<T>();
285    match name {
286        "alloy_consensus::transaction::typed::EthereumTypedTransaction<alloy_consensus::transaction::eip4844::TxEip4844>" => "Transaction".to_string(),
287        "alloy_consensus::transaction::envelope::EthereumTxEnvelope<alloy_consensus::transaction::eip4844::TxEip4844>" => "TransactionSigned".to_string(),
288        name => {
289            name.split("::").last().unwrap_or(std::any::type_name::<T>()).to_string()
290        }
291    }
292}
reth_cli_commands/test_vectors/compact.rs

reth_cli_commands/test_vectors/
compact.rs
