reth_provider/providers/static_file/
mod.rs

1mod manager;
2pub use manager::{
3    StaticFileAccess, StaticFileProvider, StaticFileProviderBuilder, StaticFileWriteCtx,
4    StaticFileWriter,
5};
6
7mod jar;
8pub use jar::StaticFileJarProvider;
9
10mod writer;
11pub use writer::{StaticFileProviderRW, StaticFileProviderRWRefMut};
12
13mod metrics;
14
15#[cfg(test)]
16mod writer_tests;
17
18use reth_nippy_jar::NippyJar;
19use reth_static_file_types::{ChangesetOffsetReader, SegmentHeader, StaticFileSegment};
20use reth_storage_errors::provider::{ProviderError, ProviderResult};
21use std::{io, ops::Deref, sync::Arc};
22
23/// Alias type for each specific `NippyJar`.
24type LoadedJarRef<'a> =
25    reth_primitives_traits::dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
26
27/// Helper type to reuse an associated static file mmap handle on created cursors.
28#[derive(Debug)]
29pub struct LoadedJar {
30    jar: NippyJar<SegmentHeader>,
31    mmap_handle: Arc<reth_nippy_jar::DataReader>,
32    csoff_reader: Option<ChangesetOffsetReader>,
33}
34
35impl LoadedJar {
36    fn new(jar: NippyJar<SegmentHeader>) -> ProviderResult<Self> {
37        match jar.open_data_reader() {
38            Ok(data_reader) => {
39                let mmap_handle = Arc::new(data_reader);
40
41                let csoff_reader = if jar.user_header().segment().is_change_based() {
42                    let csoff_path = jar.data_path().with_extension("csoff");
43                    let len = jar.user_header().changeset_offsets_len();
44                    match ChangesetOffsetReader::new(&csoff_path, len) {
45                        Ok(reader) => Some(reader),
46                        Err(err) if err.kind() == io::ErrorKind::NotFound && len == 0 => None,
47                        Err(err) => return Err(ProviderError::other(err)),
48                    }
49                } else {
50                    None
51                };
52
53                Ok(Self { jar, mmap_handle, csoff_reader })
54            }
55            Err(e) => Err(ProviderError::other(e)),
56        }
57    }
58
59    /// Returns a clone of the mmap handle that can be used to instantiate a cursor.
60    fn mmap_handle(&self) -> Arc<reth_nippy_jar::DataReader> {
61        self.mmap_handle.clone()
62    }
63
64    const fn segment(&self) -> StaticFileSegment {
65        self.jar.user_header().segment()
66    }
67
68    /// Returns the total size of the data and offsets files (from the in-memory mmap).
69    fn size(&self) -> usize {
70        self.mmap_handle.size() + self.mmap_handle.offsets_size()
71    }
72
73    /// Returns a reference to the cached changeset offset reader.
74    const fn csoff_reader(&self) -> Option<&ChangesetOffsetReader> {
75        self.csoff_reader.as_ref()
76    }
77}
78
79impl Deref for LoadedJar {
80    type Target = NippyJar<SegmentHeader>;
81    fn deref(&self) -> &Self::Target {
82        &self.jar
83    }
84}
85
86#[cfg(test)]
87mod tests {
88    use super::*;
89    use crate::{
90        providers::static_file::manager::StaticFileProviderBuilder,
91        test_utils::create_test_provider_factory, HeaderProvider, StaticFileProviderFactory,
92    };
93    use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy};
94    use alloy_primitives::{Address, BlockHash, Signature, TxNumber, B256, U160, U256};
95    use assert_matches::assert_matches;
96    use rand::seq::SliceRandom;
97    use reth_db::{
98        models::{AccountBeforeTx, StorageBeforeTx},
99        test_utils::create_test_static_files_dir,
100    };
101    use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers};
102    use reth_ethereum_primitives::{EthPrimitives, Receipt, TransactionSigned};
103    use reth_nippy_jar::NippyJarError;
104    use reth_primitives_traits::Account;
105    use reth_static_file_types::{
106        find_fixed_range, SegmentRangeInclusive, DEFAULT_BLOCKS_PER_STATIC_FILE,
107    };
108    use reth_storage_api::{
109        BlockHashReader, ChangeSetReader, ReceiptProvider, StorageChangeSetReader,
110        TransactionsProvider,
111    };
112    use reth_testing_utils::generators::{self, random_header_range};
113    use std::{collections::BTreeMap, fmt::Debug, fs, ops::Range, path::Path};
114
115    fn assert_eyre<T: PartialEq + Debug>(got: T, expected: T, msg: &str) -> eyre::Result<()> {
116        if got != expected {
117            eyre::bail!("{msg} | got: {got:?} expected: {expected:?}");
118        }
119        Ok(())
120    }
121
122    fn assert_cached_size_matches_disk(
123        sf_rw: &StaticFileProvider<EthPrimitives>,
124        segment: StaticFileSegment,
125        block: u64,
126    ) -> eyre::Result<()> {
127        let jar_provider = sf_rw.get_segment_provider_for_block(segment, block, None)?;
128        let disk_size = fs::metadata(jar_provider.data_path())?.len() +
129            fs::metadata(jar_provider.offsets_path())?.len();
130        assert_eyre(jar_provider.size() as u64, disk_size, "cached size mismatch")
131    }
132
133    #[test]
134    fn test_static_files() {
135        // Ranges
136        let row_count = 100u64;
137        let range = 0..=(row_count - 1);
138
139        // Data sources
140        let factory = create_test_provider_factory();
141        let static_files_path = tempfile::tempdir().unwrap();
142        let static_file = static_files_path.path().join(
143            StaticFileSegment::Headers
144                .filename(&find_fixed_range(*range.end(), DEFAULT_BLOCKS_PER_STATIC_FILE)),
145        );
146
147        // Setup data
148        let mut headers = random_header_range(
149            &mut generators::rng(),
150            *range.start()..(*range.end() + 1),
151            B256::random(),
152        );
153
154        let mut provider_rw = factory.provider_rw().unwrap();
155        let tx = provider_rw.tx_mut();
156        for header in headers.clone() {
157            let hash = header.hash();
158
159            tx.put::<CanonicalHeaders>(header.number, hash).unwrap();
160            tx.put::<Headers>(header.number, header.clone_header()).unwrap();
161            tx.put::<HeaderNumbers>(hash, header.number).unwrap();
162        }
163        provider_rw.commit().unwrap();
164
165        // Create StaticFile
166        {
167            let manager = factory.static_file_provider();
168            let mut writer = manager.latest_writer(StaticFileSegment::Headers).unwrap();
169
170            for header in headers.clone() {
171                let hash = header.hash();
172                writer.append_header(&header.unseal(), &hash).unwrap();
173            }
174            writer.commit().unwrap();
175        }
176
177        // Use providers to query Header data and compare if it matches
178        {
179            let db_provider = factory.provider().unwrap();
180            let manager = db_provider.static_file_provider();
181            let jar_provider = manager
182                .get_segment_provider_for_block(StaticFileSegment::Headers, 0, Some(&static_file))
183                .unwrap();
184
185            assert!(!headers.is_empty());
186
187            // Shuffled for chaos.
188            headers.shuffle(&mut generators::rng());
189
190            for header in headers {
191                let header_hash = header.hash();
192                let header = header.unseal();
193
194                // Compare Header
195                assert_eq!(header, db_provider.header(header_hash).unwrap().unwrap());
196                assert_eq!(header, jar_provider.header_by_number(header.number).unwrap().unwrap());
197            }
198        }
199    }
200
201    #[test]
202    fn test_header_truncation() {
203        let (static_dir, _) = create_test_static_files_dir();
204
205        let blocks_per_file = 10; // Number of headers per file
206        let files_per_range = 3; // Number of files per range (data/conf/offset files)
207        let file_set_count = 3; // Number of sets of files to create
208        let initial_file_count = files_per_range * file_set_count;
209        let tip = blocks_per_file * file_set_count - 1; // Initial highest block (29 in this case)
210
211        // [ Headers Creation and Commit ]
212        {
213            let sf_rw: StaticFileProvider<EthPrimitives> =
214                StaticFileProviderBuilder::read_write(&static_dir)
215                    .with_blocks_per_file(blocks_per_file)
216                    .build()
217                    .expect("Failed to build static file provider");
218
219            let mut header_writer = sf_rw.latest_writer(StaticFileSegment::Headers).unwrap();
220
221            // Append headers from 0 to the tip (29) and commit
222            let mut header = Header::default();
223            for num in 0..=tip {
224                header.number = num;
225                header_writer.append_header(&header, &BlockHash::default()).unwrap();
226            }
227            header_writer.commit().unwrap();
228        }
229
230        // Helper function to prune headers and validate truncation results
231        fn prune_and_validate(
232            writer: &mut StaticFileProviderRWRefMut<'_, EthPrimitives>,
233            sf_rw: &StaticFileProvider<EthPrimitives>,
234            static_dir: impl AsRef<Path>,
235            prune_count: u64,
236            expected_tip: Option<u64>,
237            expected_file_count: u64,
238        ) -> eyre::Result<()> {
239            writer.prune_headers(prune_count)?;
240            writer.commit()?;
241
242            // Validate the highest block after pruning
243            assert_eyre(
244                sf_rw.get_highest_static_file_block(StaticFileSegment::Headers),
245                expected_tip,
246                "block mismatch",
247            )?;
248
249            if let Some(id) = expected_tip {
250                assert_eyre(
251                    sf_rw.header_by_number(id)?.map(|h| h.number),
252                    expected_tip,
253                    "header mismatch",
254                )?;
255                assert_cached_size_matches_disk(sf_rw, StaticFileSegment::Headers, id)?;
256            }
257
258            // Validate the number of files remaining in the directory
259            assert_eyre(
260                count_files_without_lockfile(static_dir)?,
261                expected_file_count as usize,
262                "file count mismatch",
263            )?;
264
265            Ok(())
266        }
267
268        // [ Test Cases ]
269        type PruneCount = u64;
270        type ExpectedTip = u64;
271        type ExpectedFileCount = u64;
272        let mut tmp_tip = tip;
273        let test_cases: Vec<(PruneCount, Option<ExpectedTip>, ExpectedFileCount)> = vec![
274            // Case 0: Pruning 1 header
275            {
276                tmp_tip -= 1;
277                (1, Some(tmp_tip), initial_file_count)
278            },
279            // Case 1: Pruning remaining rows from file should result in its deletion
280            {
281                tmp_tip -= blocks_per_file - 1;
282                (blocks_per_file - 1, Some(tmp_tip), initial_file_count - files_per_range)
283            },
284            // Case 2: Pruning more headers than a single file has (tip reduced by
285            // blocks_per_file + 1) should result in a file set deletion
286            {
287                tmp_tip -= blocks_per_file + 1;
288                (blocks_per_file + 1, Some(tmp_tip), initial_file_count - files_per_range * 2)
289            },
290            // Case 3: Pruning all remaining headers from the file except the genesis header
291            {
292                (
293                    tmp_tip,
294                    Some(0),         // Only genesis block remains
295                    files_per_range, // The file set with block 0 should remain
296                )
297            },
298            // Case 4: Pruning the genesis header (should not delete the file set with block 0)
299            {
300                (
301                    1,
302                    None,            // No blocks left
303                    files_per_range, // The file set with block 0 remains
304                )
305            },
306        ];
307
308        // Test cases execution
309        {
310            let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
311                .with_blocks_per_file(blocks_per_file)
312                .build()
313                .expect("Failed to build static file provider");
314
315            assert_eq!(sf_rw.get_highest_static_file_block(StaticFileSegment::Headers), Some(tip));
316            assert_eq!(
317                count_files_without_lockfile(static_dir.as_ref()).unwrap(),
318                initial_file_count as usize
319            );
320
321            let mut header_writer = sf_rw.latest_writer(StaticFileSegment::Headers).unwrap();
322
323            for (case, (prune_count, expected_tip, expected_file_count)) in
324                test_cases.into_iter().enumerate()
325            {
326                prune_and_validate(
327                    &mut header_writer,
328                    &sf_rw,
329                    &static_dir,
330                    prune_count,
331                    expected_tip,
332                    expected_file_count,
333                )
334                .map_err(|err| eyre::eyre!("Test case {case}: {err}"))
335                .unwrap();
336            }
337        }
338    }
339
340    /// 3 block ranges are built
341    ///
342    /// for `blocks_per_file = 10`:
343    /// * `0..=9` : except genesis, every block has a tx/receipt
344    /// * `10..=19`: no txs/receipts
345    /// * `20..=29`: only one tx/receipt
346    fn setup_tx_based_scenario(
347        sf_rw: &StaticFileProvider<EthPrimitives>,
348        segment: StaticFileSegment,
349        blocks_per_file: u64,
350    ) {
351        fn setup_block_ranges(
352            writer: &mut StaticFileProviderRWRefMut<'_, EthPrimitives>,
353            sf_rw: &StaticFileProvider<EthPrimitives>,
354            segment: StaticFileSegment,
355            block_range: &Range<u64>,
356            mut tx_count: u64,
357            next_tx_num: &mut u64,
358        ) {
359            let mut receipt = Receipt::default();
360            let mut tx = TxLegacy::default();
361
362            for block in block_range.clone() {
363                writer.increment_block(block).unwrap();
364
365                // Append transaction/receipt if there's still a transaction count to append
366                if tx_count > 0 {
367                    match segment {
368                        StaticFileSegment::Headers |
369                        StaticFileSegment::AccountChangeSets |
370                        StaticFileSegment::StorageChangeSets => {
371                            panic!("non tx based segment")
372                        }
373                        StaticFileSegment::Transactions => {
374                            // Used as ID for validation
375                            tx.nonce = *next_tx_num;
376                            let tx: TransactionSigned =
377                                tx.clone().into_signed(Signature::test_signature()).into();
378                            writer.append_transaction(*next_tx_num, &tx).unwrap();
379                        }
380                        StaticFileSegment::Receipts => {
381                            // Used as ID for validation
382                            receipt.cumulative_gas_used = *next_tx_num;
383                            writer.append_receipt(*next_tx_num, &receipt).unwrap();
384                        }
385                        StaticFileSegment::TransactionSenders => {
386                            // Used as ID for validation
387                            let sender = Address::from(U160::from(*next_tx_num));
388                            writer.append_transaction_sender(*next_tx_num, &sender).unwrap();
389                        }
390                    }
391                    *next_tx_num += 1;
392                    tx_count -= 1;
393                }
394            }
395            writer.commit().unwrap();
396
397            // Calculate expected values based on the range and transactions
398            let expected_block = block_range.end - 1;
399            let expected_tx = if tx_count == 0 { *next_tx_num - 1 } else { *next_tx_num };
400
401            // Perform assertions after processing the blocks
402            assert_eq!(sf_rw.get_highest_static_file_block(segment), Some(expected_block),);
403            assert_eq!(sf_rw.get_highest_static_file_tx(segment), Some(expected_tx),);
404        }
405
406        // Define the block ranges and transaction counts as vectors
407        let block_ranges = [
408            0..blocks_per_file,
409            blocks_per_file..blocks_per_file * 2,
410            blocks_per_file * 2..blocks_per_file * 3,
411        ];
412
413        let tx_counts = [
414            blocks_per_file - 1, // First range: tx per block except genesis
415            0,                   // Second range: no transactions
416            1,                   // Third range: 1 transaction in the second block
417        ];
418
419        let mut writer = sf_rw.latest_writer(segment).unwrap();
420        let mut next_tx_num = 0;
421
422        // Loop through setup scenarios
423        for (block_range, tx_count) in block_ranges.iter().zip(tx_counts.iter()) {
424            setup_block_ranges(
425                &mut writer,
426                sf_rw,
427                segment,
428                block_range,
429                *tx_count,
430                &mut next_tx_num,
431            );
432        }
433
434        // Ensure that scenario was properly setup
435        let expected_tx_ranges = vec![
436            Some(SegmentRangeInclusive::new(0, 8)),
437            None,
438            Some(SegmentRangeInclusive::new(9, 9)),
439        ];
440
441        block_ranges.iter().zip(expected_tx_ranges).for_each(|(block_range, expected_tx_range)| {
442            assert_eq!(
443                sf_rw
444                    .get_segment_provider_for_block(segment, block_range.start, None)
445                    .unwrap()
446                    .user_header()
447                    .tx_range(),
448                expected_tx_range
449            );
450        });
451
452        // Ensure transaction index
453        let expected_tx_index = BTreeMap::from([
454            (8, SegmentRangeInclusive::new(0, 9)),
455            (9, SegmentRangeInclusive::new(20, 29)),
456        ]);
457        assert_eq!(
458            sf_rw.tx_index(segment),
459            (!expected_tx_index.is_empty()).then_some(expected_tx_index),
460            "tx index mismatch",
461        );
462    }
463
464    #[test]
465    fn test_tx_based_truncation() {
466        let segments = [StaticFileSegment::Transactions, StaticFileSegment::Receipts];
467        let blocks_per_file = 10; // Number of blocks per file
468        let files_per_range = 3; // Number of files per range (data/conf/offset files)
469        let file_set_count = 3; // Number of sets of files to create
470        let initial_file_count = files_per_range * file_set_count;
471
472        #[expect(clippy::too_many_arguments)]
473        fn prune_and_validate(
474            sf_rw: &StaticFileProvider<EthPrimitives>,
475            static_dir: impl AsRef<Path>,
476            segment: StaticFileSegment,
477            prune_count: u64,
478            last_block: u64,
479            expected_tx_tip: Option<u64>,
480            expected_file_count: i32,
481            expected_tx_index: BTreeMap<TxNumber, SegmentRangeInclusive>,
482        ) -> eyre::Result<()> {
483            let mut writer = sf_rw.latest_writer(segment)?;
484
485            // Prune transactions or receipts based on the segment type
486            match segment {
487                StaticFileSegment::Headers |
488                StaticFileSegment::AccountChangeSets |
489                StaticFileSegment::StorageChangeSets => {
490                    panic!("non tx based segment")
491                }
492                StaticFileSegment::Transactions => {
493                    writer.prune_transactions(prune_count, last_block)?
494                }
495                StaticFileSegment::Receipts => writer.prune_receipts(prune_count, last_block)?,
496                StaticFileSegment::TransactionSenders => {
497                    writer.prune_transaction_senders(prune_count, last_block)?
498                }
499            }
500            writer.commit()?;
501
502            // Verify the highest block and transaction tips
503            assert_eyre(
504                sf_rw.get_highest_static_file_block(segment),
505                Some(last_block),
506                "block mismatch",
507            )?;
508            assert_eyre(sf_rw.get_highest_static_file_tx(segment), expected_tx_tip, "tx mismatch")?;
509            assert_cached_size_matches_disk(sf_rw, segment, last_block)?;
510
511            // Verify that transactions and receipts are returned correctly. Uses
512            // cumulative_gas_used & nonce as ids.
513            if let Some(id) = expected_tx_tip {
514                match segment {
515                    StaticFileSegment::Headers |
516                    StaticFileSegment::AccountChangeSets |
517                    StaticFileSegment::StorageChangeSets => {
518                        panic!("non tx based segment")
519                    }
520                    StaticFileSegment::Transactions => assert_eyre(
521                        expected_tx_tip,
522                        sf_rw.transaction_by_id(id)?.map(|t| t.nonce()),
523                        "tx mismatch",
524                    )?,
525                    StaticFileSegment::Receipts => assert_eyre(
526                        expected_tx_tip,
527                        sf_rw.receipt(id)?.map(|r| r.cumulative_gas_used),
528                        "receipt mismatch",
529                    )?,
530                    StaticFileSegment::TransactionSenders => assert_eyre(
531                        expected_tx_tip,
532                        sf_rw
533                            .transaction_sender(id)?
534                            .map(|s| u64::try_from(U160::from_be_bytes(s.0.into())).unwrap()),
535                        "sender mismatch",
536                    )?,
537                }
538            }
539
540            // Ensure the file count has reduced as expected
541            assert_eyre(
542                count_files_without_lockfile(static_dir)?,
543                expected_file_count as usize,
544                "file count mismatch",
545            )?;
546
547            // Ensure that the inner tx index (max_tx -> block range) is as expected
548            assert_eyre(
549                sf_rw.tx_index(segment).map(|index| index.iter().map(|(k, v)| (*k, *v)).collect()),
550                (!expected_tx_index.is_empty()).then_some(expected_tx_index),
551                "tx index mismatch",
552            )?;
553
554            Ok(())
555        }
556
557        for segment in segments {
558            let (static_dir, _) = create_test_static_files_dir();
559
560            let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
561                .with_blocks_per_file(blocks_per_file)
562                .build()
563                .expect("Failed to build static file provider");
564
565            setup_tx_based_scenario(&sf_rw, segment, blocks_per_file);
566
567            let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
568                .with_blocks_per_file(blocks_per_file)
569                .build()
570                .expect("Failed to build static file provider");
571            let highest_tx = sf_rw.get_highest_static_file_tx(segment).unwrap();
572
573            // Test cases
574            // [prune_count, last_block, expected_tx_tip, expected_file_count, expected_tx_index)
575            let test_cases = vec![
576                // Case 0: 20..=29 has only one tx. Prune the only tx of the block range.
577                // It ensures that the file is not deleted even though there are no rows, since the
578                // `last_block` which is passed to the prune method is the first
579                // block of the range.
580                (
581                    1,
582                    blocks_per_file * 2,
583                    Some(highest_tx - 1),
584                    initial_file_count,
585                    BTreeMap::from([(highest_tx - 1, SegmentRangeInclusive::new(0, 9))]),
586                ),
587                // Case 1: 10..=19 has no txs. There are no txes in the whole block range, but want
588                // to unwind to block 9. Ensures that the 20..=29 and 10..=19 files
589                // are deleted.
590                (
591                    0,
592                    blocks_per_file - 1,
593                    Some(highest_tx - 1),
594                    files_per_range,
595                    BTreeMap::from([(highest_tx - 1, SegmentRangeInclusive::new(0, 9))]),
596                ),
597                // Case 2: Prune most txs up to block 1.
598                (
599                    highest_tx - 1,
600                    1,
601                    Some(0),
602                    files_per_range,
603                    BTreeMap::from([(0, SegmentRangeInclusive::new(0, 1))]),
604                ),
605                // Case 3: Prune remaining tx and ensure that file is not deleted.
606                (1, 0, None, files_per_range, BTreeMap::from([])),
607            ];
608
609            // Loop through test cases
610            for (
611                case,
612                (prune_count, last_block, expected_tx_tip, expected_file_count, expected_tx_index),
613            ) in test_cases.into_iter().enumerate()
614            {
615                prune_and_validate(
616                    &sf_rw,
617                    &static_dir,
618                    segment,
619                    prune_count,
620                    last_block,
621                    expected_tx_tip,
622                    expected_file_count,
623                    expected_tx_index,
624                )
625                .map_err(|err| eyre::eyre!("Test case {case}: {err}"))
626                .unwrap();
627            }
628        }
629    }
630
631    /// Returns the number of files in the provided path, excluding ".lock" files.
632    fn count_files_without_lockfile(path: impl AsRef<Path>) -> eyre::Result<usize> {
633        let is_lockfile =
634            |entry: &fs::DirEntry| entry.path().file_name().is_some_and(|name| name == "lock");
635        let count = fs::read_dir(path)?
636            .filter_map(|entry| entry.ok())
637            .filter(|entry| !is_lockfile(entry))
638            .count();
639
640        Ok(count)
641    }
642
643    #[test]
644    fn test_dynamic_size() -> eyre::Result<()> {
645        let (static_dir, _) = create_test_static_files_dir();
646
647        {
648            let sf_rw: StaticFileProvider<EthPrimitives> =
649                StaticFileProviderBuilder::read_write(&static_dir)
650                    .with_blocks_per_file(10)
651                    .build()?;
652            let mut header_writer = sf_rw.latest_writer(StaticFileSegment::Headers)?;
653
654            let mut header = Header::default();
655            for num in 0..=15 {
656                header.number = num;
657                header_writer.append_header(&header, &BlockHash::default()).unwrap();
658            }
659            header_writer.commit().unwrap();
660
661            assert_eq!(sf_rw.headers_range(0..=15)?.len(), 16);
662            assert_eq!(
663                sf_rw.expected_block_index(StaticFileSegment::Headers),
664                Some(BTreeMap::from([
665                    (9, SegmentRangeInclusive::new(0, 9)),
666                    (19, SegmentRangeInclusive::new(10, 19))
667                ])),
668            )
669        }
670
671        {
672            let sf_rw: StaticFileProvider<EthPrimitives> =
673                StaticFileProviderBuilder::read_write(&static_dir)
674                    .with_blocks_per_file(5)
675                    .build()?;
676            let mut header_writer = sf_rw.latest_writer(StaticFileSegment::Headers)?;
677
678            let mut header = Header::default();
679            for num in 16..=22 {
680                header.number = num;
681                header_writer.append_header(&header, &BlockHash::default()).unwrap();
682            }
683            header_writer.commit().unwrap();
684
685            assert_eq!(sf_rw.headers_range(0..=22)?.len(), 23);
686            assert_eq!(
687                sf_rw.expected_block_index(StaticFileSegment::Headers),
688                Some(BTreeMap::from([
689                    (9, SegmentRangeInclusive::new(0, 9)),
690                    (19, SegmentRangeInclusive::new(10, 19)),
691                    (24, SegmentRangeInclusive::new(20, 24))
692                ]))
693            )
694        }
695
696        {
697            let sf_rw: StaticFileProvider<EthPrimitives> =
698                StaticFileProviderBuilder::read_write(&static_dir)
699                    .with_blocks_per_file(15)
700                    .build()?;
701            let mut header_writer = sf_rw.latest_writer(StaticFileSegment::Headers)?;
702
703            let mut header = Header::default();
704            for num in 23..=40 {
705                header.number = num;
706                header_writer.append_header(&header, &BlockHash::default()).unwrap();
707            }
708            header_writer.commit().unwrap();
709
710            assert_eq!(sf_rw.headers_range(0..=40)?.len(), 41);
711            assert_eq!(
712                sf_rw.expected_block_index(StaticFileSegment::Headers),
713                Some(BTreeMap::from([
714                    (9, SegmentRangeInclusive::new(0, 9)),
715                    (19, SegmentRangeInclusive::new(10, 19)),
716                    (24, SegmentRangeInclusive::new(20, 24)),
717                    (39, SegmentRangeInclusive::new(25, 39)),
718                    (54, SegmentRangeInclusive::new(40, 54))
719                ]))
720            )
721        }
722
723        Ok(())
724    }
725
726    #[test]
727    fn test_account_changeset_static_files() {
728        let (static_dir, _) = create_test_static_files_dir();
729
730        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
731            .expect("Failed to create static file provider");
732
733        // Helper function to generate test changesets
734        fn generate_test_changesets(
735            block_num: u64,
736            addresses: Vec<Address>,
737        ) -> Vec<AccountBeforeTx> {
738            addresses
739                .into_iter()
740                .map(|address| AccountBeforeTx {
741                    address,
742                    info: Some(Account {
743                        nonce: block_num,
744                        balance: U256::from(block_num * 1000),
745                        ..Default::default()
746                    }),
747                })
748                .collect()
749        }
750
751        // Test writing and reading account changesets
752        {
753            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
754
755            // Create test data for multiple blocks
756            let test_blocks = 10u64;
757            let addresses_per_block = 5;
758
759            for block_num in 0..test_blocks {
760                // Generate unique addresses for each block
761                let addresses: Vec<Address> = (0..addresses_per_block)
762                    .map(|i| {
763                        let mut addr = Address::ZERO;
764                        addr.0[0] = block_num as u8;
765                        addr.0[1] = i as u8;
766                        addr
767                    })
768                    .collect();
769
770                let changeset = generate_test_changesets(block_num, addresses);
771
772                writer.append_account_changeset(changeset, block_num).unwrap();
773            }
774
775            writer.commit().unwrap();
776        }
777
778        // Verify data can be read back correctly
779        {
780            let provider = sf_rw
781                .get_segment_provider_for_block(StaticFileSegment::AccountChangeSets, 5, None)
782                .unwrap();
783
784            // Check that the segment header has changeset offsets
785            let offsets = provider.read_changeset_offsets().unwrap();
786            assert!(offsets.is_some());
787            let offsets = offsets.unwrap();
788            assert_eq!(offsets.len(), 10); // Should have 10 blocks worth of offsets
789
790            // Verify each block has the expected number of changes
791            for (i, offset) in offsets.iter().enumerate() {
792                assert_eq!(offset.num_changes(), 5, "Block {} should have 5 changes", i);
793            }
794        }
795    }
796
797    #[test]
798    fn test_get_account_before_block() {
799        let (static_dir, _) = create_test_static_files_dir();
800
801        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
802            .expect("Failed to create static file provider");
803
804        // Setup test data
805        let test_address = Address::repeat_byte(1u8);
806        let other_address = Address::repeat_byte(2u8);
807        let missing_address = Address::repeat_byte(3u8);
808
809        // Write changesets for multiple blocks
810        {
811            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
812
813            // Block 0: test_address and other_address change
814            writer
815                .append_account_changeset(
816                    vec![
817                        AccountBeforeTx {
818                            address: test_address,
819                            info: None, // Account created
820                        },
821                        AccountBeforeTx { address: other_address, info: None },
822                    ],
823                    0,
824                )
825                .unwrap();
826
827            // Block 1: only other_address changes
828            writer
829                .append_account_changeset(
830                    vec![AccountBeforeTx {
831                        address: other_address,
832                        info: Some(Account::default()),
833                    }],
834                    1,
835                )
836                .unwrap();
837
838            // Block 2: test_address changes again
839            writer
840                .append_account_changeset(
841                    vec![AccountBeforeTx {
842                        address: test_address,
843                        info: Some(Account {
844                            nonce: 1,
845                            balance: U256::from(1000),
846                            ..Default::default()
847                        }),
848                    }],
849                    2,
850                )
851                .unwrap();
852
853            writer.commit().unwrap();
854        }
855
856        // Test get_account_before_block
857        {
858            // Test retrieving account state before block 0
859            let result = sf_rw.get_account_before_block(0, test_address).unwrap();
860            assert!(result.is_some());
861            let account_before = result.unwrap();
862            assert_eq!(account_before.address, test_address);
863            assert!(account_before.info.is_none()); // Was created in block 0
864
865            // Test retrieving account state before block 2
866            let result = sf_rw.get_account_before_block(2, test_address).unwrap();
867            assert!(result.is_some());
868            let account_before = result.unwrap();
869            assert_eq!(account_before.address, test_address);
870            assert!(account_before.info.is_some());
871            let info = account_before.info.unwrap();
872            assert_eq!(info.nonce, 1);
873            assert_eq!(info.balance, U256::from(1000));
874
875            // Test retrieving account that doesn't exist in changeset for block
876            let result = sf_rw.get_account_before_block(1, test_address).unwrap();
877            assert!(result.is_none()); // test_address didn't change in block 1
878
879            // Test retrieving account that never existed
880            let result = sf_rw.get_account_before_block(2, missing_address).unwrap();
881            assert!(result.is_none());
882
883            // Test other_address changes
884            let result = sf_rw.get_account_before_block(1, other_address).unwrap();
885            assert!(result.is_some());
886            let account_before = result.unwrap();
887            assert_eq!(account_before.address, other_address);
888            assert!(account_before.info.is_some());
889        }
890    }
891
892    #[test]
893    fn test_account_changeset_truncation() {
894        let (static_dir, _) = create_test_static_files_dir();
895
896        let blocks_per_file = 10;
897        // 3 main files (jar, dat, idx) + 1 csoff sidecar file for changeset segments
898        let files_per_range = 4;
899        let file_set_count = 3;
900        let initial_file_count = files_per_range * file_set_count;
901        let tip = blocks_per_file * file_set_count - 1;
902
903        // Setup: Create account changesets for multiple blocks
904        {
905            let sf_rw: StaticFileProvider<EthPrimitives> =
906                StaticFileProviderBuilder::read_write(&static_dir)
907                    .with_blocks_per_file(blocks_per_file)
908                    .build()
909                    .expect("failed to create static file provider");
910
911            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
912
913            for block_num in 0..=tip {
914                // Create varying number of changes per block
915                let num_changes = ((block_num % 5) + 1) as usize;
916                let mut changeset = Vec::with_capacity(num_changes);
917
918                for i in 0..num_changes {
919                    let mut address = Address::ZERO;
920                    address.0[0] = block_num as u8;
921                    address.0[1] = i as u8;
922
923                    changeset.push(AccountBeforeTx {
924                        address,
925                        info: Some(Account {
926                            nonce: block_num,
927                            balance: U256::from(block_num * 1000 + i as u64),
928                            ..Default::default()
929                        }),
930                    });
931                }
932
933                writer.append_account_changeset(changeset, block_num).unwrap();
934            }
935
936            writer.commit().unwrap();
937        }
938
939        // Helper function to validate truncation
940        fn validate_truncation(
941            sf_rw: &StaticFileProvider<EthPrimitives>,
942            static_dir: impl AsRef<Path>,
943            expected_tip: Option<u64>,
944            expected_file_count: u64,
945        ) -> eyre::Result<()> {
946            // Verify highest block
947            let highest_block =
948                sf_rw.get_highest_static_file_block(StaticFileSegment::AccountChangeSets);
949            assert_eyre(highest_block, expected_tip, "block tip mismatch")?;
950
951            // Verify file count
952            assert_eyre(
953                count_files_without_lockfile(static_dir)?,
954                expected_file_count as usize,
955                "file count mismatch",
956            )?;
957
958            if let Some(tip) = expected_tip {
959                // Verify we can still read data up to the tip
960                let provider = sf_rw.get_segment_provider_for_block(
961                    StaticFileSegment::AccountChangeSets,
962                    tip,
963                    None,
964                )?;
965
966                // Check offsets are valid
967                let offsets = provider.read_changeset_offsets().unwrap();
968                assert!(offsets.is_some(), "Should have changeset offsets");
969            }
970
971            Ok(())
972        }
973
974        // Test truncation scenarios
975        let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
976            .with_blocks_per_file(blocks_per_file)
977            .build()
978            .expect("failed to create static file provider");
979
980        // Re-initialize the index to ensure it knows about the written files
981        sf_rw.initialize_index().expect("Failed to initialize index");
982
983        // Case 1: Truncate to block 20 (remove last 9 blocks)
984        {
985            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
986            writer.prune_account_changesets(20).unwrap();
987            writer.commit().unwrap();
988
989            validate_truncation(&sf_rw, &static_dir, Some(20), initial_file_count)
990                .expect("Truncation validation failed");
991        }
992
993        // Case 2: Truncate to block 9 (should remove 2 files)
994        {
995            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
996            writer.prune_account_changesets(9).unwrap();
997            writer.commit().unwrap();
998
999            validate_truncation(&sf_rw, &static_dir, Some(9), files_per_range)
1000                .expect("Truncation validation failed");
1001        }
1002
1003        // Case 3: Truncate all (should keep block 0)
1004        {
1005            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
1006            writer.prune_account_changesets(0).unwrap();
1007            writer.commit().unwrap();
1008
1009            // AccountChangeSets behaves like tx-based segments and keeps at least block 0
1010            validate_truncation(&sf_rw, &static_dir, Some(0), files_per_range)
1011                .expect("Truncation validation failed");
1012        }
1013    }
1014
1015    #[test]
1016    fn test_changeset_binary_search() {
1017        let (static_dir, _) = create_test_static_files_dir();
1018
1019        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
1020            .expect("Failed to create static file provider");
1021
1022        // Create a block with many account changes to test binary search
1023        let block_num = 0u64;
1024        let num_accounts = 100;
1025
1026        let mut addresses: Vec<Address> = Vec::with_capacity(num_accounts);
1027        for i in 0..num_accounts {
1028            let mut addr = Address::ZERO;
1029            addr.0[0] = (i / 256) as u8;
1030            addr.0[1] = (i % 256) as u8;
1031            addresses.push(addr);
1032        }
1033
1034        // Write the changeset
1035        {
1036            let mut writer = sf_rw.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
1037
1038            let changeset: Vec<AccountBeforeTx> = addresses
1039                .iter()
1040                .map(|addr| AccountBeforeTx {
1041                    address: *addr,
1042                    info: Some(Account {
1043                        nonce: 1,
1044                        balance: U256::from(1000),
1045                        ..Default::default()
1046                    }),
1047                })
1048                .collect();
1049
1050            writer.append_account_changeset(changeset, block_num).unwrap();
1051            writer.commit().unwrap();
1052        }
1053
1054        // Test binary search for various addresses
1055        {
1056            // Test finding first address
1057            let result = sf_rw.get_account_before_block(block_num, addresses[0]).unwrap();
1058            assert!(result.is_some());
1059            assert_eq!(result.unwrap().address, addresses[0]);
1060
1061            // Test finding last address
1062            let result =
1063                sf_rw.get_account_before_block(block_num, addresses[num_accounts - 1]).unwrap();
1064            assert!(result.is_some());
1065            assert_eq!(result.unwrap().address, addresses[num_accounts - 1]);
1066
1067            // Test finding middle addresses
1068            let mid = num_accounts / 2;
1069            let result = sf_rw.get_account_before_block(block_num, addresses[mid]).unwrap();
1070            assert!(result.is_some());
1071            assert_eq!(result.unwrap().address, addresses[mid]);
1072
1073            // Test not finding address that doesn't exist
1074            let mut missing_addr = Address::ZERO;
1075            missing_addr.0[0] = 255;
1076            missing_addr.0[1] = 255;
1077            let result = sf_rw.get_account_before_block(block_num, missing_addr).unwrap();
1078            assert!(result.is_none());
1079
1080            // Test multiple lookups for performance
1081            for i in (0..num_accounts).step_by(10) {
1082                let result = sf_rw.get_account_before_block(block_num, addresses[i]).unwrap();
1083                assert!(result.is_some());
1084                assert_eq!(result.unwrap().address, addresses[i]);
1085            }
1086        }
1087    }
1088
1089    #[test]
1090    fn test_storage_changeset_static_files() {
1091        let (static_dir, _) = create_test_static_files_dir();
1092
1093        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
1094            .expect("Failed to create static file provider");
1095
1096        // Test writing and reading storage changesets
1097        {
1098            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1099
1100            // Create test data for multiple blocks
1101            let test_blocks = 10u64;
1102            let entries_per_block = 5;
1103
1104            for block_num in 0..test_blocks {
1105                let changeset = (0..entries_per_block)
1106                    .map(|i| {
1107                        let mut addr = Address::ZERO;
1108                        addr.0[0] = block_num as u8;
1109                        addr.0[1] = i as u8;
1110                        StorageBeforeTx {
1111                            address: addr,
1112                            key: B256::with_last_byte(i as u8),
1113                            value: U256::from(block_num * 1000 + i as u64),
1114                        }
1115                    })
1116                    .collect::<Vec<_>>();
1117
1118                writer.append_storage_changeset(changeset, block_num).unwrap();
1119            }
1120
1121            writer.commit().unwrap();
1122        }
1123
1124        // Verify data can be read back correctly
1125        {
1126            let provider = sf_rw
1127                .get_segment_provider_for_block(StaticFileSegment::StorageChangeSets, 5, None)
1128                .unwrap();
1129
1130            // Check that the segment header has changeset offsets
1131            let offsets = provider.read_changeset_offsets().unwrap();
1132            assert!(offsets.is_some());
1133            let offsets = offsets.unwrap();
1134            assert_eq!(offsets.len(), 10); // Should have 10 blocks worth of offsets
1135
1136            // Verify each block has the expected number of changes
1137            for (i, offset) in offsets.iter().enumerate() {
1138                assert_eq!(offset.num_changes(), 5, "Block {} should have 5 changes", i);
1139            }
1140        }
1141    }
1142
1143    #[test]
1144    fn test_get_storage_before_block() {
1145        let (static_dir, _) = create_test_static_files_dir();
1146
1147        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
1148            .expect("Failed to create static file provider");
1149
1150        let test_address = Address::repeat_byte(1u8);
1151        let other_address = Address::repeat_byte(2u8);
1152        let missing_address = Address::repeat_byte(3u8);
1153        let test_key = B256::with_last_byte(1);
1154        let other_key = B256::with_last_byte(2);
1155
1156        // Write changesets for multiple blocks
1157        {
1158            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1159
1160            // Block 0: test_address and other_address change
1161            writer
1162                .append_storage_changeset(
1163                    vec![
1164                        StorageBeforeTx { address: test_address, key: test_key, value: U256::ZERO },
1165                        StorageBeforeTx {
1166                            address: other_address,
1167                            key: other_key,
1168                            value: U256::from(5),
1169                        },
1170                    ],
1171                    0,
1172                )
1173                .unwrap();
1174
1175            // Block 1: only other_address changes
1176            writer
1177                .append_storage_changeset(
1178                    vec![StorageBeforeTx {
1179                        address: other_address,
1180                        key: other_key,
1181                        value: U256::from(7),
1182                    }],
1183                    1,
1184                )
1185                .unwrap();
1186
1187            // Block 2: test_address changes again
1188            writer
1189                .append_storage_changeset(
1190                    vec![StorageBeforeTx {
1191                        address: test_address,
1192                        key: test_key,
1193                        value: U256::from(9),
1194                    }],
1195                    2,
1196                )
1197                .unwrap();
1198
1199            writer.commit().unwrap();
1200        }
1201
1202        // Test get_storage_before_block
1203        {
1204            let result = sf_rw.get_storage_before_block(0, test_address, test_key).unwrap();
1205            assert!(result.is_some());
1206            let entry = result.unwrap();
1207            assert_eq!(entry.key, test_key);
1208            assert_eq!(entry.value, U256::ZERO);
1209
1210            let result = sf_rw.get_storage_before_block(2, test_address, test_key).unwrap();
1211            assert!(result.is_some());
1212            let entry = result.unwrap();
1213            assert_eq!(entry.key, test_key);
1214            assert_eq!(entry.value, U256::from(9));
1215
1216            let result = sf_rw.get_storage_before_block(1, test_address, test_key).unwrap();
1217            assert!(result.is_none());
1218
1219            let result = sf_rw.get_storage_before_block(2, missing_address, test_key).unwrap();
1220            assert!(result.is_none());
1221
1222            let result = sf_rw.get_storage_before_block(1, other_address, other_key).unwrap();
1223            assert!(result.is_some());
1224            let entry = result.unwrap();
1225            assert_eq!(entry.key, other_key);
1226        }
1227    }
1228
1229    #[test]
1230    fn test_storage_changeset_truncation() {
1231        let (static_dir, _) = create_test_static_files_dir();
1232
1233        let blocks_per_file = 10;
1234        // 3 main files (jar, dat, idx) + 1 csoff sidecar file for changeset segments
1235        let files_per_range = 4;
1236        let file_set_count = 3;
1237        let initial_file_count = files_per_range * file_set_count;
1238        let tip = blocks_per_file * file_set_count - 1;
1239
1240        // Setup: Create storage changesets for multiple blocks
1241        {
1242            let sf_rw: StaticFileProvider<EthPrimitives> =
1243                StaticFileProviderBuilder::read_write(&static_dir)
1244                    .with_blocks_per_file(blocks_per_file)
1245                    .build()
1246                    .expect("failed to create static file provider");
1247
1248            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1249
1250            for block_num in 0..=tip {
1251                let num_changes = ((block_num % 5) + 1) as usize;
1252                let mut changeset = Vec::with_capacity(num_changes);
1253
1254                for i in 0..num_changes {
1255                    let mut address = Address::ZERO;
1256                    address.0[0] = block_num as u8;
1257                    address.0[1] = i as u8;
1258
1259                    changeset.push(StorageBeforeTx {
1260                        address,
1261                        key: B256::with_last_byte(i as u8),
1262                        value: U256::from(block_num * 1000 + i as u64),
1263                    });
1264                }
1265
1266                writer.append_storage_changeset(changeset, block_num).unwrap();
1267            }
1268
1269            writer.commit().unwrap();
1270        }
1271
1272        fn validate_truncation(
1273            sf_rw: &StaticFileProvider<EthPrimitives>,
1274            static_dir: impl AsRef<Path>,
1275            expected_tip: Option<u64>,
1276            expected_file_count: u64,
1277        ) -> eyre::Result<()> {
1278            let highest_block =
1279                sf_rw.get_highest_static_file_block(StaticFileSegment::StorageChangeSets);
1280            assert_eyre(highest_block, expected_tip, "block tip mismatch")?;
1281
1282            assert_eyre(
1283                count_files_without_lockfile(static_dir)?,
1284                expected_file_count as usize,
1285                "file count mismatch",
1286            )?;
1287
1288            if let Some(tip) = expected_tip {
1289                let provider = sf_rw.get_segment_provider_for_block(
1290                    StaticFileSegment::StorageChangeSets,
1291                    tip,
1292                    None,
1293                )?;
1294                let offsets = provider.read_changeset_offsets()?;
1295                assert!(offsets.is_some(), "Should have changeset offsets");
1296            }
1297
1298            Ok(())
1299        }
1300
1301        let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
1302            .with_blocks_per_file(blocks_per_file)
1303            .build()
1304            .expect("failed to create static file provider");
1305
1306        sf_rw.initialize_index().expect("Failed to initialize index");
1307
1308        // Case 1: Truncate to block 20
1309        {
1310            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1311            writer.prune_storage_changesets(20).unwrap();
1312            writer.commit().unwrap();
1313
1314            validate_truncation(&sf_rw, &static_dir, Some(20), initial_file_count)
1315                .expect("Truncation validation failed");
1316        }
1317
1318        // Case 2: Truncate to block 9
1319        {
1320            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1321            writer.prune_storage_changesets(9).unwrap();
1322            writer.commit().unwrap();
1323
1324            validate_truncation(&sf_rw, &static_dir, Some(9), files_per_range)
1325                .expect("Truncation validation failed");
1326        }
1327
1328        // Case 3: Truncate all (should keep block 0)
1329        {
1330            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1331            writer.prune_storage_changesets(0).unwrap();
1332            writer.commit().unwrap();
1333
1334            validate_truncation(&sf_rw, &static_dir, Some(0), files_per_range)
1335                .expect("Truncation validation failed");
1336        }
1337    }
1338
1339    #[test]
1340    fn test_storage_changeset_binary_search() {
1341        let (static_dir, _) = create_test_static_files_dir();
1342
1343        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
1344            .expect("Failed to create static file provider");
1345
1346        let block_num = 0u64;
1347        let num_slots = 100;
1348        let address = Address::repeat_byte(4u8);
1349
1350        let mut keys: Vec<B256> = Vec::with_capacity(num_slots);
1351        for i in 0..num_slots {
1352            keys.push(B256::with_last_byte(i as u8));
1353        }
1354
1355        {
1356            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1357            let changeset = keys
1358                .iter()
1359                .enumerate()
1360                .map(|(i, key)| StorageBeforeTx { address, key: *key, value: U256::from(i as u64) })
1361                .collect::<Vec<_>>();
1362
1363            writer.append_storage_changeset(changeset, block_num).unwrap();
1364            writer.commit().unwrap();
1365        }
1366
1367        {
1368            let result = sf_rw.get_storage_before_block(block_num, address, keys[0]).unwrap();
1369            assert!(result.is_some());
1370            let entry = result.unwrap();
1371            assert_eq!(entry.key, keys[0]);
1372            assert_eq!(entry.value, U256::ZERO);
1373
1374            let result =
1375                sf_rw.get_storage_before_block(block_num, address, keys[num_slots - 1]).unwrap();
1376            assert!(result.is_some());
1377            let entry = result.unwrap();
1378            assert_eq!(entry.key, keys[num_slots - 1]);
1379
1380            let mid = num_slots / 2;
1381            let result = sf_rw.get_storage_before_block(block_num, address, keys[mid]).unwrap();
1382            assert!(result.is_some());
1383            let entry = result.unwrap();
1384            assert_eq!(entry.key, keys[mid]);
1385
1386            let missing_key = B256::with_last_byte(255);
1387            let result = sf_rw.get_storage_before_block(block_num, address, missing_key).unwrap();
1388            assert!(result.is_none());
1389
1390            for i in (0..num_slots).step_by(10) {
1391                let result = sf_rw.get_storage_before_block(block_num, address, keys[i]).unwrap();
1392                assert!(result.is_some());
1393                assert_eq!(result.unwrap().key, keys[i]);
1394            }
1395        }
1396    }
1397
1398    #[test]
1399    fn test_last_block_flushed_on_commit() {
1400        let (static_dir, _) = create_test_static_files_dir();
1401
1402        let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
1403            .expect("Failed to create static file provider");
1404
1405        let address = Address::repeat_byte(5u8);
1406        let key = B256::with_last_byte(1);
1407
1408        // Write changes for a single block without calling increment_block explicitly
1409        // (append_storage_changeset calls it internally), then commit
1410        {
1411            let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
1412
1413            // Append a single block's changeset (block 0)
1414            writer
1415                .append_storage_changeset(
1416                    vec![StorageBeforeTx { address, key, value: U256::from(42) }],
1417                    0,
1418                )
1419                .unwrap();
1420
1421            // Commit without any subsequent block - the current block's offset should be flushed
1422            writer.commit().unwrap();
1423        }
1424
1425        // Verify highest block is 0
1426        let highest = sf_rw.get_highest_static_file_block(StaticFileSegment::StorageChangeSets);
1427        assert_eq!(highest, Some(0), "Should have block 0 after commit");
1428
1429        // Verify the data is actually readable via the high-level API
1430        let result = sf_rw.get_storage_before_block(0, address, key).unwrap();
1431        assert!(result.is_some(), "Should be able to read the changeset entry");
1432        let entry = result.unwrap();
1433        assert_eq!(entry.value, U256::from(42));
1434    }
1435
1436    #[test]
1437    fn test_read_headers_while_appending() {
1438        let (static_dir, _) = create_test_static_files_dir();
1439        let sf_rw: StaticFileProvider<EthPrimitives> =
1440            StaticFileProviderBuilder::read_write(&static_dir)
1441                .build()
1442                .expect("Failed to build static file provider");
1443        let hash = |num: u64| B256::with_last_byte(num as u8 + 1);
1444
1445        let mut header_writer = sf_rw.latest_writer(StaticFileSegment::Headers).unwrap();
1446        let mut header = Header::default();
1447        for num in 0..=2 {
1448            header.number = num;
1449            header_writer.append_header(&header, &hash(num)).unwrap();
1450        }
1451        header_writer.commit().unwrap();
1452
1453        // Writes the next header to disk without committing it, as persistence does before the
1454        // database commit.
1455        header.number = 3;
1456        header_writer.append_header(&header, &hash(3)).unwrap();
1457        header_writer.sync_all().unwrap();
1458
1459        // Reloads the jar from disk meanwhile, as every static file deletion by the pruner does.
1460        sf_rw.initialize_index().unwrap();
1461
1462        assert_eq!(sf_rw.block_hash(2).unwrap(), Some(hash(2)));
1463        assert_eq!(sf_rw.block_hash(3).unwrap(), None);
1464    }
1465
1466    #[test]
1467    fn test_header_read_errors_are_not_missing_blocks() {
1468        for corrupt_offsets in [true, false] {
1469            let (static_dir, _) = create_test_static_files_dir();
1470            let provider = StaticFileProvider::<EthPrimitives>::read_write(&static_dir).unwrap();
1471            let header = Header::default();
1472            let hash = header.hash_slow();
1473            {
1474                let mut writer = provider.latest_writer(StaticFileSegment::Headers).unwrap();
1475                writer.append_header(&header, &hash).unwrap();
1476                writer.commit().unwrap();
1477            }
1478
1479            assert_eq!(provider.header_by_number(0).unwrap(), Some(header));
1480            assert_eq!(provider.block_hash(0).unwrap(), Some(hash));
1481            assert_eq!(provider.header_by_number(1).unwrap(), None);
1482            assert_eq!(provider.block_hash(1).unwrap(), None);
1483
1484            let jar = provider
1485                .get_segment_provider_for_block(StaticFileSegment::Headers, 0, None)
1486                .unwrap();
1487            let path =
1488                if corrupt_offsets { jar.offsets_path() } else { jar.data_path().to_path_buf() };
1489            drop(jar);
1490            drop(provider);
1491
1492            // Corrupt the files before opening any readers, retaining the committed row count.
1493            let mut bytes = reth_fs_util::read(&path).unwrap();
1494            if corrupt_offsets {
1495                // Retain the offset-size byte, but point every offset beyond the data file.
1496                bytes[1..].fill(u8::MAX);
1497            } else {
1498                // An unterminated LZ4 literal-length sequence cannot be decompressed.
1499                bytes.fill(u8::MAX);
1500            }
1501            reth_fs_util::write(&path, bytes).unwrap();
1502
1503            let provider = StaticFileProvider::<EthPrimitives>::read_only(&static_dir).unwrap();
1504            for error in [
1505                provider.header_by_number(0).unwrap_err(),
1506                provider.block_hash(0).unwrap_err(),
1507                provider.sealed_header(0).unwrap_err(),
1508                provider.headers_range(0..1).unwrap_err(),
1509            ] {
1510                let error = error.downcast_other_ref::<NippyJarError>().unwrap();
1511                if corrupt_offsets {
1512                    assert_matches!(error, NippyJarError::InconsistentState);
1513                } else {
1514                    assert_matches!(error, NippyJarError::OutputTooSmall);
1515                }
1516            }
1517            assert_eq!(provider.header_by_number(1).unwrap(), None);
1518            assert_eq!(provider.block_hash(1).unwrap(), None);
1519        }
1520    }
1521}
reth_provider/providers/static_file/mod.rs

reth_provider/providers/static_file/
mod.rs
