1mod bodies;
3mod era;
4mod execution;
6mod finish;
8mod hashing_account;
10mod hashing_storage;
12mod headers;
14mod index_account_history;
16mod index_storage_history;
18mod merkle;
20mod prune;
21mod sender_recovery;
23mod tx_lookup;
25
26pub use bodies::*;
27pub use era::*;
28pub use execution::*;
29pub use finish::*;
30pub use hashing_account::*;
31pub use hashing_storage::*;
32pub use headers::*;
33pub use index_account_history::*;
34pub use index_storage_history::*;
35pub use merkle::*;
36pub use prune::*;
37pub use sender_recovery::*;
38pub use tx_lookup::*;
39
40mod utils;
41use utils::*;
42
43#[cfg(test)]
44mod tests {
45 use super::*;
46 use crate::test_utils::{StorageKind, TestStageDB};
47 use alloy_consensus::{SignableTransaction, TxLegacy};
48 use alloy_primitives::{
49 address, hex_literal::hex, keccak256, BlockNumber, Signature, B256, U256,
50 };
51 use alloy_rlp::Decodable;
52 use reth_chainspec::ChainSpecBuilder;
53 use reth_db::mdbx::{cursor::Cursor, RW};
54 use reth_db_api::{
55 cursor::{DbCursorRO, DbCursorRW},
56 models::StorageSettings,
57 table::Table,
58 tables,
59 transaction::{DbTx, DbTxMut},
60 AccountsHistory,
61 };
62 use reth_ethereum_consensus::EthBeaconConsensus;
63 use reth_ethereum_primitives::Block;
64 use reth_evm_ethereum::EthEvmConfig;
65 use reth_exex::ExExManagerHandle;
66 use reth_primitives_traits::{Account, Bytecode, SealedBlock};
67 use reth_provider::{
68 providers::{StaticFileProvider, StaticFileWriter},
69 test_utils::MockNodeTypesWithDB,
70 AccountExtReader, BlockBodyIndicesProvider, BlockWriter, DatabaseProviderFactory,
71 ProviderFactory, ProviderResult, PruneCheckpointWriter, ReceiptProvider,
72 StageCheckpointWriter, StaticFileProviderFactory, StorageReader,
73 };
74 use reth_prune_types::{PruneCheckpoint, PruneMode, PruneModes, PruneSegment};
75 use reth_stages_api::{
76 ExecInput, ExecutionStageThresholds, PipelineTarget, Stage, StageCheckpoint, StageId,
77 };
78 use reth_static_file_types::StaticFileSegment;
79 use reth_storage_api::StorageSettingsCache;
80 use reth_testing_utils::generators::{
81 self, random_block, random_block_range, random_receipt, BlockRangeParams,
82 };
83 use std::{io::Write, sync::Arc};
84
85 #[tokio::test]
86 #[ignore]
87 async fn test_prune() {
88 let test_db = TestStageDB::default();
89
90 let provider_rw = test_db.factory.provider_rw().unwrap();
91 let tip = 66;
92 let input = ExecInput { target: Some(tip), checkpoint: None };
93 let mut genesis_rlp = hex!("f901faf901f5a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa045571b40ae66ca7480791bbb2887286e4e4c4b1b298b191c889d6959023a32eda056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000808502540be400808000a00000000000000000000000000000000000000000000000000000000000000000880000000000000000c0c0").as_slice();
94 let genesis = SealedBlock::<Block>::decode(&mut genesis_rlp).unwrap();
95 let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice();
96 let block = SealedBlock::<Block>::decode(&mut block_rlp).unwrap();
97 let mut head = block.hash();
98 provider_rw.insert_block(&genesis.try_recover().unwrap()).unwrap();
99 provider_rw.insert_block(&block.try_recover().unwrap()).unwrap();
100
101 let mut rng = generators::rng();
103 for block_number in 2..=tip {
104 let nblock = random_block(
105 &mut rng,
106 block_number,
107 generators::BlockParams { parent: Some(head), ..Default::default() },
108 );
109 head = nblock.hash();
110 provider_rw.insert_block(&nblock.try_recover().unwrap()).unwrap();
111 }
112 provider_rw
113 .static_file_provider()
114 .latest_writer(StaticFileSegment::Headers)
115 .unwrap()
116 .commit()
117 .unwrap();
118 provider_rw.commit().unwrap();
119
120 let provider_rw = test_db.factory.provider_rw().unwrap();
122 let code = hex!("5a465a905090036002900360015500");
123 let code_hash = keccak256(hex!("5a465a905090036002900360015500"));
124 provider_rw
125 .tx_ref()
126 .put::<tables::PlainAccountState>(
127 address!("0x1000000000000000000000000000000000000000"),
128 Account { nonce: 0, balance: U256::ZERO, bytecode_hash: Some(code_hash) },
129 )
130 .unwrap();
131 provider_rw
132 .tx_ref()
133 .put::<tables::PlainAccountState>(
134 address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"),
135 Account {
136 nonce: 0,
137 balance: U256::from(0x3635c9adc5dea00000u128),
138 bytecode_hash: None,
139 },
140 )
141 .unwrap();
142 provider_rw
143 .tx_ref()
144 .put::<tables::Bytecodes>(code_hash, Bytecode::new_raw(code.to_vec().into()))
145 .unwrap();
146 provider_rw.commit().unwrap();
147
148 let check_pruning = |factory: ProviderFactory<MockNodeTypesWithDB>,
149 prune_modes: PruneModes,
150 expect_num_receipts: usize,
151 expect_num_acc_changesets: usize,
152 expect_num_storage_changesets: usize| async move {
153 let provider = factory.database_provider_rw().unwrap();
154
155 let mut execution_stage = ExecutionStage::new(
158 EthEvmConfig::ethereum(Arc::new(
159 ChainSpecBuilder::mainnet().berlin_activated().build(),
160 )),
161 Arc::new(EthBeaconConsensus::new(Arc::new(
162 ChainSpecBuilder::mainnet().berlin_activated().build(),
163 ))),
164 ExecutionStageThresholds {
165 max_blocks: Some(100),
166 max_changes: None,
167 max_cumulative_gas: None,
168 max_duration: None,
169 },
170 MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD,
171 ExExManagerHandle::empty(),
172 );
173
174 execution_stage.execute(&provider, input).unwrap();
175 assert_eq!(
176 provider.receipts_by_block(1.into()).unwrap().unwrap().len(),
177 expect_num_receipts
178 );
179
180 assert_eq!(
181 provider.changed_storages_and_blocks_with_range(0..=1000).unwrap().len(),
182 expect_num_storage_changesets
183 );
184
185 assert_eq!(
186 provider.changed_accounts_and_blocks_with_range(0..=1000).unwrap().len(),
187 expect_num_acc_changesets
188 );
189
190 let mut acc_indexing_stage = IndexAccountHistoryStage {
192 prune_mode: prune_modes.account_history,
193 ..Default::default()
194 };
195
196 if prune_modes.account_history == Some(PruneMode::Full) {
197 assert!(acc_indexing_stage.execute(&provider, input).is_err());
199 } else {
200 acc_indexing_stage.execute(&provider, input).unwrap();
201 let mut account_history: Cursor<RW, AccountsHistory> =
202 provider.tx_ref().cursor_read::<tables::AccountsHistory>().unwrap();
203 assert_eq!(account_history.walk(None).unwrap().count(), expect_num_acc_changesets);
204 }
205
206 let mut storage_indexing_stage = IndexStorageHistoryStage {
208 prune_mode: prune_modes.storage_history,
209 ..Default::default()
210 };
211
212 if prune_modes.storage_history == Some(PruneMode::Full) {
213 assert!(storage_indexing_stage.execute(&provider, input).is_err());
215 } else {
216 storage_indexing_stage.execute(&provider, input).unwrap();
217
218 let mut storage_history =
219 provider.tx_ref().cursor_read::<tables::StoragesHistory>().unwrap();
220 assert_eq!(
221 storage_history.walk(None).unwrap().count(),
222 expect_num_storage_changesets
223 );
224 }
225 };
226
227 let mut prune = PruneModes::default();
230 check_pruning(test_db.factory.clone(), prune.clone(), 1, 3, 1).await;
231
232 prune.receipts = Some(PruneMode::Full);
233 prune.account_history = Some(PruneMode::Full);
234 prune.storage_history = Some(PruneMode::Full);
235 check_pruning(test_db.factory.clone(), prune.clone(), 0, 0, 0).await;
237
238 prune.receipts = Some(PruneMode::Before(1));
239 prune.account_history = Some(PruneMode::Before(1));
240 prune.storage_history = Some(PruneMode::Before(1));
241 check_pruning(test_db.factory.clone(), prune.clone(), 1, 3, 1).await;
242
243 prune.receipts = Some(PruneMode::Before(2));
244 prune.account_history = Some(PruneMode::Before(2));
245 prune.storage_history = Some(PruneMode::Before(2));
246 check_pruning(test_db.factory.clone(), prune.clone(), 0, 1, 0).await;
248
249 prune.receipts = Some(PruneMode::Distance(66));
250 prune.account_history = Some(PruneMode::Distance(66));
251 prune.storage_history = Some(PruneMode::Distance(66));
252 check_pruning(test_db.factory.clone(), prune.clone(), 1, 3, 1).await;
253
254 prune.receipts = Some(PruneMode::Distance(64));
255 prune.account_history = Some(PruneMode::Distance(64));
256 prune.storage_history = Some(PruneMode::Distance(64));
257 check_pruning(test_db.factory.clone(), prune.clone(), 0, 1, 0).await;
259 }
260
261 fn seed_data(num_blocks: usize) -> ProviderResult<TestStageDB> {
264 let db = TestStageDB::default();
265 let mut rng = generators::rng();
266 let genesis_hash = B256::ZERO;
267 let tip = (num_blocks - 1) as u64;
268
269 let blocks = random_block_range(
270 &mut rng,
271 0..=tip,
272 BlockRangeParams { parent: Some(genesis_hash), tx_count: 2..3, ..Default::default() },
273 );
274 db.insert_blocks(blocks.iter(), StorageKind::Static)?;
275
276 let mut receipts = Vec::with_capacity(blocks.len());
277 let mut tx_num = 0u64;
278 for block in &blocks {
279 let mut block_receipts = Vec::with_capacity(block.transaction_count());
280 for transaction in &block.body().transactions {
281 block_receipts.push((tx_num, random_receipt(&mut rng, transaction, Some(0), None)));
282 tx_num += 1;
283 }
284 receipts.push((block.number, block_receipts));
285 }
286 db.insert_receipts_by_block(receipts, StorageKind::Static)?;
287
288 let provider_rw = db.factory.provider_rw()?;
290 for stage in StageId::ALL {
291 provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(tip))?;
292 }
293 provider_rw.commit()?;
294
295 Ok(db)
296 }
297
298 fn seed_v2_data() -> TestStageDB {
299 let db = seed_data(90).unwrap();
300 db.factory.set_storage_settings_cache(StorageSettings::v2());
301 db
302 }
303
304 fn save_prune_checkpoints(
305 db: &TestStageDB,
306 checkpoints: impl IntoIterator<Item = (PruneSegment, BlockNumber, PruneMode)>,
307 ) {
308 let provider_rw = db.factory.provider_rw().unwrap();
309 for (segment, block_number, prune_mode) in checkpoints {
310 provider_rw
311 .save_prune_checkpoint(
312 segment,
313 PruneCheckpoint {
314 block_number: Some(block_number),
315 tx_number: None,
316 prune_mode,
317 },
318 )
319 .unwrap();
320 }
321 provider_rw.commit().unwrap();
322 }
323
324 fn assert_consistency(db: &TestStageDB, expected: Option<PipelineTarget>) {
325 assert_eq!(
326 db.factory
327 .static_file_provider()
328 .check_consistency(&db.factory.database_provider_ro().unwrap())
329 .unwrap(),
330 expected
331 );
332 }
333
334 fn simulate_behind_checkpoint_corruption(
337 db: &TestStageDB,
338 prune_count: usize,
339 segment: StaticFileSegment,
340 expected: Option<PipelineTarget>,
341 ) {
342 let mut static_file_provider = db.factory.static_file_provider();
345 static_file_provider = StaticFileProvider::read_write(static_file_provider.path()).unwrap();
346
347 {
350 let mut headers_writer = static_file_provider.latest_writer(segment).unwrap();
351 let reader = headers_writer.inner().jar().open_data_reader().unwrap();
352 let columns = headers_writer.inner().jar().columns();
353 let data_file = headers_writer.inner().data_file();
354 let last_offset = reader.reverse_offset(prune_count * columns).unwrap();
355 data_file.get_mut().set_len(last_offset).unwrap();
356 data_file.flush().unwrap();
357 data_file.get_ref().sync_all().unwrap();
358 }
359
360 let mut static_file_provider = db.factory.static_file_provider();
363 static_file_provider = StaticFileProvider::read_write(static_file_provider.path()).unwrap();
364 assert!(matches!(
365 static_file_provider
366 .check_consistency(&db.factory.database_provider_ro().unwrap()),
367 Ok(e) if e == expected
368 ));
369 }
370
371 fn save_checkpoint_and_check(
374 db: &TestStageDB,
375 stage_id: StageId,
376 checkpoint_block_number: BlockNumber,
377 expected: Option<PipelineTarget>,
378 ) {
379 let provider_rw = db.factory.provider_rw().unwrap();
380 provider_rw
381 .save_stage_checkpoint(stage_id, StageCheckpoint::new(checkpoint_block_number))
382 .unwrap();
383 provider_rw.commit().unwrap();
384
385 assert!(matches!(
386 db.factory
387 .static_file_provider()
388 .check_consistency(&db.factory.database_provider_ro().unwrap(),),
389 Ok(e) if e == expected
390 ));
391 }
392
393 fn update_db_and_check<T: Table<Key = u64>>(
396 db: &TestStageDB,
397 key: u64,
398 expected: Option<PipelineTarget>,
399 ) where
400 <T as Table>::Value: Default,
401 {
402 update_db_with_and_check::<T>(db, key, expected, &Default::default());
403 }
404
405 fn update_db_with_and_check<T: Table<Key = u64>>(
408 db: &TestStageDB,
409 key: u64,
410 expected: Option<PipelineTarget>,
411 value: &T::Value,
412 ) {
413 let provider_rw = db.factory.provider_rw().unwrap();
414 let mut cursor = provider_rw.tx_ref().cursor_write::<T>().unwrap();
415 cursor.insert(key, value).unwrap();
416 provider_rw.commit().unwrap();
417
418 assert!(matches!(
419 db.factory
420 .static_file_provider()
421 .check_consistency(&db.factory.database_provider_ro().unwrap()),
422 Ok(e) if e == expected
423 ));
424 }
425
426 #[test]
427 fn test_consistency() {
428 let db = seed_data(90).unwrap();
429 let db_provider = db.factory.database_provider_ro().unwrap();
430
431 assert!(matches!(
432 db.factory.static_file_provider().check_consistency(&db_provider),
433 Ok(None)
434 ));
435 }
436
437 #[test]
438 fn test_consistency_no_commit_prune() {
439 let mut db_full = seed_data(90).unwrap();
441 db_full.factory = db_full.factory.with_prune_modes(PruneModes {
442 receipts: Some(PruneMode::Before(1)),
443 ..Default::default()
444 });
445
446 simulate_behind_checkpoint_corruption(&db_full, 1, StaticFileSegment::Receipts, None);
449
450 let db_archive = seed_data(90).unwrap();
452
453 simulate_behind_checkpoint_corruption(
456 &db_archive,
457 1,
458 StaticFileSegment::Receipts,
459 Some(PipelineTarget::Unwind(88)),
460 );
461
462 simulate_behind_checkpoint_corruption(
463 &db_archive,
464 3,
465 StaticFileSegment::Headers,
466 Some(PipelineTarget::Unwind(86)),
467 );
468 }
469
470 #[test]
471 fn test_consistency_checkpoints() {
472 let db = seed_data(90).unwrap();
473
474 let block = 87;
476 save_checkpoint_and_check(&db, StageId::Bodies, block, None);
477 assert_eq!(
478 db.factory
479 .static_file_provider()
480 .get_highest_static_file_block(StaticFileSegment::Transactions),
481 Some(block)
482 );
483 assert_eq!(
484 db.factory
485 .static_file_provider()
486 .get_highest_static_file_tx(StaticFileSegment::Transactions),
487 db.factory.block_body_indices(block).unwrap().map(|b| b.last_tx_num())
488 );
489
490 let block = 86;
491 save_checkpoint_and_check(&db, StageId::Execution, block, None);
492 assert_eq!(
493 db.factory
494 .static_file_provider()
495 .get_highest_static_file_block(StaticFileSegment::Receipts),
496 Some(block)
497 );
498 assert_eq!(
499 db.factory
500 .static_file_provider()
501 .get_highest_static_file_tx(StaticFileSegment::Receipts),
502 db.factory.block_body_indices(block).unwrap().map(|b| b.last_tx_num())
503 );
504
505 let block = 80;
506 save_checkpoint_and_check(&db, StageId::Headers, block, None);
507 assert_eq!(
508 db.factory
509 .static_file_provider()
510 .get_highest_static_file_block(StaticFileSegment::Headers),
511 Some(block)
512 );
513
514 save_checkpoint_and_check(&db, StageId::Headers, 91, Some(PipelineTarget::Unwind(block)));
516 }
517
518 #[test]
519 fn test_consistency_headers_gap() {
520 let db = seed_data(90).unwrap();
521 let current = db
522 .factory
523 .static_file_provider()
524 .get_highest_static_file_block(StaticFileSegment::Headers)
525 .unwrap();
526
527 update_db_and_check::<tables::Headers>(&db, current + 2, Some(PipelineTarget::Unwind(89)));
529
530 update_db_and_check::<tables::Headers>(&db, current + 1, None);
532 }
533
534 #[test]
535 fn test_consistency_tx_gap() {
536 let db = seed_data(90).unwrap();
537 let current = db
538 .factory
539 .static_file_provider()
540 .get_highest_static_file_tx(StaticFileSegment::Transactions)
541 .unwrap();
542
543 update_db_with_and_check::<tables::Transactions>(
545 &db,
546 current + 2,
547 Some(PipelineTarget::Unwind(89)),
548 &TxLegacy::default().into_signed(Signature::test_signature()).into(),
549 );
550
551 update_db_with_and_check::<tables::Transactions>(
553 &db,
554 current + 1,
555 None,
556 &TxLegacy::default().into_signed(Signature::test_signature()).into(),
557 );
558 }
559
560 #[test]
561 fn test_consistency_receipt_gap() {
562 let db = seed_data(90).unwrap();
563 let current = db
564 .factory
565 .static_file_provider()
566 .get_highest_static_file_tx(StaticFileSegment::Receipts)
567 .unwrap();
568
569 update_db_and_check::<tables::Receipts>(&db, current + 2, Some(PipelineTarget::Unwind(89)));
571
572 update_db_and_check::<tables::Receipts>(&db, current + 1, None);
574 }
575
576 #[test]
577 fn test_consistency_pruned_v2_segments() {
578 for prune_mode in [PruneMode::Distance(2_000_000), PruneMode::Before(90)] {
579 let db = seed_v2_data();
580
581 assert_consistency(&db, Some(PipelineTarget::Unwind(0)));
583
584 save_prune_checkpoints(
585 &db,
586 [
587 PruneSegment::SenderRecovery,
588 PruneSegment::AccountHistory,
589 PruneSegment::StorageHistory,
590 ]
591 .map(|segment| (segment, 89, prune_mode)),
592 );
593 assert_consistency(&db, None);
594 }
595 }
596
597 #[test]
598 fn test_consistency_unwind_bounded_by_prune_checkpoint() {
599 let db = seed_v2_data();
600
601 save_prune_checkpoints(
605 &db,
606 [
607 (PruneSegment::SenderRecovery, 50, PruneMode::Distance(39)),
608 (PruneSegment::AccountHistory, 89, PruneMode::Distance(2_000_000)),
609 (PruneSegment::StorageHistory, 89, PruneMode::Distance(2_000_000)),
610 ],
611 );
612 assert_consistency(&db, Some(PipelineTarget::Unwind(50)));
613 }
614
615 #[test]
616 fn test_consistency_receipts_distance_prune_checkpoint() {
617 let db = seed_data(90).unwrap();
618 let static_file_provider = db.factory.static_file_provider();
619
620 while let Some(block) =
623 static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts)
624 {
625 static_file_provider.delete_jar(StaticFileSegment::Receipts, block).unwrap();
626 }
627
628 assert_consistency(&db, Some(PipelineTarget::Unwind(0)));
629
630 save_prune_checkpoints(&db, [(PruneSegment::Receipts, 89, PruneMode::Distance(2_000_000))]);
631 assert_consistency(&db, None);
632 }
633}