1use alloy_consensus::BlockHeader;
4use alloy_genesis::GenesisAccount;
5use alloy_primitives::{map::HashMap, Address, B256, U256};
6use reth_chainspec::EthChainSpec;
7use reth_codecs::Compact;
8use reth_config::config::EtlConfig;
9use reth_db_api::{tables, transaction::DbTxMut, DatabaseError};
10use reth_etl::Collector;
11use reth_primitives_traits::{Account, Bytecode, GotExpected, NodePrimitives, StorageEntry};
12use reth_provider::{
13 errors::provider::ProviderResult, providers::StaticFileWriter, writer::UnifiedStorageWriter,
14 BlockHashReader, BlockNumReader, BundleStateInit, ChainSpecProvider, DBProvider,
15 DatabaseProviderFactory, ExecutionOutcome, HashingWriter, HeaderProvider, HistoryWriter,
16 OriginalValuesKnown, ProviderError, RevertsInit, StageCheckpointReader, StageCheckpointWriter,
17 StateWriter, StaticFileProviderFactory, StorageLocation, TrieWriter,
18};
19use reth_stages_types::{StageCheckpoint, StageId};
20use reth_static_file_types::StaticFileSegment;
21use reth_trie::{IntermediateStateRootState, StateRoot as StateRootComputer, StateRootProgress};
22use reth_trie_db::DatabaseStateRoot;
23use serde::{Deserialize, Serialize};
24use std::io::BufRead;
25use tracing::{debug, error, info, trace};
26
27pub const DEFAULT_SOFT_LIMIT_BYTE_LEN_ACCOUNTS_CHUNK: usize = 1_000_000_000;
32
33pub const AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP: usize = 285_228;
40
41const SOFT_LIMIT_COUNT_FLUSHED_UPDATES: usize = 1_000_000;
43
44#[derive(Debug, thiserror::Error, Clone)]
46pub enum InitStorageError {
47 #[error("static files found, but the database is uninitialized. If attempting to re-syncing, delete both.")]
49 UninitializedDatabase,
50 #[error("genesis hash in the storage does not match the specified chainspec: chainspec is {chainspec_hash}, database is {storage_hash}")]
53 GenesisHashMismatch {
54 chainspec_hash: B256,
56 storage_hash: B256,
58 },
59 #[error(transparent)]
61 Provider(#[from] ProviderError),
62 #[error("state root mismatch: {_0}")]
64 StateRootMismatch(GotExpected<B256>),
65}
66
67impl From<DatabaseError> for InitStorageError {
68 fn from(error: DatabaseError) -> Self {
69 Self::Provider(ProviderError::Database(error))
70 }
71}
72
73pub fn init_genesis<PF>(factory: &PF) -> Result<B256, InitStorageError>
75where
76 PF: DatabaseProviderFactory
77 + StaticFileProviderFactory<Primitives: NodePrimitives<BlockHeader: Compact>>
78 + ChainSpecProvider
79 + StageCheckpointReader
80 + BlockHashReader,
81 PF::ProviderRW: StaticFileProviderFactory<Primitives = PF::Primitives>
82 + StageCheckpointWriter
83 + HistoryWriter
84 + HeaderProvider
85 + HashingWriter
86 + StateWriter
87 + AsRef<PF::ProviderRW>,
88 PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
89{
90 let chain = factory.chain_spec();
91
92 let genesis = chain.genesis();
93 let hash = chain.genesis_hash();
94
95 match factory.block_hash(0) {
97 Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {}
98 Ok(Some(block_hash)) => {
99 if block_hash == hash {
100 if factory.get_stage_checkpoint(StageId::Headers)?.is_none() {
104 error!(target: "reth::storage", "Genesis header found on static files, but database is uninitialized.");
105 return Err(InitStorageError::UninitializedDatabase)
106 }
107
108 debug!("Genesis already written, skipping.");
109 return Ok(hash)
110 }
111
112 return Err(InitStorageError::GenesisHashMismatch {
113 chainspec_hash: hash,
114 storage_hash: block_hash,
115 })
116 }
117 Err(e) => {
118 debug!(?e);
119 return Err(e.into());
120 }
121 }
122
123 debug!("Writing genesis block.");
124
125 let alloc = &genesis.alloc;
126
127 let provider_rw = factory.database_provider_rw()?;
129 insert_genesis_hashes(&provider_rw, alloc.iter())?;
130 insert_genesis_history(&provider_rw, alloc.iter())?;
131
132 insert_genesis_header(&provider_rw, &chain)?;
134
135 insert_genesis_state(&provider_rw, alloc.iter())?;
136
137 for stage in StageId::ALL {
139 provider_rw.save_stage_checkpoint(stage, Default::default())?;
140 }
141
142 let static_file_provider = provider_rw.static_file_provider();
143 let segment = StaticFileSegment::Receipts;
145 static_file_provider.latest_writer(segment)?.increment_block(0)?;
146
147 let segment = StaticFileSegment::Transactions;
148 static_file_provider.latest_writer(segment)?.increment_block(0)?;
149
150 UnifiedStorageWriter::commit_unwind(provider_rw)?;
153
154 Ok(hash)
155}
156
157pub fn insert_genesis_state<'a, 'b, Provider>(
159 provider: &Provider,
160 alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)>,
161) -> ProviderResult<()>
162where
163 Provider: StaticFileProviderFactory
164 + DBProvider<Tx: DbTxMut>
165 + HeaderProvider
166 + StateWriter
167 + AsRef<Provider>,
168{
169 insert_state(provider, alloc, 0)
170}
171
172pub fn insert_state<'a, 'b, Provider>(
174 provider: &Provider,
175 alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)>,
176 block: u64,
177) -> ProviderResult<()>
178where
179 Provider: StaticFileProviderFactory
180 + DBProvider<Tx: DbTxMut>
181 + HeaderProvider
182 + StateWriter
183 + AsRef<Provider>,
184{
185 let capacity = alloc.size_hint().1.unwrap_or(0);
186 let mut state_init: BundleStateInit =
187 HashMap::with_capacity_and_hasher(capacity, Default::default());
188 let mut reverts_init = HashMap::with_capacity_and_hasher(capacity, Default::default());
189 let mut contracts: HashMap<B256, Bytecode> =
190 HashMap::with_capacity_and_hasher(capacity, Default::default());
191
192 for (address, account) in alloc {
193 let bytecode_hash = if let Some(code) = &account.code {
194 match Bytecode::new_raw_checked(code.clone()) {
195 Ok(bytecode) => {
196 let hash = bytecode.hash_slow();
197 contracts.insert(hash, bytecode);
198 Some(hash)
199 }
200 Err(err) => {
201 error!(%address, %err, "Failed to decode genesis bytecode.");
202 return Err(DatabaseError::Other(err.to_string()).into());
203 }
204 }
205 } else {
206 None
207 };
208
209 let storage = account
211 .storage
212 .as_ref()
213 .map(|m| {
214 m.iter()
215 .map(|(key, value)| {
216 let value = U256::from_be_bytes(value.0);
217 (*key, (U256::ZERO, value))
218 })
219 .collect::<HashMap<_, _>>()
220 })
221 .unwrap_or_default();
222
223 reverts_init.insert(
224 *address,
225 (Some(None), storage.keys().map(|k| StorageEntry::new(*k, U256::ZERO)).collect()),
226 );
227
228 state_init.insert(
229 *address,
230 (
231 None,
232 Some(Account {
233 nonce: account.nonce.unwrap_or_default(),
234 balance: account.balance,
235 bytecode_hash,
236 }),
237 storage,
238 ),
239 );
240 }
241 let all_reverts_init: RevertsInit = HashMap::from_iter([(block, reverts_init)]);
242
243 let execution_outcome = ExecutionOutcome::new_init(
244 state_init,
245 all_reverts_init,
246 contracts,
247 Vec::default(),
248 block,
249 Vec::new(),
250 );
251
252 provider.write_state(
253 &execution_outcome,
254 OriginalValuesKnown::Yes,
255 StorageLocation::Database,
256 )?;
257
258 trace!(target: "reth::cli", "Inserted state");
259
260 Ok(())
261}
262
263pub fn insert_genesis_hashes<'a, 'b, Provider>(
265 provider: &Provider,
266 alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
267) -> ProviderResult<()>
268where
269 Provider: DBProvider<Tx: DbTxMut> + HashingWriter,
270{
271 let alloc_accounts = alloc.clone().map(|(addr, account)| (*addr, Some(Account::from(account))));
273 provider.insert_account_for_hashing(alloc_accounts)?;
274
275 trace!(target: "reth::cli", "Inserted account hashes");
276
277 let alloc_storage = alloc.filter_map(|(addr, account)| {
278 account.storage.as_ref().map(|storage| {
280 (*addr, storage.iter().map(|(&key, &value)| StorageEntry { key, value: value.into() }))
281 })
282 });
283 provider.insert_storage_for_hashing(alloc_storage)?;
284
285 trace!(target: "reth::cli", "Inserted storage hashes");
286
287 Ok(())
288}
289
290pub fn insert_genesis_history<'a, 'b, Provider>(
292 provider: &Provider,
293 alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
294) -> ProviderResult<()>
295where
296 Provider: DBProvider<Tx: DbTxMut> + HistoryWriter,
297{
298 insert_history(provider, alloc, 0)
299}
300
301pub fn insert_history<'a, 'b, Provider>(
303 provider: &Provider,
304 alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
305 block: u64,
306) -> ProviderResult<()>
307where
308 Provider: DBProvider<Tx: DbTxMut> + HistoryWriter,
309{
310 let account_transitions = alloc.clone().map(|(addr, _)| (*addr, [block]));
311 provider.insert_account_history_index(account_transitions)?;
312
313 trace!(target: "reth::cli", "Inserted account history");
314
315 let storage_transitions = alloc
316 .filter_map(|(addr, account)| account.storage.as_ref().map(|storage| (addr, storage)))
317 .flat_map(|(addr, storage)| storage.iter().map(|(key, _)| ((*addr, *key), [block])));
318 provider.insert_storage_history_index(storage_transitions)?;
319
320 trace!(target: "reth::cli", "Inserted storage history");
321
322 Ok(())
323}
324
325pub fn insert_genesis_header<Provider, Spec>(
327 provider: &Provider,
328 chain: &Spec,
329) -> ProviderResult<()>
330where
331 Provider: StaticFileProviderFactory<Primitives: NodePrimitives<BlockHeader: Compact>>
332 + DBProvider<Tx: DbTxMut>,
333 Spec: EthChainSpec<Header = <Provider::Primitives as NodePrimitives>::BlockHeader>,
334{
335 let (header, block_hash) = (chain.genesis_header(), chain.genesis_hash());
336 let static_file_provider = provider.static_file_provider();
337
338 match static_file_provider.block_hash(0) {
339 Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {
340 let (difficulty, hash) = (header.difficulty(), block_hash);
341 let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
342 writer.append_header(header, difficulty, &hash)?;
343 }
344 Ok(Some(_)) => {}
345 Err(e) => return Err(e),
346 }
347
348 provider.tx_ref().put::<tables::HeaderNumbers>(block_hash, 0)?;
349 provider.tx_ref().put::<tables::BlockBodyIndices>(0, Default::default())?;
350
351 Ok(())
352}
353
354pub fn init_from_state_dump<Provider>(
361 mut reader: impl BufRead,
362 provider_rw: &Provider,
363 etl_config: EtlConfig,
364) -> eyre::Result<B256>
365where
366 Provider: StaticFileProviderFactory
367 + DBProvider<Tx: DbTxMut>
368 + BlockNumReader
369 + BlockHashReader
370 + ChainSpecProvider
371 + StageCheckpointWriter
372 + HistoryWriter
373 + HeaderProvider
374 + HashingWriter
375 + TrieWriter
376 + StateWriter
377 + AsRef<Provider>,
378{
379 if etl_config.file_size == 0 {
380 return Err(eyre::eyre!("ETL file size cannot be zero"))
381 }
382
383 let block = provider_rw.last_block_number()?;
384 let hash = provider_rw.block_hash(block)?.unwrap();
385 let expected_state_root = provider_rw
386 .header_by_number(block)?
387 .ok_or_else(|| ProviderError::HeaderNotFound(block.into()))?
388 .state_root();
389
390 let dump_state_root = parse_state_root(&mut reader)?;
392 if expected_state_root != dump_state_root {
393 error!(target: "reth::cli",
394 ?dump_state_root,
395 ?expected_state_root,
396 "State root from state dump does not match state root in current header."
397 );
398 return Err(InitStorageError::StateRootMismatch(GotExpected {
399 got: dump_state_root,
400 expected: expected_state_root,
401 })
402 .into())
403 }
404
405 debug!(target: "reth::cli",
406 block,
407 chain=%provider_rw.chain_spec().chain(),
408 "Initializing state at block"
409 );
410
411 let collector = parse_accounts(&mut reader, etl_config)?;
413
414 dump_state(collector, provider_rw, block)?;
416
417 let computed_state_root = compute_state_root(provider_rw)?;
419 if computed_state_root == expected_state_root {
420 info!(target: "reth::cli",
421 ?computed_state_root,
422 "Computed state root matches state root in state dump"
423 );
424 } else {
425 error!(target: "reth::cli",
426 ?computed_state_root,
427 ?expected_state_root,
428 "Computed state root does not match state root in state dump"
429 );
430
431 return Err(InitStorageError::StateRootMismatch(GotExpected {
432 got: computed_state_root,
433 expected: expected_state_root,
434 })
435 .into())
436 }
437
438 for stage in StageId::STATE_REQUIRED {
440 provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(block))?;
441 }
442
443 Ok(hash)
444}
445
446fn parse_state_root(reader: &mut impl BufRead) -> eyre::Result<B256> {
448 let mut line = String::new();
449 reader.read_line(&mut line)?;
450
451 let expected_state_root = serde_json::from_str::<StateRoot>(&line)?.root;
452 trace!(target: "reth::cli",
453 root=%expected_state_root,
454 "Read state root from file"
455 );
456 Ok(expected_state_root)
457}
458
459fn parse_accounts(
461 mut reader: impl BufRead,
462 etl_config: EtlConfig,
463) -> Result<Collector<Address, GenesisAccount>, eyre::Error> {
464 let mut line = String::new();
465 let mut collector = Collector::new(etl_config.file_size, etl_config.dir);
466
467 while let Ok(n) = reader.read_line(&mut line) {
468 if n == 0 {
469 break
470 }
471
472 let GenesisAccountWithAddress { genesis_account, address } = serde_json::from_str(&line)?;
473 collector.insert(address, genesis_account)?;
474
475 if !collector.is_empty() && collector.len() % AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP == 0
476 {
477 info!(target: "reth::cli",
478 parsed_new_accounts=collector.len(),
479 );
480 }
481
482 line.clear();
483 }
484
485 Ok(collector)
486}
487
488fn dump_state<Provider>(
490 mut collector: Collector<Address, GenesisAccount>,
491 provider_rw: &Provider,
492 block: u64,
493) -> Result<(), eyre::Error>
494where
495 Provider: StaticFileProviderFactory
496 + DBProvider<Tx: DbTxMut>
497 + HeaderProvider
498 + HashingWriter
499 + HistoryWriter
500 + StateWriter
501 + AsRef<Provider>,
502{
503 let accounts_len = collector.len();
504 let mut accounts = Vec::with_capacity(AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP);
505 let mut total_inserted_accounts = 0;
506
507 for (index, entry) in collector.iter()?.enumerate() {
508 let (address, account) = entry?;
509 let (address, _) = Address::from_compact(address.as_slice(), address.len());
510 let (account, _) = GenesisAccount::from_compact(account.as_slice(), account.len());
511
512 accounts.push((address, account));
513
514 if (index > 0 && index % AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP == 0) ||
515 index == accounts_len - 1
516 {
517 total_inserted_accounts += accounts.len();
518
519 info!(target: "reth::cli",
520 total_inserted_accounts,
521 "Writing accounts to db"
522 );
523
524 insert_genesis_hashes(
526 provider_rw,
527 accounts.iter().map(|(address, account)| (address, account)),
528 )?;
529
530 insert_history(
531 provider_rw,
532 accounts.iter().map(|(address, account)| (address, account)),
533 block,
534 )?;
535
536 insert_state(
538 provider_rw,
539 accounts.iter().map(|(address, account)| (address, account)),
540 block,
541 )?;
542
543 accounts.clear();
544 }
545 }
546 Ok(())
547}
548
549fn compute_state_root<Provider>(provider: &Provider) -> eyre::Result<B256>
552where
553 Provider: DBProvider<Tx: DbTxMut> + TrieWriter,
554{
555 trace!(target: "reth::cli", "Computing state root");
556
557 let tx = provider.tx_ref();
558 let mut intermediate_state: Option<IntermediateStateRootState> = None;
559 let mut total_flushed_updates = 0;
560
561 loop {
562 match StateRootComputer::from_tx(tx)
563 .with_intermediate_state(intermediate_state)
564 .root_with_progress()?
565 {
566 StateRootProgress::Progress(state, _, updates) => {
567 let updated_len = provider.write_trie_updates(&updates)?;
568 total_flushed_updates += updated_len;
569
570 trace!(target: "reth::cli",
571 last_account_key = %state.last_account_key,
572 updated_len,
573 total_flushed_updates,
574 "Flushing trie updates"
575 );
576
577 intermediate_state = Some(*state);
578
579 if total_flushed_updates % SOFT_LIMIT_COUNT_FLUSHED_UPDATES == 0 {
580 info!(target: "reth::cli",
581 total_flushed_updates,
582 "Flushing trie updates"
583 );
584 }
585 }
586 StateRootProgress::Complete(root, _, updates) => {
587 let updated_len = provider.write_trie_updates(&updates)?;
588 total_flushed_updates += updated_len;
589
590 trace!(target: "reth::cli",
591 %root,
592 updated_len,
593 total_flushed_updates,
594 "State root has been computed"
595 );
596
597 return Ok(root)
598 }
599 }
600 }
601}
602
603#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
605struct StateRoot {
606 root: B256,
607}
608
609#[derive(Debug, Serialize, Deserialize)]
612struct GenesisAccountWithAddress {
613 #[serde(flatten)]
615 genesis_account: GenesisAccount,
616 address: Address,
618}
619
620#[cfg(test)]
621mod tests {
622 use super::*;
623 use alloy_consensus::constants::{
624 HOLESKY_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH,
625 };
626 use alloy_genesis::Genesis;
627 use reth_chainspec::{Chain, ChainSpec, HOLESKY, MAINNET, SEPOLIA};
628 use reth_db::DatabaseEnv;
629 use reth_db_api::{
630 cursor::DbCursorRO,
631 models::{storage_sharded_key::StorageShardedKey, IntegerList, ShardedKey},
632 table::{Table, TableRow},
633 transaction::DbTx,
634 Database,
635 };
636 use reth_provider::{
637 test_utils::{create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB},
638 ProviderFactory,
639 };
640 use std::{collections::BTreeMap, sync::Arc};
641
642 fn collect_table_entries<DB, T>(
643 tx: &<DB as Database>::TX,
644 ) -> Result<Vec<TableRow<T>>, InitStorageError>
645 where
646 DB: Database,
647 T: Table,
648 {
649 Ok(tx.cursor_read::<T>()?.walk_range(..)?.collect::<Result<Vec<_>, _>>()?)
650 }
651
652 #[test]
653 fn success_init_genesis_mainnet() {
654 let genesis_hash =
655 init_genesis(&create_test_provider_factory_with_chain_spec(MAINNET.clone())).unwrap();
656
657 assert_eq!(genesis_hash, MAINNET_GENESIS_HASH);
659 }
660
661 #[test]
662 fn success_init_genesis_sepolia() {
663 let genesis_hash =
664 init_genesis(&create_test_provider_factory_with_chain_spec(SEPOLIA.clone())).unwrap();
665
666 assert_eq!(genesis_hash, SEPOLIA_GENESIS_HASH);
668 }
669
670 #[test]
671 fn success_init_genesis_holesky() {
672 let genesis_hash =
673 init_genesis(&create_test_provider_factory_with_chain_spec(HOLESKY.clone())).unwrap();
674
675 assert_eq!(genesis_hash, HOLESKY_GENESIS_HASH);
677 }
678
679 #[test]
680 fn fail_init_inconsistent_db() {
681 let factory = create_test_provider_factory_with_chain_spec(SEPOLIA.clone());
682 let static_file_provider = factory.static_file_provider();
683 init_genesis(&factory).unwrap();
684
685 let genesis_hash = init_genesis(&ProviderFactory::<MockNodeTypesWithDB>::new(
687 factory.into_db(),
688 MAINNET.clone(),
689 static_file_provider,
690 ));
691
692 assert!(matches!(
693 genesis_hash.unwrap_err(),
694 InitStorageError::GenesisHashMismatch {
695 chainspec_hash: MAINNET_GENESIS_HASH,
696 storage_hash: SEPOLIA_GENESIS_HASH
697 }
698 ))
699 }
700
701 #[test]
702 fn init_genesis_history() {
703 let address_with_balance = Address::with_last_byte(1);
704 let address_with_storage = Address::with_last_byte(2);
705 let storage_key = B256::with_last_byte(1);
706 let chain_spec = Arc::new(ChainSpec {
707 chain: Chain::from_id(1),
708 genesis: Genesis {
709 alloc: BTreeMap::from([
710 (
711 address_with_balance,
712 GenesisAccount { balance: U256::from(1), ..Default::default() },
713 ),
714 (
715 address_with_storage,
716 GenesisAccount {
717 storage: Some(BTreeMap::from([(storage_key, B256::random())])),
718 ..Default::default()
719 },
720 ),
721 ]),
722 ..Default::default()
723 },
724 hardforks: Default::default(),
725 paris_block_and_final_difficulty: None,
726 deposit_contract: None,
727 ..Default::default()
728 });
729
730 let factory = create_test_provider_factory_with_chain_spec(chain_spec);
731 init_genesis(&factory).unwrap();
732
733 let provider = factory.provider().unwrap();
734
735 let tx = provider.tx_ref();
736
737 assert_eq!(
738 collect_table_entries::<Arc<DatabaseEnv>, tables::AccountsHistory>(tx)
739 .expect("failed to collect"),
740 vec![
741 (ShardedKey::new(address_with_balance, u64::MAX), IntegerList::new([0]).unwrap()),
742 (ShardedKey::new(address_with_storage, u64::MAX), IntegerList::new([0]).unwrap())
743 ],
744 );
745
746 assert_eq!(
747 collect_table_entries::<Arc<DatabaseEnv>, tables::StoragesHistory>(tx)
748 .expect("failed to collect"),
749 vec![(
750 StorageShardedKey::new(address_with_storage, storage_key, u64::MAX),
751 IntegerList::new([0]).unwrap()
752 )],
753 );
754 }
755}