1mod archive;
81pub mod config_gen;
82mod extract;
83mod fetch;
84pub mod manifest;
85pub mod manifest_cmd;
86mod planning;
87mod progress;
88mod session;
89mod source;
90mod tui;
91mod verify;
92
93pub use planning::{DownloadPlan, DownloadPlanArchive};
94
95use crate::common::EnvironmentArgs;
96use archive::run_modular_downloads;
97use clap::{builder::RangedU64ValueParser, Parser};
98use config_gen::{config_for_selections, write_config};
99use extract::stream_and_extract;
100use eyre::Result;
101use manifest::{ComponentSelection, SnapshotComponentType, SnapshotManifest};
102use planning::{collect_planned_archives, summarize_download_startup, PlannedDownloads};
103use progress::{DownloadProgress, DownloadRequestLimiter};
104use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks, MAINNET};
105use reth_cli::chainspec::ChainSpecParser;
106use reth_cli_util::cancellation::CancellationToken;
107use reth_db::{init_db, Database};
108use reth_db_api::transaction::DbTx;
109use reth_fs_util as fs;
110use reth_node_core::args::DefaultPruningValues;
111use reth_prune_types::PruneMode;
112use source::{
113 discover_manifest_url, fetch_manifest_from_source, fetch_snapshot_api_entries,
114 print_snapshot_listing, resolve_manifest_base_url,
115};
116use std::{
117 borrow::Cow,
118 collections::BTreeMap,
119 path::{Path, PathBuf},
120 sync::{Arc, OnceLock},
121};
122use tracing::info;
123use tui::{run_selector, SelectorOutput};
124
125const RETH_SNAPSHOTS_BASE_URL: &str = "https://snapshots-r2.reth.rs";
126const RETH_SNAPSHOTS_API_URL: &str = "https://snapshots.reth.rs/api/snapshots";
127const RETH_SNAPSHOTS_SOURCE: &str = "https://snapshots.reth.rs (default)";
128const SNAPSHOT_API_PATH: &str = "/api/snapshots";
129const FORCE_REMOVED_DATADIR_PATHS: &[&str] = &["db", "rocksdb", "static_files", "reth.toml"];
130
131const MAX_CONCURRENT_DOWNLOADS: usize = 8;
133
134#[derive(Debug, Clone, Copy, PartialEq, Eq)]
136pub(crate) enum SelectionPreset {
137 Minimal,
139 Full,
141 Archive,
143}
144
145struct ResolvedComponents {
146 selections: BTreeMap<SnapshotComponentType, ComponentSelection>,
147 preset: Option<SelectionPreset>,
148}
149
150static DOWNLOAD_DEFAULTS: OnceLock<DownloadDefaults> = OnceLock::new();
152
153#[derive(Debug, Clone)]
157pub struct DownloadDefaults {
158 pub available_snapshots: Vec<Cow<'static, str>>,
160 pub default_base_url: Cow<'static, str>,
162 pub default_chain_aware_base_url: Option<Cow<'static, str>>,
170 pub snapshot_api_url: Cow<'static, str>,
174 pub long_help: Option<String>,
176}
177
178impl DownloadDefaults {
179 pub fn try_init(self) -> Result<(), Self> {
181 DOWNLOAD_DEFAULTS.set(self)
182 }
183
184 pub fn get_global() -> &'static DownloadDefaults {
186 DOWNLOAD_DEFAULTS.get_or_init(DownloadDefaults::default_download_defaults)
187 }
188
189 pub fn default_download_defaults() -> Self {
191 Self {
192 available_snapshots: vec![
193 Cow::Borrowed(RETH_SNAPSHOTS_SOURCE),
194 Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"),
195 ],
196 default_base_url: Cow::Borrowed(RETH_SNAPSHOTS_BASE_URL),
197 default_chain_aware_base_url: None,
198 snapshot_api_url: Cow::Borrowed(RETH_SNAPSHOTS_API_URL),
199 long_help: None,
200 }
201 }
202
203 pub fn long_help(&self) -> String {
208 if let Some(ref custom_help) = self.long_help {
209 return custom_help.clone();
210 }
211
212 let implicit_download_help = if self.mainnet_only_discovery() {
213 "\nIf no URL is provided, the latest archive snapshot will only be proposed\nfor Ethereum mainnet. For other chains, provide --manifest-url, --manifest-path,\nor -u explicitly."
214 } else {
215 "\nIf no URL is provided, the latest archive snapshot for the selected chain\nwill be proposed for download from "
216 };
217
218 let mut help = format!(
219 "Specify a snapshot URL or let the command propose a default one.\n\n\
220 Browse available snapshots at {}\n\
221 or use --list-snapshots to see them from the CLI.\n\nAvailable snapshot sources:\n",
222 self.snapshot_source_url(),
223 );
224
225 for source in &self.available_snapshots {
226 help.push_str("- ");
227 help.push_str(source);
228 help.push('\n');
229 }
230
231 help.push_str(implicit_download_help);
232 if !self.mainnet_only_discovery() {
233 help.push_str(
234 self.default_chain_aware_base_url.as_deref().unwrap_or(&self.default_base_url),
235 );
236 help.push('.');
237 }
238 help.push_str(
239 "\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
240 );
241 help
242 }
243
244 fn mainnet_only_discovery(&self) -> bool {
245 self.snapshot_api_url.trim_end_matches('/') == RETH_SNAPSHOTS_API_URL
246 }
247
248 fn snapshot_source_url(&self) -> &str {
249 snapshot_source_url_from_api(&self.snapshot_api_url)
250 }
251
252 pub fn with_snapshot(mut self, source: impl Into<Cow<'static, str>>) -> Self {
254 self.available_snapshots.push(source.into());
255 self
256 }
257
258 pub fn with_snapshots(mut self, sources: Vec<Cow<'static, str>>) -> Self {
260 self.available_snapshots = sources;
261 self
262 }
263
264 pub fn with_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
266 self.default_base_url = url.into();
267 self
268 }
269
270 pub fn with_chain_aware_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
272 self.default_chain_aware_base_url = Some(url.into());
273 self
274 }
275
276 pub fn with_snapshot_api_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
281 self.snapshot_api_url = url.into();
282
283 let source_url = self.snapshot_source_url().to_string();
284 if self.default_chain_aware_base_url.is_none() {
285 self.default_chain_aware_base_url = Some(Cow::Owned(source_url.clone()));
286 }
287 for source in &mut self.available_snapshots {
288 if source.as_ref() == RETH_SNAPSHOTS_SOURCE {
289 *source = Cow::Owned(format!("{source_url} (default)"));
290 }
291 }
292
293 self
294 }
295
296 pub fn with_snapshot_source_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
301 let source_url = normalize_snapshot_source_url(url.into());
302 self.available_snapshots = vec![Cow::Owned(format!("{} (default)", source_url.as_ref()))];
303 self.default_base_url = source_url.clone();
304 self.default_chain_aware_base_url = Some(source_url.clone());
305 self.snapshot_api_url = Cow::Owned(format!("{}{SNAPSHOT_API_PATH}", source_url.as_ref()));
306 self
307 }
308
309 pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
311 self.long_help = Some(help.into());
312 self
313 }
314}
315
316fn snapshot_source_url_from_api(api_url: &str) -> &str {
317 api_url.trim_end_matches('/').trim_end_matches(SNAPSHOT_API_PATH)
318}
319
320fn normalize_snapshot_source_url(url: Cow<'static, str>) -> Cow<'static, str> {
321 match url {
322 Cow::Borrowed(url) => Cow::Borrowed(snapshot_source_url_from_api(url)),
323 Cow::Owned(url) => Cow::Owned(snapshot_source_url_from_api(&url).to_string()),
324 }
325}
326
327impl Default for DownloadDefaults {
328 fn default() -> Self {
330 Self::default_download_defaults()
331 }
332}
333
334#[derive(Debug, Parser)]
336pub struct DownloadCommand<C: ChainSpecParser> {
337 #[command(flatten)]
338 env: EnvironmentArgs<C>,
339
340 #[arg(long, short, long_help = DownloadDefaults::get_global().long_help())]
345 url: Option<String>,
346
347 #[arg(long, value_name = "URL", conflicts_with = "url")]
352 manifest_url: Option<String>,
353
354 #[arg(long, value_name = "PATH", conflicts_with_all = ["url", "manifest_url"])]
356 manifest_path: Option<PathBuf>,
357
358 #[arg(long, conflicts_with_all = ["with_txs_since", "with_txs_distance", "minimal", "full", "archive"])]
360 with_txs: bool,
361
362 #[arg(long, value_name = "BLOCK_NUMBER", conflicts_with_all = ["with_txs", "with_txs_distance", "minimal", "full", "archive"])]
364 with_txs_since: Option<u64>,
365
366 #[arg(long, value_name = "BLOCKS", value_parser = RangedU64ValueParser::<u64>::new().range(1..), conflicts_with_all = ["with_txs", "with_txs_since", "minimal", "full", "archive"])]
368 with_txs_distance: Option<u64>,
369
370 #[arg(long, conflicts_with_all = ["with_receipts_since", "with_receipts_distance", "minimal", "full", "archive"])]
372 with_receipts: bool,
373
374 #[arg(long, value_name = "BLOCK_NUMBER", conflicts_with_all = ["with_receipts", "with_receipts_distance", "minimal", "full", "archive"])]
376 with_receipts_since: Option<u64>,
377
378 #[arg(long, value_name = "BLOCKS", value_parser = RangedU64ValueParser::<u64>::new().range(1..), conflicts_with_all = ["with_receipts", "with_receipts_since", "minimal", "full", "archive"])]
380 with_receipts_distance: Option<u64>,
381
382 #[arg(long, alias = "with-changesets", conflicts_with_all = ["with_state_history_since", "with_state_history_distance", "minimal", "full", "archive"])]
384 with_state_history: bool,
385
386 #[arg(long, value_name = "BLOCK_NUMBER", conflicts_with_all = ["with_state_history", "with_state_history_distance", "minimal", "full", "archive"])]
388 with_state_history_since: Option<u64>,
389
390 #[arg(long, value_name = "BLOCKS", value_parser = RangedU64ValueParser::<u64>::new().range(1..), conflicts_with_all = ["with_state_history", "with_state_history_since", "minimal", "full", "archive"])]
392 with_state_history_distance: Option<u64>,
393
394 #[arg(long, requires = "with_txs", conflicts_with_all = ["minimal", "full", "archive"])]
396 with_senders: bool,
397
398 #[arg(long, conflicts_with_all = ["minimal", "full", "archive", "without_rocksdb"])]
400 with_rocksdb: bool,
401
402 #[arg(long, alias = "all", conflicts_with_all = ["with_txs", "with_txs_since", "with_txs_distance", "with_receipts", "with_receipts_since", "with_receipts_distance", "with_state_history", "with_state_history_since", "with_state_history_distance", "with_senders", "with_rocksdb", "minimal", "full"])]
404 archive: bool,
405
406 #[arg(long, conflicts_with_all = ["with_txs", "with_txs_since", "with_txs_distance", "with_receipts", "with_receipts_since", "with_receipts_distance", "with_state_history", "with_state_history_since", "with_state_history_distance", "with_senders", "with_rocksdb", "archive", "full"])]
408 minimal: bool,
409
410 #[arg(long, conflicts_with_all = ["with_txs", "with_txs_since", "with_txs_distance", "with_receipts", "with_receipts_since", "with_receipts_distance", "with_state_history", "with_state_history_since", "with_state_history_distance", "with_senders", "with_rocksdb", "archive", "minimal"])]
412 full: bool,
413
414 #[arg(long, conflicts_with_all = ["url", "with_rocksdb"])]
418 without_rocksdb: bool,
419
420 #[arg(long, short = 'y')]
423 non_interactive: bool,
424
425 #[arg(long, conflicts_with = "list")]
427 force: bool,
428
429 #[arg(long, default_value_t = true, num_args = 0..=1, default_missing_value = "true")]
436 resumable: bool,
437
438 #[arg(long, default_value_t = MAX_CONCURRENT_DOWNLOADS)]
443 download_concurrency: usize,
444
445 #[arg(long, alias = "list-snapshots", conflicts_with_all = ["url", "manifest_url", "manifest_path"])]
450 list: bool,
451
452 #[arg(long, conflicts_with_all = ["url", "list"])]
454 print_plan_json: bool,
455}
456
457impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCommand<C> {
458 pub async fn execute<N>(self) -> Result<Option<PreparedSnapshotDownload>> {
460 let chain = self.env.chain.chain();
461
462 if self.list {
464 let entries = fetch_snapshot_api_entries(chain.id()).await?;
465 print_snapshot_listing(&entries, chain.id());
466 return Ok(None);
467 }
468
469 if let Some(ref url) = self.url {
471 let cancel_token = CancellationToken::new();
472 let _cancel_guard = cancel_token.drop_guard();
473 let data_dir = self.env.datadir.clone().resolve_datadir(chain);
474 let target_dir = data_dir.data_dir();
475 if self.force {
476 clear_existing_datadir(target_dir)?;
477 }
478 fs::create_dir_all(target_dir)?;
479
480 let request_limiter = DownloadRequestLimiter::new(self.download_concurrency.max(1));
481 info!(target: "reth::cli",
482 dir = ?data_dir.data_dir(),
483 url = %url,
484 "Starting snapshot download and extraction"
485 );
486
487 stream_and_extract(
488 url,
489 data_dir.data_dir(),
490 None,
491 self.resumable,
492 Some(request_limiter),
493 cancel_token.clone(),
494 )
495 .await?;
496 info!(target: "reth::cli", "Snapshot downloaded and extracted successfully");
497
498 return Ok(None);
499 }
500
501 let ResolvedDownload { manifest, selections, preset, planned } =
502 self.resolve_download(chain.id()).await?;
503 let data_dir = self.env.datadir.clone().resolve_datadir(chain).data_dir().to_path_buf();
504 let prepared = PreparedSnapshotDownload { manifest, data_dir };
505 if self.print_plan_json {
506 DownloadPlan::from_planned(&prepared.manifest, &planned)
507 .write_json(std::io::stdout().lock())?;
508 return Ok(Some(prepared))
509 }
510
511 let target_dir = prepared.data_dir.as_path();
512 let cancel_token = CancellationToken::new();
513 let _cancel_guard = cancel_token.drop_guard();
514 if self.force {
515 clear_existing_datadir(target_dir)?;
516 }
517 fs::create_dir_all(target_dir)?;
518 let startup_summary = summarize_download_startup(&planned.archives, target_dir)?;
519 info!(target: "reth::cli",
520 reusable = startup_summary.reusable,
521 needs_download = startup_summary.needs_download,
522 "Startup integrity summary (plain output files)"
523 );
524
525 info!(target: "reth::cli",
526 archives = planned.total_archives(),
527 download_total = %DownloadProgress::format_size(planned.total_download_size),
528 output_total = %DownloadProgress::format_size(planned.total_output_size),
529 "Downloading all archives"
530 );
531
532 run_modular_downloads(
533 planned,
534 target_dir,
535 self.download_concurrency.max(1),
536 cancel_token.clone(),
537 )
538 .await?;
539
540 self.finalize_modular_download(
541 &selections,
542 &prepared.manifest,
543 preset,
544 target_dir,
545 &target_dir.join("db"),
546 )?;
547
548 Ok(Some(prepared))
549 }
550
551 pub async fn plan(&self) -> Result<(DownloadPlan, PreparedSnapshotDownload)> {
554 let chain = self.env.chain.chain();
555 let resolved = self.resolve_download(chain.id()).await?;
556 let plan = DownloadPlan::from_planned(&resolved.manifest, &resolved.planned);
557 let prepared = PreparedSnapshotDownload {
558 manifest: resolved.manifest,
559 data_dir: self.env.datadir.clone().resolve_datadir(chain).data_dir().to_path_buf(),
560 };
561 Ok((plan, prepared))
562 }
563
564 async fn resolve_download(&self, chain_id: u64) -> Result<ResolvedDownload> {
565 let manifest = self.load_manifest(chain_id).await?;
566 let ResolvedComponents { mut selections, preset } = self.resolve_components(&manifest)?;
567
568 if matches!(preset, Some(SelectionPreset::Archive)) {
569 inject_archive_only_components(&mut selections, &manifest, !self.without_rocksdb);
570 }
571
572 let planned = collect_planned_archives(&manifest, &selections)?;
573 Ok(ResolvedDownload { manifest, selections, preset, planned })
574 }
575
576 async fn load_manifest(&self, chain_id: u64) -> Result<SnapshotManifest> {
578 let manifest_source = self.resolve_manifest_source(chain_id).await?;
579
580 info!(target: "reth::cli", source = %manifest_source, "Fetching snapshot manifest");
581 let mut manifest = fetch_manifest_from_source(&manifest_source).await?;
582 manifest.base_url = Some(resolve_manifest_base_url(&manifest, &manifest_source)?);
583
584 info!(target: "reth::cli",
585 block = manifest.block,
586 chain_id = manifest.chain_id,
587 storage_version = %manifest.storage_version,
588 components = manifest.components.len(),
589 "Loaded snapshot manifest"
590 );
591
592 Ok(manifest)
593 }
594
595 fn finalize_modular_download(
597 &self,
598 selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
599 manifest: &SnapshotManifest,
600 preset: Option<SelectionPreset>,
601 target_dir: &Path,
602 db_path: &Path,
603 ) -> Result<()> {
604 let config =
605 config_for_selections(selections, manifest, preset, Some(self.env.chain.as_ref()));
606 if write_config(&config, target_dir)? {
607 let desc = config_gen::describe_prune_config(&config);
608 info!(target: "reth::cli", "{}", desc.join(", "));
609 }
610
611 let db = init_db(db_path, self.env.db.database_args())?;
612 let should_write_prune = config.prune.segments != Default::default();
613 let should_reset_indices = should_reset_index_stage_checkpoints(selections);
614 if should_write_prune || should_reset_indices {
615 let tx = db.tx_mut()?;
616
617 if should_write_prune {
618 config_gen::write_prune_checkpoints_tx(&tx, &config, manifest.block)?;
619 }
620
621 if should_reset_indices {
622 config_gen::reset_index_stage_checkpoints_tx(&tx)?;
623 }
624
625 tx.commit()?;
626 }
627
628 let start_command = startup_node_command::<C>(self.env.chain.as_ref());
629 info!(target: "reth::cli", "Snapshot download complete. Run `{}` to start syncing.", start_command);
630
631 Ok(())
632 }
633
634 fn resolve_components(&self, manifest: &SnapshotManifest) -> Result<ResolvedComponents> {
636 let available = |ty: SnapshotComponentType| manifest.component(ty).is_some();
637
638 if self.archive {
640 return Ok(ResolvedComponents {
641 selections: SnapshotComponentType::ALL
642 .iter()
643 .copied()
644 .filter(|ty| available(*ty))
645 .filter(|ty| {
646 !self.without_rocksdb || *ty != SnapshotComponentType::RocksdbIndices
647 })
648 .map(|ty| (ty, ComponentSelection::All))
649 .collect(),
650 preset: Some(SelectionPreset::Archive),
651 });
652 }
653
654 if self.full {
655 return Ok(ResolvedComponents {
656 selections: self.full_preset_selections(manifest),
657 preset: Some(SelectionPreset::Full),
658 });
659 }
660
661 if self.minimal {
662 return Ok(ResolvedComponents {
663 selections: self.minimal_preset_selections(manifest),
664 preset: Some(SelectionPreset::Minimal),
665 });
666 }
667
668 let has_explicit_flags = self.with_txs ||
669 self.with_txs_since.is_some() ||
670 self.with_txs_distance.is_some() ||
671 self.with_receipts ||
672 self.with_receipts_since.is_some() ||
673 self.with_receipts_distance.is_some() ||
674 self.with_state_history ||
675 self.with_state_history_since.is_some() ||
676 self.with_state_history_distance.is_some() ||
677 self.with_senders ||
678 self.with_rocksdb;
679
680 if has_explicit_flags {
681 let mut selections = BTreeMap::new();
682 let tx_selection = explicit_component_selection(
683 self.with_txs,
684 self.with_txs_since,
685 self.with_txs_distance,
686 manifest.block,
687 );
688 let receipt_selection = explicit_component_selection(
689 self.with_receipts,
690 self.with_receipts_since,
691 self.with_receipts_distance,
692 manifest.block,
693 );
694 let state_history_selection = explicit_component_selection(
695 self.with_state_history,
696 self.with_state_history_since,
697 self.with_state_history_distance,
698 manifest.block,
699 );
700
701 if available(SnapshotComponentType::State) {
703 selections.insert(SnapshotComponentType::State, ComponentSelection::All);
704 }
705 if available(SnapshotComponentType::Headers) {
706 selections.insert(SnapshotComponentType::Headers, ComponentSelection::All);
707 }
708 if let Some(selection) = tx_selection &&
709 available(SnapshotComponentType::Transactions)
710 {
711 selections.insert(SnapshotComponentType::Transactions, selection);
712 }
713 if let Some(selection) = receipt_selection &&
714 available(SnapshotComponentType::Receipts)
715 {
716 selections.insert(SnapshotComponentType::Receipts, selection);
717 }
718 if let Some(selection) = state_history_selection {
719 if available(SnapshotComponentType::AccountChangesets) {
720 selections.insert(SnapshotComponentType::AccountChangesets, selection);
721 }
722 if available(SnapshotComponentType::StorageChangesets) {
723 selections.insert(SnapshotComponentType::StorageChangesets, selection);
724 }
725 }
726 if self.with_senders && available(SnapshotComponentType::TransactionSenders) {
727 selections
728 .insert(SnapshotComponentType::TransactionSenders, ComponentSelection::All);
729 }
730 if self.with_rocksdb && available(SnapshotComponentType::RocksdbIndices) {
731 selections.insert(SnapshotComponentType::RocksdbIndices, ComponentSelection::All);
732 }
733 return Ok(ResolvedComponents { selections, preset: None });
734 }
735
736 if self.non_interactive {
737 return Ok(ResolvedComponents {
738 selections: self.minimal_preset_selections(manifest),
739 preset: Some(SelectionPreset::Minimal),
740 });
741 }
742
743 let full_preset = self.full_preset_selections(manifest);
745 let SelectorOutput { selections, preset } = run_selector(manifest.clone(), &full_preset)?;
746 let selected =
747 selections.into_iter().filter(|(_, sel)| *sel != ComponentSelection::None).collect();
748
749 Ok(ResolvedComponents { selections: selected, preset })
750 }
751
752 fn minimal_preset_selections(
754 &self,
755 manifest: &SnapshotManifest,
756 ) -> BTreeMap<SnapshotComponentType, ComponentSelection> {
757 SnapshotComponentType::ALL
758 .iter()
759 .copied()
760 .filter(|ty| manifest.component(*ty).is_some())
761 .map(|ty| (ty, ty.minimal_selection()))
762 .collect()
763 }
764
765 fn full_preset_selections(
767 &self,
768 manifest: &SnapshotManifest,
769 ) -> BTreeMap<SnapshotComponentType, ComponentSelection> {
770 let mut selections = BTreeMap::new();
771
772 for ty in [
773 SnapshotComponentType::State,
774 SnapshotComponentType::Headers,
775 SnapshotComponentType::Transactions,
776 SnapshotComponentType::Receipts,
777 SnapshotComponentType::AccountChangesets,
778 SnapshotComponentType::StorageChangesets,
779 SnapshotComponentType::TransactionSenders,
780 SnapshotComponentType::RocksdbIndices,
781 ] {
782 if manifest.component(ty).is_none() {
783 continue;
784 }
785
786 let selection = self.full_selection_for_component(ty, manifest.block);
787 if selection != ComponentSelection::None {
788 selections.insert(ty, selection);
789 }
790 }
791
792 selections
793 }
794
795 fn full_selection_for_component(
797 &self,
798 ty: SnapshotComponentType,
799 snapshot_block: u64,
800 ) -> ComponentSelection {
801 let defaults = DefaultPruningValues::get_global();
802 match ty {
803 SnapshotComponentType::State | SnapshotComponentType::Headers => {
804 ComponentSelection::All
805 }
806 SnapshotComponentType::Transactions => {
807 if defaults.full_bodies_history_use_pre_merge {
808 match self
809 .env
810 .chain
811 .ethereum_fork_activation(EthereumHardfork::Paris)
812 .block_number()
813 {
814 Some(paris) if snapshot_block >= paris => ComponentSelection::Since(paris),
815 Some(_) => ComponentSelection::None,
816 None => ComponentSelection::All,
817 }
818 } else {
819 selection_from_prune_mode(
820 defaults.full_prune_modes.bodies_history,
821 snapshot_block,
822 )
823 }
824 }
825 SnapshotComponentType::Receipts => {
826 selection_from_prune_mode(defaults.full_prune_modes.receipts, snapshot_block)
827 }
828 SnapshotComponentType::AccountChangesets => {
829 selection_from_prune_mode(defaults.full_prune_modes.account_history, snapshot_block)
830 }
831 SnapshotComponentType::StorageChangesets => {
832 selection_from_prune_mode(defaults.full_prune_modes.storage_history, snapshot_block)
833 }
834 SnapshotComponentType::TransactionSenders => {
835 selection_from_prune_mode(defaults.full_prune_modes.sender_recovery, snapshot_block)
836 }
837 SnapshotComponentType::RocksdbIndices => ComponentSelection::None,
839 }
840 }
841
842 async fn resolve_manifest_source(&self, chain_id: u64) -> Result<String> {
844 if let Some(path) = &self.manifest_path {
845 return Ok(path.display().to_string());
846 }
847
848 match &self.manifest_url {
849 Some(url) => Ok(url.clone()),
850 None => {
851 let defaults = DownloadDefaults::get_global();
852 if defaults.mainnet_only_discovery() && chain_id != MAINNET.chain.id() {
853 eyre::bail!(
854 "Snapshots are only auto-discovered for Ethereum mainnet.\n\n\
855 Chain {chain_id} requires an explicit source:\n\
856 \t--manifest-url <URL>\n\
857 \t--manifest-path <PATH>\n\
858 \t-u <SNAPSHOT-URL>\n\n\
859 Use --list to inspect snapshots exposed by {}.",
860 defaults.snapshot_source_url(),
861 );
862 }
863
864 discover_manifest_url(chain_id).await
865 }
866 }
867 }
868}
869
870fn explicit_component_selection(
873 all: bool,
874 since: Option<u64>,
875 distance: Option<u64>,
876 snapshot_block: u64,
877) -> Option<ComponentSelection> {
878 if all {
879 Some(ComponentSelection::All)
880 } else if let Some(block) = since {
881 (block <= snapshot_block).then_some(ComponentSelection::Since(block))
882 } else {
883 distance.map(ComponentSelection::Distance)
884 }
885}
886
887fn selection_from_prune_mode(mode: Option<PruneMode>, snapshot_block: u64) -> ComponentSelection {
889 match mode {
890 None => ComponentSelection::All,
891 Some(PruneMode::Full) => ComponentSelection::None,
892 Some(PruneMode::Distance(d)) => ComponentSelection::Distance(d),
893 Some(PruneMode::Before(block)) => {
894 if snapshot_block >= block {
895 ComponentSelection::Since(block)
896 } else {
897 ComponentSelection::None
898 }
899 }
900 }
901}
902
903fn clear_existing_datadir(target_dir: &Path) -> Result<()> {
905 if !target_dir.try_exists()? {
906 return Ok(());
907 }
908
909 info!(target: "reth::cli", dir = ?target_dir, "Clearing existing snapshot data");
910 for entry in FORCE_REMOVED_DATADIR_PATHS {
911 let path = target_dir.join(entry);
912 if !path.try_exists()? {
913 continue;
914 }
915
916 let metadata = fs::metadata(&path)?;
917 if metadata.is_dir() {
918 fs::remove_dir_all(&path)?;
919 } else if metadata.is_file() {
920 fs::remove_file(&path)?;
921 }
922 }
923
924 Ok(())
925}
926
927fn inject_archive_only_components(
930 selections: &mut BTreeMap<SnapshotComponentType, ComponentSelection>,
931 manifest: &SnapshotManifest,
932 include_rocksdb: bool,
933) {
934 let is_all =
935 |ty: SnapshotComponentType| selections.get(&ty).copied() == Some(ComponentSelection::All);
936
937 let is_archive = is_all(SnapshotComponentType::Transactions) &&
938 is_all(SnapshotComponentType::Receipts) &&
939 is_all(SnapshotComponentType::AccountChangesets) &&
940 is_all(SnapshotComponentType::StorageChangesets);
941
942 if !is_archive {
943 return;
944 }
945
946 for component in
947 [SnapshotComponentType::TransactionSenders, SnapshotComponentType::RocksdbIndices]
948 {
949 if component == SnapshotComponentType::RocksdbIndices && !include_rocksdb {
950 continue;
951 }
952
953 if manifest.component(component).is_some() {
954 selections.insert(component, ComponentSelection::All);
955 }
956 }
957}
958
959fn should_reset_index_stage_checkpoints(
961 selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
962) -> bool {
963 !matches!(selections.get(&SnapshotComponentType::RocksdbIndices), Some(ComponentSelection::All))
964}
965
966fn startup_node_command<C>(chain_spec: &C::ChainSpec) -> String
967where
968 C: ChainSpecParser,
969 C::ChainSpec: EthChainSpec,
970{
971 startup_node_command_for_binary::<C>(¤t_binary_name(), chain_spec)
972}
973
974fn startup_node_command_for_binary<C>(binary_name: &str, chain_spec: &C::ChainSpec) -> String
975where
976 C: ChainSpecParser,
977 C::ChainSpec: EthChainSpec,
978{
979 let mut command = format!("{binary_name} node");
980
981 if let Some(chain_arg) = startup_chain_arg::<C>(chain_spec) {
982 command.push_str(" --chain ");
983 command.push_str(&chain_arg);
984 }
985
986 command
987}
988
989fn current_binary_name() -> String {
990 std::env::args_os()
991 .next()
992 .map(PathBuf::from)
993 .and_then(|path| path.file_stem().map(|name| name.to_owned()))
994 .and_then(|name| name.into_string().ok())
995 .filter(|name| !name.is_empty())
996 .unwrap_or_else(|| "reth".to_string())
997}
998
999fn download_command() -> String {
1000 download_command_for_binary(¤t_binary_name())
1001}
1002
1003fn download_command_for_binary(binary_name: &str) -> String {
1004 format!("{binary_name} download")
1005}
1006
1007fn startup_chain_arg<C>(chain_spec: &C::ChainSpec) -> Option<String>
1008where
1009 C: ChainSpecParser,
1010 C::ChainSpec: EthChainSpec,
1011{
1012 let current_chain = chain_spec.chain();
1013 let current_genesis_hash = chain_spec.genesis_hash();
1014 let default_chain = C::default_value().and_then(|chain_name| C::parse(chain_name).ok());
1015
1016 if default_chain.as_ref().is_some_and(|default_chain| {
1017 default_chain.chain() == current_chain &&
1018 default_chain.genesis_hash() == current_genesis_hash
1019 }) {
1020 return None;
1021 }
1022
1023 C::SUPPORTED_CHAINS
1024 .iter()
1025 .find_map(|chain_name| {
1026 let parsed_chain = C::parse(chain_name).ok()?;
1027 (parsed_chain.chain() == current_chain &&
1028 parsed_chain.genesis_hash() == current_genesis_hash)
1029 .then(|| (*chain_name).to_string())
1030 })
1031 .or_else(|| Some("<chain-or-chainspec>".to_string()))
1032}
1033
1034impl<C: ChainSpecParser> DownloadCommand<C> {
1035 pub const fn env(&self) -> &EnvironmentArgs<C> {
1037 &self.env
1038 }
1039
1040 pub fn chain_spec(&self) -> Option<&Arc<C::ChainSpec>> {
1042 Some(&self.env.chain)
1043 }
1044
1045 pub const fn prints_plan_json(&self) -> bool {
1047 self.print_plan_json
1048 }
1049}
1050
1051#[derive(Debug)]
1053pub struct PreparedSnapshotDownload {
1054 pub manifest: SnapshotManifest,
1056 pub data_dir: PathBuf,
1058}
1059
1060struct ResolvedDownload {
1061 manifest: SnapshotManifest,
1062 selections: BTreeMap<SnapshotComponentType, ComponentSelection>,
1063 preset: Option<SelectionPreset>,
1064 planned: PlannedDownloads,
1065}
1066
1067const MAX_DOWNLOAD_RETRIES: u32 = 10;
1068const RETRY_BACKOFF_SECS: u64 = 5;
1069
1070#[cfg(test)]
1071mod tests {
1072 use super::*;
1073 use clap::{Args, Parser};
1074 use extract::CompressionFormat;
1075 use manifest::{ComponentManifest, SingleArchive};
1076 use reth_chainspec::{HOLESKY, MAINNET};
1077 use reth_ethereum_cli::chainspec::EthereumChainSpecParser;
1078
1079 #[derive(Parser)]
1080 struct CommandParser<T: Args> {
1081 #[command(flatten)]
1082 args: T,
1083 }
1084
1085 fn manifest_with_archive_only_components() -> SnapshotManifest {
1086 let mut components = BTreeMap::new();
1087 components.insert(
1088 SnapshotComponentType::TransactionSenders.key().to_string(),
1089 ComponentManifest::Single(SingleArchive {
1090 file: "transaction_senders.tar.zst".to_string(),
1091 size: 1,
1092 decompressed_size: 0,
1093 blake3: None,
1094 output_files: vec![],
1095 }),
1096 );
1097 components.insert(
1098 SnapshotComponentType::RocksdbIndices.key().to_string(),
1099 ComponentManifest::Single(SingleArchive {
1100 file: "rocksdb_indices.tar.zst".to_string(),
1101 size: 1,
1102 decompressed_size: 0,
1103 blake3: None,
1104 output_files: vec![],
1105 }),
1106 );
1107 SnapshotManifest {
1108 block: 0,
1109 chain_id: 1,
1110 storage_version: 2,
1111 timestamp: 0,
1112 base_url: Some("https://example.com".to_string()),
1113 reth_version: None,
1114 components,
1115 extensions: Default::default(),
1116 }
1117 }
1118
1119 #[test]
1120 fn test_download_defaults_builder() {
1121 let defaults = DownloadDefaults::default()
1122 .with_snapshot("https://example.com/snapshots (example)")
1123 .with_base_url("https://example.com");
1124
1125 assert_eq!(defaults.default_base_url, "https://example.com");
1126 assert_eq!(defaults.available_snapshots.len(), 3); }
1128
1129 #[test]
1130 fn test_download_defaults_replace_snapshots() {
1131 let defaults = DownloadDefaults::default().with_snapshots(vec![
1132 Cow::Borrowed("https://custom1.com"),
1133 Cow::Borrowed("https://custom2.com"),
1134 ]);
1135
1136 assert_eq!(defaults.available_snapshots.len(), 2);
1137 assert_eq!(defaults.available_snapshots[0], "https://custom1.com");
1138 }
1139
1140 #[test]
1141 fn test_long_help_generation() {
1142 let defaults = DownloadDefaults::default();
1143 let help = defaults.long_help();
1144
1145 assert!(help.contains("Available snapshot sources:"));
1146 assert!(help.contains("Ethereum mainnet"));
1147 assert!(help.contains("snapshots.reth.rs"));
1148 assert!(help.contains("publicnode.com"));
1149 assert!(help.contains("file://"));
1150 }
1151
1152 #[test]
1153 fn test_custom_snapshot_api_keeps_selected_chain_help() {
1154 let defaults = DownloadDefaults::default()
1155 .with_snapshot_api_url("https://snapshots.tempoxyz.dev/api/snapshots");
1156 let help = defaults.long_help();
1157
1158 assert_eq!(
1159 defaults.default_chain_aware_base_url.as_deref(),
1160 Some("https://snapshots.tempoxyz.dev")
1161 );
1162 assert!(help.contains("Browse available snapshots at https://snapshots.tempoxyz.dev"));
1163 assert!(help.contains("- https://snapshots.tempoxyz.dev (default)"));
1164 assert!(help.contains("selected chain"));
1165 assert!(!help.contains("Ethereum mainnet"));
1166 assert!(!help.contains("snapshots.reth.rs"));
1167 }
1168
1169 #[test]
1170 fn test_snapshot_source_url_sets_generated_references() {
1171 let defaults =
1172 DownloadDefaults::default().with_snapshot_source_url("https://snapshots.tempoxyz.dev/");
1173 let help = defaults.long_help();
1174
1175 assert_eq!(defaults.snapshot_api_url, "https://snapshots.tempoxyz.dev/api/snapshots");
1176 assert_eq!(defaults.default_base_url, "https://snapshots.tempoxyz.dev");
1177 assert_eq!(
1178 defaults.default_chain_aware_base_url.as_deref(),
1179 Some("https://snapshots.tempoxyz.dev")
1180 );
1181 assert_eq!(
1182 defaults.available_snapshots.iter().map(|source| source.as_ref()).collect::<Vec<_>>(),
1183 vec!["https://snapshots.tempoxyz.dev (default)"]
1184 );
1185 assert!(!defaults.mainnet_only_discovery());
1186 assert!(help.contains("Browse available snapshots at https://snapshots.tempoxyz.dev"));
1187 assert!(help.contains("from https://snapshots.tempoxyz.dev."));
1188 }
1189
1190 #[test]
1191 fn test_snapshot_api_url_trailing_slash_sets_source_url() {
1192 let defaults = DownloadDefaults::default()
1193 .with_snapshot_api_url("https://snapshots.tempoxyz.dev/api/snapshots/");
1194 let help = defaults.long_help();
1195
1196 assert_eq!(
1197 defaults.default_chain_aware_base_url.as_deref(),
1198 Some("https://snapshots.tempoxyz.dev")
1199 );
1200 assert!(help.contains("Browse available snapshots at https://snapshots.tempoxyz.dev"));
1201 assert!(help.contains("- https://snapshots.tempoxyz.dev (default)"));
1202 }
1203
1204 #[test]
1205 fn test_long_help_override() {
1206 let custom_help = "This is custom help text for downloading snapshots.";
1207 let defaults = DownloadDefaults::default().with_long_help(custom_help);
1208
1209 let help = defaults.long_help();
1210 assert_eq!(help, custom_help);
1211 assert!(!help.contains("Available snapshot sources:"));
1212 }
1213
1214 #[test]
1215 fn test_builder_chaining() {
1216 let defaults = DownloadDefaults::default()
1217 .with_base_url("https://custom.example.com")
1218 .with_snapshot("https://snapshot1.com")
1219 .with_snapshot("https://snapshot2.com")
1220 .with_long_help("Custom help for snapshots");
1221
1222 assert_eq!(defaults.default_base_url, "https://custom.example.com");
1223 assert_eq!(defaults.available_snapshots.len(), 4); assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string()));
1225 }
1226
1227 #[test]
1228 fn test_download_resumable_defaults_to_true() {
1229 let args =
1230 CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from(["reth"]).args;
1231
1232 assert!(args.resumable);
1233 }
1234
1235 #[test]
1236 fn test_download_resumable_implicit_true() {
1237 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1238 "reth",
1239 "--resumable",
1240 ])
1241 .args;
1242
1243 assert!(args.resumable);
1244 }
1245
1246 #[test]
1247 fn test_download_resumable_explicit_false() {
1248 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1249 "reth",
1250 "--resumable=false",
1251 ])
1252 .args;
1253
1254 assert!(!args.resumable);
1255 }
1256
1257 #[test]
1258 fn test_download_print_plan_json_parses() {
1259 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1260 "reth",
1261 "--manifest-path",
1262 "manifest.json",
1263 "--minimal",
1264 "--print-plan-json",
1265 ])
1266 .args;
1267
1268 assert!(args.prints_plan_json());
1269 }
1270
1271 #[test]
1272 fn test_download_print_plan_json_rejects_single_archive() {
1273 let result = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::try_parse_from([
1274 "reth",
1275 "--url",
1276 "https://example.com/snapshot.tar.zst",
1277 "--print-plan-json",
1278 ]);
1279
1280 assert!(result.is_err());
1281 }
1282
1283 #[test]
1284 fn resolve_manifest_source_requires_explicit_source_for_non_mainnet_defaults() {
1285 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1286 "reth", "--chain", "holesky",
1287 ])
1288 .args;
1289
1290 let err = tokio::runtime::Runtime::new()
1291 .unwrap()
1292 .block_on(args.resolve_manifest_source(HOLESKY.chain.id()))
1293 .unwrap_err();
1294
1295 let message = err.to_string();
1296 assert!(message.contains("only auto-discovered for Ethereum mainnet"));
1297 assert!(message.contains("--manifest-url <URL>"));
1298 assert!(message.contains("-u <SNAPSHOT-URL>"));
1299 }
1300
1301 #[test]
1302 fn resolve_manifest_source_allows_manifest_path_for_non_mainnet_defaults() {
1303 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1304 "reth",
1305 "--chain",
1306 "holesky",
1307 "--manifest-path",
1308 "./manifest.json",
1309 ])
1310 .args;
1311
1312 let source = tokio::runtime::Runtime::new()
1313 .unwrap()
1314 .block_on(args.resolve_manifest_source(HOLESKY.chain.id()))
1315 .unwrap();
1316
1317 assert_eq!(source, "./manifest.json");
1318 }
1319
1320 #[test]
1321 fn test_compression_format_detection() {
1322 assert!(matches!(
1323 CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
1324 Ok(CompressionFormat::Lz4)
1325 ));
1326 assert!(matches!(
1327 CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
1328 Ok(CompressionFormat::Zstd)
1329 ));
1330 assert!(matches!(
1331 CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
1332 Ok(CompressionFormat::Lz4)
1333 ));
1334 assert!(matches!(
1335 CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
1336 Ok(CompressionFormat::Zstd)
1337 ));
1338 assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
1339 }
1340
1341 #[test]
1342 fn inject_archive_only_components_for_archive_selection() {
1343 let manifest = manifest_with_archive_only_components();
1344 let mut selections = BTreeMap::new();
1345 selections.insert(SnapshotComponentType::Transactions, ComponentSelection::All);
1346 selections.insert(SnapshotComponentType::Receipts, ComponentSelection::All);
1347 selections.insert(SnapshotComponentType::AccountChangesets, ComponentSelection::All);
1348 selections.insert(SnapshotComponentType::StorageChangesets, ComponentSelection::All);
1349
1350 inject_archive_only_components(&mut selections, &manifest, true);
1351
1352 assert_eq!(
1353 selections.get(&SnapshotComponentType::TransactionSenders),
1354 Some(&ComponentSelection::All)
1355 );
1356 assert_eq!(
1357 selections.get(&SnapshotComponentType::RocksdbIndices),
1358 Some(&ComponentSelection::All)
1359 );
1360 }
1361
1362 #[test]
1363 fn inject_archive_only_components_without_rocksdb() {
1364 let manifest = manifest_with_archive_only_components();
1365 let mut selections = BTreeMap::new();
1366 selections.insert(SnapshotComponentType::Transactions, ComponentSelection::All);
1367 selections.insert(SnapshotComponentType::Receipts, ComponentSelection::All);
1368 selections.insert(SnapshotComponentType::AccountChangesets, ComponentSelection::All);
1369 selections.insert(SnapshotComponentType::StorageChangesets, ComponentSelection::All);
1370
1371 inject_archive_only_components(&mut selections, &manifest, false);
1372
1373 assert_eq!(
1374 selections.get(&SnapshotComponentType::TransactionSenders),
1375 Some(&ComponentSelection::All)
1376 );
1377 assert_eq!(selections.get(&SnapshotComponentType::RocksdbIndices), None);
1378 }
1379
1380 #[test]
1381 fn should_reset_index_stage_checkpoints_without_rocksdb_indices() {
1382 let mut selections = BTreeMap::new();
1383 selections.insert(SnapshotComponentType::Transactions, ComponentSelection::All);
1384 assert!(should_reset_index_stage_checkpoints(&selections));
1385
1386 selections.insert(SnapshotComponentType::RocksdbIndices, ComponentSelection::All);
1387 assert!(!should_reset_index_stage_checkpoints(&selections));
1388 }
1389
1390 #[test]
1391 fn startup_node_command_omits_default_chain_arg() {
1392 let command =
1393 startup_node_command_for_binary::<EthereumChainSpecParser>("reth", MAINNET.as_ref());
1394
1395 assert_eq!(command, "reth node");
1396 }
1397
1398 #[test]
1399 fn startup_node_command_includes_non_default_chain_arg() {
1400 let command =
1401 startup_node_command_for_binary::<EthereumChainSpecParser>("reth", HOLESKY.as_ref());
1402
1403 assert_eq!(command, "reth node --chain holesky");
1404 }
1405
1406 #[test]
1407 fn startup_node_command_uses_running_binary_name() {
1408 let command =
1409 startup_node_command_for_binary::<EthereumChainSpecParser>("tempo", HOLESKY.as_ref());
1410
1411 assert_eq!(command, "tempo node --chain holesky");
1412 }
1413
1414 #[test]
1415 fn download_command_uses_binary_name() {
1416 assert_eq!(download_command_for_binary("tempo"), "tempo download");
1417 }
1418}