1mod archive;
81pub mod config_gen;
82mod extract;
83mod fetch;
84pub mod manifest;
85pub mod manifest_cmd;
86mod planning;
87mod progress;
88mod session;
89mod source;
90mod tui;
91mod verify;
92
93pub use planning::{DownloadPlan, DownloadPlanArchive};
94
95use crate::common::EnvironmentArgs;
96use archive::run_modular_downloads;
97use clap::{builder::RangedU64ValueParser, Parser};
98use config_gen::{config_for_selections, write_config};
99use extract::stream_and_extract;
100use eyre::Result;
101use manifest::{ComponentSelection, SnapshotComponentType, SnapshotManifest};
102use planning::{collect_planned_archives, summarize_download_startup, PlannedDownloads};
103use progress::{DownloadProgress, DownloadRequestLimiter};
104use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks, MAINNET};
105use reth_cli::chainspec::ChainSpecParser;
106use reth_cli_util::cancellation::CancellationToken;
107use reth_db::{init_db, Database};
108use reth_db_api::transaction::DbTx;
109use reth_fs_util as fs;
110use reth_node_core::args::DefaultPruningValues;
111use reth_prune_types::PruneMode;
112use source::{
113 discover_manifest_url, fetch_manifest_from_source, fetch_snapshot_api_entries,
114 print_snapshot_listing, resolve_manifest_base_url,
115};
116use std::{
117 borrow::Cow,
118 collections::BTreeMap,
119 path::{Path, PathBuf},
120 sync::{Arc, OnceLock},
121};
122use tracing::info;
123use tui::{run_selector, SelectorOutput};
124
125const RETH_SNAPSHOTS_BASE_URL: &str = "https://snapshots-r2.reth.rs";
126const RETH_SNAPSHOTS_API_URL: &str = "https://snapshots.reth.rs/api/snapshots";
127const RETH_SNAPSHOTS_SOURCE: &str = "https://snapshots.reth.rs (default)";
128const SNAPSHOT_API_PATH: &str = "/api/snapshots";
129const FORCE_REMOVED_DATADIR_PATHS: &[&str] = &["db", "rocksdb", "static_files", "reth.toml"];
130
131const MAX_CONCURRENT_DOWNLOADS: usize = 8;
133
134#[derive(Debug, Clone, Copy, PartialEq, Eq)]
136pub(crate) enum SelectionPreset {
137 Minimal,
139 Full,
141 Archive,
143}
144
145struct ResolvedComponents {
146 selections: BTreeMap<SnapshotComponentType, ComponentSelection>,
147 preset: Option<SelectionPreset>,
148}
149
150static DOWNLOAD_DEFAULTS: OnceLock<DownloadDefaults> = OnceLock::new();
152
153#[derive(Debug, Clone)]
157pub struct DownloadDefaults {
158 pub available_snapshots: Vec<Cow<'static, str>>,
160 pub default_base_url: Cow<'static, str>,
162 pub default_chain_aware_base_url: Option<Cow<'static, str>>,
170 pub snapshot_api_url: Cow<'static, str>,
174 pub long_help: Option<String>,
176}
177
178impl DownloadDefaults {
179 pub fn try_init(self) -> Result<(), Self> {
181 DOWNLOAD_DEFAULTS.set(self)
182 }
183
184 pub fn get_global() -> &'static DownloadDefaults {
186 DOWNLOAD_DEFAULTS.get_or_init(DownloadDefaults::default_download_defaults)
187 }
188
189 pub fn default_download_defaults() -> Self {
191 Self {
192 available_snapshots: vec![
193 Cow::Borrowed(RETH_SNAPSHOTS_SOURCE),
194 Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"),
195 ],
196 default_base_url: Cow::Borrowed(RETH_SNAPSHOTS_BASE_URL),
197 default_chain_aware_base_url: None,
198 snapshot_api_url: Cow::Borrowed(RETH_SNAPSHOTS_API_URL),
199 long_help: None,
200 }
201 }
202
203 pub fn long_help(&self) -> String {
208 if let Some(ref custom_help) = self.long_help {
209 return custom_help.clone();
210 }
211
212 let implicit_download_help = if self.mainnet_only_discovery() {
213 "\nIf no URL is provided, the latest archive snapshot will only be proposed\nfor Ethereum mainnet. For other chains, provide --manifest-url, --manifest-path,\nor -u explicitly."
214 } else {
215 "\nIf no URL is provided, the latest archive snapshot for the selected chain\nwill be proposed for download from "
216 };
217
218 let mut help = format!(
219 "Specify a snapshot URL or let the command propose a default one.\n\n\
220 Browse available snapshots at {}\n\
221 or use --list-snapshots to see them from the CLI.\n\nAvailable snapshot sources:\n",
222 self.snapshot_source_url(),
223 );
224
225 for source in &self.available_snapshots {
226 help.push_str("- ");
227 help.push_str(source);
228 help.push('\n');
229 }
230
231 help.push_str(implicit_download_help);
232 if !self.mainnet_only_discovery() {
233 help.push_str(
234 self.default_chain_aware_base_url.as_deref().unwrap_or(&self.default_base_url),
235 );
236 help.push('.');
237 }
238 help.push_str(
239 "\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
240 );
241 help
242 }
243
244 fn mainnet_only_discovery(&self) -> bool {
245 self.snapshot_api_url.trim_end_matches('/') == RETH_SNAPSHOTS_API_URL
246 }
247
248 fn snapshot_source_url(&self) -> &str {
249 snapshot_source_url_from_api(&self.snapshot_api_url)
250 }
251
252 pub fn with_snapshot(mut self, source: impl Into<Cow<'static, str>>) -> Self {
254 self.available_snapshots.push(source.into());
255 self
256 }
257
258 pub fn with_snapshots(mut self, sources: Vec<Cow<'static, str>>) -> Self {
260 self.available_snapshots = sources;
261 self
262 }
263
264 pub fn with_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
266 self.default_base_url = url.into();
267 self
268 }
269
270 pub fn with_chain_aware_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
272 self.default_chain_aware_base_url = Some(url.into());
273 self
274 }
275
276 pub fn with_snapshot_api_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
281 self.snapshot_api_url = url.into();
282
283 let source_url = self.snapshot_source_url().to_string();
284 if self.default_chain_aware_base_url.is_none() {
285 self.default_chain_aware_base_url = Some(Cow::Owned(source_url.clone()));
286 }
287 for source in &mut self.available_snapshots {
288 if source.as_ref() == RETH_SNAPSHOTS_SOURCE {
289 *source = Cow::Owned(format!("{source_url} (default)"));
290 }
291 }
292
293 self
294 }
295
296 pub fn with_snapshot_source_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
301 let source_url = normalize_snapshot_source_url(url.into());
302 self.available_snapshots = vec![Cow::Owned(format!("{} (default)", source_url.as_ref()))];
303 self.default_base_url = source_url.clone();
304 self.default_chain_aware_base_url = Some(source_url.clone());
305 self.snapshot_api_url = Cow::Owned(format!("{}{SNAPSHOT_API_PATH}", source_url.as_ref()));
306 self
307 }
308
309 pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
311 self.long_help = Some(help.into());
312 self
313 }
314}
315
316fn snapshot_source_url_from_api(api_url: &str) -> &str {
317 api_url.trim_end_matches('/').trim_end_matches(SNAPSHOT_API_PATH)
318}
319
320fn normalize_snapshot_source_url(url: Cow<'static, str>) -> Cow<'static, str> {
321 match url {
322 Cow::Borrowed(url) => Cow::Borrowed(snapshot_source_url_from_api(url)),
323 Cow::Owned(url) => Cow::Owned(snapshot_source_url_from_api(&url).to_string()),
324 }
325}
326
327impl Default for DownloadDefaults {
328 fn default() -> Self {
330 Self::default_download_defaults()
331 }
332}
333
334#[derive(Debug, Parser)]
336pub struct DownloadCommand<C: ChainSpecParser> {
337 #[command(flatten)]
338 env: EnvironmentArgs<C>,
339
340 #[arg(long, short, long_help = DownloadDefaults::get_global().long_help())]
345 url: Option<String>,
346
347 #[arg(long, value_name = "URL", conflicts_with = "url")]
352 manifest_url: Option<String>,
353
354 #[arg(long, value_name = "PATH", conflicts_with_all = ["url", "manifest_url"])]
356 manifest_path: Option<PathBuf>,
357
358 #[arg(long, conflicts_with_all = ["with_txs_since", "with_txs_distance", "minimal", "full", "archive"])]
360 with_txs: bool,
361
362 #[arg(long, value_name = "BLOCK_NUMBER", conflicts_with_all = ["with_txs", "with_txs_distance", "minimal", "full", "archive"])]
364 with_txs_since: Option<u64>,
365
366 #[arg(long, value_name = "BLOCKS", value_parser = RangedU64ValueParser::<u64>::new().range(1..), conflicts_with_all = ["with_txs", "with_txs_since", "minimal", "full", "archive"])]
368 with_txs_distance: Option<u64>,
369
370 #[arg(long, conflicts_with_all = ["with_receipts_since", "with_receipts_distance", "minimal", "full", "archive"])]
372 with_receipts: bool,
373
374 #[arg(long, value_name = "BLOCK_NUMBER", conflicts_with_all = ["with_receipts", "with_receipts_distance", "minimal", "full", "archive"])]
376 with_receipts_since: Option<u64>,
377
378 #[arg(long, value_name = "BLOCKS", value_parser = RangedU64ValueParser::<u64>::new().range(1..), conflicts_with_all = ["with_receipts", "with_receipts_since", "minimal", "full", "archive"])]
380 with_receipts_distance: Option<u64>,
381
382 #[arg(long, alias = "with-changesets", conflicts_with_all = ["with_state_history_since", "with_state_history_distance", "minimal", "full", "archive"])]
384 with_state_history: bool,
385
386 #[arg(long, value_name = "BLOCK_NUMBER", conflicts_with_all = ["with_state_history", "with_state_history_distance", "minimal", "full", "archive"])]
388 with_state_history_since: Option<u64>,
389
390 #[arg(long, value_name = "BLOCKS", value_parser = RangedU64ValueParser::<u64>::new().range(1..), conflicts_with_all = ["with_state_history", "with_state_history_since", "minimal", "full", "archive"])]
392 with_state_history_distance: Option<u64>,
393
394 #[arg(long, requires = "with_txs", conflicts_with_all = ["minimal", "full", "archive"])]
396 with_senders: bool,
397
398 #[arg(long, conflicts_with_all = ["minimal", "full", "archive", "without_rocksdb"])]
400 with_rocksdb: bool,
401
402 #[arg(long, alias = "all", conflicts_with_all = ["with_txs", "with_txs_since", "with_txs_distance", "with_receipts", "with_receipts_since", "with_receipts_distance", "with_state_history", "with_state_history_since", "with_state_history_distance", "with_senders", "with_rocksdb", "minimal", "full"])]
404 archive: bool,
405
406 #[arg(long, conflicts_with_all = ["with_txs", "with_txs_since", "with_txs_distance", "with_receipts", "with_receipts_since", "with_receipts_distance", "with_state_history", "with_state_history_since", "with_state_history_distance", "with_senders", "with_rocksdb", "archive", "full"])]
408 minimal: bool,
409
410 #[arg(long, conflicts_with_all = ["with_txs", "with_txs_since", "with_txs_distance", "with_receipts", "with_receipts_since", "with_receipts_distance", "with_state_history", "with_state_history_since", "with_state_history_distance", "with_senders", "with_rocksdb", "archive", "minimal"])]
412 full: bool,
413
414 #[arg(long, conflicts_with_all = ["url", "with_rocksdb"])]
418 without_rocksdb: bool,
419
420 #[arg(long, short = 'y')]
423 non_interactive: bool,
424
425 #[arg(long, conflicts_with = "list")]
427 force: bool,
428
429 #[arg(long, default_value_t = true, num_args = 0..=1, default_missing_value = "true")]
436 resumable: bool,
437
438 #[arg(long, default_value_t = MAX_CONCURRENT_DOWNLOADS)]
443 download_concurrency: usize,
444
445 #[arg(long, alias = "list-snapshots", conflicts_with_all = ["url", "manifest_url", "manifest_path"])]
450 list: bool,
451
452 #[arg(long, conflicts_with_all = ["url", "list"])]
454 print_plan_json: bool,
455}
456
457impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCommand<C> {
458 pub async fn execute<N>(self) -> Result<()> {
460 let chain = self.env.chain.chain();
461 let chain_id = chain.id();
462
463 if self.list {
465 let entries = fetch_snapshot_api_entries(chain_id).await?;
466 print_snapshot_listing(&entries, chain_id);
467 return Ok(());
468 }
469
470 let data_dir = self.env.datadir.clone().resolve_datadir(chain);
471
472 let cancel_token = CancellationToken::new();
473 let _cancel_guard = cancel_token.drop_guard();
474
475 if let Some(ref url) = self.url {
477 let target_dir = data_dir.data_dir();
478 if self.force {
479 clear_existing_datadir(target_dir)?;
480 }
481 fs::create_dir_all(target_dir)?;
482
483 let request_limiter = DownloadRequestLimiter::new(self.download_concurrency.max(1));
484 info!(target: "reth::cli",
485 dir = ?data_dir.data_dir(),
486 url = %url,
487 "Starting snapshot download and extraction"
488 );
489
490 stream_and_extract(
491 url,
492 data_dir.data_dir(),
493 None,
494 self.resumable,
495 Some(request_limiter),
496 cancel_token.clone(),
497 )
498 .await?;
499 info!(target: "reth::cli", "Snapshot downloaded and extracted successfully");
500
501 return Ok(());
502 }
503
504 let ResolvedDownload { manifest, selections, preset, planned } =
505 self.resolve_download(chain_id).await?;
506 if self.print_plan_json {
507 DownloadPlan::from_planned(&manifest, &planned).write_json(std::io::stdout().lock())?;
508 return Ok(())
509 }
510
511 let target_dir = data_dir.data_dir();
512 if self.force {
513 clear_existing_datadir(target_dir)?;
514 }
515 fs::create_dir_all(target_dir)?;
516 let startup_summary = summarize_download_startup(&planned.archives, target_dir)?;
517 info!(target: "reth::cli",
518 reusable = startup_summary.reusable,
519 needs_download = startup_summary.needs_download,
520 "Startup integrity summary (plain output files)"
521 );
522
523 info!(target: "reth::cli",
524 archives = planned.total_archives(),
525 download_total = %DownloadProgress::format_size(planned.total_download_size),
526 output_total = %DownloadProgress::format_size(planned.total_output_size),
527 "Downloading all archives"
528 );
529
530 run_modular_downloads(
531 planned,
532 target_dir,
533 self.download_concurrency.max(1),
534 cancel_token.clone(),
535 )
536 .await?;
537
538 self.finalize_modular_download(&selections, &manifest, preset, target_dir, &data_dir.db())?;
539
540 Ok(())
541 }
542
543 pub async fn plan(&self) -> Result<DownloadPlan> {
545 let chain_id = self.env.chain.chain().id();
546 let resolved = self.resolve_download(chain_id).await?;
547 Ok(DownloadPlan::from_planned(&resolved.manifest, &resolved.planned))
548 }
549
550 async fn resolve_download(&self, chain_id: u64) -> Result<ResolvedDownload> {
551 let manifest = self.load_manifest(chain_id).await?;
552 let ResolvedComponents { mut selections, preset } = self.resolve_components(&manifest)?;
553
554 if matches!(preset, Some(SelectionPreset::Archive)) {
555 inject_archive_only_components(&mut selections, &manifest, !self.without_rocksdb);
556 }
557
558 let planned = collect_planned_archives(&manifest, &selections)?;
559 Ok(ResolvedDownload { manifest, selections, preset, planned })
560 }
561
562 async fn load_manifest(&self, chain_id: u64) -> Result<SnapshotManifest> {
564 let manifest_source = self.resolve_manifest_source(chain_id).await?;
565
566 info!(target: "reth::cli", source = %manifest_source, "Fetching snapshot manifest");
567 let mut manifest = fetch_manifest_from_source(&manifest_source).await?;
568 manifest.base_url = Some(resolve_manifest_base_url(&manifest, &manifest_source)?);
569
570 info!(target: "reth::cli",
571 block = manifest.block,
572 chain_id = manifest.chain_id,
573 storage_version = %manifest.storage_version,
574 components = manifest.components.len(),
575 "Loaded snapshot manifest"
576 );
577
578 Ok(manifest)
579 }
580
581 fn finalize_modular_download(
583 &self,
584 selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
585 manifest: &SnapshotManifest,
586 preset: Option<SelectionPreset>,
587 target_dir: &Path,
588 db_path: &Path,
589 ) -> Result<()> {
590 let config =
591 config_for_selections(selections, manifest, preset, Some(self.env.chain.as_ref()));
592 if write_config(&config, target_dir)? {
593 let desc = config_gen::describe_prune_config(&config);
594 info!(target: "reth::cli", "{}", desc.join(", "));
595 }
596
597 let db = init_db(db_path, self.env.db.database_args())?;
598 let should_write_prune = config.prune.segments != Default::default();
599 let should_reset_indices = should_reset_index_stage_checkpoints(selections);
600 if should_write_prune || should_reset_indices {
601 let tx = db.tx_mut()?;
602
603 if should_write_prune {
604 config_gen::write_prune_checkpoints_tx(&tx, &config, manifest.block)?;
605 }
606
607 if should_reset_indices {
608 config_gen::reset_index_stage_checkpoints_tx(&tx)?;
609 }
610
611 tx.commit()?;
612 }
613
614 let start_command = startup_node_command::<C>(self.env.chain.as_ref());
615 info!(target: "reth::cli", "Snapshot download complete. Run `{}` to start syncing.", start_command);
616
617 Ok(())
618 }
619
620 fn resolve_components(&self, manifest: &SnapshotManifest) -> Result<ResolvedComponents> {
622 let available = |ty: SnapshotComponentType| manifest.component(ty).is_some();
623
624 if self.archive {
626 return Ok(ResolvedComponents {
627 selections: SnapshotComponentType::ALL
628 .iter()
629 .copied()
630 .filter(|ty| available(*ty))
631 .filter(|ty| {
632 !self.without_rocksdb || *ty != SnapshotComponentType::RocksdbIndices
633 })
634 .map(|ty| (ty, ComponentSelection::All))
635 .collect(),
636 preset: Some(SelectionPreset::Archive),
637 });
638 }
639
640 if self.full {
641 return Ok(ResolvedComponents {
642 selections: self.full_preset_selections(manifest),
643 preset: Some(SelectionPreset::Full),
644 });
645 }
646
647 if self.minimal {
648 return Ok(ResolvedComponents {
649 selections: self.minimal_preset_selections(manifest),
650 preset: Some(SelectionPreset::Minimal),
651 });
652 }
653
654 let has_explicit_flags = self.with_txs ||
655 self.with_txs_since.is_some() ||
656 self.with_txs_distance.is_some() ||
657 self.with_receipts ||
658 self.with_receipts_since.is_some() ||
659 self.with_receipts_distance.is_some() ||
660 self.with_state_history ||
661 self.with_state_history_since.is_some() ||
662 self.with_state_history_distance.is_some() ||
663 self.with_senders ||
664 self.with_rocksdb;
665
666 if has_explicit_flags {
667 let mut selections = BTreeMap::new();
668 let tx_selection = explicit_component_selection(
669 self.with_txs,
670 self.with_txs_since,
671 self.with_txs_distance,
672 manifest.block,
673 );
674 let receipt_selection = explicit_component_selection(
675 self.with_receipts,
676 self.with_receipts_since,
677 self.with_receipts_distance,
678 manifest.block,
679 );
680 let state_history_selection = explicit_component_selection(
681 self.with_state_history,
682 self.with_state_history_since,
683 self.with_state_history_distance,
684 manifest.block,
685 );
686
687 if available(SnapshotComponentType::State) {
689 selections.insert(SnapshotComponentType::State, ComponentSelection::All);
690 }
691 if available(SnapshotComponentType::Headers) {
692 selections.insert(SnapshotComponentType::Headers, ComponentSelection::All);
693 }
694 if let Some(selection) = tx_selection &&
695 available(SnapshotComponentType::Transactions)
696 {
697 selections.insert(SnapshotComponentType::Transactions, selection);
698 }
699 if let Some(selection) = receipt_selection &&
700 available(SnapshotComponentType::Receipts)
701 {
702 selections.insert(SnapshotComponentType::Receipts, selection);
703 }
704 if let Some(selection) = state_history_selection {
705 if available(SnapshotComponentType::AccountChangesets) {
706 selections.insert(SnapshotComponentType::AccountChangesets, selection);
707 }
708 if available(SnapshotComponentType::StorageChangesets) {
709 selections.insert(SnapshotComponentType::StorageChangesets, selection);
710 }
711 }
712 if self.with_senders && available(SnapshotComponentType::TransactionSenders) {
713 selections
714 .insert(SnapshotComponentType::TransactionSenders, ComponentSelection::All);
715 }
716 if self.with_rocksdb && available(SnapshotComponentType::RocksdbIndices) {
717 selections.insert(SnapshotComponentType::RocksdbIndices, ComponentSelection::All);
718 }
719 return Ok(ResolvedComponents { selections, preset: None });
720 }
721
722 if self.non_interactive {
723 return Ok(ResolvedComponents {
724 selections: self.minimal_preset_selections(manifest),
725 preset: Some(SelectionPreset::Minimal),
726 });
727 }
728
729 let full_preset = self.full_preset_selections(manifest);
731 let SelectorOutput { selections, preset } = run_selector(manifest.clone(), &full_preset)?;
732 let selected =
733 selections.into_iter().filter(|(_, sel)| *sel != ComponentSelection::None).collect();
734
735 Ok(ResolvedComponents { selections: selected, preset })
736 }
737
738 fn minimal_preset_selections(
740 &self,
741 manifest: &SnapshotManifest,
742 ) -> BTreeMap<SnapshotComponentType, ComponentSelection> {
743 SnapshotComponentType::ALL
744 .iter()
745 .copied()
746 .filter(|ty| manifest.component(*ty).is_some())
747 .map(|ty| (ty, ty.minimal_selection()))
748 .collect()
749 }
750
751 fn full_preset_selections(
753 &self,
754 manifest: &SnapshotManifest,
755 ) -> BTreeMap<SnapshotComponentType, ComponentSelection> {
756 let mut selections = BTreeMap::new();
757
758 for ty in [
759 SnapshotComponentType::State,
760 SnapshotComponentType::Headers,
761 SnapshotComponentType::Transactions,
762 SnapshotComponentType::Receipts,
763 SnapshotComponentType::AccountChangesets,
764 SnapshotComponentType::StorageChangesets,
765 SnapshotComponentType::TransactionSenders,
766 SnapshotComponentType::RocksdbIndices,
767 ] {
768 if manifest.component(ty).is_none() {
769 continue;
770 }
771
772 let selection = self.full_selection_for_component(ty, manifest.block);
773 if selection != ComponentSelection::None {
774 selections.insert(ty, selection);
775 }
776 }
777
778 selections
779 }
780
781 fn full_selection_for_component(
783 &self,
784 ty: SnapshotComponentType,
785 snapshot_block: u64,
786 ) -> ComponentSelection {
787 let defaults = DefaultPruningValues::get_global();
788 match ty {
789 SnapshotComponentType::State | SnapshotComponentType::Headers => {
790 ComponentSelection::All
791 }
792 SnapshotComponentType::Transactions => {
793 if defaults.full_bodies_history_use_pre_merge {
794 match self
795 .env
796 .chain
797 .ethereum_fork_activation(EthereumHardfork::Paris)
798 .block_number()
799 {
800 Some(paris) if snapshot_block >= paris => ComponentSelection::Since(paris),
801 Some(_) => ComponentSelection::None,
802 None => ComponentSelection::All,
803 }
804 } else {
805 selection_from_prune_mode(
806 defaults.full_prune_modes.bodies_history,
807 snapshot_block,
808 )
809 }
810 }
811 SnapshotComponentType::Receipts => {
812 selection_from_prune_mode(defaults.full_prune_modes.receipts, snapshot_block)
813 }
814 SnapshotComponentType::AccountChangesets => {
815 selection_from_prune_mode(defaults.full_prune_modes.account_history, snapshot_block)
816 }
817 SnapshotComponentType::StorageChangesets => {
818 selection_from_prune_mode(defaults.full_prune_modes.storage_history, snapshot_block)
819 }
820 SnapshotComponentType::TransactionSenders => {
821 selection_from_prune_mode(defaults.full_prune_modes.sender_recovery, snapshot_block)
822 }
823 SnapshotComponentType::RocksdbIndices => ComponentSelection::None,
825 }
826 }
827
828 async fn resolve_manifest_source(&self, chain_id: u64) -> Result<String> {
830 if let Some(path) = &self.manifest_path {
831 return Ok(path.display().to_string());
832 }
833
834 match &self.manifest_url {
835 Some(url) => Ok(url.clone()),
836 None => {
837 let defaults = DownloadDefaults::get_global();
838 if defaults.mainnet_only_discovery() && chain_id != MAINNET.chain.id() {
839 eyre::bail!(
840 "Snapshots are only auto-discovered for Ethereum mainnet.\n\n\
841 Chain {chain_id} requires an explicit source:\n\
842 \t--manifest-url <URL>\n\
843 \t--manifest-path <PATH>\n\
844 \t-u <SNAPSHOT-URL>\n\n\
845 Use --list to inspect snapshots exposed by {}.",
846 defaults.snapshot_source_url(),
847 );
848 }
849
850 discover_manifest_url(chain_id).await
851 }
852 }
853 }
854}
855
856fn explicit_component_selection(
859 all: bool,
860 since: Option<u64>,
861 distance: Option<u64>,
862 snapshot_block: u64,
863) -> Option<ComponentSelection> {
864 if all {
865 Some(ComponentSelection::All)
866 } else if let Some(block) = since {
867 (block <= snapshot_block).then_some(ComponentSelection::Since(block))
868 } else {
869 distance.map(ComponentSelection::Distance)
870 }
871}
872
873fn selection_from_prune_mode(mode: Option<PruneMode>, snapshot_block: u64) -> ComponentSelection {
875 match mode {
876 None => ComponentSelection::All,
877 Some(PruneMode::Full) => ComponentSelection::None,
878 Some(PruneMode::Distance(d)) => ComponentSelection::Distance(d),
879 Some(PruneMode::Before(block)) => {
880 if snapshot_block >= block {
881 ComponentSelection::Since(block)
882 } else {
883 ComponentSelection::None
884 }
885 }
886 }
887}
888
889fn clear_existing_datadir(target_dir: &Path) -> Result<()> {
891 if !target_dir.try_exists()? {
892 return Ok(());
893 }
894
895 info!(target: "reth::cli", dir = ?target_dir, "Clearing existing snapshot data");
896 for entry in FORCE_REMOVED_DATADIR_PATHS {
897 let path = target_dir.join(entry);
898 if !path.try_exists()? {
899 continue;
900 }
901
902 let metadata = fs::metadata(&path)?;
903 if metadata.is_dir() {
904 fs::remove_dir_all(&path)?;
905 } else if metadata.is_file() {
906 fs::remove_file(&path)?;
907 }
908 }
909
910 Ok(())
911}
912
913fn inject_archive_only_components(
916 selections: &mut BTreeMap<SnapshotComponentType, ComponentSelection>,
917 manifest: &SnapshotManifest,
918 include_rocksdb: bool,
919) {
920 let is_all =
921 |ty: SnapshotComponentType| selections.get(&ty).copied() == Some(ComponentSelection::All);
922
923 let is_archive = is_all(SnapshotComponentType::Transactions) &&
924 is_all(SnapshotComponentType::Receipts) &&
925 is_all(SnapshotComponentType::AccountChangesets) &&
926 is_all(SnapshotComponentType::StorageChangesets);
927
928 if !is_archive {
929 return;
930 }
931
932 for component in
933 [SnapshotComponentType::TransactionSenders, SnapshotComponentType::RocksdbIndices]
934 {
935 if component == SnapshotComponentType::RocksdbIndices && !include_rocksdb {
936 continue;
937 }
938
939 if manifest.component(component).is_some() {
940 selections.insert(component, ComponentSelection::All);
941 }
942 }
943}
944
945fn should_reset_index_stage_checkpoints(
947 selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
948) -> bool {
949 !matches!(selections.get(&SnapshotComponentType::RocksdbIndices), Some(ComponentSelection::All))
950}
951
952fn startup_node_command<C>(chain_spec: &C::ChainSpec) -> String
953where
954 C: ChainSpecParser,
955 C::ChainSpec: EthChainSpec,
956{
957 startup_node_command_for_binary::<C>(¤t_binary_name(), chain_spec)
958}
959
960fn startup_node_command_for_binary<C>(binary_name: &str, chain_spec: &C::ChainSpec) -> String
961where
962 C: ChainSpecParser,
963 C::ChainSpec: EthChainSpec,
964{
965 let mut command = format!("{binary_name} node");
966
967 if let Some(chain_arg) = startup_chain_arg::<C>(chain_spec) {
968 command.push_str(" --chain ");
969 command.push_str(&chain_arg);
970 }
971
972 command
973}
974
975fn current_binary_name() -> String {
976 std::env::args_os()
977 .next()
978 .map(PathBuf::from)
979 .and_then(|path| path.file_stem().map(|name| name.to_owned()))
980 .and_then(|name| name.into_string().ok())
981 .filter(|name| !name.is_empty())
982 .unwrap_or_else(|| "reth".to_string())
983}
984
985fn download_command() -> String {
986 download_command_for_binary(¤t_binary_name())
987}
988
989fn download_command_for_binary(binary_name: &str) -> String {
990 format!("{binary_name} download")
991}
992
993fn startup_chain_arg<C>(chain_spec: &C::ChainSpec) -> Option<String>
994where
995 C: ChainSpecParser,
996 C::ChainSpec: EthChainSpec,
997{
998 let current_chain = chain_spec.chain();
999 let current_genesis_hash = chain_spec.genesis_hash();
1000 let default_chain = C::default_value().and_then(|chain_name| C::parse(chain_name).ok());
1001
1002 if default_chain.as_ref().is_some_and(|default_chain| {
1003 default_chain.chain() == current_chain &&
1004 default_chain.genesis_hash() == current_genesis_hash
1005 }) {
1006 return None;
1007 }
1008
1009 C::SUPPORTED_CHAINS
1010 .iter()
1011 .find_map(|chain_name| {
1012 let parsed_chain = C::parse(chain_name).ok()?;
1013 (parsed_chain.chain() == current_chain &&
1014 parsed_chain.genesis_hash() == current_genesis_hash)
1015 .then(|| (*chain_name).to_string())
1016 })
1017 .or_else(|| Some("<chain-or-chainspec>".to_string()))
1018}
1019
1020impl<C: ChainSpecParser> DownloadCommand<C> {
1021 pub const fn env(&self) -> &EnvironmentArgs<C> {
1023 &self.env
1024 }
1025
1026 pub fn chain_spec(&self) -> Option<&Arc<C::ChainSpec>> {
1028 Some(&self.env.chain)
1029 }
1030
1031 pub const fn prints_plan_json(&self) -> bool {
1033 self.print_plan_json
1034 }
1035}
1036
1037struct ResolvedDownload {
1038 manifest: SnapshotManifest,
1039 selections: BTreeMap<SnapshotComponentType, ComponentSelection>,
1040 preset: Option<SelectionPreset>,
1041 planned: PlannedDownloads,
1042}
1043
1044const MAX_DOWNLOAD_RETRIES: u32 = 10;
1045const RETRY_BACKOFF_SECS: u64 = 5;
1046
1047#[cfg(test)]
1048mod tests {
1049 use super::*;
1050 use clap::{Args, Parser};
1051 use extract::CompressionFormat;
1052 use manifest::{ComponentManifest, SingleArchive};
1053 use reth_chainspec::{HOLESKY, MAINNET};
1054 use reth_ethereum_cli::chainspec::EthereumChainSpecParser;
1055
1056 #[derive(Parser)]
1057 struct CommandParser<T: Args> {
1058 #[command(flatten)]
1059 args: T,
1060 }
1061
1062 fn manifest_with_archive_only_components() -> SnapshotManifest {
1063 let mut components = BTreeMap::new();
1064 components.insert(
1065 SnapshotComponentType::TransactionSenders.key().to_string(),
1066 ComponentManifest::Single(SingleArchive {
1067 file: "transaction_senders.tar.zst".to_string(),
1068 size: 1,
1069 decompressed_size: 0,
1070 blake3: None,
1071 output_files: vec![],
1072 }),
1073 );
1074 components.insert(
1075 SnapshotComponentType::RocksdbIndices.key().to_string(),
1076 ComponentManifest::Single(SingleArchive {
1077 file: "rocksdb_indices.tar.zst".to_string(),
1078 size: 1,
1079 decompressed_size: 0,
1080 blake3: None,
1081 output_files: vec![],
1082 }),
1083 );
1084 SnapshotManifest {
1085 block: 0,
1086 chain_id: 1,
1087 storage_version: 2,
1088 timestamp: 0,
1089 base_url: Some("https://example.com".to_string()),
1090 reth_version: None,
1091 components,
1092 }
1093 }
1094
1095 #[test]
1096 fn test_download_defaults_builder() {
1097 let defaults = DownloadDefaults::default()
1098 .with_snapshot("https://example.com/snapshots (example)")
1099 .with_base_url("https://example.com");
1100
1101 assert_eq!(defaults.default_base_url, "https://example.com");
1102 assert_eq!(defaults.available_snapshots.len(), 3); }
1104
1105 #[test]
1106 fn test_download_defaults_replace_snapshots() {
1107 let defaults = DownloadDefaults::default().with_snapshots(vec![
1108 Cow::Borrowed("https://custom1.com"),
1109 Cow::Borrowed("https://custom2.com"),
1110 ]);
1111
1112 assert_eq!(defaults.available_snapshots.len(), 2);
1113 assert_eq!(defaults.available_snapshots[0], "https://custom1.com");
1114 }
1115
1116 #[test]
1117 fn test_long_help_generation() {
1118 let defaults = DownloadDefaults::default();
1119 let help = defaults.long_help();
1120
1121 assert!(help.contains("Available snapshot sources:"));
1122 assert!(help.contains("Ethereum mainnet"));
1123 assert!(help.contains("snapshots.reth.rs"));
1124 assert!(help.contains("publicnode.com"));
1125 assert!(help.contains("file://"));
1126 }
1127
1128 #[test]
1129 fn test_custom_snapshot_api_keeps_selected_chain_help() {
1130 let defaults = DownloadDefaults::default()
1131 .with_snapshot_api_url("https://snapshots.tempoxyz.dev/api/snapshots");
1132 let help = defaults.long_help();
1133
1134 assert_eq!(
1135 defaults.default_chain_aware_base_url.as_deref(),
1136 Some("https://snapshots.tempoxyz.dev")
1137 );
1138 assert!(help.contains("Browse available snapshots at https://snapshots.tempoxyz.dev"));
1139 assert!(help.contains("- https://snapshots.tempoxyz.dev (default)"));
1140 assert!(help.contains("selected chain"));
1141 assert!(!help.contains("Ethereum mainnet"));
1142 assert!(!help.contains("snapshots.reth.rs"));
1143 }
1144
1145 #[test]
1146 fn test_snapshot_source_url_sets_generated_references() {
1147 let defaults =
1148 DownloadDefaults::default().with_snapshot_source_url("https://snapshots.tempoxyz.dev/");
1149 let help = defaults.long_help();
1150
1151 assert_eq!(defaults.snapshot_api_url, "https://snapshots.tempoxyz.dev/api/snapshots");
1152 assert_eq!(defaults.default_base_url, "https://snapshots.tempoxyz.dev");
1153 assert_eq!(
1154 defaults.default_chain_aware_base_url.as_deref(),
1155 Some("https://snapshots.tempoxyz.dev")
1156 );
1157 assert_eq!(
1158 defaults.available_snapshots.iter().map(|source| source.as_ref()).collect::<Vec<_>>(),
1159 vec!["https://snapshots.tempoxyz.dev (default)"]
1160 );
1161 assert!(!defaults.mainnet_only_discovery());
1162 assert!(help.contains("Browse available snapshots at https://snapshots.tempoxyz.dev"));
1163 assert!(help.contains("from https://snapshots.tempoxyz.dev."));
1164 }
1165
1166 #[test]
1167 fn test_snapshot_api_url_trailing_slash_sets_source_url() {
1168 let defaults = DownloadDefaults::default()
1169 .with_snapshot_api_url("https://snapshots.tempoxyz.dev/api/snapshots/");
1170 let help = defaults.long_help();
1171
1172 assert_eq!(
1173 defaults.default_chain_aware_base_url.as_deref(),
1174 Some("https://snapshots.tempoxyz.dev")
1175 );
1176 assert!(help.contains("Browse available snapshots at https://snapshots.tempoxyz.dev"));
1177 assert!(help.contains("- https://snapshots.tempoxyz.dev (default)"));
1178 }
1179
1180 #[test]
1181 fn test_long_help_override() {
1182 let custom_help = "This is custom help text for downloading snapshots.";
1183 let defaults = DownloadDefaults::default().with_long_help(custom_help);
1184
1185 let help = defaults.long_help();
1186 assert_eq!(help, custom_help);
1187 assert!(!help.contains("Available snapshot sources:"));
1188 }
1189
1190 #[test]
1191 fn test_builder_chaining() {
1192 let defaults = DownloadDefaults::default()
1193 .with_base_url("https://custom.example.com")
1194 .with_snapshot("https://snapshot1.com")
1195 .with_snapshot("https://snapshot2.com")
1196 .with_long_help("Custom help for snapshots");
1197
1198 assert_eq!(defaults.default_base_url, "https://custom.example.com");
1199 assert_eq!(defaults.available_snapshots.len(), 4); assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string()));
1201 }
1202
1203 #[test]
1204 fn test_download_resumable_defaults_to_true() {
1205 let args =
1206 CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from(["reth"]).args;
1207
1208 assert!(args.resumable);
1209 }
1210
1211 #[test]
1212 fn test_download_resumable_implicit_true() {
1213 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1214 "reth",
1215 "--resumable",
1216 ])
1217 .args;
1218
1219 assert!(args.resumable);
1220 }
1221
1222 #[test]
1223 fn test_download_resumable_explicit_false() {
1224 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1225 "reth",
1226 "--resumable=false",
1227 ])
1228 .args;
1229
1230 assert!(!args.resumable);
1231 }
1232
1233 #[test]
1234 fn test_download_print_plan_json_parses() {
1235 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1236 "reth",
1237 "--manifest-path",
1238 "manifest.json",
1239 "--minimal",
1240 "--print-plan-json",
1241 ])
1242 .args;
1243
1244 assert!(args.prints_plan_json());
1245 }
1246
1247 #[test]
1248 fn test_download_print_plan_json_rejects_single_archive() {
1249 let result = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::try_parse_from([
1250 "reth",
1251 "--url",
1252 "https://example.com/snapshot.tar.zst",
1253 "--print-plan-json",
1254 ]);
1255
1256 assert!(result.is_err());
1257 }
1258
1259 #[test]
1260 fn resolve_manifest_source_requires_explicit_source_for_non_mainnet_defaults() {
1261 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1262 "reth", "--chain", "holesky",
1263 ])
1264 .args;
1265
1266 let err = tokio::runtime::Runtime::new()
1267 .unwrap()
1268 .block_on(args.resolve_manifest_source(HOLESKY.chain.id()))
1269 .unwrap_err();
1270
1271 let message = err.to_string();
1272 assert!(message.contains("only auto-discovered for Ethereum mainnet"));
1273 assert!(message.contains("--manifest-url <URL>"));
1274 assert!(message.contains("-u <SNAPSHOT-URL>"));
1275 }
1276
1277 #[test]
1278 fn resolve_manifest_source_allows_manifest_path_for_non_mainnet_defaults() {
1279 let args = CommandParser::<DownloadCommand<EthereumChainSpecParser>>::parse_from([
1280 "reth",
1281 "--chain",
1282 "holesky",
1283 "--manifest-path",
1284 "./manifest.json",
1285 ])
1286 .args;
1287
1288 let source = tokio::runtime::Runtime::new()
1289 .unwrap()
1290 .block_on(args.resolve_manifest_source(HOLESKY.chain.id()))
1291 .unwrap();
1292
1293 assert_eq!(source, "./manifest.json");
1294 }
1295
1296 #[test]
1297 fn test_compression_format_detection() {
1298 assert!(matches!(
1299 CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
1300 Ok(CompressionFormat::Lz4)
1301 ));
1302 assert!(matches!(
1303 CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
1304 Ok(CompressionFormat::Zstd)
1305 ));
1306 assert!(matches!(
1307 CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
1308 Ok(CompressionFormat::Lz4)
1309 ));
1310 assert!(matches!(
1311 CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
1312 Ok(CompressionFormat::Zstd)
1313 ));
1314 assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
1315 }
1316
1317 #[test]
1318 fn inject_archive_only_components_for_archive_selection() {
1319 let manifest = manifest_with_archive_only_components();
1320 let mut selections = BTreeMap::new();
1321 selections.insert(SnapshotComponentType::Transactions, ComponentSelection::All);
1322 selections.insert(SnapshotComponentType::Receipts, ComponentSelection::All);
1323 selections.insert(SnapshotComponentType::AccountChangesets, ComponentSelection::All);
1324 selections.insert(SnapshotComponentType::StorageChangesets, ComponentSelection::All);
1325
1326 inject_archive_only_components(&mut selections, &manifest, true);
1327
1328 assert_eq!(
1329 selections.get(&SnapshotComponentType::TransactionSenders),
1330 Some(&ComponentSelection::All)
1331 );
1332 assert_eq!(
1333 selections.get(&SnapshotComponentType::RocksdbIndices),
1334 Some(&ComponentSelection::All)
1335 );
1336 }
1337
1338 #[test]
1339 fn inject_archive_only_components_without_rocksdb() {
1340 let manifest = manifest_with_archive_only_components();
1341 let mut selections = BTreeMap::new();
1342 selections.insert(SnapshotComponentType::Transactions, ComponentSelection::All);
1343 selections.insert(SnapshotComponentType::Receipts, ComponentSelection::All);
1344 selections.insert(SnapshotComponentType::AccountChangesets, ComponentSelection::All);
1345 selections.insert(SnapshotComponentType::StorageChangesets, ComponentSelection::All);
1346
1347 inject_archive_only_components(&mut selections, &manifest, false);
1348
1349 assert_eq!(
1350 selections.get(&SnapshotComponentType::TransactionSenders),
1351 Some(&ComponentSelection::All)
1352 );
1353 assert_eq!(selections.get(&SnapshotComponentType::RocksdbIndices), None);
1354 }
1355
1356 #[test]
1357 fn should_reset_index_stage_checkpoints_without_rocksdb_indices() {
1358 let mut selections = BTreeMap::new();
1359 selections.insert(SnapshotComponentType::Transactions, ComponentSelection::All);
1360 assert!(should_reset_index_stage_checkpoints(&selections));
1361
1362 selections.insert(SnapshotComponentType::RocksdbIndices, ComponentSelection::All);
1363 assert!(!should_reset_index_stage_checkpoints(&selections));
1364 }
1365
1366 #[test]
1367 fn startup_node_command_omits_default_chain_arg() {
1368 let command =
1369 startup_node_command_for_binary::<EthereumChainSpecParser>("reth", MAINNET.as_ref());
1370
1371 assert_eq!(command, "reth node");
1372 }
1373
1374 #[test]
1375 fn startup_node_command_includes_non_default_chain_arg() {
1376 let command =
1377 startup_node_command_for_binary::<EthereumChainSpecParser>("reth", HOLESKY.as_ref());
1378
1379 assert_eq!(command, "reth node --chain holesky");
1380 }
1381
1382 #[test]
1383 fn startup_node_command_uses_running_binary_name() {
1384 let command =
1385 startup_node_command_for_binary::<EthereumChainSpecParser>("tempo", HOLESKY.as_ref());
1386
1387 assert_eq!(command, "tempo node --chain holesky");
1388 }
1389
1390 #[test]
1391 fn download_command_uses_binary_name() {
1392 assert_eq!(download_command_for_binary("tempo"), "tempo download");
1393 }
1394}