reth_cli_commands/download/
source.rs

1use super::{
2    download_command, manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults,
3};
4use eyre::{Result, WrapErr};
5use reqwest::Client;
6use reth_fs_util as fs;
7use std::path::{Path, PathBuf};
8use tracing::info;
9use url::Url;
10
11/// An entry from the snapshot discovery API listing.
12#[derive(serde::Deserialize)]
13#[serde(rename_all = "camelCase")]
14pub(crate) struct SnapshotApiEntry {
15    #[serde(deserialize_with = "deserialize_string_or_u64")]
16    chain_id: u64,
17    #[serde(deserialize_with = "deserialize_string_or_u64")]
18    block: u64,
19    #[serde(default)]
20    date: Option<String>,
21    #[serde(default)]
22    profile: Option<String>,
23    metadata_url: String,
24    #[serde(default)]
25    size: u64,
26}
27
28impl SnapshotApiEntry {
29    /// Returns whether this discovery entry points to a modular manifest.
30    fn is_modular(&self) -> bool {
31        self.metadata_url.ends_with("manifest.json")
32    }
33}
34
35/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
36///
37/// Queries the configured snapshot API and returns the manifest URL for the most
38/// recent modular snapshot matching the requested chain.
39pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
40    let defaults = DownloadDefaults::get_global();
41    let api_url = &*defaults.snapshot_api_url;
42    let snapshot_source_url = defaults.snapshot_source_url();
43
44    info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
45
46    let entries = fetch_snapshot_api_entries(chain_id).await?;
47    let entry =
48        entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
49            eyre::eyre!(
50                "No modular snapshot manifest found for chain \
51             {chain_id} at {api_url}\n\n\
52             You can provide a manifest URL directly with --manifest-url, or\n\
53             use a direct snapshot URL with -u from:\n\
54             \t- {}\n\n\
55             Use --list to see all available snapshots.",
56                snapshot_source_url,
57            )
58        })?;
59
60    info!(target: "reth::cli",
61        block = entry.block,
62        url = %entry.metadata_url,
63        "Found latest snapshot manifest"
64    );
65
66    Ok(entry.metadata_url.clone())
67}
68
69/// Deserializes a JSON value that may be either a number or a string-encoded number.
70fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
71where
72    D: serde::Deserializer<'de>,
73{
74    use serde::Deserialize;
75    let value = serde_json::Value::deserialize(deserializer)?;
76    match &value {
77        serde_json::Value::Number(n) => {
78            n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
79        }
80        serde_json::Value::String(s) => {
81            s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
82        }
83        _ => Err(serde::de::Error::custom("expected number or string")),
84    }
85}
86
87/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
88pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
89    let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
90
91    let entries: Vec<SnapshotApiEntry> = Client::new()
92        .get(api_url)
93        .send()
94        .await
95        .and_then(|r| r.error_for_status())
96        .wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
97        .json()
98        .await?;
99
100    Ok(entries.into_iter().filter(|entry| entry.chain_id == chain_id).collect())
101}
102
103/// Prints a formatted table of available modular snapshots.
104pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
105    let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
106
107    let snapshot_source_url = DownloadDefaults::get_global().snapshot_source_url();
108    println!("Available snapshots for chain {chain_id} ({}):\n", snapshot_source_url,);
109    println!("{:<12}  {:>10}  {:<10}  {:>10}  MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
110    println!("{}", "-".repeat(100));
111
112    for entry in &modular {
113        let date = entry.date.as_deref().unwrap_or("-");
114        let profile = entry.profile.as_deref().unwrap_or("-");
115        let size = if entry.size > 0 {
116            DownloadProgress::format_size(entry.size)
117        } else {
118            "-".to_string()
119        };
120
121        println!(
122            "{date:<12}  {:>10}  {profile:<10}  {size:>10}  {}",
123            entry.block, entry.metadata_url
124        );
125    }
126
127    if modular.is_empty() {
128        println!("  (no modular snapshots found)");
129    }
130
131    println!(
132        "\nTo download a specific snapshot, copy its manifest URL and run:\n  \
133         {} --manifest-url <URL>",
134        download_command()
135    );
136}
137
138/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path.
139pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
140    if let Ok(parsed) = Url::parse(source) {
141        return match parsed.scheme() {
142            "http" | "https" => {
143                let response = Client::new()
144                    .get(source)
145                    .send()
146                    .await
147                    .and_then(|r| r.error_for_status())
148                    .wrap_err_with(|| {
149                        let sources = DownloadDefaults::get_global()
150                            .available_snapshots
151                            .iter()
152                            .map(|snapshot| format!("\t- {snapshot}"))
153                            .collect::<Vec<_>>()
154                            .join("\n");
155                        format!(
156                            "Failed to fetch snapshot manifest from {source}\n\n\
157                             The manifest endpoint may not be available for this snapshot source.\n\
158                             You can use a direct snapshot URL instead:\n\n\
159                             \t{} -u <snapshot-url>\n\n\
160                             Available snapshot sources:\n{sources}",
161                            download_command()
162                        )
163                    })?;
164                Ok(response.json().await?)
165            }
166            "file" => {
167                let path = parsed
168                    .to_file_path()
169                    .map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
170                let content = fs::read_to_string(path)?;
171                Ok(serde_json::from_str(&content)?)
172            }
173            _ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
174        };
175    }
176
177    let content = fs::read_to_string(source)?;
178    Ok(serde_json::from_str(&content)?)
179}
180
181/// Resolves the base URL used to join relative archive paths in a manifest.
182pub(crate) fn resolve_manifest_base_url(
183    manifest: &SnapshotManifest,
184    source: &str,
185) -> Result<String> {
186    if let Some(base_url) = manifest.base_url.as_deref() &&
187        !base_url.is_empty()
188    {
189        return Ok(base_url.trim_end_matches('/').to_string());
190    }
191
192    if let Ok(mut url) = Url::parse(source) {
193        if url.scheme() == "file" {
194            let mut path = url
195                .to_file_path()
196                .map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
197            path.pop();
198            let mut base = Url::from_directory_path(path)
199                .map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
200                .to_string();
201            if base.ends_with('/') {
202                base.pop();
203            }
204            return Ok(base);
205        }
206
207        {
208            let mut segments = url
209                .path_segments_mut()
210                .map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
211            segments.pop_if_empty();
212            segments.pop();
213        }
214        return Ok(url.as_str().trim_end_matches('/').to_string());
215    }
216
217    let path = Path::new(source);
218    let manifest_dir = if path.is_absolute() {
219        path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
220    } else {
221        let joined = std::env::current_dir()?.join(path);
222        joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
223    };
224
225    let mut base = Url::from_directory_path(&manifest_dir)
226        .map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
227        .to_string();
228    if base.ends_with('/') {
229        base.pop();
230    }
231    Ok(base)
232}
reth_cli_commands/download/source.rs

reth_cli_commands/download/
source.rs
