reth_cli_commands/download/
source.rs

1use super::{
2    download_command, manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults,
3};
4use eyre::{Result, WrapErr};
5use reqwest::Client;
6use reth_fs_util as fs;
7use serde::Deserialize;
8use std::path::{Path, PathBuf};
9use tracing::info;
10use url::Url;
11
12/// An entry from the snapshot discovery API listing.
13#[derive(serde::Deserialize)]
14#[serde(rename_all = "camelCase")]
15pub(crate) struct SnapshotApiEntry {
16    #[serde(deserialize_with = "deserialize_string_or_u64")]
17    chain_id: u64,
18    #[serde(deserialize_with = "deserialize_string_or_u64")]
19    block: u64,
20    #[serde(default)]
21    date: Option<String>,
22    #[serde(default)]
23    profile: Option<String>,
24    metadata_url: String,
25    #[serde(default)]
26    size: u64,
27}
28
29impl SnapshotApiEntry {
30    /// Returns whether this discovery entry points to a modular manifest.
31    fn is_modular(&self) -> bool {
32        self.metadata_url.ends_with("manifest.json")
33    }
34
35    /// Strips control characters from the strings supplied by the API, so they can't inject
36    /// terminal escape sequences when printed or logged.
37    fn strip_control_chars(mut self) -> Self {
38        for s in [Some(&mut self.metadata_url), self.date.as_mut(), self.profile.as_mut()]
39            .into_iter()
40            .flatten()
41        {
42            s.retain(|c| !c.is_control());
43        }
44        self
45    }
46}
47
48/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
49///
50/// Queries the configured snapshot API and returns the manifest URL for the most
51/// recent modular snapshot matching the requested chain.
52pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
53    let defaults = DownloadDefaults::get_global();
54    let api_url = &*defaults.snapshot_api_url;
55    let snapshot_source_url = defaults.snapshot_source_url();
56
57    info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
58
59    let entries = fetch_snapshot_api_entries(chain_id).await?;
60    let entry =
61        entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
62            eyre::eyre!(
63                "No modular snapshot manifest found for chain \
64             {chain_id} at {api_url}\n\n\
65             You can provide a manifest URL directly with --manifest-url, or\n\
66             use a direct snapshot URL with -u from:\n\
67             \t- {}\n\n\
68             Use --list to see all available snapshots.",
69                snapshot_source_url,
70            )
71        })?;
72
73    info!(target: "reth::cli",
74        block = entry.block,
75        url = %entry.metadata_url,
76        "Found latest snapshot manifest"
77    );
78
79    Ok(entry.metadata_url.clone())
80}
81
82/// Deserializes a JSON value that may be either a number or a string-encoded number.
83fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
84where
85    D: serde::Deserializer<'de>,
86{
87    let value = serde_json::Value::deserialize(deserializer)?;
88    match &value {
89        serde_json::Value::Number(n) => {
90            n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
91        }
92        serde_json::Value::String(s) => {
93            s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
94        }
95        _ => Err(serde::de::Error::custom("expected number or string")),
96    }
97}
98
99/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
100pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
101    let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
102
103    let entries: Vec<SnapshotApiEntry> = Client::new()
104        .get(api_url)
105        .send()
106        .await
107        .and_then(|r| r.error_for_status())
108        .wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
109        .json()
110        .await?;
111
112    Ok(entries
113        .into_iter()
114        .filter(|entry| entry.chain_id == chain_id)
115        .map(SnapshotApiEntry::strip_control_chars)
116        .collect())
117}
118
119/// Prints a formatted table of available modular snapshots.
120pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
121    let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
122
123    let snapshot_source_url = DownloadDefaults::get_global().snapshot_source_url();
124    println!("Available snapshots for chain {chain_id} ({}):\n", snapshot_source_url,);
125    println!("{:<12}  {:>10}  {:<10}  {:>10}  MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
126    println!("{}", "-".repeat(100));
127
128    for entry in &modular {
129        let date = entry.date.as_deref().unwrap_or("-");
130        let profile = entry.profile.as_deref().unwrap_or("-");
131        let size = if entry.size > 0 {
132            DownloadProgress::format_size(entry.size)
133        } else {
134            "-".to_string()
135        };
136
137        println!(
138            "{date:<12}  {:>10}  {profile:<10}  {size:>10}  {}",
139            entry.block, entry.metadata_url
140        );
141    }
142
143    if modular.is_empty() {
144        println!("  (no modular snapshots found)");
145    }
146
147    println!(
148        "\nTo download a specific snapshot, copy its manifest URL and run:\n  \
149         {} --manifest-url <URL>",
150        download_command()
151    );
152}
153
154/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path, rejecting zero
155/// `blocks_per_file` and output file paths outside the datadir.
156pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
157    let manifest = read_manifest_source(source).await?;
158    manifest.validate_blocks_per_file()?;
159    manifest.validate_output_paths()?;
160    Ok(manifest)
161}
162
163async fn read_manifest_source(source: &str) -> Result<SnapshotManifest> {
164    if let Ok(parsed) = Url::parse(source) {
165        return match parsed.scheme() {
166            "http" | "https" => {
167                let response = Client::new()
168                    .get(source)
169                    .send()
170                    .await
171                    .and_then(|r| r.error_for_status())
172                    .wrap_err_with(|| {
173                        let sources = DownloadDefaults::get_global()
174                            .available_snapshots
175                            .iter()
176                            .map(|snapshot| format!("\t- {snapshot}"))
177                            .collect::<Vec<_>>()
178                            .join("\n");
179                        format!(
180                            "Failed to fetch snapshot manifest from {source}\n\n\
181                             The manifest endpoint may not be available for this snapshot source.\n\
182                             You can use a direct snapshot URL instead:\n\n\
183                             \t{} -u <snapshot-url>\n\n\
184                             Available snapshot sources:\n{sources}",
185                            download_command()
186                        )
187                    })?;
188                Ok(response.json().await?)
189            }
190            "file" => {
191                let path = parsed
192                    .to_file_path()
193                    .map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
194                let content = fs::read_to_string(path)?;
195                Ok(serde_json::from_str(&content)?)
196            }
197            _ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
198        };
199    }
200
201    let content = fs::read_to_string(source)?;
202    Ok(serde_json::from_str(&content)?)
203}
204
205/// Resolves the base URL used to join relative archive paths in a manifest.
206pub(crate) fn resolve_manifest_base_url(
207    manifest: &SnapshotManifest,
208    source: &str,
209) -> Result<String> {
210    if let Some(base_url) = manifest.base_url.as_deref() &&
211        !base_url.is_empty()
212    {
213        return Ok(base_url.trim_end_matches('/').to_string());
214    }
215
216    if let Ok(mut url) = Url::parse(source) {
217        if url.scheme() == "file" {
218            let mut path = url
219                .to_file_path()
220                .map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
221            path.pop();
222            let mut base = Url::from_directory_path(path)
223                .map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
224                .to_string();
225            if base.ends_with('/') {
226                base.pop();
227            }
228            return Ok(base);
229        }
230
231        {
232            let mut segments = url
233                .path_segments_mut()
234                .map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
235            segments.pop_if_empty();
236            segments.pop();
237        }
238        return Ok(url.as_str().trim_end_matches('/').to_string());
239    }
240
241    let path = Path::new(source);
242    let manifest_dir = if path.is_absolute() {
243        path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
244    } else {
245        let joined = std::env::current_dir()?.join(path);
246        joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
247    };
248
249    let mut base = Url::from_directory_path(&manifest_dir)
250        .map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
251        .to_string();
252    if base.ends_with('/') {
253        base.pop();
254    }
255    Ok(base)
256}
257
258#[cfg(test)]
259mod tests {
260    use super::*;
261
262    #[test]
263    fn strip_control_chars_removes_terminal_escapes() {
264        let entry = serde_json::from_str::<SnapshotApiEntry>(
265            r#"{
266                "chainId": "1",
267                "block": 100,
268                "date": "2026-\u009b01-01",
269                "profile": "x\u001b]0;title\u0007y",
270                "metadataUrl": "https://example.com/\u001b[2Jmanifest.json"
271            }"#,
272        )
273        .unwrap()
274        .strip_control_chars();
275
276        assert_eq!(entry.date.as_deref(), Some("2026-01-01"));
277        assert_eq!(entry.profile.as_deref(), Some("x]0;titley"));
278        assert_eq!(entry.metadata_url, "https://example.com/[2Jmanifest.json");
279    }
280}
reth_cli_commands/download/source.rs

reth_cli_commands/download/
source.rs
