1use super::{
2 fetch::{ArchiveFetcher, DownloadedArchive},
3 progress::{
4 ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress,
5 DownloadRequestLimiter, ProgressReader, SharedProgressReader,
6 },
7 session::DownloadSession,
8 MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
9};
10use eyre::{Result, WrapErr};
11use lz4::Decoder;
12use reqwest::blocking::Client as BlockingClient;
13use reth_cli_util::cancellation::CancellationToken;
14use reth_fs_util as fs;
15use std::{
16 io::Read,
17 path::{Component, Path, PathBuf},
18 sync::{
19 atomic::{AtomicBool, Ordering},
20 Arc,
21 },
22 thread,
23 time::{Duration, Instant},
24};
25use tar::Archive;
26use tokio::task;
27use tracing::{info, warn};
28use url::Url;
29use zstd::stream::read::Decoder as ZstdDecoder;
30
31const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
32const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
33const STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE: u64 = 64 * 1024 * 1024;
34const EXTRACTION_PROGRESS_POLL_INTERVAL: Duration = Duration::from_millis(100);
35
36#[derive(Debug, Clone, Copy)]
38pub(crate) enum CompressionFormat {
39 Lz4,
41 Zstd,
43}
44
45impl CompressionFormat {
46 pub(crate) fn from_url(url: &str) -> Result<Self> {
48 let path =
49 Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
50
51 if path.ends_with(EXTENSION_TAR_LZ4) {
52 Ok(Self::Lz4)
53 } else if path.ends_with(EXTENSION_TAR_ZSTD) {
54 Ok(Self::Zstd)
55 } else {
56 Err(eyre::eyre!(
57 "Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
58 path
59 ))
60 }
61 }
62}
63
64fn extract_archive<R: Read>(
66 reader: R,
67 total_size: u64,
68 format: CompressionFormat,
69 target_dir: &Path,
70 static_files_dir: Option<&Path>,
71 cancel_token: CancellationToken,
72) -> Result<()> {
73 let progress_reader = ProgressReader::new(reader, total_size, cancel_token);
74
75 match format {
76 CompressionFormat::Lz4 => {
77 let decoder = Decoder::new(progress_reader)?;
78 unpack_archive(Archive::new(decoder), target_dir, static_files_dir, None)?;
79 }
80 CompressionFormat::Zstd => {
81 let decoder = ZstdDecoder::new(progress_reader)?;
82 unpack_archive(Archive::new(decoder), target_dir, static_files_dir, None)?;
83 }
84 }
85
86 println!();
87 Ok(())
88}
89
90pub(crate) fn extract_archive_raw<R: Read>(
92 reader: R,
93 format: CompressionFormat,
94 target_dir: &Path,
95 static_files_dir: Option<&Path>,
96 progress: Option<&mut ArchiveExtractionProgress>,
97) -> Result<()> {
98 match format {
99 CompressionFormat::Lz4 => {
100 unpack_archive(
101 Archive::new(Decoder::new(reader)?),
102 target_dir,
103 static_files_dir,
104 progress,
105 )?;
106 }
107 CompressionFormat::Zstd => {
108 unpack_archive(
109 Archive::new(ZstdDecoder::new(reader)?),
110 target_dir,
111 static_files_dir,
112 progress,
113 )?;
114 }
115 }
116
117 Ok(())
118}
119
120fn unpack_archive<R: Read>(
121 mut archive: Archive<R>,
122 target_dir: &Path,
123 static_files_dir: Option<&Path>,
124 mut progress: Option<&mut ArchiveExtractionProgress>,
125) -> Result<()> {
126 if static_files_dir.is_none() && progress.is_none() {
127 archive.unpack(target_dir)?;
128 return Ok(())
129 }
130 let entries = archive.entries().wrap_err_with(|| {
131 format!("failed to read archive entries for `{}`", target_dir.display())
132 })?;
133
134 for entry in entries {
135 let mut entry = entry.wrap_err_with(|| {
136 format!("failed to read archive entry for `{}`", target_dir.display())
137 })?;
138 extract_entry_with_progress(
139 &mut entry,
140 target_dir,
141 static_files_dir,
142 progress.as_deref_mut(),
143 )?;
144 }
145
146 Ok(())
147}
148
149pub(crate) fn static_file_relative_path(path: &Path) -> Option<&Path> {
151 path.strip_prefix(".").unwrap_or(path).strip_prefix("static_files").ok()
152}
153
154fn unpack_static_file<R: Read>(
157 entry: &mut tar::Entry<'_, R>,
158 static_files_dir: &Path,
159 relative_path: &Path,
160) -> Result<()> {
161 eyre::ensure!(
162 relative_path
163 .components()
164 .all(|part| matches!(part, Component::Normal(_) | Component::CurDir)),
165 "Invalid static file archive path: {}",
166 relative_path.display()
167 );
168 let entry_type = entry.header().entry_type();
169 eyre::ensure!(
170 entry_type.is_file() || entry_type.is_dir(),
171 "Unsupported static file archive entry"
172 );
173 fs::create_dir_all(static_files_dir)?;
174 let root = static_files_dir.to_path_buf();
175 let dest = root.join(relative_path);
176 let mut current = root.clone();
178 for part in relative_path.components() {
179 current.push(part);
180 if let Ok(metadata) = std::fs::symlink_metadata(¤t) {
181 eyre::ensure!(
182 !metadata.file_type().is_symlink(),
183 "Static file archive path contains a symlink"
184 );
185 }
186 }
187 if relative_path.as_os_str().is_empty() {
188 return Ok(())
189 }
190 if let Some(parent) = dest.parent() {
191 fs::create_dir_all(parent)?;
192 }
193 entry.unpack(dest)?;
194 Ok(())
195}
196
197fn unpack_entry<R: Read>(
198 entry: &mut tar::Entry<'_, R>,
199 target_dir: &Path,
200 static_files_dir: Option<&Path>,
201) -> Result<()> {
202 eyre::ensure!(
205 static_files_dir.is_none() || !entry.header().entry_type().is_symlink(),
206 "Archive symlinks are unsupported with a custom static files directory"
207 );
208 let path = entry.path()?.into_owned();
209 if let Some(static_files_dir) = static_files_dir &&
210 let Some(relative_path) = static_file_relative_path(&path)
211 {
212 unpack_static_file(entry, static_files_dir, relative_path)
213 } else {
214 entry.unpack_in(target_dir)?;
215 Ok(())
216 }
217}
218
219fn extract_entry_with_progress<R: Read>(
220 entry: &mut tar::Entry<'_, R>,
221 target_dir: &Path,
222 static_files_dir: Option<&Path>,
223 progress: Option<&mut ArchiveExtractionProgress>,
224) -> Result<()> {
225 let size = entry.header().entry_size().unwrap_or(0);
226 let entry_type = entry.header().entry_type();
227
228 if !entry_type.is_file() || size == 0 {
229 unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| {
230 format!("failed to extract archive into `{}`", target_dir.display())
231 })?;
232 return Ok(())
233 }
234
235 if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE {
236 unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| {
237 format!("failed to extract archive into `{}`", target_dir.display())
238 })?;
239 if let Some(progress) = progress {
240 progress.record_extracted(size);
241 }
242 return Ok(())
243 }
244
245 let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else {
246 unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| {
247 format!("failed to extract archive into `{}`", target_dir.display())
248 })?;
249 return Ok(())
250 };
251
252 let Some(entry_path) = entry_destination_path(entry, target_dir, static_files_dir)? else {
253 unpack_entry(entry, target_dir, static_files_dir).wrap_err_with(|| {
254 format!("failed to extract archive into `{}`", target_dir.display())
255 })?;
256 return Ok(())
257 };
258
259 let stop = Arc::new(AtomicBool::new(false));
260 let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop));
261 let unpack_result = unpack_entry(entry, target_dir, static_files_dir)
262 .wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display()));
263 stop.store(true, Ordering::Relaxed);
264
265 let monitor_result = monitor.join();
266 unpack_result?;
267
268 monitor_result.map_err(|_| eyre::eyre!("extraction progress monitor panicked"))?;
269 Ok(())
270}
271
272fn entry_destination_path<R: Read>(
273 entry: &tar::Entry<'_, R>,
274 target_dir: &Path,
275 static_files_dir: Option<&Path>,
276) -> Result<Option<PathBuf>> {
277 let mut file_dst = target_dir.to_path_buf();
278 let path = entry.path().wrap_err("invalid path in archive entry")?;
279 let path = if let Some(static_files_dir) = static_files_dir &&
280 let Some(relative_path) = static_file_relative_path(&path)
281 {
282 file_dst = static_files_dir.to_path_buf();
283 relative_path
284 } else {
285 path.as_ref()
286 };
287
288 for part in path.components() {
289 match part {
290 Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
291 Component::ParentDir => return Ok(None),
292 Component::Normal(part) => file_dst.push(part),
293 }
294 }
295
296 if file_dst == target_dir {
297 return Ok(None)
298 }
299
300 Ok(Some(file_dst))
301}
302
303fn spawn_extraction_progress_monitor(
304 entry_path: PathBuf,
305 progress: ArchiveExtractionProgressHandle,
306 stop: Arc<AtomicBool>,
307) -> thread::JoinHandle<()> {
308 thread::spawn(move || {
309 let mut extracted = 0_u64;
310
311 loop {
312 record_extracted_file_bytes(&entry_path, &progress, &mut extracted);
313 if stop.load(Ordering::Relaxed) {
314 break;
315 }
316 thread::sleep(EXTRACTION_PROGRESS_POLL_INTERVAL);
317 }
318 })
319}
320
321fn record_extracted_file_bytes(
322 entry_path: &Path,
323 progress: &ArchiveExtractionProgressHandle,
324 extracted: &mut u64,
325) {
326 let Ok(meta) = fs::metadata(entry_path) else { return };
327 let len = meta.len();
328 if len > *extracted {
329 progress.record_extracted(len - *extracted);
330 *extracted = len;
331 }
332}
333
334fn extract_from_file(
336 path: &Path,
337 format: CompressionFormat,
338 target_dir: &Path,
339 static_files_dir: Option<&Path>,
340) -> Result<()> {
341 let file = std::fs::File::open(path)?;
342 let total_size = file.metadata()?.len();
343 info!(target: "reth::cli",
344 file = %path.display(),
345 size = %DownloadProgress::format_size(total_size),
346 "Extracting local archive"
347 );
348 let start = Instant::now();
349 extract_archive(
350 file,
351 total_size,
352 format,
353 target_dir,
354 static_files_dir,
355 CancellationToken::new(),
356 )?;
357 info!(target: "reth::cli",
358 file = %path.display(),
359 elapsed = %DownloadProgress::format_duration(start.elapsed()),
360 "Local extraction complete"
361 );
362 Ok(())
363}
364
365pub(crate) fn streaming_download_and_extract(
369 url: &str,
370 format: CompressionFormat,
371 target_dir: &Path,
372 static_files_dir: Option<&Path>,
373 session: &DownloadSession,
374) -> Result<()> {
375 if let Some(path) = archive_file_url_path(url)? {
376 let size = path.metadata()?.len();
377 extract_from_file(&path, format, target_dir, static_files_dir)?;
378 session.record_archive_output_complete(size);
379 return Ok(())
380 }
381
382 let shared = session.progress();
383 let quiet = session.progress().is_some();
384 let mut last_error: Option<eyre::Error> = None;
385
386 for attempt in 1..=MAX_DOWNLOAD_RETRIES {
387 if attempt > 1 {
388 info!(target: "reth::cli",
389 url = %url,
390 attempt,
391 max = MAX_DOWNLOAD_RETRIES,
392 "Retrying streaming download from scratch"
393 );
394 }
395
396 let client = BlockingClient::builder().connect_timeout(Duration::from_secs(30)).build()?;
397 let _request_permit = session
398 .request_limiter()
399 .map(|limiter| limiter.acquire(session.progress(), session.cancel_token()))
400 .transpose()?;
401
402 let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
403 Ok(r) => r,
404 Err(error) => {
405 let err = eyre::Error::from(error);
406 if attempt < MAX_DOWNLOAD_RETRIES {
407 warn!(target: "reth::cli",
408 url = %url,
409 attempt,
410 max = MAX_DOWNLOAD_RETRIES,
411 err = %err,
412 "Streaming request failed, retrying"
413 );
414 }
415 last_error = Some(err);
416 if attempt < MAX_DOWNLOAD_RETRIES {
417 std::thread::sleep(
418 session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)),
419 );
420 }
421 continue;
422 }
423 };
424
425 if !quiet && let Some(size) = response.content_length() {
426 info!(target: "reth::cli",
427 url = %url,
428 size = %DownloadProgress::format_size(size),
429 "Streaming archive"
430 );
431 }
432
433 let result = if let Some(progress) = shared {
434 let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) };
435 extract_archive_raw(reader, format, target_dir, static_files_dir, None)
436 } else {
437 let total_size = response.content_length().unwrap_or(0);
438 extract_archive(
439 response,
440 total_size,
441 format,
442 target_dir,
443 static_files_dir,
444 session.cancel_token().clone(),
445 )
446 };
447
448 match result {
449 Ok(()) => return Ok(()),
450 Err(error) => {
451 if attempt < MAX_DOWNLOAD_RETRIES {
452 warn!(target: "reth::cli",
453 url = %url,
454 attempt,
455 max = MAX_DOWNLOAD_RETRIES,
456 err = %error,
457 "Streaming extraction failed, retrying"
458 );
459 }
460 last_error = Some(error);
461 if attempt < MAX_DOWNLOAD_RETRIES {
462 std::thread::sleep(
463 session.retry_delay(Duration::from_secs(RETRY_BACKOFF_SECS)),
464 );
465 }
466 }
467 }
468 }
469
470 Err(last_error.unwrap_or_else(|| {
471 eyre::eyre!("Streaming download failed after {MAX_DOWNLOAD_RETRIES} attempts")
472 }))
473}
474
475fn archive_file_url_path(url: &str) -> Result<Option<PathBuf>> {
477 let Ok(parsed) = Url::parse(url) else { return Ok(None) };
478 if parsed.scheme() != "file" {
479 return Ok(None)
480 }
481
482 parsed
483 .to_file_path()
484 .map(Some)
485 .map_err(|_| eyre::eyre!("Invalid file:// archive URL path: {url}"))
486}
487
488fn download_and_extract(
490 url: &str,
491 format: CompressionFormat,
492 target_dir: &Path,
493 static_files_dir: Option<&Path>,
494 session: DownloadSession,
495) -> Result<()> {
496 let quiet = session.progress().is_some();
497 let fetcher = ArchiveFetcher::new(url.to_string(), target_dir, session.clone());
498 let DownloadedArchive { path: downloaded_path, size: total_size } = fetcher.download(None)?;
499
500 let file_name =
501 downloaded_path.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
502
503 if !quiet {
504 info!(target: "reth::cli",
505 file = %file_name,
506 size = %DownloadProgress::format_size(total_size),
507 "Extracting archive"
508 );
509 }
510 let file = fs::open(&downloaded_path)?;
511
512 if quiet {
513 extract_archive_raw(file, format, target_dir, static_files_dir, None)?;
514 } else {
515 extract_archive(
516 file,
517 total_size,
518 format,
519 target_dir,
520 static_files_dir,
521 session.cancel_token().clone(),
522 )?;
523 info!(target: "reth::cli",
524 file = %file_name,
525 "Extraction complete"
526 );
527 }
528
529 fetcher.cleanup_downloaded_files();
530 session.record_archive_output_complete(total_size);
531
532 Ok(())
533}
534
535fn blocking_download_and_extract(
541 url: &str,
542 target_dir: &Path,
543 static_files_dir: Option<&Path>,
544 resumable: bool,
545 request_limiter: Option<Arc<DownloadRequestLimiter>>,
546 cancel_token: CancellationToken,
547 retry_backoff: Option<Duration>,
548) -> Result<()> {
549 let format = CompressionFormat::from_url(url)?;
550
551 if let Ok(parsed_url) = Url::parse(url) &&
552 parsed_url.scheme() == "file"
553 {
554 let session = DownloadSession::new(None, request_limiter, cancel_token)
555 .with_retry_backoff(retry_backoff);
556 let file_path = parsed_url
557 .to_file_path()
558 .map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
559 let result = extract_from_file(&file_path, format, target_dir, static_files_dir);
560 if result.is_ok() {
561 session.record_archive_output_complete(file_path.metadata()?.len());
562 }
563 result
564 } else if let Some(request_limiter) = request_limiter {
565 download_and_extract(
566 url,
567 format,
568 target_dir,
569 static_files_dir,
570 DownloadSession::new(None, Some(request_limiter), cancel_token)
571 .with_retry_backoff(retry_backoff),
572 )
573 } else if resumable {
574 let session =
575 DownloadSession::new(None, Some(DownloadRequestLimiter::new(1)), cancel_token)
576 .with_retry_backoff(retry_backoff);
577 download_and_extract(url, format, target_dir, static_files_dir, session)
578 } else {
579 let session =
580 DownloadSession::new(None, None, cancel_token).with_retry_backoff(retry_backoff);
581 let result =
582 streaming_download_and_extract(url, format, target_dir, static_files_dir, &session);
583 if result.is_ok() {
584 session.record_archive_output_complete(0);
585 }
586 result
587 }
588}
589
590pub(crate) async fn stream_and_extract(
595 url: &str,
596 target_dir: &Path,
597 static_files_dir: Option<&Path>,
598 resumable: bool,
599 request_limiter: Option<Arc<DownloadRequestLimiter>>,
600 cancel_token: CancellationToken,
601 retry_backoff: Option<Duration>,
602) -> Result<()> {
603 let target_dir = target_dir.to_path_buf();
604 let static_files_dir = static_files_dir.map(Path::to_path_buf);
605 let url = url.to_string();
606 task::spawn_blocking(move || {
607 blocking_download_and_extract(
608 &url,
609 &target_dir,
610 static_files_dir.as_deref(),
611 resumable,
612 request_limiter,
613 cancel_token,
614 retry_backoff,
615 )
616 })
617 .await??;
618
619 Ok(())
620}
621
622#[cfg(test)]
623mod tests {
624 use super::*;
625
626 #[test]
627 fn custom_static_root_cannot_be_redirected_by_archive_symlinks() {
628 for custom_path in ["custom", "custom/nested"] {
629 let target = tempfile::tempdir().unwrap();
630 let outside = tempfile::tempdir().unwrap();
631 fs::create_dir_all(outside.path().join("nested")).unwrap();
632 for path in ["headers", "nested/headers"] {
633 fs::write(outside.path().join(path), b"keep").unwrap();
634 }
635 let custom = target.path().join(custom_path);
636 let mut archive = tar::Builder::new(Vec::new());
637 let mut link = tar::Header::new_gnu();
638 link.set_entry_type(tar::EntryType::Symlink);
639 link.set_size(0);
640 link.set_mode(0o777);
641 archive.append_link(&mut link, "custom", outside.path()).unwrap();
642 let mut file = tar::Header::new_gnu();
643 file.set_size(4);
644 file.set_mode(0o644);
645 file.set_cksum();
646 archive.append_data(&mut file, "static_files/headers", b"data".as_slice()).unwrap();
647 let tar = archive.into_inner().unwrap();
648 let err =
649 unpack_archive(Archive::new(tar.as_slice()), target.path(), Some(&custom), None)
650 .unwrap_err();
651 assert!(format!("{err:#}").contains("Archive symlinks are unsupported"));
652 let outputs = [super::super::manifest::OutputFileChecksum {
653 path: "static_files/headers".into(),
654 size: 4,
655 blake3: String::new(),
656 }];
657 super::super::verify::OutputVerifier::new(target.path(), Some(&custom))
658 .cleanup(&outputs);
659 for path in ["headers", "nested/headers"] {
660 assert_eq!(fs::read(outside.path().join(path)).unwrap(), b"keep");
661 }
662 }
663 }
664
665 #[test]
666 fn remap_static_files_in_both_compression_formats() {
667 let mut archive = tar::Builder::new(Vec::new());
668 let mut directory = tar::Header::new_gnu();
669 directory.set_entry_type(tar::EntryType::Directory);
670 directory.set_size(0);
671 directory.set_mode(0o755);
672 directory.set_cksum();
673 archive.append_data(&mut directory, "./static_files/", std::io::empty()).unwrap();
674 for path in ["./static_files/nested/headers", "db/data"] {
675 let mut header = tar::Header::new_gnu();
676 header.set_size(4);
677 header.set_mode(0o644);
678 header.set_cksum();
679 archive.append_data(&mut header, path, b"data".as_slice()).unwrap();
680 }
681 let tar = archive.into_inner().unwrap();
682 for format in [CompressionFormat::Lz4, CompressionFormat::Zstd] {
683 let bytes = match format {
684 CompressionFormat::Lz4 => {
685 let mut encoder = lz4::EncoderBuilder::new().build(Vec::new()).unwrap();
686 std::io::copy(&mut tar.as_slice(), &mut encoder).unwrap();
687 let (bytes, result) = encoder.finish();
688 result.unwrap();
689 bytes
690 }
691 CompressionFormat::Zstd => zstd::encode_all(tar.as_slice(), 0).unwrap(),
692 };
693 let target = tempfile::tempdir().unwrap();
694 let custom = tempfile::tempdir().unwrap();
695 extract_archive_raw(bytes.as_slice(), format, target.path(), Some(custom.path()), None)
696 .unwrap();
697 assert_eq!(fs::read(custom.path().join("nested/headers")).unwrap(), b"data");
698 assert_eq!(fs::read(target.path().join("db/data")).unwrap(), b"data");
699 assert!(!target.path().join("static_files").exists());
700 }
701 }
702
703 #[test]
704 fn test_compression_format_detection() {
705 assert!(matches!(
706 CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
707 Ok(CompressionFormat::Lz4)
708 ));
709 assert!(matches!(
710 CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
711 Ok(CompressionFormat::Zstd)
712 ));
713 assert!(matches!(
714 CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
715 Ok(CompressionFormat::Lz4)
716 ));
717 assert!(matches!(
718 CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
719 Ok(CompressionFormat::Zstd)
720 ));
721 assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
722 }
723}