From 9366186245e07049190ee609cf7c8feee6361ae5 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 10 Sep 2026 20:13:23 +0200 Subject: [PATCH] mirror: true overall progress via --check-first + visible scan phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With SFTP remotes rclone lists the tree while transferring, so the `Transferred:` total grows during the run: the first ticks showed a single file's percentage (e.g. 2.9 MiB / 8.6 MiB, 35%) which then collapsed to 0% once the real 113 GiB total appeared. That is what looked like "progress per album". - pass --check-first: rclone finishes the remote listing before any transfer starts, so the byte total is final and the gauge is a true overall percentage for the whole mirror - while scanning, no bytes move yet, so the panel now shows `Scanning remote · 11,810 items listed` and `0 B / 113 GiB to transfer` (parsed from rclone's `Checks: … Listed N`) instead of a bogus 0% bar - once transfers start the display switches back to percentage + bytes + current file Verified against the real storagebox remote (capped with RCLONE_MAX_TRANSFER): scan ticks show listed counts climbing to 11,810 with the pending total, then transfers show `9.984 MiB / 113.477 GiB` with the final denominator and no per-file percentage jumps. Bump to 0.3.2. --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 5 ++++- src/engine.rs | 37 ++++++++++++++++++++++++++++++---- src/progress.rs | 24 ++++++++++++++++++++++ src/ui/mod.rs | 39 ++++++++++++++++++++++++++++++++++++ src/ui/tabs/sync.rs | 49 +++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 147 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 121d216..dc52005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,7 +253,7 @@ dependencies = [ [[package]] name = "dap-tui" -version = "0.3.1" +version = "0.3.2" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 336fa4f..fa462b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dap-tui" -version = "0.3.1" +version = "0.3.2" edition = "2021" description = "Device sync for portable players — TUI edition (ratatui)" license = "MIT" diff --git a/README.md b/README.md index 1c7efc0..5ed0f92 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,10 @@ Firmware types: `apple` (podkit command), `rockbox`, `android`, `sony` ## How the sync pipeline works 1. **Mirror** — `rclone sync` (or local `rsync`) pulls the StorageBox source - into the local library. + into the local library. rclone runs with `--check-first`, so the remote + listing completes before transfers start: the gauge then shows a true + overall percentage instead of the growing-total estimate, and the listing + phase is displayed as `Scanning remote · N items listed`. 2. **Diff** — the device is scanned and compared to the source with NFC-normalised paths, size, and 2-second-rounded mtimes (FAT32 friendly). 3. **Sync** — stale files are removed first, then `rsync --files-from` copies diff --git a/src/engine.rs b/src/engine.rs index 5b008e7..eb195d5 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,5 +1,5 @@ use crate::config::{self, Config, Device, Firmware}; -use crate::progress::{parse_rclone_current, parse_rclone_stats, parse_rsync_line, RsyncLine}; +use crate::progress::{parse_rclone_current, parse_rclone_listed, parse_rclone_stats, parse_rsync_line, RsyncLine}; use chrono::Local; use std::collections::HashMap; use std::fs; @@ -49,6 +49,9 @@ pub struct ProgressInfo { pub speed: String, pub eta: String, pub current: String, + /// rclone only: number of remote items listed so far (during the scan + /// phase before `--check-first` lets transfers start). + pub listed: u64, } #[derive(Debug, Clone)] @@ -137,6 +140,13 @@ impl WorkerCtx { self.changed(); } + fn set_mirror_listed(&self, listed: u64) { + if let Ok(mut s) = self.state.lock() { + s.mirror.listed = listed; + } + self.changed(); + } + fn set_device_progress(&self, done: u64, total: u64, speed: &str, eta: &str, current: &str, pct: Option) { if let Ok(mut s) = self.state.lock() { let p = &mut s.device_prog; @@ -420,10 +430,23 @@ fn run_mirror(config: &Config, device: &Device, ctx: &WorkerCtx) -> Result<(), S // multi-line `Transferred: … / …, N%, speed, ETA …` block newline-terminated, // so it streams live through the pipe and we can update the gauge in real // time while a cloud→local mirror is running. + // + // `--check-first` completes the remote listing before any transfer starts. + // Without it (e.g. over SFTP) rclone lists while transferring, so the + // `Transferred:` total keeps growing and the percentage stuck near 0% — + // early ticks even showed a single file's percentage, which looked like + // per-album progress. With the full total known up front the gauge is a + // true overall percentage; the initial listing shows as a scan phase. args.extend( - ["--create-empty-src-dirs", "--progress", "--stats", "1s"] - .iter() - .map(|s| s.to_string()), + [ + "--create-empty-src-dirs", + "--check-first", + "--progress", + "--stats", + "1s", + ] + .iter() + .map(|s| s.to_string()), ); stream(ctx, "rclone", &args, |line| { @@ -431,6 +454,11 @@ fn run_mirror(config: &Config, device: &Device, ctx: &WorkerCtx) -> Result<(), S ctx.set_mirror(p); return true; } + // `Checks: … Listed N` — the remote scan phase before transfers start. + if let Some(listed) = parse_rclone_listed(line) { + ctx.set_mirror_listed(listed); + return true; + } if let Some(file) = parse_rclone_current(line) { // Per-file `Transferring:` entry — show it as the current file // instead of logging it once per second. @@ -947,6 +975,7 @@ mod tests { + #[cfg(unix)] fn process_is_alive(pid: i32) -> bool { match std::fs::read_to_string(format!("/proc/{pid}/stat")) { diff --git a/src/progress.rs b/src/progress.rs index dd8f93b..da20a93 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -234,6 +234,16 @@ pub fn parse_rclone_stats(line: &str) -> Option { }) } +/// Parse the "Listed N" counter from an rclone `Checks:` progress line: +/// `Checks: 0 / 0, -, Listed 11810`. +/// rclone prints this while scanning the remote tree (before `--check-first` +/// lets any transfer start), so it is the only progress signal in that phase. +pub fn parse_rclone_listed(line: &str) -> Option { + let (_, rest) = line.trim().split_once("Listed")?; + let n = rest.split_whitespace().next()?; + n.replace(',', "").parse().ok() +} + /// A size like `80 MiB` / `0 B` — but not a plain file count like `6`. fn has_size_unit(s: &str) -> bool { s.chars().any(|c| c.is_ascii_alphabetic()) @@ -339,6 +349,20 @@ mod tests { assert_eq!(s.eta, "2s"); } + #[test] + fn parses_rclone_listed_count() { + assert_eq!( + parse_rclone_listed("Checks: 0 / 0, -, Listed 11810"), + Some(11810) + ); + assert_eq!( + parse_rclone_listed("Checks: 5 / 10, 50%, Listed 1,234"), + Some(1234) + ); + assert!(parse_rclone_listed("Transferred: 1 MiB / 2 MiB, 50%").is_none()); + assert!(parse_rclone_listed("Elapsed time: 5.0s").is_none()); + } + #[test] fn parses_rclone_current_file() { assert_eq!( diff --git a/src/ui/mod.rs b/src/ui/mod.rs index 879b1dd..d74354e 100644 --- a/src/ui/mod.rs +++ b/src/ui/mod.rs @@ -356,4 +356,43 @@ mod tests { assert!(out.contains("32.082 MiB"), "transferred bytes missing"); assert!(out.contains("sub/file3.bin"), "current file missing"); } + + #[test] + fn renders_mirror_scanning_phase() { + use crate::engine::{StepState, StepStatus}; + + let mut app = test_app(); + { + let mut s = app.engine.state.lock().unwrap(); + s.running = true; + s.device = Some("Test Player".into()); + s.steps = vec![ + StepState { + title: "Mirror StorageBox".into(), + detail: "refreshing local library…".into(), + status: StepStatus::Active, + }, + StepState { + title: "Sync Test Player".into(), + detail: "rockbox · music".into(), + status: StepStatus::Pending, + }, + ]; + // --check-first listing phase: no bytes moved, total still growing. + s.mirror.pct = 0; + s.mirror.done = "0 B".into(); + s.mirror.total = "113.477 GiB".into(); + s.mirror.listed = 11810; + } + app.tab = crate::app::Tab::Sync; + + let out = render(&app, 100, 30); + assert!(out.contains("Scanning remote"), "scan label missing"); + assert!(out.contains("11,810"), "listed count missing"); + assert!(out.contains("113.477 GiB"), "pending total missing"); + assert!( + !out.contains("Mirroring local library"), + "scan phase must replace the 0% mirror label" + ); + } } diff --git a/src/ui/tabs/sync.rs b/src/ui/tabs/sync.rs index 7429b2f..7d654ba 100644 --- a/src/ui/tabs/sync.rs +++ b/src/ui/tabs/sync.rs @@ -258,7 +258,7 @@ fn draw_progress(f: &mut Frame, area: Rect, state: &SyncState) { .position(|s| s.status == StepStatus::Active) .unwrap_or(usize::MAX); - let (pct, done, total, speed, eta, current, label) = if active == 0 { + let (pct, done, total, speed, eta, current, listed, label) = if active == 0 { ( state.mirror.pct, state.mirror.done.clone(), @@ -266,6 +266,7 @@ fn draw_progress(f: &mut Frame, area: Rect, state: &SyncState) { state.mirror.speed.clone(), state.mirror.eta.clone(), state.mirror.current.clone(), + state.mirror.listed, "Mirroring local library", ) } else if active == 1 { @@ -276,20 +277,39 @@ fn draw_progress(f: &mut Frame, area: Rect, state: &SyncState) { state.device_prog.speed.clone(), state.device_prog.eta.clone(), state.device_prog.current.clone(), + 0, "Syncing device", ) } else { - (0, String::new(), String::new(), String::new(), String::new(), String::new(), "Finalizing") + ( + 0, + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + 0, + "Finalizing", + ) }; + // With --check-first, rclone lists the whole remote before transferring: + // no bytes move yet, so show the scan instead of a meaningless 0% gauge. + let scanning = listed > 0 && (done.is_empty() || done == "0 B"); + let [gauge_row, stats] = Layout::vertical([Constraint::Length(2), Constraint::Min(0)]).areas(inner); + let gauge_label = if scanning { + format!(" Scanning remote · {} items listed ", fmt_count(listed)) + } else { + format!(" {label} · {pct}% ") + }; let g = Gauge::default() .block(Block::default().borders(Borders::BOTTOM).border_style(Style::default().fg(theme::BORDER))) .gauge_style(Style::default().fg(theme::ACCENT).bg(theme::BG_ALT)) .ratio((pct as f64) / 100.0) .label(Span::styled( - format!(" {label} · {pct}% "), + gauge_label, // No explicit colour: Gauge swaps fg/bg under the label so text // stays readable over both the filled and empty bar. Style::default().add_modifier(Modifier::BOLD), @@ -309,6 +329,9 @@ fn draw_progress(f: &mut Frame, area: Rect, state: &SyncState) { if total.is_empty() { "…".to_string() } else { total }, Style::default().fg(theme::CYAN), )); + if scanning { + left.push(Span::styled(" to transfer", Style::default().fg(theme::DIM))); + } let mut right: Vec = Vec::new(); if !speed.is_empty() { @@ -326,10 +349,15 @@ fn draw_progress(f: &mut Frame, area: Rect, state: &SyncState) { left.extend(right); let mut lines = vec![Line::from(left)]; + let current_line = if scanning && current.is_empty() { + "building the file list…".to_string() + } else { + current.clone() + }; lines.push(Line::from(vec![ Span::raw(" "), Span::styled( - truncate(¤t, inner.width.saturating_sub(2) as usize), + truncate(¤t_line, inner.width.saturating_sub(2) as usize), Style::default().fg(theme::DIM), ), ])); @@ -338,3 +366,16 @@ fn draw_progress(f: &mut Frame, area: Rect, state: &SyncState) { stats, ); } + +/// `11810` → `11,810` +fn fmt_count(n: u64) -> String { + let s = n.to_string(); + let mut out = String::with_capacity(s.len() + s.len() / 3); + for (i, c) in s.chars().enumerate() { + if i > 0 && (s.len() - i) % 3 == 0 { + out.push(','); + } + out.push(c); + } + out +}