From 04a0c45abce5ce6fc884808e1c5da0edc2233ab2 Mon Sep 17 00:00:00 2001 From: shuki Date: Sat, 7 Mar 2026 03:26:13 +0200 Subject: [PATCH] Add rsync auto-retry on partial transfer and show running tasks in header Rsync exit 23 (partial transfer) now triggers an automatic retry to recover failed files before accepting with a warning. Also adds a task counter next to the clock in the header bar showing running job count. Co-Authored-By: Claude Opus 4.6 --- lib/source.sh | 31 +++++++++++++++++++--- lib/targets.sh | 4 +++ lib/transfer.sh | 45 ++++++++++++++++++++++++++----- tui/screens/backup.py | 1 + tui/screens/logs.py | 1 + tui/screens/main_menu.py | 1 + tui/screens/remote_edit.py | 1 + tui/screens/remotes.py | 1 + tui/screens/restore.py | 1 + tui/screens/retention.py | 1 + tui/screens/running_tasks.py | 1 + tui/screens/schedule.py | 1 + tui/screens/schedule_edit.py | 1 + tui/screens/settings.py | 1 + tui/screens/snapshots.py | 1 + tui/screens/target_edit.py | 1 + tui/screens/targets.py | 1 + tui/screens/wizard.py | 1 + tui/widgets/__init__.py | 1 + tui/widgets/header.py | 51 ++++++++++++++++++++++++++++++++++++ 20 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 tui/widgets/header.py diff --git a/lib/source.sh b/lib/source.sh index 3157c69..d611191 100755 --- a/lib/source.sh +++ b/lib/source.sh @@ -22,10 +22,16 @@ pull_from_source() { s3) _build_source_rclone_config "s3" _rclone_from_source "$remote_path" "$local_dir" + local rc=$? + rm -f "${_SOURCE_RCLONE_CONF:-}"; _SOURCE_RCLONE_CONF="" + return $rc ;; gdrive) _build_source_rclone_config "gdrive" _rclone_from_source "$remote_path" "$local_dir" + local rc=$? + rm -f "${_SOURCE_RCLONE_CONF:-}"; _SOURCE_RCLONE_CONF="" + return $rc ;; *) log_error "Unknown source type: ${TARGET_SOURCE_TYPE}" @@ -82,14 +88,33 @@ _rsync_from_source_ssh() { else "${rsync_cmd[@]}" || rc=$? fi + unset SSHPASS if (( rc == 0 )); then log_debug "rsync (source pull) succeeded on attempt $attempt" return 0 fi - if (( rc == 23 || rc == 24 )); then - log_warn "rsync (source pull) completed with warnings (exit $rc): some files could not be transferred" + if (( rc == 23 )); then + log_warn "rsync (source pull) partial transfer (exit 23): retrying to pick up failed files..." + sleep 2 + local rc2=0 + if [[ -n "${_TRANSFER_LOG:-}" ]]; then + echo "=== rsync (source pull retry): $source_spec -> $local_dir ===" >> "$_TRANSFER_LOG" + "${rsync_cmd[@]}" > >(_snaplog_tee) 2>&1 || rc2=$? + else + "${rsync_cmd[@]}" || rc2=$? + fi + unset SSHPASS + if (( rc2 == 0 )); then + log_info "rsync (source pull) retry succeeded — all files transferred" + return 0 + fi + log_warn "rsync (source pull) retry completed (exit $rc2): some files could not be transferred" + return 0 + fi + if (( rc == 24 )); then + log_warn "rsync (source pull) completed with warnings (exit $rc): vanished source files" return 0 fi @@ -110,7 +135,7 @@ _rsync_from_source_ssh() { # Usage: _build_source_rclone_config _build_source_rclone_config() { local src_type="$1" - _SOURCE_RCLONE_CONF=$(mktemp /tmp/gniza-source-rclone-XXXXXX.conf) + _SOURCE_RCLONE_CONF=$(mktemp "${WORK_DIR:-/tmp}/gniza-source-rclone-XXXXXX.conf") if [[ "$src_type" == "s3" ]]; then cat > "$_SOURCE_RCLONE_CONF" < ${REMOTE_USER}@${REMOTE_HOST}:${remote_dest} ===" >> "$_TRANSFER_LOG" + "${rsync_cmd[@]}" > >(_snaplog_tee) 2>&1 || rc2=$? + else + "${rsync_cmd[@]}" || rc2=$? + fi + if (( rc2 == 0 )); then + log_info "rsync retry succeeded — all files transferred" + return 0 + fi + log_warn "rsync retry completed (exit $rc2): some files could not be transferred" + return 0 + fi + if (( rc == 24 )); then + log_warn "rsync completed with warnings (exit $rc): vanished source files" return 0 fi @@ -160,8 +176,25 @@ rsync_local() { return 0 fi - if (( rc == 23 || rc == 24 )); then - log_warn "rsync (local) completed with warnings (exit $rc): some files could not be transferred" + if (( rc == 23 )); then + log_warn "rsync (local) partial transfer (exit 23): retrying to pick up failed files..." + sleep 2 + local rc2=0 + if [[ -n "${_TRANSFER_LOG:-}" ]]; then + echo "=== rsync (local retry): $source_dir -> $local_dest ===" >> "$_TRANSFER_LOG" + rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" > >(_snaplog_tee) 2>&1 || rc2=$? + else + rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" || rc2=$? + fi + if (( rc2 == 0 )); then + log_info "rsync (local) retry succeeded — all files transferred" + return 0 + fi + log_warn "rsync (local) retry completed (exit $rc2): some files could not be transferred" + return 0 + fi + if (( rc == 24 )); then + log_warn "rsync (local) completed with warnings (exit $rc): vanished source files" return 0 fi diff --git a/tui/screens/backup.py b/tui/screens/backup.py index de1d468..0d7f322 100644 --- a/tui/screens/backup.py +++ b/tui/screens/backup.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Select +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import list_conf_dir, has_targets, has_remotes from tui.jobs import job_manager diff --git a/tui/screens/logs.py b/tui/screens/logs.py index 6e64c1f..c3cf1d8 100644 --- a/tui/screens/logs.py +++ b/tui/screens/logs.py @@ -4,6 +4,7 @@ from pathlib import Path from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, DataTable, RichLog +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import LOG_DIR diff --git a/tui/screens/main_menu.py b/tui/screens/main_menu.py index b7c14a0..fafdd14 100644 --- a/tui/screens/main_menu.py +++ b/tui/screens/main_menu.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, OptionList +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.widgets.option_list import Option from textual.containers import Horizontal, Vertical diff --git a/tui/screens/remote_edit.py b/tui/screens/remote_edit.py index 5f0f48b..2eb33bd 100644 --- a/tui/screens/remote_edit.py +++ b/tui/screens/remote_edit.py @@ -3,6 +3,7 @@ from pathlib import Path from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Input, Select, RadioSet, RadioButton +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import parse_conf, write_conf, CONFIG_DIR diff --git a/tui/screens/remotes.py b/tui/screens/remotes.py index 54e0f9d..718ccc2 100644 --- a/tui/screens/remotes.py +++ b/tui/screens/remotes.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, DataTable +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from textual import work diff --git a/tui/screens/restore.py b/tui/screens/restore.py index 3a7fd58..60242c6 100644 --- a/tui/screens/restore.py +++ b/tui/screens/restore.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Select, Input, RadioSet, RadioButton, Switch +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from textual import work, on diff --git a/tui/screens/retention.py b/tui/screens/retention.py index e49cf35..1b60ae4 100644 --- a/tui/screens/retention.py +++ b/tui/screens/retention.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Select, Input +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from textual import work diff --git a/tui/screens/running_tasks.py b/tui/screens/running_tasks.py index 56ea827..fc236c4 100644 --- a/tui/screens/running_tasks.py +++ b/tui/screens/running_tasks.py @@ -5,6 +5,7 @@ from pathlib import Path from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, DataTable, RichLog, ProgressBar +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from textual.timer import Timer diff --git a/tui/screens/schedule.py b/tui/screens/schedule.py index 1493909..ac88f66 100644 --- a/tui/screens/schedule.py +++ b/tui/screens/schedule.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, DataTable +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from textual import work diff --git a/tui/screens/schedule_edit.py b/tui/screens/schedule_edit.py index d035265..3b1811f 100644 --- a/tui/screens/schedule_edit.py +++ b/tui/screens/schedule_edit.py @@ -2,6 +2,7 @@ import re from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Input, Select, SelectionList +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import list_conf_dir, parse_conf, write_conf, CONFIG_DIR diff --git a/tui/screens/settings.py b/tui/screens/settings.py index 0d287b9..1c9f8cd 100644 --- a/tui/screens/settings.py +++ b/tui/screens/settings.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Input, Select +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import parse_conf, write_conf, CONFIG_DIR diff --git a/tui/screens/snapshots.py b/tui/screens/snapshots.py index 6dba1c3..39c005d 100644 --- a/tui/screens/snapshots.py +++ b/tui/screens/snapshots.py @@ -4,6 +4,7 @@ from datetime import datetime from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Select, DataTable +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from textual import work diff --git a/tui/screens/target_edit.py b/tui/screens/target_edit.py index ec781a7..1fe2255 100644 --- a/tui/screens/target_edit.py +++ b/tui/screens/target_edit.py @@ -2,6 +2,7 @@ import re from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, Input, Select +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import parse_conf, write_conf, CONFIG_DIR, list_conf_dir diff --git a/tui/screens/targets.py b/tui/screens/targets.py index e92701f..ba8ed4b 100644 --- a/tui/screens/targets.py +++ b/tui/screens/targets.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button, DataTable +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Horizontal from tui.config import list_conf_dir, parse_conf, CONFIG_DIR diff --git a/tui/screens/wizard.py b/tui/screens/wizard.py index e798e2a..e2085d2 100644 --- a/tui/screens/wizard.py +++ b/tui/screens/wizard.py @@ -1,6 +1,7 @@ from textual.app import ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Static, Button +from tui.widgets.header import GnizaHeader as Header # noqa: F811 from textual.containers import Vertical, Center from tui.config import has_remotes, has_targets diff --git a/tui/widgets/__init__.py b/tui/widgets/__init__.py index a9cbf52..aaa17c7 100644 --- a/tui/widgets/__init__.py +++ b/tui/widgets/__init__.py @@ -4,3 +4,4 @@ from tui.widgets.confirm_dialog import ConfirmDialog from tui.widgets.operation_log import OperationLog from tui.widgets.snapshot_browser import SnapshotBrowser from tui.widgets.docs_panel import DocsPanel, HelpModal +from tui.widgets.header import GnizaHeader diff --git a/tui/widgets/header.py b/tui/widgets/header.py new file mode 100644 index 0000000..20d0e06 --- /dev/null +++ b/tui/widgets/header.py @@ -0,0 +1,51 @@ +from datetime import datetime + +from rich.text import Text +from textual.reactive import Reactive +from textual.widgets import Header, Static +from textual.widgets._header import HeaderIcon, HeaderTitle, HeaderClock, HeaderClockSpace + +from tui.jobs import job_manager + + +class HeaderTaskClock(HeaderClock): + """Clock widget that also shows running task count.""" + + DEFAULT_CSS = """ + HeaderTaskClock { + background: $foreground-darken-1 5%; + color: $foreground; + text-opacity: 85%; + content-align: center middle; + dock: right; + width: auto; + min-width: 10; + padding: 0 1; + } + """ + + time_format: Reactive[str] = Reactive("%X") + + def render(self): + clock = datetime.now().time().strftime(self.time_format) + count = job_manager.running_count() + if count > 0: + return Text.assemble( + ("Tasks ", "bold"), + (f"({count})", "bold yellow"), + " ", + clock, + ) + return Text(clock) + + +class GnizaHeader(Header): + + def compose(self): + yield HeaderIcon().data_bind(Header.icon) + yield HeaderTitle() + yield ( + HeaderTaskClock().data_bind(Header.time_format) + if self._show_clock + else HeaderClockSpace() + )