Add rsync auto-retry on partial transfer and show running tasks in header

Rsync exit 23 (partial transfer) now triggers an automatic retry to recover
failed files before accepting with a warning. Also adds a task counter next
to the clock in the header bar showing running job count.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

lib/source.sh

@@ -22,10 +22,16 @@ pull_from_source() {
         s3)
             _build_source_rclone_config "s3"
             _rclone_from_source "$remote_path" "$local_dir"
+            local rc=$?
+            rm -f "${_SOURCE_RCLONE_CONF:-}"; _SOURCE_RCLONE_CONF=""
+            return $rc
             ;;
         gdrive)
             _build_source_rclone_config "gdrive"
             _rclone_from_source "$remote_path" "$local_dir"
+            local rc=$?
+            rm -f "${_SOURCE_RCLONE_CONF:-}"; _SOURCE_RCLONE_CONF=""
+            return $rc
             ;;
         *)
             log_error "Unknown source type: ${TARGET_SOURCE_TYPE}"
@@ -82,14 +88,33 @@ _rsync_from_source_ssh() {
         else
             "${rsync_cmd[@]}" || rc=$?
         fi
+        unset SSHPASS
 
         if (( rc == 0 )); then
             log_debug "rsync (source pull) succeeded on attempt $attempt"
             return 0
         fi
 
-        if (( rc == 23 || rc == 24 )); then
+        if (( rc == 23 )); then
-            log_warn "rsync (source pull) completed with warnings (exit $rc): some files could not be transferred"
+            log_warn "rsync (source pull) partial transfer (exit 23): retrying to pick up failed files..."
+            sleep 2
+            local rc2=0
+            if [[ -n "${_TRANSFER_LOG:-}" ]]; then
+                echo "=== rsync (source pull retry): $source_spec -> $local_dir ===" >> "$_TRANSFER_LOG"
+                "${rsync_cmd[@]}" > >(_snaplog_tee) 2>&1 || rc2=$?
+            else
+                "${rsync_cmd[@]}" || rc2=$?
+            fi
+            unset SSHPASS
+            if (( rc2 == 0 )); then
+                log_info "rsync (source pull) retry succeeded — all files transferred"
+                return 0
+            fi
+            log_warn "rsync (source pull) retry completed (exit $rc2): some files could not be transferred"
+            return 0
+        fi
+        if (( rc == 24 )); then
+            log_warn "rsync (source pull) completed with warnings (exit $rc): vanished source files"
             return 0
         fi
 
@@ -110,7 +135,7 @@ _rsync_from_source_ssh() {
 # Usage: _build_source_rclone_config <type>
 _build_source_rclone_config() {
     local src_type="$1"
-    _SOURCE_RCLONE_CONF=$(mktemp /tmp/gniza-source-rclone-XXXXXX.conf)
+    _SOURCE_RCLONE_CONF=$(mktemp "${WORK_DIR:-/tmp}/gniza-source-rclone-XXXXXX.conf")
 
     if [[ "$src_type" == "s3" ]]; then
         cat > "$_SOURCE_RCLONE_CONF" <<EOF

lib/targets.sh

@@ -142,6 +142,10 @@ validate_target() {
                         ((errors++)) || true
                     fi
                     ;;
+                *)
+                    log_error "Target '$name': unknown TARGET_SOURCE_TYPE: ${TARGET_SOURCE_TYPE}"
+                    ((errors++)) || true
+                    ;;
             esac
             # Validate paths are absolute (even on remote)
             local -a folders

lib/transfer.sh

@@ -80,11 +80,27 @@ rsync_to_remote() {
             return 0
         fi
 
-        # Exit 23 = partial transfer (permission denied on some files)
+        # Exit 23 = partial transfer (some files failed)
         # Exit 24 = vanished source files (deleted during transfer)
-        # Both are expected in non-root backups — treat as success with warning
+        if (( rc == 23 )); then
-        if (( rc == 23 || rc == 24 )); then
+            log_warn "rsync partial transfer (exit 23): retrying to pick up failed files..."
-            log_warn "rsync completed with warnings (exit $rc): some files could not be transferred"
+            sleep 2
+            local rc2=0
+            if [[ -n "${_TRANSFER_LOG:-}" ]]; then
+                echo "=== rsync (retry): $source_dir -> ${REMOTE_USER}@${REMOTE_HOST}:${remote_dest} ===" >> "$_TRANSFER_LOG"
+                "${rsync_cmd[@]}" > >(_snaplog_tee) 2>&1 || rc2=$?
+            else
+                "${rsync_cmd[@]}" || rc2=$?
+            fi
+            if (( rc2 == 0 )); then
+                log_info "rsync retry succeeded — all files transferred"
+                return 0
+            fi
+            log_warn "rsync retry completed (exit $rc2): some files could not be transferred"
+            return 0
+        fi
+        if (( rc == 24 )); then
+            log_warn "rsync completed with warnings (exit $rc): vanished source files"
             return 0
         fi
 
@@ -160,8 +176,25 @@ rsync_local() {
             return 0
         fi
 
-        if (( rc == 23 || rc == 24 )); then
+        if (( rc == 23 )); then
-            log_warn "rsync (local) completed with warnings (exit $rc): some files could not be transferred"
+            log_warn "rsync (local) partial transfer (exit 23): retrying to pick up failed files..."
+            sleep 2
+            local rc2=0
+            if [[ -n "${_TRANSFER_LOG:-}" ]]; then
+                echo "=== rsync (local retry): $source_dir -> $local_dest ===" >> "$_TRANSFER_LOG"
+                rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" > >(_snaplog_tee) 2>&1 || rc2=$?
+            else
+                rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" || rc2=$?
+            fi
+            if (( rc2 == 0 )); then
+                log_info "rsync (local) retry succeeded — all files transferred"
+                return 0
+            fi
+            log_warn "rsync (local) retry completed (exit $rc2): some files could not be transferred"
+            return 0
+        fi
+        if (( rc == 24 )); then
+            log_warn "rsync (local) completed with warnings (exit $rc): vanished source files"
             return 0
         fi
 

tui/screens/backup.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Select
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from tui.config import list_conf_dir, has_targets, has_remotes
 from tui.jobs import job_manager

tui/screens/logs.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, DataTable, RichLog
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 
 from tui.config import LOG_DIR

tui/screens/main_menu.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, OptionList
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.widgets.option_list import Option
 from textual.containers import Horizontal, Vertical
 

tui/screens/remote_edit.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Input, Select, RadioSet, RadioButton
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 
 from tui.config import parse_conf, write_conf, CONFIG_DIR

tui/screens/remotes.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, DataTable
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from textual import work
 

tui/screens/restore.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Select, Input, RadioSet, RadioButton, Switch
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from textual import work, on
 

tui/screens/retention.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Select, Input
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from textual import work
 

tui/screens/running_tasks.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, DataTable, RichLog, ProgressBar
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from textual.timer import Timer
 

tui/screens/schedule.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, DataTable
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from textual import work
 

tui/screens/schedule_edit.py

@@ -2,6 +2,7 @@ import re
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Input, Select, SelectionList
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 
 from tui.config import list_conf_dir, parse_conf, write_conf, CONFIG_DIR

tui/screens/settings.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Input, Select
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 
 from tui.config import parse_conf, write_conf, CONFIG_DIR

tui/screens/snapshots.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Select, DataTable
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 from textual import work
 

tui/screens/target_edit.py

@@ -2,6 +2,7 @@ import re
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, Input, Select
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 
 from tui.config import parse_conf, write_conf, CONFIG_DIR, list_conf_dir

tui/screens/targets.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button, DataTable
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Horizontal
 
 from tui.config import list_conf_dir, parse_conf, CONFIG_DIR

tui/screens/wizard.py

@@ -1,6 +1,7 @@
 from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.widgets import Header, Footer, Static, Button
+from tui.widgets.header import GnizaHeader as Header  # noqa: F811
 from textual.containers import Vertical, Center
 
 from tui.config import has_remotes, has_targets

tui/widgets/__init__.py

@@ -4,3 +4,4 @@ from tui.widgets.confirm_dialog import ConfirmDialog
 from tui.widgets.operation_log import OperationLog
 from tui.widgets.snapshot_browser import SnapshotBrowser
 from tui.widgets.docs_panel import DocsPanel, HelpModal
+from tui.widgets.header import GnizaHeader

tui/widgets/header.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+
+from rich.text import Text
+from textual.reactive import Reactive
+from textual.widgets import Header, Static
+from textual.widgets._header import HeaderIcon, HeaderTitle, HeaderClock, HeaderClockSpace
+
+from tui.jobs import job_manager
+
+
+class HeaderTaskClock(HeaderClock):
+    """Clock widget that also shows running task count."""
+
+    DEFAULT_CSS = """
+    HeaderTaskClock {
+        background: $foreground-darken-1 5%;
+        color: $foreground;
+        text-opacity: 85%;
+        content-align: center middle;
+        dock: right;
+        width: auto;
+        min-width: 10;
+        padding: 0 1;
+    }
+    """
+
+    time_format: Reactive[str] = Reactive("%X")
+
+    def render(self):
+        clock = datetime.now().time().strftime(self.time_format)
+        count = job_manager.running_count()
+        if count > 0:
+            return Text.assemble(
+                ("Tasks ", "bold"),
+                (f"({count})", "bold yellow"),
+                "  ",
+                clock,
+            )
+        return Text(clock)
+
+
+class GnizaHeader(Header):
+
+    def compose(self):
+        yield HeaderIcon().data_bind(Header.icon)
+        yield HeaderTitle()
+        yield (
+            HeaderTaskClock().data_bind(Header.time_format)
+            if self._show_clock
+            else HeaderClockSpace()
+        )