diff --git a/bin/gniza b/bin/gniza index bef59b6..b8d9b72 100755 --- a/bin/gniza +++ b/bin/gniza @@ -24,6 +24,7 @@ source "$GNIZA_DIR/lib/ssh.sh" source "$GNIZA_DIR/lib/rclone.sh" source "$GNIZA_DIR/lib/snapshot.sh" source "$GNIZA_DIR/lib/transfer.sh" +source "$GNIZA_DIR/lib/snaplog.sh" # ── Help ───────────────────────────────────────────────────── show_help() { diff --git a/lib/backup.sh b/lib/backup.sh index 167ce16..7c07dff 100644 --- a/lib/backup.sh +++ b/lib/backup.sh @@ -92,6 +92,9 @@ _backup_target_impl() { # 5. Get timestamp local ts; ts=$(timestamp) + # 5.5. Initialize snapshot logging + snaplog_init + # 6. Get previous snapshot for --link-dest local prev; prev=$(get_latest_snapshot "$target_name") || prev="" if [[ -n "$prev" ]]; then @@ -106,6 +109,7 @@ _backup_target_impl() { log_info "Running pre-hook for $target_name..." if ! bash -c "$TARGET_PRE_HOOK"; then log_error "Pre-hook failed for $target_name" + snaplog_cleanup return 1 fi fi @@ -120,6 +124,7 @@ _backup_target_impl() { else log_error "MySQL dump failed for $target_name" mysql_cleanup_dump + snaplog_cleanup return 1 fi fi @@ -149,9 +154,15 @@ _backup_target_impl() { if [[ "$transfer_failed" == "true" ]]; then log_error "One or more folder transfers failed for $target_name" + snaplog_generate "$target_name" "$remote_name" "$ts" "$start_time" "Failed" + snaplog_upload "$target_name" "$ts" + snaplog_cleanup return 1 fi + # 9.9. Generate snapshot logs + snaplog_generate "$target_name" "$remote_name" "$ts" "$start_time" "Success" + # 10. Generate meta.json local end_time; end_time=$(date +%s) local duration=$(( end_time - start_time )) @@ -198,9 +209,13 @@ METAEOF remote_exec "find '${sq_partial}' -type f > '${sq_partial}/manifest.txt'" 2>/dev/null || log_warn "Failed to write manifest.txt" fi + # 11.5. Upload snapshot logs + snaplog_upload "$target_name" "$ts" + # 12. Finalize snapshot if ! finalize_snapshot "$target_name" "$ts"; then log_error "Failed to finalize snapshot for $target_name" + snaplog_cleanup return 1 fi @@ -228,6 +243,7 @@ METAEOF # 14. Enforce retention enforce_retention "$target_name" + snaplog_cleanup return 0 } diff --git a/lib/rclone.sh b/lib/rclone.sh index 4705799..96375ff 100644 --- a/lib/rclone.sh +++ b/lib/rclone.sh @@ -96,7 +96,12 @@ _rclone_cmd() { log_debug "rclone $subcmd ${rclone_opts[*]} $*" local rc=0 - rclone "$subcmd" "${rclone_opts[@]}" "$@" || rc=$? + if [[ -n "${_TRANSFER_LOG:-}" && "$subcmd" == "copy" ]]; then + echo "=== rclone copy $* ===" >> "$_TRANSFER_LOG" + rclone "$subcmd" "${rclone_opts[@]}" --verbose "$@" > >(tee -a "$_TRANSFER_LOG") 2>&1 || rc=$? + else + rclone "$subcmd" "${rclone_opts[@]}" "$@" || rc=$? + fi _cleanup_rclone_config "$conf" return "$rc" diff --git a/lib/snaplog.sh b/lib/snaplog.sh new file mode 100644 index 0000000..7cffc2d --- /dev/null +++ b/lib/snaplog.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# gniza4linux/lib/snaplog.sh — Per-snapshot backup logs (dirvish-style) + +[[ -n "${_GNIZA4LINUX_SNAPLOG_LOADED:-}" ]] && return 0 +_GNIZA4LINUX_SNAPLOG_LOADED=1 + +# Initialize snapshot log directory and transfer log file. +snaplog_init() { + _SNAP_LOG_DIR=$(mktemp -d "${WORK_DIR}/gniza-snaplog-XXXXXX") + _TRANSFER_LOG="$_SNAP_LOG_DIR/rsync_raw.log" + touch "$_TRANSFER_LOG" +} + +# Generate snapshot log files (log, rsync_error, summary, index). +# Usage: snaplog_generate +snaplog_generate() { + local target="$1" + local remote="$2" + local ts="$3" + local start_time="$4" + local status="$5" + + # Copy raw transfer log + cp "$_TRANSFER_LOG" "$_SNAP_LOG_DIR/log" + + # Extract errors/warnings + grep -iE '(error|warning|failed|cannot|denied|vanished|rsync:)' "$_TRANSFER_LOG" > "$_SNAP_LOG_DIR/rsync_error" || true + + # Generate summary + local end_time; end_time=$(date +%s) + local duration=$(( end_time - start_time )) + local start_fmt; start_fmt=$(date -u -d "@$start_time" "+%Y-%m-%d %H:%M:%S UTC") + local end_fmt; end_fmt=$(date -u -d "@$end_time" "+%Y-%m-%d %H:%M:%S UTC") + local mysql_flag="no" + [[ "${TARGET_MYSQL_ENABLED:-no}" == "yes" ]] && mysql_flag="yes" + + cat > "$_SNAP_LOG_DIR/summary" < +snaplog_generate_index() { + local target="$1" + local ts="$2" + + if _is_rclone_mode; then + _rclone_cmd lsl "$(_rclone_remote_path "targets/${target}/snapshots/${ts}")" 2>/dev/null > "$_SNAP_LOG_DIR/index" || true + elif [[ "${REMOTE_TYPE:-ssh}" == "local" ]]; then + local snap_dir; snap_dir=$(get_snapshot_dir "$target") + find "$snap_dir/${ts}.partial" -printf '%M %n %u %g %8s %T+ %P\n' 2>/dev/null | sort > "$_SNAP_LOG_DIR/index" + else + local snap_dir; snap_dir=$(get_snapshot_dir "$target") + remote_exec "find '$(shquote "$snap_dir/${ts}.partial")' -printf '%M %n %u %g %8s %T+ %P\n' 2>/dev/null | sort" > "$_SNAP_LOG_DIR/index" + fi +} + +# Upload snapshot logs to the remote. +# Usage: snaplog_upload +snaplog_upload() { + local target="$1" + local ts="$2" + + if _is_rclone_mode; then + local snap_subpath="targets/${target}/snapshots/${ts}" + for f in log rsync_error summary index; do + local remote_dest; remote_dest=$(_rclone_remote_path "${snap_subpath}/${f}") + _rclone_cmd copyto "$_SNAP_LOG_DIR/$f" "$remote_dest" || log_warn "Failed to upload $f" + done + elif [[ "${REMOTE_TYPE:-ssh}" == "local" ]]; then + local snap_dir; snap_dir=$(get_snapshot_dir "$target") + cp "$_SNAP_LOG_DIR"/{log,rsync_error,summary,index} "$snap_dir/${ts}.partial/" || log_warn "Failed to copy snapshot logs" + else + local snap_dir; snap_dir=$(get_snapshot_dir "$target") + local rsync_ssh; rsync_ssh=$(build_rsync_ssh_cmd) + local rsync_cmd=(rsync -e "$rsync_ssh" "$_SNAP_LOG_DIR/log" "$_SNAP_LOG_DIR/rsync_error" "$_SNAP_LOG_DIR/summary" "$_SNAP_LOG_DIR/index" "${REMOTE_USER}@${REMOTE_HOST}:${snap_dir}/${ts}.partial/") + if _is_password_mode; then + export SSHPASS="$REMOTE_PASSWORD" + rsync_cmd=(sshpass -e "${rsync_cmd[@]}") + fi + "${rsync_cmd[@]}" || log_warn "Failed to upload snapshot logs" + fi +} + +# Clean up temporary snapshot log directory. +snaplog_cleanup() { + [[ -n "${_SNAP_LOG_DIR:-}" && -d "$_SNAP_LOG_DIR" ]] && rm -rf "$_SNAP_LOG_DIR" + _SNAP_LOG_DIR="" + _TRANSFER_LOG="" +} diff --git a/lib/transfer.sh b/lib/transfer.sh index be63739..2fec6fa 100644 --- a/lib/transfer.sh +++ b/lib/transfer.sh @@ -35,6 +35,10 @@ rsync_to_remote() { rsync_opts+=("${extra_filter_opts[@]}") fi + if [[ -n "${_TRANSFER_LOG:-}" ]]; then + rsync_opts+=(--verbose --stats) + fi + rsync_opts+=(-e "$rsync_ssh") # Ensure source ends with / @@ -51,7 +55,12 @@ rsync_to_remote() { rsync_cmd=(sshpass -e "${rsync_cmd[@]}") fi local rc=0 - "${rsync_cmd[@]}" || rc=$? + if [[ -n "${_TRANSFER_LOG:-}" ]]; then + echo "=== rsync: $source_dir -> ${REMOTE_USER}@${REMOTE_HOST}:${remote_dest} ===" >> "$_TRANSFER_LOG" + "${rsync_cmd[@]}" > >(tee -a "$_TRANSFER_LOG") 2>&1 || rc=$? + else + "${rsync_cmd[@]}" || rc=$? + fi if (( rc == 0 )); then log_debug "rsync succeeded on attempt $attempt" return 0 @@ -110,6 +119,10 @@ rsync_local() { rsync_opts+=("${extra_filter_opts[@]}") fi + if [[ -n "${_TRANSFER_LOG:-}" ]]; then + rsync_opts+=(--verbose --stats) + fi + # Ensure source ends with / [[ "$source_dir" != */ ]] && source_dir="$source_dir/" @@ -118,7 +131,12 @@ rsync_local() { log_debug "rsync (local) attempt $attempt/$max_retries: $source_dir -> $local_dest" local rc=0 - rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" || rc=$? + if [[ -n "${_TRANSFER_LOG:-}" ]]; then + echo "=== rsync (local): $source_dir -> $local_dest ===" >> "$_TRANSFER_LOG" + rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" > >(tee -a "$_TRANSFER_LOG") 2>&1 || rc=$? + else + rsync "${rsync_opts[@]}" "$source_dir" "$local_dest" || rc=$? + fi if (( rc == 0 )); then log_debug "rsync (local) succeeded on attempt $attempt" return 0