#!/usr/bin/env bash # gniza/lib/rclone.sh — Rclone transport layer for S3 and Google Drive remotes [[ -n "${_GNIZA_RCLONE_LOADED:-}" ]] && return 0 _GNIZA_RCLONE_LOADED=1 # ── Mode Detection ──────────────────────────────────────────── _is_rclone_mode() { [[ "${REMOTE_TYPE:-ssh}" == "s3" || "${REMOTE_TYPE:-ssh}" == "gdrive" ]] } # ── Rclone Config Generation ───────────────────────────────── _build_rclone_config() { local tmpfile tmpfile=$(mktemp /tmp/gniza-rclone-XXXXXX.conf) || { log_error "Failed to create temp rclone config" return 1 } chmod 600 "$tmpfile" case "${REMOTE_TYPE}" in s3) cat > "$tmpfile" <> "$tmpfile" fi ;; gdrive) cat > "$tmpfile" <> "$tmpfile" fi ;; *) rm -f "$tmpfile" log_error "Unknown REMOTE_TYPE for rclone: ${REMOTE_TYPE}" return 1 ;; esac echo "$tmpfile" } _cleanup_rclone_config() { local path="$1" [[ -n "$path" && -f "$path" ]] && rm -f "$path" } # ── Path Construction ───────────────────────────────────────── _rclone_remote_path() { local subpath="${1:-}" local hostname; hostname=$(hostname -f) case "${REMOTE_TYPE}" in s3) echo "remote:${S3_BUCKET}${REMOTE_BASE}/${hostname}${subpath:+/$subpath}" ;; gdrive) echo "remote:${REMOTE_BASE}/${hostname}${subpath:+/$subpath}" ;; esac } # ── Core Command Runner ────────────────────────────────────── # Run an rclone subcommand with auto config lifecycle. # Usage: _rclone_cmd [args...] _rclone_cmd() { local subcmd="$1"; shift local conf conf=$(_build_rclone_config) || return 1 local rclone_opts=(--config "$conf") if [[ "${BWLIMIT:-0}" -gt 0 ]]; then rclone_opts+=(--bwlimit "${BWLIMIT}k") fi log_debug "rclone $subcmd ${rclone_opts[*]} $*" local rc=0 rclone "$subcmd" "${rclone_opts[@]}" "$@" || rc=$? _cleanup_rclone_config "$conf" return "$rc" } # ── Transfer Functions ──────────────────────────────────────── rclone_to_remote() { local source_dir="$1" local remote_subpath="$2" local attempt=0 local max_retries="${SSH_RETRIES:-$DEFAULT_SSH_RETRIES}" local remote_dest; remote_dest=$(_rclone_remote_path "$remote_subpath") [[ "$source_dir" != */ ]] && source_dir="$source_dir/" while (( attempt < max_retries )); do ((attempt++)) || true log_debug "rclone copy attempt $attempt/$max_retries: $source_dir -> $remote_dest" if _rclone_cmd copy "$source_dir" "$remote_dest"; then log_debug "rclone copy succeeded on attempt $attempt" return 0 fi log_warn "rclone copy failed, attempt $attempt/$max_retries" if (( attempt < max_retries )); then local backoff=$(( attempt * 10 )) log_info "Retrying in ${backoff}s..." sleep "$backoff" fi done log_error "rclone copy failed after $max_retries attempts" return 1 } rclone_from_remote() { local remote_subpath="$1" local local_dir="$2" local attempt=0 local max_retries="${SSH_RETRIES:-$DEFAULT_SSH_RETRIES}" local remote_src; remote_src=$(_rclone_remote_path "$remote_subpath") mkdir -p "$local_dir" || { log_error "Failed to create local dir: $local_dir" return 1 } while (( attempt < max_retries )); do ((attempt++)) || true log_debug "rclone copy attempt $attempt/$max_retries: $remote_src -> $local_dir" if _rclone_cmd copy "$remote_src" "$local_dir"; then log_debug "rclone download succeeded on attempt $attempt" return 0 fi log_warn "rclone download failed, attempt $attempt/$max_retries" if (( attempt < max_retries )); then local backoff=$(( attempt * 10 )) log_info "Retrying in ${backoff}s..." sleep "$backoff" fi done log_error "rclone download failed after $max_retries attempts" return 1 } # Like rclone_from_remote but passes extra args (e.g. --exclude) to rclone copy. # Usage: rclone_from_remote_filtered [extra_args...] rclone_from_remote_filtered() { local remote_subpath="$1" local local_dir="$2" shift 2 local -a extra_args=("$@") local attempt=0 local max_retries="${SSH_RETRIES:-$DEFAULT_SSH_RETRIES}" local remote_src; remote_src=$(_rclone_remote_path "$remote_subpath") mkdir -p "$local_dir" || { log_error "Failed to create local dir: $local_dir" return 1 } while (( attempt < max_retries )); do ((attempt++)) || true log_debug "rclone copy (filtered) attempt $attempt/$max_retries: $remote_src -> $local_dir" if _rclone_cmd copy "$remote_src" "$local_dir" "${extra_args[@]}"; then log_debug "rclone download succeeded on attempt $attempt" return 0 fi log_warn "rclone download failed, attempt $attempt/$max_retries" if (( attempt < max_retries )); then local backoff=$(( attempt * 10 )) log_info "Retrying in ${backoff}s..." sleep "$backoff" fi done log_error "rclone download failed after $max_retries attempts" return 1 } # ── Snapshot Management ─────────────────────────────────────── rclone_list_dirs() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd lsf --dirs-only "$remote_path" 2>/dev/null | sed 's|/$||' } rclone_list_remote_snapshots() { local user="$1" local snap_subpath="accounts/${user}/snapshots" local all_dirs; all_dirs=$(rclone_list_dirs "$snap_subpath") || true [[ -z "$all_dirs" ]] && return 0 # Filter to dirs with .complete marker, sorted newest first local completed="" while IFS= read -r dir; do [[ -z "$dir" ]] && continue if rclone_exists "${snap_subpath}/${dir}/.complete"; then completed+="${dir}"$'\n' fi done <<< "$all_dirs" [[ -n "$completed" ]] && echo "$completed" | sort -r } rclone_get_latest_snapshot() { local user="$1" local snap_subpath="accounts/${user}/snapshots" # Try reading latest.txt first local latest; latest=$(rclone_cat "${snap_subpath}/latest.txt" 2>/dev/null) || true if [[ -n "$latest" ]]; then # Verify it still exists with .complete marker if rclone_exists "${snap_subpath}/${latest}/.complete"; then echo "$latest" return 0 fi fi # Fall back to sorted list rclone_list_remote_snapshots "$user" | head -1 } rclone_clean_partial_snapshots() { local user="$1" local snap_subpath="accounts/${user}/snapshots" local all_dirs; all_dirs=$(rclone_list_dirs "$snap_subpath") || true [[ -z "$all_dirs" ]] && return 0 while IFS= read -r dir; do [[ -z "$dir" ]] && continue if ! rclone_exists "${snap_subpath}/${dir}/.complete"; then log_info "Purging incomplete snapshot for $user: $dir" rclone_purge "${snap_subpath}/${dir}" || { log_warn "Failed to purge incomplete snapshot: $dir" } fi done <<< "$all_dirs" } rclone_finalize_snapshot() { local user="$1" local ts="$2" local snap_subpath="accounts/${user}/snapshots" # Create .complete marker rclone_rcat "${snap_subpath}/${ts}/.complete" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" || { log_error "Failed to create .complete marker for $user/$ts" return 1 } # Update latest.txt rclone_update_latest "$user" "$ts" } rclone_update_latest() { local user="$1" local ts="$2" local snap_subpath="accounts/${user}/snapshots" rclone_rcat "${snap_subpath}/latest.txt" "$ts" || { log_warn "Failed to update latest.txt for $user" return 1 } log_debug "Updated latest.txt for $user -> $ts" } rclone_resolve_snapshot() { local user="$1" local requested="$2" local snap_subpath="accounts/${user}/snapshots" if rclone_exists "${snap_subpath}/${requested}/.complete"; then echo "$requested" else log_error "Snapshot not found or incomplete for $user: $requested" return 1 fi } # ── Remote Operations ───────────────────────────────────────── rclone_ensure_dir() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd mkdir "$remote_path" } rclone_purge() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd purge "$remote_path" } rclone_exists() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd lsf "$remote_path" &>/dev/null } rclone_size() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd size --json "$remote_path" 2>/dev/null } rclone_list_files() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd lsf "$remote_path" 2>/dev/null } rclone_cat() { local remote_subpath="$1" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") _rclone_cmd cat "$remote_path" 2>/dev/null } rclone_rcat() { local remote_subpath="$1" local content="$2" local remote_path; remote_path=$(_rclone_remote_path "$remote_subpath") echo -n "$content" | _rclone_cmd rcat "$remote_path" } test_rclone_connection() { local remote_path case "${REMOTE_TYPE}" in s3) remote_path="remote:${S3_BUCKET}" ;; gdrive) remote_path="remote:${REMOTE_BASE}" ;; *) log_error "Unknown REMOTE_TYPE: ${REMOTE_TYPE}" return 1 ;; esac log_debug "Testing rclone connection to ${REMOTE_TYPE}..." if _rclone_cmd lsd "$remote_path" &>/dev/null; then log_debug "Rclone connection test passed" return 0 else log_error "Rclone connection test failed for ${REMOTE_TYPE}" return 1 fi }