diff --git a/README.md b/README.md index 309e9b1..ad5d93d 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,12 @@ crontab -e You should automatically get emails when a backup fails or fails to run at all. Check the logs to see what went wrong. +## Stale locks + +If a restic process dies mid-operation (e.g. the SFTP connection drops), it can leave a lock behind in the repo, which blocks `forget --prune` and `check` until it is removed. When a restic command fails because the repo is locked (exit code 11), both scripts run a plain `restic unlock` and retry once. Without `--remove-all`, restic only removes locks that are actually stale: created on this host by a process that no longer exists, or not refreshed in 30 minutes (running restic processes refresh their locks every 5 minutes). Locks held by running backups or checks are left alone, and the operation fails instead. + +If the integrity check reports that it "could not run" because the repo is locked, that is not corruption. Check the active locks with `restic -r list locks --no-lock` and re-run the check after the other operation finishes. + ## Stats To see how much space a repo is taking up in the remote, run: diff --git a/backup.sh b/backup.sh index ef34294..425b41c 100755 --- a/backup.sh +++ b/backup.sh @@ -20,6 +20,9 @@ EXCLUDE_CACHES=true ONE_FILE_SYSTEM=true COMPRESSION="auto" +# Restic exit code for "failed to lock repository" (restic >= 0.17.0) +RESTIC_EXIT_LOCK_FAILED=11 + # Function to display usage usage() { cat <&1); then + log_message "Unlock output: $unlock_output" + else + log_message "WARNING: Failed to remove stale locks: $unlock_output" + fi +} + +# Function to run a command, retrying once after removing stale locks if it +# failed only because the repository was locked +with_stale_lock_retry() { + local status=0 + "$@" || status=$? + if [ "$status" -eq "$RESTIC_EXIT_LOCK_FAILED" ]; then + remove_stale_locks + status=0 + "$@" || status=$? + fi + return "$status" +} + # Parse command line arguments CUSTOM_TAGS=() EXCLUDE_PATTERNS=() @@ -294,7 +325,7 @@ send_healthcheck "START" "Starting backup of $BACKUP_NAME" # Pre-backup: Check repository connectivity log_message "Checking repository connectivity..." CONNECTIVITY_OUTPUT="" -if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --latest 1" 2>&1); then +if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --no-lock --latest 1" 2>&1); then log_message "WARNING: Could not connect to repository or no snapshots exist yet" if [ "$VERBOSE" = true ]; then log_message "Connectivity check output: $CONNECTIVITY_OUTPUT" @@ -314,16 +345,18 @@ log_message "Running command: $RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} BACKUP_OUTPUT="" BACKUP_SUCCESS=false -if [ "$VERBOSE" = true ]; then - # Show progress in verbose mode - if eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}"; then - BACKUP_SUCCESS=true - fi -else - # Capture output for logging - if BACKUP_OUTPUT=$(eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" 2>&1); then - BACKUP_SUCCESS=true +run_backup() { + if [ "$VERBOSE" = true ]; then + # Show progress in verbose mode + eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" + else + # Capture output for logging + BACKUP_OUTPUT=$(eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" 2>&1) fi +} + +if with_stale_lock_retry run_backup; then + BACKUP_SUCCESS=true fi if [ "$BACKUP_SUCCESS" = true ]; then @@ -335,7 +368,10 @@ if [ "$BACKUP_SUCCESS" = true ]; then # Clean up old snapshots according to retention policy log_message "Applying retention policy..." FORGET_OUTPUT="" - if FORGET_OUTPUT=$(eval "$RESTIC_CMD forget --tag $BACKUP_NAME --keep-daily $KEEP_DAILY --keep-weekly $KEEP_WEEKLY --keep-monthly $KEEP_MONTHLY --keep-yearly $KEEP_YEARLY --prune" 2>&1); then + run_forget() { + FORGET_OUTPUT=$(eval "$RESTIC_CMD forget --tag $BACKUP_NAME --keep-daily $KEEP_DAILY --keep-weekly $KEEP_WEEKLY --keep-monthly $KEEP_MONTHLY --keep-yearly $KEEP_YEARLY --prune" 2>&1) + } + if with_stale_lock_retry run_forget; then log_message "Retention policy applied successfully" # Get backup stats for healthcheck message diff --git a/restic-integrity-check.sh b/restic-integrity-check.sh index 22fe309..8953fe7 100755 --- a/restic-integrity-check.sh +++ b/restic-integrity-check.sh @@ -13,6 +13,9 @@ READ_DATA=true VERBOSE=false DRY_RUN=false +# Restic exit code for "failed to lock repository" (restic >= 0.17.0) +RESTIC_EXIT_LOCK_FAILED=11 + # Function to display usage usage() { cat << EOF @@ -65,7 +68,11 @@ NOTES: and verifying all data blobs. This is slower but more comprehensive. - Use --no-read-data for faster checks that only verify repository structure and metadata. - - Exit codes: 0 = success, 1 = check failed or error occurred + - If the repository is locked, only stale locks (from dead processes or + not refreshed for 30 minutes) are removed before retrying once. Locks + held by running restic processes are never removed. + - Exit codes: 0 = success, 1 = check failed or error occurred, + 11 = check could not run because the repository is locked EOF } @@ -108,6 +115,34 @@ send_healthcheck() { [ "$VERBOSE" = true ] && echo "Healthcheck sent: $status - $message" } +# Function to remove stale repository locks +# Plain `restic unlock` (never --remove-all) only removes locks that restic +# considers stale: created on this host by a process that no longer exists, or +# not refreshed for 30 minutes. Running restic processes refresh their locks +# every 5 minutes, so locks held by live operations are never removed. +remove_stale_locks() { + log_message "Repository is locked, removing stale locks (if any)..." + local unlock_output="" + if unlock_output=$(eval "$RESTIC_CMD unlock" 2>&1); then + log_message "Unlock output: $unlock_output" + else + log_message "WARNING: Failed to remove stale locks: $unlock_output" + fi +} + +# Function to run a command, retrying once after removing stale locks if it +# failed only because the repository was locked +with_stale_lock_retry() { + local status=0 + "$@" || status=$? + if [ "$status" -eq "$RESTIC_EXIT_LOCK_FAILED" ]; then + remove_stale_locks + status=0 + "$@" || status=$? + fi + return "$status" +} + # Parse command line arguments POSITIONAL_ARGS=() @@ -215,7 +250,7 @@ fi # Test repository connectivity first log_message "Testing repository connectivity..." CONNECTIVITY_OUTPUT="" -if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --last" 2>&1); then +if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --no-lock --latest 1" 2>&1); then log_message "ERROR: Cannot connect to repository or repository is empty" if [ "$VERBOSE" = true ]; then log_message "Connectivity error: $CONNECTIVITY_OUTPUT" @@ -235,21 +270,21 @@ log_message "Running repository integrity check..." START_TIME=$(date +%s) CHECK_OUTPUT="" -CHECK_SUCCESS=false +CHECK_EXIT=0 -if [ "$VERBOSE" = true ]; then - # Show progress in verbose mode - if eval "$RESTIC_CMD check ${CHECK_ARGS[*]}"; then - CHECK_SUCCESS=true +run_check() { + if [ "$VERBOSE" = true ]; then + # Show progress in verbose mode + eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" + else + # Capture output for logging + CHECK_OUTPUT=$(eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" 2>&1) fi -else - # Capture output for logging - if CHECK_OUTPUT=$(eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" 2>&1); then - CHECK_SUCCESS=true - fi -fi +} -if [ "$CHECK_SUCCESS" = true ]; then +with_stale_lock_retry run_check || CHECK_EXIT=$? + +if [ "$CHECK_EXIT" -eq 0 ]; then END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) @@ -281,6 +316,21 @@ if [ "$CHECK_SUCCESS" = true ]; then fi [ -n "$CHECK_OUTPUT" ] && echo -e "\nDetailed output:\n$CHECK_OUTPUT" fi +elif [ "$CHECK_EXIT" -eq "$RESTIC_EXIT_LOCK_FAILED" ]; then + # The check never ran, so this says nothing about repository integrity + ERROR_MSG="Integrity check for $REPO_NAME could not run: repository is locked by another active restic process (not a corruption issue). Re-run once it finishes." + log_message "$ERROR_MSG" + send_healthcheck "FAILED" "$ERROR_MSG" + + if [ "$VERBOSE" = true ]; then + echo "=== Check Summary ===" + echo "Repository: $REPOSITORY" + echo "Result: NOT RUN (repository locked)" + echo "Check active locks with: restic -r $REPOSITORY list locks --no-lock" + [ -n "$CHECK_OUTPUT" ] && echo -e "\nError output:\n$CHECK_OUTPUT" + fi + + exit "$RESTIC_EXIT_LOCK_FAILED" else END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME))