Remove stale restic locks and retry when a repo is locked

If a restic process dies mid-operation (e.g. the SFTP connection drops)
it leaves a lock in the repo that blocks every later forget --prune and
check until it is manually removed.

When a restic command exits with code 11 (failed to lock repository),
both scripts now run a plain `restic unlock` and retry once. Without
--remove-all, restic only removes locks from dead processes on this
host or locks not refreshed in 30 minutes, so locks held by running
operations are never removed.

The integrity check now reports a lock conflict as "could not run"
instead of claiming the repository may be corrupted, and the
connectivity checks use --no-lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

README.md

@@ -45,6 +45,12 @@ crontab -e
 
 You should automatically get emails when a backup fails or fails to run at all. Check the logs to see what went wrong.
 
+## Stale locks
+
+If a restic process dies mid-operation (e.g. the SFTP connection drops), it can leave a lock behind in the repo, which blocks `forget --prune` and `check` until it is removed. When a restic command fails because the repo is locked (exit code 11), both scripts run a plain `restic unlock` and retry once. Without `--remove-all`, restic only removes locks that are actually stale: created on this host by a process that no longer exists, or not refreshed in 30 minutes (running restic processes refresh their locks every 5 minutes). Locks held by running backups or checks are left alone, and the operation fails instead.
+
+If the integrity check reports that it "could not run" because the repo is locked, that is not corruption. Check the active locks with `restic -r <repo> list locks --no-lock` and re-run the check after the other operation finishes.
+
 ## Stats
 
 To see how much space a repo is taking up in the remote, run:

backup.sh

@@ -20,6 +20,9 @@ EXCLUDE_CACHES=true
 ONE_FILE_SYSTEM=true
 COMPRESSION="auto"
 
+# Restic exit code for "failed to lock repository" (restic >= 0.17.0)
+RESTIC_EXIT_LOCK_FAILED=11
+
 # Function to display usage
 usage() {
     cat <<EOF
@@ -119,6 +122,34 @@ send_healthcheck() {
     [ "$VERBOSE" = true ] && echo "Healthcheck sent: $status - $message"
 }
 
+# Function to remove stale repository locks
+# Plain `restic unlock` (never --remove-all) only removes locks that restic
+# considers stale: created on this host by a process that no longer exists, or
+# not refreshed for 30 minutes. Running restic processes refresh their locks
+# every 5 minutes, so locks held by live operations are never removed.
+remove_stale_locks() {
+    log_message "Repository is locked, removing stale locks (if any)..."
+    local unlock_output=""
+    if unlock_output=$(eval "$RESTIC_CMD unlock" 2>&1); then
+        log_message "Unlock output: $unlock_output"
+    else
+        log_message "WARNING: Failed to remove stale locks: $unlock_output"
+    fi
+}
+
+# Function to run a command, retrying once after removing stale locks if it
+# failed only because the repository was locked
+with_stale_lock_retry() {
+    local status=0
+    "$@" || status=$?
+    if [ "$status" -eq "$RESTIC_EXIT_LOCK_FAILED" ]; then
+        remove_stale_locks
+        status=0
+        "$@" || status=$?
+    fi
+    return "$status"
+}
+
 # Parse command line arguments
 CUSTOM_TAGS=()
 EXCLUDE_PATTERNS=()
@@ -294,7 +325,7 @@ send_healthcheck "START" "Starting backup of $BACKUP_NAME"
 # Pre-backup: Check repository connectivity
 log_message "Checking repository connectivity..."
 CONNECTIVITY_OUTPUT=""
-if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --latest 1" 2>&1); then
+if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --no-lock --latest 1" 2>&1); then
     log_message "WARNING: Could not connect to repository or no snapshots exist yet"
     if [ "$VERBOSE" = true ]; then
         log_message "Connectivity check output: $CONNECTIVITY_OUTPUT"
@@ -314,16 +345,18 @@ log_message "Running command: $RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]}
 BACKUP_OUTPUT=""
 BACKUP_SUCCESS=false
 
-if [ "$VERBOSE" = true ]; then
+run_backup() {
+    if [ "$VERBOSE" = true ]; then
         # Show progress in verbose mode
-    if eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}"; then
-        BACKUP_SUCCESS=true
-    fi
-else
+        eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}"
+    else
         # Capture output for logging
-    if BACKUP_OUTPUT=$(eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" 2>&1); then
-        BACKUP_SUCCESS=true
+        BACKUP_OUTPUT=$(eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" 2>&1)
     fi
+}
+
+if with_stale_lock_retry run_backup; then
+    BACKUP_SUCCESS=true
 fi
 
 if [ "$BACKUP_SUCCESS" = true ]; then
@@ -335,7 +368,10 @@ if [ "$BACKUP_SUCCESS" = true ]; then
     # Clean up old snapshots according to retention policy
     log_message "Applying retention policy..."
     FORGET_OUTPUT=""
-    if FORGET_OUTPUT=$(eval "$RESTIC_CMD forget --tag $BACKUP_NAME --keep-daily $KEEP_DAILY --keep-weekly $KEEP_WEEKLY --keep-monthly $KEEP_MONTHLY --keep-yearly $KEEP_YEARLY --prune" 2>&1); then
+    run_forget() {
+        FORGET_OUTPUT=$(eval "$RESTIC_CMD forget --tag $BACKUP_NAME --keep-daily $KEEP_DAILY --keep-weekly $KEEP_WEEKLY --keep-monthly $KEEP_MONTHLY --keep-yearly $KEEP_YEARLY --prune" 2>&1)
+    }
+    if with_stale_lock_retry run_forget; then
         log_message "Retention policy applied successfully"
 
         # Get backup stats for healthcheck message

restic-integrity-check.sh

@@ -13,6 +13,9 @@ READ_DATA=true
 VERBOSE=false
 DRY_RUN=false
 
+# Restic exit code for "failed to lock repository" (restic >= 0.17.0)
+RESTIC_EXIT_LOCK_FAILED=11
+
 # Function to display usage
 usage() {
     cat << EOF
@@ -65,7 +68,11 @@ NOTES:
       and verifying all data blobs. This is slower but more comprehensive.
     - Use --no-read-data for faster checks that only verify repository
       structure and metadata.
-    - Exit codes: 0 = success, 1 = check failed or error occurred
+    - If the repository is locked, only stale locks (from dead processes or
+      not refreshed for 30 minutes) are removed before retrying once. Locks
+      held by running restic processes are never removed.
+    - Exit codes: 0 = success, 1 = check failed or error occurred,
+      11 = check could not run because the repository is locked
 
 EOF
 }
@@ -108,6 +115,34 @@ send_healthcheck() {
     [ "$VERBOSE" = true ] && echo "Healthcheck sent: $status - $message"
 }
 
+# Function to remove stale repository locks
+# Plain `restic unlock` (never --remove-all) only removes locks that restic
+# considers stale: created on this host by a process that no longer exists, or
+# not refreshed for 30 minutes. Running restic processes refresh their locks
+# every 5 minutes, so locks held by live operations are never removed.
+remove_stale_locks() {
+    log_message "Repository is locked, removing stale locks (if any)..."
+    local unlock_output=""
+    if unlock_output=$(eval "$RESTIC_CMD unlock" 2>&1); then
+        log_message "Unlock output: $unlock_output"
+    else
+        log_message "WARNING: Failed to remove stale locks: $unlock_output"
+    fi
+}
+
+# Function to run a command, retrying once after removing stale locks if it
+# failed only because the repository was locked
+with_stale_lock_retry() {
+    local status=0
+    "$@" || status=$?
+    if [ "$status" -eq "$RESTIC_EXIT_LOCK_FAILED" ]; then
+        remove_stale_locks
+        status=0
+        "$@" || status=$?
+    fi
+    return "$status"
+}
+
 # Parse command line arguments
 POSITIONAL_ARGS=()
 
@@ -215,7 +250,7 @@ fi
 # Test repository connectivity first
 log_message "Testing repository connectivity..."
 CONNECTIVITY_OUTPUT=""
-if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --last" 2>&1); then
+if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --no-lock --latest 1" 2>&1); then
     log_message "ERROR: Cannot connect to repository or repository is empty"
     if [ "$VERBOSE" = true ]; then
         log_message "Connectivity error: $CONNECTIVITY_OUTPUT"
@@ -235,21 +270,21 @@ log_message "Running repository integrity check..."
 START_TIME=$(date +%s)
 
 CHECK_OUTPUT=""
-CHECK_SUCCESS=false
+CHECK_EXIT=0
 
-if [ "$VERBOSE" = true ]; then
+run_check() {
+    if [ "$VERBOSE" = true ]; then
         # Show progress in verbose mode
-    if eval "$RESTIC_CMD check ${CHECK_ARGS[*]}"; then
-        CHECK_SUCCESS=true
-    fi
-else
+        eval "$RESTIC_CMD check ${CHECK_ARGS[*]}"
+    else
         # Capture output for logging
-    if CHECK_OUTPUT=$(eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" 2>&1); then
-        CHECK_SUCCESS=true
+        CHECK_OUTPUT=$(eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" 2>&1)
     fi
-fi
+}
 
-if [ "$CHECK_SUCCESS" = true ]; then
+with_stale_lock_retry run_check || CHECK_EXIT=$?
+
+if [ "$CHECK_EXIT" -eq 0 ]; then
     END_TIME=$(date +%s)
     DURATION=$((END_TIME - START_TIME))
     
@@ -281,6 +316,21 @@ if [ "$CHECK_SUCCESS" = true ]; then
         fi
         [ -n "$CHECK_OUTPUT" ] && echo -e "\nDetailed output:\n$CHECK_OUTPUT"
     fi
+elif [ "$CHECK_EXIT" -eq "$RESTIC_EXIT_LOCK_FAILED" ]; then
+    # The check never ran, so this says nothing about repository integrity
+    ERROR_MSG="Integrity check for $REPO_NAME could not run: repository is locked by another active restic process (not a corruption issue). Re-run once it finishes."
+    log_message "$ERROR_MSG"
+    send_healthcheck "FAILED" "$ERROR_MSG"
+
+    if [ "$VERBOSE" = true ]; then
+        echo "=== Check Summary ==="
+        echo "Repository: $REPOSITORY"
+        echo "Result: NOT RUN (repository locked)"
+        echo "Check active locks with: restic -r $REPOSITORY list locks --no-lock"
+        [ -n "$CHECK_OUTPUT" ] && echo -e "\nError output:\n$CHECK_OUTPUT"
+    fi
+
+    exit "$RESTIC_EXIT_LOCK_FAILED"
 else
     END_TIME=$(date +%s)
     DURATION=$((END_TIME - START_TIME))