Initial commit: backup and integrity check scripts
This commit is contained in:
305
restic-integrity-check.sh
Executable file
305
restic-integrity-check.sh
Executable file
@@ -0,0 +1,305 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Generic Restic Repository Integrity Check Script
|
||||
# Usage: ./restic-integrity-check.sh [OPTIONS] <repository>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Default values
|
||||
RESTIC_PASSWORD_FILE="$HOME/.restic-password"
|
||||
LOG_FILE=""
|
||||
HEALTHCHECK_URL=""
|
||||
READ_DATA=true
|
||||
VERBOSE=false
|
||||
DRY_RUN=false
|
||||
|
||||
# Function to display usage
|
||||
usage() {
|
||||
cat << EOF
|
||||
Usage: $0 [OPTIONS] <repository>
|
||||
|
||||
Generic integrity check script for Restic repositories.
|
||||
|
||||
REQUIRED ARGUMENTS:
|
||||
repository Restic repository URL (e.g., sftp:user@host:/path)
|
||||
|
||||
OPTIONS:
|
||||
-p, --password-file FILE Password file path (default: ~/.restic-password)
|
||||
-l, --log-file FILE Log file path (default: no logging)
|
||||
-h, --healthcheck URL Healthchecks.io ping URL for monitoring
|
||||
|
||||
CHECK OPTIONS:
|
||||
--no-read-data Skip reading and verifying data blobs (faster)
|
||||
--read-data Verify data blobs (default, more thorough)
|
||||
|
||||
OTHER OPTIONS:
|
||||
--dry-run Show what would be checked without doing it
|
||||
-v, --verbose Enable verbose output
|
||||
--help Show this help message
|
||||
|
||||
EXAMPLES:
|
||||
# Basic integrity check
|
||||
$0 sftp:user@host:/backups/music
|
||||
|
||||
# With healthcheck monitoring and custom log
|
||||
$0 --healthcheck https://hc-ping.com/uuid \\
|
||||
--log-file /var/log/integrity-check.log \\
|
||||
sftp:user@host:/backups/documents
|
||||
|
||||
# Quick check without reading data (faster)
|
||||
$0 --no-read-data --verbose \\
|
||||
local:/path/to/repo
|
||||
|
||||
# Multiple repositories check (run separately)
|
||||
for repo in repo1 repo2 repo3; do
|
||||
$0 --verbose "sftp:user@host:/backups/\$repo"
|
||||
done
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
RESTIC_PASSWORD_FILE Alternative to --password-file
|
||||
RESTIC_PASSWORD Direct password (not recommended)
|
||||
RESTIC_REPOSITORY Alternative to repository argument
|
||||
|
||||
NOTES:
|
||||
- The --read-data option (default) performs a thorough check by reading
|
||||
and verifying all data blobs. This is slower but more comprehensive.
|
||||
- Use --no-read-data for faster checks that only verify repository
|
||||
structure and metadata.
|
||||
- Exit codes: 0 = success, 1 = check failed or error occurred
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# Function to log messages
|
||||
log_message() {
|
||||
local message="$(date '+%Y-%m-%d %H:%M:%S') - $1"
|
||||
|
||||
if [ "$VERBOSE" = true ]; then
|
||||
echo "$message"
|
||||
fi
|
||||
|
||||
if [ -n "$LOG_FILE" ]; then
|
||||
echo "$message" >> "$LOG_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to send healthcheck ping
|
||||
send_healthcheck() {
|
||||
local status="$1"
|
||||
local message="$2"
|
||||
|
||||
if [ -z "$HEALTHCHECK_URL" ]; then
|
||||
[ "$VERBOSE" = true ] && echo "Healthcheck: $status - $message (no URL configured)"
|
||||
return
|
||||
fi
|
||||
|
||||
case "$status" in
|
||||
"START")
|
||||
curl -fsS -m 10 --retry 3 "$HEALTHCHECK_URL/start" >/dev/null 2>&1 || true
|
||||
;;
|
||||
"SUCCESS")
|
||||
curl -fsS -m 10 --retry 3 --data-raw "$message" "$HEALTHCHECK_URL" >/dev/null 2>&1 || true
|
||||
;;
|
||||
"FAILED")
|
||||
curl -fsS -m 10 --retry 3 --data-raw "$message" "$HEALTHCHECK_URL/fail" >/dev/null 2>&1 || true
|
||||
;;
|
||||
esac
|
||||
|
||||
[ "$VERBOSE" = true ] && echo "Healthcheck sent: $status - $message"
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
POSITIONAL_ARGS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-p|--password-file)
|
||||
RESTIC_PASSWORD_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-l|--log-file)
|
||||
LOG_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--healthcheck)
|
||||
HEALTHCHECK_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--no-read-data)
|
||||
READ_DATA=false
|
||||
shift
|
||||
;;
|
||||
--read-data)
|
||||
READ_DATA=true
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
-v|--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "Unknown option $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
POSITIONAL_ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Restore positional parameters
|
||||
set -- "${POSITIONAL_ARGS[@]}"
|
||||
|
||||
# Check required arguments
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Error: Missing required repository argument"
|
||||
echo
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPOSITORY="$1"
|
||||
|
||||
# Use environment variable as fallback for repository
|
||||
if [ -n "${RESTIC_REPOSITORY:-}" ] && [ "$REPOSITORY" = "${RESTIC_REPOSITORY}" ]; then
|
||||
REPOSITORY="$RESTIC_REPOSITORY"
|
||||
fi
|
||||
|
||||
# Use environment variable as fallback for password file
|
||||
if [ -n "${RESTIC_PASSWORD_FILE:-}" ]; then
|
||||
RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE}"
|
||||
fi
|
||||
|
||||
# Validation
|
||||
if [ ! -f "$RESTIC_PASSWORD_FILE" ] && [ -z "${RESTIC_PASSWORD:-}" ]; then
|
||||
echo "ERROR: Restic password file not found: $RESTIC_PASSWORD_FILE"
|
||||
echo " Set RESTIC_PASSWORD environment variable or provide valid password file"
|
||||
send_healthcheck "FAILED" "Restic password file not found: $RESTIC_PASSWORD_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build restic command base
|
||||
RESTIC_CMD="restic -r $REPOSITORY"
|
||||
if [ -f "$RESTIC_PASSWORD_FILE" ]; then
|
||||
RESTIC_CMD="$RESTIC_CMD --password-file $RESTIC_PASSWORD_FILE"
|
||||
fi
|
||||
|
||||
# Generate repository name for logging/monitoring
|
||||
REPO_NAME=$(echo "$REPOSITORY" | sed 's|.*/||' | sed 's|:.*||')
|
||||
if [ -z "$REPO_NAME" ]; then
|
||||
REPO_NAME="repository"
|
||||
fi
|
||||
|
||||
log_message "Starting integrity check for repository: $REPOSITORY"
|
||||
send_healthcheck "START" "Starting integrity check for $REPO_NAME"
|
||||
|
||||
# Build check command arguments
|
||||
CHECK_ARGS=()
|
||||
if [ "$READ_DATA" = true ]; then
|
||||
CHECK_ARGS+=("--read-data")
|
||||
log_message "Performing thorough check with data verification (this may take a while)"
|
||||
else
|
||||
log_message "Performing quick check without data verification"
|
||||
fi
|
||||
|
||||
# Test repository connectivity first
|
||||
log_message "Testing repository connectivity..."
|
||||
CONNECTIVITY_OUTPUT=""
|
||||
if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --last" 2>&1); then
|
||||
log_message "ERROR: Cannot connect to repository or repository is empty"
|
||||
if [ "$VERBOSE" = true ]; then
|
||||
log_message "Connectivity error: $CONNECTIVITY_OUTPUT"
|
||||
fi
|
||||
send_healthcheck "FAILED" "Cannot connect to repository $REPO_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Perform integrity check
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
log_message "DRY RUN: Would check repository $REPOSITORY"
|
||||
echo "Would run: $RESTIC_CMD check ${CHECK_ARGS[*]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log_message "Running repository integrity check..."
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
CHECK_OUTPUT=""
|
||||
CHECK_SUCCESS=false
|
||||
|
||||
if [ "$VERBOSE" = true ]; then
|
||||
# Show progress in verbose mode
|
||||
if eval "$RESTIC_CMD check ${CHECK_ARGS[*]}"; then
|
||||
CHECK_SUCCESS=true
|
||||
fi
|
||||
else
|
||||
# Capture output for logging
|
||||
if CHECK_OUTPUT=$(eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" 2>&1); then
|
||||
CHECK_SUCCESS=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$CHECK_SUCCESS" = true ]; then
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
|
||||
log_message "Repository integrity check completed successfully in ${DURATION} seconds"
|
||||
|
||||
# Get repository stats for the success message
|
||||
REPO_STATS=""
|
||||
if STATS_OUTPUT=$(eval "$RESTIC_CMD stats" 2>/dev/null); then
|
||||
REPO_STATS=$(echo "$STATS_OUTPUT" | grep -E "(Total Size|Total File Count)" | head -2 | tr '\n' ', ' | sed 's/, $//' || echo "Repository stats available")
|
||||
if [ -z "$REPO_STATS" ]; then
|
||||
REPO_STATS="Repository stats available"
|
||||
fi
|
||||
else
|
||||
REPO_STATS="Repository stats unavailable"
|
||||
fi
|
||||
|
||||
SUCCESS_MSG="Integrity check passed for $REPO_NAME in ${DURATION}s. $REPO_STATS"
|
||||
send_healthcheck "SUCCESS" "$SUCCESS_MSG"
|
||||
|
||||
# Additional verbose output
|
||||
if [ "$VERBOSE" = true ]; then
|
||||
echo "=== Check Summary ==="
|
||||
echo "Repository: $REPOSITORY"
|
||||
echo "Duration: ${DURATION} seconds"
|
||||
echo "Data verification: $([ "$READ_DATA" = true ] && echo "enabled" || echo "disabled")"
|
||||
echo "Result: PASSED"
|
||||
if [ -n "$REPO_STATS" ] && [ "$REPO_STATS" != "Repository stats unavailable" ]; then
|
||||
echo "Stats: $REPO_STATS"
|
||||
fi
|
||||
[ -n "$CHECK_OUTPUT" ] && echo -e "\nDetailed output:\n$CHECK_OUTPUT"
|
||||
fi
|
||||
else
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
|
||||
ERROR_MSG="CRITICAL: Repository integrity check failed for $REPO_NAME after ${DURATION}s - backup repository may be corrupted!"
|
||||
log_message "$ERROR_MSG"
|
||||
send_healthcheck "FAILED" "$ERROR_MSG"
|
||||
|
||||
if [ "$VERBOSE" = true ]; then
|
||||
echo "=== Check Summary ==="
|
||||
echo "Repository: $REPOSITORY"
|
||||
echo "Duration: ${DURATION} seconds"
|
||||
echo "Data verification: $([ "$READ_DATA" = true ] && echo "enabled" || echo "disabled")"
|
||||
echo "Result: FAILED"
|
||||
echo "WARNING: Repository may be corrupted! Check logs and consider running 'restic repair' if needed."
|
||||
[ -n "$CHECK_OUTPUT" ] && echo -e "\nError output:\n$CHECK_OUTPUT"
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_message "Integrity check process completed successfully"
|
||||
Reference in New Issue
Block a user