From 428c4416c90cdc18bf842403a68d587c579fc23f Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Fri, 10 Oct 2025 14:53:08 -0400 Subject: [PATCH] Initial commit: backup and integrity check scripts --- backup.sh | 347 ++++++++++++++++++++++++++++++++++++++ restic-integrity-check.sh | 305 +++++++++++++++++++++++++++++++++ 2 files changed, 652 insertions(+) create mode 100755 backup.sh create mode 100755 restic-integrity-check.sh diff --git a/backup.sh b/backup.sh new file mode 100755 index 0000000..66c8a47 --- /dev/null +++ b/backup.sh @@ -0,0 +1,347 @@ +#!/bin/bash + +# Generic Restic Backup Script +# Usage: ./backup.sh [OPTIONS] + +set -euo pipefail + +# Default values +RESTIC_PASSWORD_FILE="$HOME/.restic-password" +LOG_FILE="" +HEALTHCHECK_URL="" +KEEP_DAILY=7 +KEEP_WEEKLY=4 +KEEP_MONTHLY=6 +KEEP_YEARLY=2 +TAG_PREFIX="" +DRY_RUN=false +VERBOSE=false +EXCLUDE_CACHES=true +ONE_FILE_SYSTEM=true +COMPRESSION="auto" + +# Function to display usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Generic backup script using Restic for any folder and repository. + +REQUIRED ARGUMENTS: + source_path Path to backup (e.g., /home/user/documents) + repository Restic repository URL (e.g., sftp:user@host:/path) + +OPTIONS: + -p, --password-file FILE Password file path (default: ~/.restic-password) + -l, --log-file FILE Log file path (default: no logging) + -h, --healthcheck URL Healthchecks.io ping URL + -t, --tag TAG Additional tag for backup (can be used multiple times) + --tag-prefix PREFIX Prefix for auto-generated tags (default: none) + + RETENTION POLICY: + --keep-daily N Keep N daily snapshots (default: 7) + --keep-weekly N Keep N weekly snapshots (default: 4) + --keep-monthly N Keep N monthly snapshots (default: 6) + --keep-yearly N Keep N yearly snapshots (default: 2) + + BACKUP OPTIONS: + --no-exclude-caches Don't exclude cache directories + --no-one-file-system Allow crossing filesystem boundaries + --compression LEVEL Compression level: auto|max|off|fastest|better (default: auto) + + OTHER OPTIONS: + --dry-run Show what would be backed up without doing it + -v, --verbose Enable verbose output + --help Show this help message + +EXAMPLES: + # Basic backup + $0 /home/user/music sftp:user@host:/backups/music + + # With custom retention and healthcheck + $0 --keep-daily 14 --healthcheck https://hc-ping.com/uuid \\ + /var/www sftp:user@host:/backups/www + + # Multiple tags and custom log file + $0 -t production -t database --tag-prefix server1 \\ + --log-file /var/log/backup.log \\ + /var/lib/mysql sftp:user@host:/backups/mysql + +ENVIRONMENT VARIABLES: + RESTIC_PASSWORD_FILE Alternative to --password-file + RESTIC_PASSWORD Direct password (not recommended) + RESTIC_REPOSITORY Alternative to repository argument + +EOF +} + +# Function to log messages +log_message() { + local message="$(date '+%Y-%m-%d %H:%M:%S') - $1" + + if [ "$VERBOSE" = true ]; then + echo "$message" + fi + + if [ -n "$LOG_FILE" ]; then + echo "$message" >> "$LOG_FILE" + fi +} + +# Function to send healthcheck ping +send_healthcheck() { + local status="$1" + local message="$2" + + if [ -z "$HEALTHCHECK_URL" ]; then + [ "$VERBOSE" = true ] && echo "Healthcheck: $status - $message (no URL configured)" + return + fi + + case "$status" in + "START") + curl -fsS -m 10 --retry 3 "$HEALTHCHECK_URL/start" >/dev/null 2>&1 || true + ;; + "SUCCESS") + curl -fsS -m 10 --retry 3 --data-raw "$message" "$HEALTHCHECK_URL" >/dev/null 2>&1 || true + ;; + "FAILED"|"WARNING") + curl -fsS -m 10 --retry 3 --data-raw "$message" "$HEALTHCHECK_URL/fail" >/dev/null 2>&1 || true + ;; + esac + + [ "$VERBOSE" = true ] && echo "Healthcheck sent: $status - $message" +} + +# Parse command line arguments +CUSTOM_TAGS=() +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -p|--password-file) + RESTIC_PASSWORD_FILE="$2" + shift 2 + ;; + -l|--log-file) + LOG_FILE="$2" + shift 2 + ;; + -h|--healthcheck) + HEALTHCHECK_URL="$2" + shift 2 + ;; + -t|--tag) + CUSTOM_TAGS+=("$2") + shift 2 + ;; + --tag-prefix) + TAG_PREFIX="$2" + shift 2 + ;; + --keep-daily) + KEEP_DAILY="$2" + shift 2 + ;; + --keep-weekly) + KEEP_WEEKLY="$2" + shift 2 + ;; + --keep-monthly) + KEEP_MONTHLY="$2" + shift 2 + ;; + --keep-yearly) + KEEP_YEARLY="$2" + shift 2 + ;; + --no-exclude-caches) + EXCLUDE_CACHES=false + shift + ;; + --no-one-file-system) + ONE_FILE_SYSTEM=false + shift + ;; + --compression) + case "$2" in + auto|max|off|fastest|better) + COMPRESSION="$2" + ;; + *) + echo "Error: Invalid compression value '$2'. Valid values: auto, max, off, fastest, better" + exit 1 + ;; + esac + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + --help) + usage + exit 0 + ;; + -*) + echo "Unknown option $1" + usage + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done + +# Restore positional parameters +set -- "${POSITIONAL_ARGS[@]}" + +# Check required arguments +if [ $# -lt 2 ]; then + echo "Error: Missing required arguments" + echo + usage + exit 1 +fi + +SOURCE_PATH="$1" +REPOSITORY="$2" + +# Use environment variable as fallback for repository +if [ -n "${RESTIC_REPOSITORY:-}" ] && [ "$REPOSITORY" = "${RESTIC_REPOSITORY}" ]; then + REPOSITORY="$RESTIC_REPOSITORY" +fi + +# Use environment variable as fallback for password file +if [ -n "${RESTIC_PASSWORD_FILE:-}" ]; then + RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE}" +fi + +# Validation +if [ ! -d "$SOURCE_PATH" ]; then + echo "ERROR: Source path does not exist: $SOURCE_PATH" + send_healthcheck "FAILED" "Source path not found: $SOURCE_PATH" + exit 1 +fi + +if [ ! -f "$RESTIC_PASSWORD_FILE" ] && [ -z "${RESTIC_PASSWORD:-}" ]; then + echo "ERROR: Restic password file not found: $RESTIC_PASSWORD_FILE" + echo " Set RESTIC_PASSWORD environment variable or provide valid password file" + send_healthcheck "FAILED" "Restic password file not found: $RESTIC_PASSWORD_FILE" + exit 1 +fi + +# Build restic command base +RESTIC_CMD="restic -r $REPOSITORY" +if [ -f "$RESTIC_PASSWORD_FILE" ]; then + RESTIC_CMD="$RESTIC_CMD --password-file $RESTIC_PASSWORD_FILE" +fi +# Add compression as a global flag +RESTIC_CMD="$RESTIC_CMD --compression $COMPRESSION" + +# Generate backup name from source path for tagging +BACKUP_NAME=$(basename "$SOURCE_PATH") +if [ -n "$TAG_PREFIX" ]; then + BACKUP_NAME="${TAG_PREFIX}-${BACKUP_NAME}" +fi + +# Build tag arguments +TAG_ARGS=() +TAG_ARGS+=("--tag" "$BACKUP_NAME") +TAG_ARGS+=("--tag" "$(date +%Y-%m)") + +# Add custom tags +for tag in "${CUSTOM_TAGS[@]}"; do + if [ -n "$TAG_PREFIX" ]; then + TAG_ARGS+=("--tag" "${TAG_PREFIX}-${tag}") + else + TAG_ARGS+=("--tag" "$tag") + fi +done + +# Build backup command arguments +BACKUP_ARGS=() +if [ "$EXCLUDE_CACHES" = true ]; then + BACKUP_ARGS+=("--exclude-caches") +fi +if [ "$ONE_FILE_SYSTEM" = true ]; then + BACKUP_ARGS+=("--one-file-system") +fi + +log_message "Starting backup of $SOURCE_PATH to $REPOSITORY" +send_healthcheck "START" "Starting backup of $BACKUP_NAME" + +# Pre-backup: Check repository connectivity +log_message "Checking repository connectivity..." +CONNECTIVITY_OUTPUT="" +if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --last" 2>&1); then + log_message "WARNING: Could not connect to repository or no snapshots exist yet" + if [ "$VERBOSE" = true ]; then + log_message "Connectivity check output: $CONNECTIVITY_OUTPUT" + fi +fi + +# Perform backup +if [ "$DRY_RUN" = true ]; then + log_message "DRY RUN: Would backup $SOURCE_PATH" + echo "Would run: $RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" + exit 0 +fi + +log_message "Creating backup snapshot..." +log_message "Running command: $RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" + +BACKUP_OUTPUT="" +BACKUP_SUCCESS=false + +if [ "$VERBOSE" = true ]; then + # Show progress in verbose mode + if eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}"; then + BACKUP_SUCCESS=true + fi +else + # Capture output for logging + if BACKUP_OUTPUT=$(eval "$RESTIC_CMD backup \"$SOURCE_PATH\" ${TAG_ARGS[*]} ${BACKUP_ARGS[*]}" 2>&1); then + BACKUP_SUCCESS=true + fi +fi + +if [ "$BACKUP_SUCCESS" = true ]; then + log_message "Backup completed successfully" + if [ "$VERBOSE" = true ] && [ -n "${BACKUP_OUTPUT:-}" ]; then + echo "$BACKUP_OUTPUT" + fi + + # Clean up old snapshots according to retention policy + log_message "Applying retention policy..." + FORGET_OUTPUT="" + if FORGET_OUTPUT=$(eval "$RESTIC_CMD forget --tag $BACKUP_NAME --keep-daily $KEEP_DAILY --keep-weekly $KEEP_WEEKLY --keep-monthly $KEEP_MONTHLY --keep-yearly $KEEP_YEARLY --prune" 2>&1); then + log_message "Retention policy applied successfully" + + # Get backup stats for healthcheck message + BACKUP_STATS="" + BACKUP_STATS=$(eval "$RESTIC_CMD stats --mode raw-data" 2>/dev/null | tail -n 3 | head -n 1 2>/dev/null || echo "Stats unavailable") + send_healthcheck "SUCCESS" "Backup of $BACKUP_NAME completed successfully. Repository: $BACKUP_STATS" + else + log_message "WARNING: Backup succeeded but retention cleanup failed" + if [ "$VERBOSE" = true ] && [ -n "$FORGET_OUTPUT" ]; then + log_message "Retention error output: $FORGET_OUTPUT" + fi + send_healthcheck "WARNING" "Backup of $BACKUP_NAME succeeded but retention cleanup failed" + fi +else + log_message "ERROR: Backup failed" + if [ -n "${BACKUP_OUTPUT:-}" ]; then + log_message "Backup error output: $BACKUP_OUTPUT" + fi + send_healthcheck "FAILED" "Backup of $BACKUP_NAME failed - check logs for details" + exit 1 +fi + +log_message "Backup process completed" diff --git a/restic-integrity-check.sh b/restic-integrity-check.sh new file mode 100755 index 0000000..22fe309 --- /dev/null +++ b/restic-integrity-check.sh @@ -0,0 +1,305 @@ +#!/bin/bash + +# Generic Restic Repository Integrity Check Script +# Usage: ./restic-integrity-check.sh [OPTIONS] + +set -euo pipefail + +# Default values +RESTIC_PASSWORD_FILE="$HOME/.restic-password" +LOG_FILE="" +HEALTHCHECK_URL="" +READ_DATA=true +VERBOSE=false +DRY_RUN=false + +# Function to display usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Generic integrity check script for Restic repositories. + +REQUIRED ARGUMENTS: + repository Restic repository URL (e.g., sftp:user@host:/path) + +OPTIONS: + -p, --password-file FILE Password file path (default: ~/.restic-password) + -l, --log-file FILE Log file path (default: no logging) + -h, --healthcheck URL Healthchecks.io ping URL for monitoring + + CHECK OPTIONS: + --no-read-data Skip reading and verifying data blobs (faster) + --read-data Verify data blobs (default, more thorough) + + OTHER OPTIONS: + --dry-run Show what would be checked without doing it + -v, --verbose Enable verbose output + --help Show this help message + +EXAMPLES: + # Basic integrity check + $0 sftp:user@host:/backups/music + + # With healthcheck monitoring and custom log + $0 --healthcheck https://hc-ping.com/uuid \\ + --log-file /var/log/integrity-check.log \\ + sftp:user@host:/backups/documents + + # Quick check without reading data (faster) + $0 --no-read-data --verbose \\ + local:/path/to/repo + + # Multiple repositories check (run separately) + for repo in repo1 repo2 repo3; do + $0 --verbose "sftp:user@host:/backups/\$repo" + done + +ENVIRONMENT VARIABLES: + RESTIC_PASSWORD_FILE Alternative to --password-file + RESTIC_PASSWORD Direct password (not recommended) + RESTIC_REPOSITORY Alternative to repository argument + +NOTES: + - The --read-data option (default) performs a thorough check by reading + and verifying all data blobs. This is slower but more comprehensive. + - Use --no-read-data for faster checks that only verify repository + structure and metadata. + - Exit codes: 0 = success, 1 = check failed or error occurred + +EOF +} + +# Function to log messages +log_message() { + local message="$(date '+%Y-%m-%d %H:%M:%S') - $1" + + if [ "$VERBOSE" = true ]; then + echo "$message" + fi + + if [ -n "$LOG_FILE" ]; then + echo "$message" >> "$LOG_FILE" + fi +} + +# Function to send healthcheck ping +send_healthcheck() { + local status="$1" + local message="$2" + + if [ -z "$HEALTHCHECK_URL" ]; then + [ "$VERBOSE" = true ] && echo "Healthcheck: $status - $message (no URL configured)" + return + fi + + case "$status" in + "START") + curl -fsS -m 10 --retry 3 "$HEALTHCHECK_URL/start" >/dev/null 2>&1 || true + ;; + "SUCCESS") + curl -fsS -m 10 --retry 3 --data-raw "$message" "$HEALTHCHECK_URL" >/dev/null 2>&1 || true + ;; + "FAILED") + curl -fsS -m 10 --retry 3 --data-raw "$message" "$HEALTHCHECK_URL/fail" >/dev/null 2>&1 || true + ;; + esac + + [ "$VERBOSE" = true ] && echo "Healthcheck sent: $status - $message" +} + +# Parse command line arguments +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -p|--password-file) + RESTIC_PASSWORD_FILE="$2" + shift 2 + ;; + -l|--log-file) + LOG_FILE="$2" + shift 2 + ;; + -h|--healthcheck) + HEALTHCHECK_URL="$2" + shift 2 + ;; + --no-read-data) + READ_DATA=false + shift + ;; + --read-data) + READ_DATA=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + --help) + usage + exit 0 + ;; + -*) + echo "Unknown option $1" + usage + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done + +# Restore positional parameters +set -- "${POSITIONAL_ARGS[@]}" + +# Check required arguments +if [ $# -lt 1 ]; then + echo "Error: Missing required repository argument" + echo + usage + exit 1 +fi + +REPOSITORY="$1" + +# Use environment variable as fallback for repository +if [ -n "${RESTIC_REPOSITORY:-}" ] && [ "$REPOSITORY" = "${RESTIC_REPOSITORY}" ]; then + REPOSITORY="$RESTIC_REPOSITORY" +fi + +# Use environment variable as fallback for password file +if [ -n "${RESTIC_PASSWORD_FILE:-}" ]; then + RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE}" +fi + +# Validation +if [ ! -f "$RESTIC_PASSWORD_FILE" ] && [ -z "${RESTIC_PASSWORD:-}" ]; then + echo "ERROR: Restic password file not found: $RESTIC_PASSWORD_FILE" + echo " Set RESTIC_PASSWORD environment variable or provide valid password file" + send_healthcheck "FAILED" "Restic password file not found: $RESTIC_PASSWORD_FILE" + exit 1 +fi + +# Build restic command base +RESTIC_CMD="restic -r $REPOSITORY" +if [ -f "$RESTIC_PASSWORD_FILE" ]; then + RESTIC_CMD="$RESTIC_CMD --password-file $RESTIC_PASSWORD_FILE" +fi + +# Generate repository name for logging/monitoring +REPO_NAME=$(echo "$REPOSITORY" | sed 's|.*/||' | sed 's|:.*||') +if [ -z "$REPO_NAME" ]; then + REPO_NAME="repository" +fi + +log_message "Starting integrity check for repository: $REPOSITORY" +send_healthcheck "START" "Starting integrity check for $REPO_NAME" + +# Build check command arguments +CHECK_ARGS=() +if [ "$READ_DATA" = true ]; then + CHECK_ARGS+=("--read-data") + log_message "Performing thorough check with data verification (this may take a while)" +else + log_message "Performing quick check without data verification" +fi + +# Test repository connectivity first +log_message "Testing repository connectivity..." +CONNECTIVITY_OUTPUT="" +if ! CONNECTIVITY_OUTPUT=$(eval "$RESTIC_CMD snapshots --last" 2>&1); then + log_message "ERROR: Cannot connect to repository or repository is empty" + if [ "$VERBOSE" = true ]; then + log_message "Connectivity error: $CONNECTIVITY_OUTPUT" + fi + send_healthcheck "FAILED" "Cannot connect to repository $REPO_NAME" + exit 1 +fi + +# Perform integrity check +if [ "$DRY_RUN" = true ]; then + log_message "DRY RUN: Would check repository $REPOSITORY" + echo "Would run: $RESTIC_CMD check ${CHECK_ARGS[*]}" + exit 0 +fi + +log_message "Running repository integrity check..." +START_TIME=$(date +%s) + +CHECK_OUTPUT="" +CHECK_SUCCESS=false + +if [ "$VERBOSE" = true ]; then + # Show progress in verbose mode + if eval "$RESTIC_CMD check ${CHECK_ARGS[*]}"; then + CHECK_SUCCESS=true + fi +else + # Capture output for logging + if CHECK_OUTPUT=$(eval "$RESTIC_CMD check ${CHECK_ARGS[*]}" 2>&1); then + CHECK_SUCCESS=true + fi +fi + +if [ "$CHECK_SUCCESS" = true ]; then + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + log_message "Repository integrity check completed successfully in ${DURATION} seconds" + + # Get repository stats for the success message + REPO_STATS="" + if STATS_OUTPUT=$(eval "$RESTIC_CMD stats" 2>/dev/null); then + REPO_STATS=$(echo "$STATS_OUTPUT" | grep -E "(Total Size|Total File Count)" | head -2 | tr '\n' ', ' | sed 's/, $//' || echo "Repository stats available") + if [ -z "$REPO_STATS" ]; then + REPO_STATS="Repository stats available" + fi + else + REPO_STATS="Repository stats unavailable" + fi + + SUCCESS_MSG="Integrity check passed for $REPO_NAME in ${DURATION}s. $REPO_STATS" + send_healthcheck "SUCCESS" "$SUCCESS_MSG" + + # Additional verbose output + if [ "$VERBOSE" = true ]; then + echo "=== Check Summary ===" + echo "Repository: $REPOSITORY" + echo "Duration: ${DURATION} seconds" + echo "Data verification: $([ "$READ_DATA" = true ] && echo "enabled" || echo "disabled")" + echo "Result: PASSED" + if [ -n "$REPO_STATS" ] && [ "$REPO_STATS" != "Repository stats unavailable" ]; then + echo "Stats: $REPO_STATS" + fi + [ -n "$CHECK_OUTPUT" ] && echo -e "\nDetailed output:\n$CHECK_OUTPUT" + fi +else + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + + ERROR_MSG="CRITICAL: Repository integrity check failed for $REPO_NAME after ${DURATION}s - backup repository may be corrupted!" + log_message "$ERROR_MSG" + send_healthcheck "FAILED" "$ERROR_MSG" + + if [ "$VERBOSE" = true ]; then + echo "=== Check Summary ===" + echo "Repository: $REPOSITORY" + echo "Duration: ${DURATION} seconds" + echo "Data verification: $([ "$READ_DATA" = true ] && echo "enabled" || echo "disabled")" + echo "Result: FAILED" + echo "WARNING: Repository may be corrupted! Check logs and consider running 'restic repair' if needed." + [ -n "$CHECK_OUTPUT" ] && echo -e "\nError output:\n$CHECK_OUTPUT" + fi + + exit 1 +fi + +log_message "Integrity check process completed successfully"