Add monitoring of scripts through healthchecks.io

Updates the README with more cron script setup instructions.

README.md

@@ -66,3 +66,21 @@ For these scripts to run successfully you will need to install rclone and setup
 - `STATIC_SERVER_FILES_BUCKET`
 - `BACKUP_SERVER_REMOTE`
 - `BACKUP_SERVER_BUCKET`
+
+## Healthchecks.io Monitoring
+
+The scripts are monitored via [healthchecks.io](https://healthchecks.io). It is free to set up, create an account and add a new check for each script with these slugs:
+
+- `modmapper-update`
+- `modmapper-sync`
+- `modmapper-backup`
+
+Then add the project's [ping key which you can generate in the settings](https://healthchecks.io/docs/slug_urls/) to the `.env` file under `HEALTHCHECKS_PING_KEY`.
+
+## Crontab Setup
+
+The scripts can be run on a schedule via cron. To edit the crontab for the current user run `crontab -e` and add these lines to run the scripts at the specified intervals:
+
+```# Run modmapper update script every day at 2 am
+0 2 * * * cd /path/to/modmapper && ./scripts/update.sh && ./scripts/sync.sh && ./scripts/backup.sh
+```

scripts/backup.sh

@@ -1,7 +1,45 @@
 #!/bin/bash
+set -e
+set -o pipefail
+
+# Load environment variables
 export $(grep -v '^#' .env | xargs -d '\n')
+
+# Generate UUID for this run
+RID=$(cat /proc/sys/kernel/random/uuid)
+
+# Healthchecks.io ping function
+ping_healthcheck() {
+    local endpoint="$1"
+    local data="$2"
+    if [ -n "$data" ]; then
+        curl -fsS -m 10 --retry 5 --data-raw "$data" "https://hc-ping.com/${HEALTHCHECK_PING_KEY}/modmapper-backup${endpoint}?rid=${RID}" >/dev/null || true
+    else
+        curl -fsS -m 10 --retry 5 "https://hc-ping.com/${HEALTHCHECK_PING_KEY}/modmapper-backup${endpoint}?rid=${RID}" >/dev/null || true
+    fi
+}
+
+# Send failure notification with logs
+send_failure() {
+    local logs=""
+    if [ -f logs/modmapper.log ]; then
+        logs=$(tail --bytes=100000 logs/modmapper.log)
+    fi
+    ping_healthcheck "/fail" "$logs"
+    exit 1
+}
+
+# Trap to catch failures
+trap send_failure ERR
+
+# Send start ping
+ping_healthcheck "/start"
+
 mkdir -p backups
 zip -r -9 backups/plugins.zip plugins -DF --out backups/plugins-update.zip
 pg_dump -h localhost -U modmapper -Fc modmapper > backups/modmapper-$(date +'%Y-%m-%d').dump
 find backups/modmapper-*.dump -mtime +30 -type f -delete
-rclone sync backups ${BACKUP_SERVER_REMOTE}:${BACKUP_SERVER_BUCKET}
+rclone sync backups ${BACKUP_SERVER_REMOTE}:${BACKUP_SERVER_BUCKET}
+
+# Send success ping
+ping_healthcheck ""

scripts/sync.sh

@@ -1,6 +1,44 @@
 #!/bin/bash
+set -e
+set -o pipefail
+
+# Load environment variables
 export $(grep -v '^#' .env | xargs -d '\n')
+
+# Generate UUID for this run
+RID=$(cat /proc/sys/kernel/random/uuid)
+
+# Healthchecks.io ping function
+ping_healthcheck() {
+    local endpoint="$1"
+    local data="$2"
+    if [ -n "$data" ]; then
+        curl -fsS -m 10 --retry 5 --data-raw "$data" "https://hc-ping.com/${HEALTHCHECK_PING_KEY}/modmapper-sync${endpoint}?rid=${RID}" >/dev/null || true
+    else
+        curl -fsS -m 10 --retry 5 "https://hc-ping.com/${HEALTHCHECK_PING_KEY}/modmapper-sync${endpoint}?rid=${RID}" >/dev/null || true
+    fi
+}
+
+# Send failure notification with logs
+send_failure() {
+    local logs=""
+    if [ -f logs/modmapper.log ]; then
+        logs=$(tail --bytes=100000 logs/modmapper.log)
+    fi
+    ping_healthcheck "/fail" "$logs"
+    exit 1
+}
+
+# Trap to catch failures
+trap send_failure ERR
+
+# Send start ping
+ping_healthcheck "/start"
+
 rclone sync --fast-list --checksum cells ${STATIC_SERVER_REMOTE}:${STATIC_SERVER_CELLS_BUCKET}
 rclone sync --fast-list --checksum mods ${STATIC_SERVER_REMOTE}:${STATIC_SERVER_MODS_BUCKET}
 rclone sync --fast-list --checksum plugins_data ${STATIC_SERVER_REMOTE}:${STATIC_SERVER_PLUGINS_BUCKET}
-rclone sync --fast-list --checksum files ${STATIC_SERVER_REMOTE}:${STATIC_SERVER_FILES_BUCKET}
+rclone sync --fast-list --checksum files ${STATIC_SERVER_REMOTE}:${STATIC_SERVER_FILES_BUCKET}
+
+# Send success ping
+ping_healthcheck ""

scripts/update.sh

@@ -1,4 +1,40 @@
 #!/bin/bash
+set -e
+set -o pipefail
+
+# Load environment variables
+export $(grep -v '^#' .env | xargs -d '\n')
+
+# Generate UUID for this run
+RID=$(cat /proc/sys/kernel/random/uuid)
+
+# Healthchecks.io ping function
+ping_healthcheck() {
+    local endpoint="$1"
+    local data="$2"
+    if [ -n "$data" ]; then
+        curl -fsS -m 10 --retry 5 --data-raw "$data" "https://hc-ping.com/${HEALTHCHECK_PING_KEY}/modmapper-update${endpoint}?rid=${RID}" >/dev/null || true
+    else
+        curl -fsS -m 10 --retry 5 "https://hc-ping.com/${HEALTHCHECK_PING_KEY}/modmapper-update${endpoint}?rid=${RID}" >/dev/null || true
+    fi
+}
+
+# Send failure notification with logs
+send_failure() {
+    local logs=""
+    if [ -f logs/modmapper.log ]; then
+        logs=$(tail --bytes=100000 logs/modmapper.log)
+    fi
+    ping_healthcheck "/fail" "$logs"
+    exit 1
+}
+
+# Trap to catch failures
+trap send_failure ERR
+
+# Send start ping
+ping_healthcheck "/start"
+
 if [ -f cells/edits.json ]; then
     last_update_time=$(date -r cells/edits.json +'%Y-%m-%dT%H:%M:%S')
 fi
@@ -29,4 +65,7 @@ else
     ./target/release/mod-mapper -m mods &>> logs/modmapper.log
     ./target/release/mod-mapper -P plugins_data &>> logs/modmapper.log
     ./target/release/mod-mapper -F files &>> logs/modmapper.log
-fi
+fi
+
+# Send success ping
+ping_healthcheck ""