Enhance backup scripts with error handling and directory checks

backups/scripts/gitea-dump.sh

@@ -1,12 +1,29 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -Eeuo pipefail
+
+trap 'echo "Error on line $LINENO while writing Gitea dump" >&2' ERR
+
+ensure_writable_dir() {
+	local dir="$1"
+	mkdir -p "$dir"
+	if [[ ! -d "$dir" || ! -w "$dir" ]]; then
+		echo "Output directory is not writable: $dir" >&2
+		exit 1
+	fi
+}
 
 TS="$(date +%F_%H%M%S)"
 OUT="/srv/backups/gitea/dumps"
-mkdir -p "$OUT"
+ensure_writable_dir "$OUT"
+artifact="$OUT/gitea-dump-$TS.zip"
 
 docker exec gitea sh -lc "gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump-$TS.zip"
 docker cp "gitea:/tmp/gitea-dump-$TS.zip" "$OUT/"
 docker exec gitea rm -f "/tmp/gitea-dump-$TS.zip"
 
-echo "Wrote: $OUT/gitea-dump-$TS.zip"
+if [[ ! -s "$artifact" ]]; then
+	echo "Backup artifact missing or empty: $artifact" >&2
+	exit 1
+fi
+
+echo "Wrote: $artifact"

backups/scripts/gitea-mirror-export.sh

@@ -1,5 +1,16 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -Eeuo pipefail
+
+trap 'echo "Error on line $LINENO during mirror export" >&2' ERR
+
+ensure_writable_dir() {
+  local dir="$1"
+  mkdir -p "$dir"
+  if [[ ! -d "$dir" || ! -w "$dir" ]]; then
+    echo "Output directory is not writable: $dir" >&2
+    exit 1
+  fi
+}
 
 GITEA_URL="${GITEA_URL:-https://git.sketchferret.com}"
 TOKEN_FILE="${TOKEN_FILE:-/srv/secrets/gitea_token}"
@@ -12,7 +23,7 @@ if [[ ! -f "$TOKEN_FILE" ]]; then
 fi
 
 TOKEN="$(cat "$TOKEN_FILE")"
-mkdir -p "$OUT/$OWNER"
+ensure_writable_dir "$OUT/$OWNER"
 
 repos_json="$(curl -fsSL -H "Authorization: token $TOKEN" "$GITEA_URL/api/v1/users/$OWNER/repos?limit=1000")"
 
@@ -23,6 +34,11 @@ for repo in json.loads(sys.argv[1]):
 PY
 )
 
+if [[ "${#urls[@]}" -eq 0 ]]; then
+  echo "No repositories returned for owner '$OWNER' from $GITEA_URL" >&2
+  exit 1
+fi
+
 for url in "${urls[@]}"; do
   name="$(basename "$url" .git)"
   target="$OUT/$OWNER/$name.git"

backups/scripts/ops-bundle.sh

@@ -1,22 +1,40 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -Eeuo pipefail
+
+trap 'echo "Error on line $LINENO while creating ops bundle" >&2' ERR
+
+ensure_writable_dir() {
+  local dir="$1"
+  mkdir -p "$dir"
+  if [[ ! -d "$dir" || ! -w "$dir" ]]; then
+    echo "Output directory is not writable: $dir" >&2
+    exit 1
+  fi
+}
 
 OPS_REPO_PATH="${OPS_REPO_PATH:-/srv/ops}"
 OUT_BASE="${OUT_BASE:-/srv/backups/ops}"
 TS="$(date +%F_%H%M%S)"
 OUT_DIR="$OUT_BASE/$TS"
 LATEST_DIR="$OUT_BASE/latest"
+artifact="$OUT_DIR/ops.bundle"
+latest_artifact="$LATEST_DIR/ops.bundle"
 
 if [[ ! -d "$OPS_REPO_PATH/.git" ]]; then
   echo "Missing git repo at $OPS_REPO_PATH"
   exit 1
 fi
 
-mkdir -p "$OUT_DIR"
+ensure_writable_dir "$OUT_DIR"
-git -C "$OPS_REPO_PATH" bundle create "$OUT_DIR/ops.bundle" --all
+git -C "$OPS_REPO_PATH" bundle create "$artifact" --all
 
-mkdir -p "$LATEST_DIR"
+ensure_writable_dir "$LATEST_DIR"
-cp "$OUT_DIR/ops.bundle" "$LATEST_DIR/ops.bundle"
+cp "$artifact" "$latest_artifact"
 
-echo "Wrote bundle: $OUT_DIR/ops.bundle"
+if [[ ! -s "$artifact" || ! -s "$latest_artifact" ]]; then
-echo "Updated latest: $LATEST_DIR/ops.bundle"
+  echo "Bundle artifact missing or empty" >&2
+  exit 1
+fi
+
+echo "Wrote bundle: $artifact"
+echo "Updated latest: $latest_artifact"

backups/scripts/pg-dump.sh

@@ -1,9 +1,21 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -Eeuo pipefail
+
+trap 'echo "Error on line $LINENO while writing Postgres dump" >&2' ERR
+
+ensure_writable_dir() {
+  local dir="$1"
+  mkdir -p "$dir"
+  if [[ ! -d "$dir" || ! -w "$dir" ]]; then
+    echo "Output directory is not writable: $dir" >&2
+    exit 1
+  fi
+}
 
 TS="$(date +%F_%H%M%S)"
 OUT="/srv/backups/gitea/pg"
-mkdir -p "$OUT"
+ensure_writable_dir "$OUT"
+artifact="$OUT/gitea-pg-$TS.sql"
 
 PGUSER="${PGUSER:-gitea}"
 PGDATABASE="${PGDATABASE:-gitea}"
@@ -15,6 +27,11 @@ if [[ ! -f "$PGPASSFILE" ]]; then
 fi
 
 export PGPASSWORD="$(cat "$PGPASSFILE")"
-docker exec -e PGPASSWORD="$PGPASSWORD" gitea-db sh -lc "pg_dump -U '$PGUSER' -d '$PGDATABASE'" > "$OUT/gitea-pg-$TS.sql"
+docker exec -e PGPASSWORD="$PGPASSWORD" gitea-db sh -lc "pg_dump -U '$PGUSER' -d '$PGDATABASE'" > "$artifact"
 
-echo "Wrote: $OUT/gitea-pg-$TS.sql"
+if [[ ! -s "$artifact" ]]; then
+  echo "Backup artifact missing or empty: $artifact" >&2
+  exit 1
+fi
+
+echo "Wrote: $artifact"

backups/scripts/retention.sh

@@ -62,15 +62,41 @@ for path, _, _, age_days, preserve_latest in files:
 		to_delete.add(path)
 
 deleted = 0
+failed = 0
+considered = 0
 for path in sorted(to_delete):
+	considered += 1
 	try:
 		os.remove(path)
 		print(path)
 		deleted += 1
-	except OSError:
+	except OSError as exc:
-		pass
+		print(f"ERROR deleting {path}: {exc}", file=sys.stderr)
+		failed += 1
 
-print(f"Retention complete for {base} (deleted {deleted} files)")
+# Remove now-empty directories except anything under a latest/ subtree.
+removed_dirs = 0
+for root, dirs, _ in os.walk(base, topdown=False):
+	parts = set(os.path.normpath(root).split(os.sep))
+	if "latest" in parts:
+		continue
+	if os.path.normpath(root) == os.path.normpath(base):
+		continue
+	for d in list(dirs):
+		dir_path = os.path.join(root, d)
+		if "latest" in set(os.path.normpath(dir_path).split(os.sep)):
+			continue
+		try:
+			if not os.listdir(dir_path):
+				os.rmdir(dir_path)
+				removed_dirs += 1
+		except OSError:
+			pass
+
+print(f"Retention complete for {base} (considered {considered}, deleted {deleted}, removed_empty_dirs {removed_dirs})")
+if failed:
+	print(f"Retention encountered {failed} delete errors", file=sys.stderr)
+	sys.exit(1)
 PY
 
 echo "Policy: daily <= ${KEEP_DAILY_DAYS}d, weekly <= ${KEEP_WEEKLY_DAYS}d"