Initial commit of MyWorkspace - contains multiple projects and global workspace configuration
This commit is contained in:
DavidSaylor
319
BackupScript/backup.sh
Normal file
319
BackupScript/backup.sh
Normal file
@@ -0,0 +1,319 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Prevent set -e from causing issues in testing
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_NAME=$(basename "$0")
|
||||
|
||||
# Log to stderr with timestamp
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $SCRIPT_NAME: $*" >&2
|
||||
}
|
||||
|
||||
# Get current timestamp in seconds since epoch
|
||||
get_timestamp() {
|
||||
date +%s
|
||||
}
|
||||
|
||||
# Set file modification time to a specific date
|
||||
set_file_mtime() {
|
||||
local file="$1"
|
||||
local new_date="$2"
|
||||
touch -d "$new_date" "$file"
|
||||
}
|
||||
|
||||
# --- Testing Functions ---
|
||||
|
||||
# Function to create test backup files with manipulated timestamps
|
||||
create_test_backups() {
|
||||
local dest_dir="${1:-/mnt/Backup/Ubuntu/Backups}"
|
||||
local use_test_timestamp="${2:-now}"
|
||||
|
||||
log "Creating test backup files in $dest_dir (using timestamp: $use_test_timestamp)"
|
||||
|
||||
# Create the directory
|
||||
mkdir -p "$dest_dir"
|
||||
|
||||
local now_ts=$(get_timestamp)
|
||||
local now_date=$(date '+%Y-%m-%d')
|
||||
|
||||
# Convert relative days to timestamp
|
||||
days_ago() {
|
||||
local d="$1"
|
||||
date -d "${d} days ago" +%s 2>/dev/null || echo ""
|
||||
}
|
||||
|
||||
case "$use_test_timestamp" in
|
||||
"today")
|
||||
set_file_mtime "$dest_dir/test-today.tgz" "$now_date"
|
||||
log "Created test file: test-today.tgz (today)"
|
||||
;;
|
||||
"yesterday")
|
||||
set_file_mtime "$dest_dir/test-yesterday.tgz" "$(days_ago 1)"
|
||||
log "Created test file: test-yesterday.tgz (1 day ago)"
|
||||
;;
|
||||
"3days")
|
||||
set_file_mtime "$dest_dir/test-3days.tgz" "$(days_ago 3)"
|
||||
log "Created test file: test-3days.tgz (3 days ago)"
|
||||
;;
|
||||
"6days")
|
||||
set_file_mtime "$dest_dir/test-6days.tgz" "$(days_ago 6)"
|
||||
log "Created test file: test-6days.tgz (6 days ago)"
|
||||
;;
|
||||
"7days")
|
||||
set_file_mtime "$dest_dir/test-7days.tgz" "$(days_ago 7)"
|
||||
log "Created test file: test-7days.tgz (7 days ago)"
|
||||
;;
|
||||
"14days")
|
||||
set_file_mtime "$dest_dir/test-14days.tgz" "$(days_ago 14)"
|
||||
log "Created test file: test-14days.tgz (14 days ago)"
|
||||
;;
|
||||
"29days")
|
||||
set_file_mtime "$dest_dir/test-29days.tgz" "$(days_ago 29)"
|
||||
log "Created test file: test-29days.tgz (29 days ago)"
|
||||
;;
|
||||
"60days")
|
||||
set_file_mtime "$dest_dir/test-60days.tgz" "$(days_ago 60)"
|
||||
log "Created test file: test-60days.tgz (60 days ago)"
|
||||
;;
|
||||
"90days")
|
||||
set_file_mtime "$dest_dir/test-90days.tgz" "$(days_ago 90)"
|
||||
log "Created test file: test-90days.tgz (90 days ago)"
|
||||
;;
|
||||
"monday")
|
||||
# Get yesterday's date, check if Monday, otherwise find next Monday
|
||||
local check_date=$(date -d "yesterday" '+%A')
|
||||
if [ "$check_date" = "Monday" ]; then
|
||||
set_file_mtime "$dest_dir/test-monday.tgz" "$(days_ago 1)"
|
||||
log "Created test file: test-monday.tgz (yesterday/Monday)"
|
||||
else
|
||||
# Find next Monday (2 days from now if today is Saturday, etc.)
|
||||
local days_to_monday=$(( (1 + (6 - $(date +%w))) % 7 ))
|
||||
set_file_mtime "$dest_dir/test-monday.tgz" "$(days_ago $days_to_monday)"
|
||||
log "Created test file: test-monday.tgz (previous Monday)"
|
||||
fi
|
||||
;;
|
||||
"sunday")
|
||||
# Find the most recent Sunday
|
||||
local days_to_sunday=$(( (date +%w) % 7 ))
|
||||
set_file_mtime "$dest_dir/test-sunday.tgz" "$(days_ago $days_to_sunday)"
|
||||
log "Created test file: test-sunday.tgz (most recent Sunday)"
|
||||
;;
|
||||
"1st")
|
||||
# Get the 1st day of current month
|
||||
current_month=$(date +%m)
|
||||
set_file_mtime "$dest_dir/test-1st.tgz" "$(date -d "${current_month}-01" '+%s')"
|
||||
log "Created test file: test-1st.tgz (1st of this month)"
|
||||
;;
|
||||
*)
|
||||
log "Unknown timestamp option: $use_test_timestamp"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Create test files for multiple scenarios
|
||||
create_all_test_backups() {
|
||||
local dest_dir="${1:-/mnt/Backup/Ubuntu/Backups}"
|
||||
log "Creating comprehensive test backup files in $dest_dir"
|
||||
|
||||
create_test_backups "$dest_dir" "today"
|
||||
create_test_backups "$dest_dir" "yesterday"
|
||||
create_test_backups "$dest_dir" "3days"
|
||||
create_test_backups "$dest_dir" "6days"
|
||||
create_test_backups "$dest_dir" "7days"
|
||||
create_test_backups "$dest_dir" "14days"
|
||||
create_test_backups "$dest_dir" "29days"
|
||||
create_test_backups "$dest_dir" "60days"
|
||||
create_test_backups "$dest_dir" "90days"
|
||||
create_test_backups "$dest_dir" "monday"
|
||||
create_test_backups "$dest_dir" "sunday"
|
||||
create_test_backups "$dest_dir" "1st"
|
||||
|
||||
log "Test backup files created. Run ./backup.sh to test retention logic."
|
||||
}
|
||||
|
||||
# --- Configuration ---
|
||||
|
||||
# Backup 1: /mnt/Content → /mnt/Backup (rsync sync)
|
||||
CONTENT_SRC="/mnt/Content"
|
||||
CONTENT_DEST="/mnt/Backup"
|
||||
CONTENT_EXCLUDES="/etc/backup-excludes/content.exclude"
|
||||
|
||||
# Backup 2: / (root filesystem) → /mnt/Backup/Ubuntu/Backups (compressed tar)
|
||||
UBUNTU_DEST="/mnt/Backup/Ubuntu/Backups"
|
||||
UBUNTU_EXCLUDES="/etc/backup-excludes/ubuntu.exclude"
|
||||
|
||||
# === Helper Functions ===
|
||||
|
||||
# Run rsync with low CPU and I/O priority
|
||||
run_rsync() {
|
||||
local label="$1"
|
||||
shift
|
||||
log "Starting low-priority rsync ($label)"
|
||||
nice -n 19 ionice -c 3 rsync -avHAX "$@"
|
||||
}
|
||||
|
||||
# Run a command with low CPU and I/O priority
|
||||
run_low_priority() {
|
||||
local label="$1"
|
||||
shift
|
||||
log "Starting low-priority $label..."
|
||||
nice -n 19 ionice -c 3 "$@"
|
||||
}
|
||||
|
||||
# === Task 1: Sync /mnt/Content → /mnt/Backup ===
|
||||
|
||||
if [ -d "$CONTENT_SRC" ]; then
|
||||
log "Starting rsync of $CONTENT_SRC → $CONTENT_DEST (with excludes)"
|
||||
|
||||
# Ensure destination directory exists
|
||||
run_low_priority "Create destination" \
|
||||
mkdir -p "$CONTENT_DEST"
|
||||
|
||||
# Use --delete to sync deletions - frees space at destination
|
||||
run_rsync "Content -> Backup (media)" \
|
||||
--delete-before \
|
||||
--exclude-from="$CONTENT_EXCLUDES" \
|
||||
"$CONTENT_SRC"/ "$CONTENT_DEST"/
|
||||
|
||||
log "Finished $CONTENT_SRC → $CONTENT_DEST backup."
|
||||
else
|
||||
log "WARNING: $CONTENT_SRC not found or not a directory; skipping."
|
||||
fi
|
||||
|
||||
# === Task 2: Full filesystem backup with compression ===
|
||||
|
||||
log "Starting tar backup of / → $UBUNTU_DEST (excluding system paths)"
|
||||
|
||||
# Ensure destination directory exists
|
||||
run_low_priority "Create destination" \
|
||||
mkdir -p "$UBUNTU_DEST"
|
||||
|
||||
# Generate timestamp for backup filename
|
||||
timestamp=$(date +%Y-%m-%d)
|
||||
|
||||
# Create backup filename with timestamp
|
||||
tar_output="$UBUNTU_DEST/backup-full-$timestamp.tgz"
|
||||
|
||||
log "Starting compressed backup of /..."
|
||||
|
||||
# Run tar with low priority (compression is I/O intensive)
|
||||
run_low_priority "Full system backup" \
|
||||
tar -czf "$tar_output" \
|
||||
--exclude-from="$UBUNTU_EXCLUDES" \
|
||||
/
|
||||
|
||||
log "Finished / → $UBUNTU_DEST backup."
|
||||
|
||||
# === Retention Management ===
|
||||
|
||||
log "Managing backup retention in $UBUNTU_DEST..."
|
||||
|
||||
# Create the directory if it doesn't exist (already done above, but for safety)
|
||||
[ ! -d "$UBUNTU_DEST" ] && sudo mkdir -p "$UBUNTU_DEST"
|
||||
|
||||
# Count current totals
|
||||
daily_count=0
|
||||
weekly_count=0
|
||||
monthly_count=0
|
||||
|
||||
if [ -n "$(ls -A "$UBUNTU_DEST"/*.tgz 2>/dev/null)" ]; then
|
||||
# First pass: count existing files by type
|
||||
for file in "$UBUNTU_DEST"/*.tgz; do
|
||||
[ -f "$file" ] || continue
|
||||
mtime=$(stat -c %Y "$file")
|
||||
age=$(( ($(date +%s) - $mtime) / (3600 * 24) ))
|
||||
backup_date=$(date -d "@$mtime" '+%Y-%m-%d')
|
||||
day=$(date -d "$backup_date" +%d)
|
||||
weekday=$(date -d "$backup_date" +%w) # 0=Sunday
|
||||
|
||||
if [ $age -lt 7 ]; then
|
||||
((daily_count++))
|
||||
fi
|
||||
if [ $weekday -eq 0 ]; then
|
||||
((weekly_count++))
|
||||
fi
|
||||
if [ $day -eq 01 ]; then
|
||||
((monthly_count++))
|
||||
fi
|
||||
done
|
||||
|
||||
log "Current counts - Daily: $daily_count, Weekly: $weekly_count, Monthly: $monthly_count"
|
||||
fi
|
||||
|
||||
# Process each backup file
|
||||
if [ -n "$(ls -A "$UBUNTU_DEST"/*.tgz 2>/dev/null)" ]; then
|
||||
for backup_file in $(ls -1 "$UBUNTU_DEST"/*.tgz | sort --key=1.2 --reverse); do
|
||||
filename=$(basename "$backup_file")
|
||||
mtime=$(stat -c %Y "$backup_file")
|
||||
age=$(( ($(date +%s) - $mtime) / (3600 * 24) ))
|
||||
|
||||
# Calculate date of the backup
|
||||
backup_date=$(date -d "@$mtime" '+%Y-%m-%d')
|
||||
day=$(date -d "$backup_date" +%d)
|
||||
month=$(date -d "$backup_date" +%m)
|
||||
weekday=$(date -d "$backup_date" +%w) # 0=Sunday
|
||||
|
||||
# Calculate months since backup
|
||||
backup_year=$(date -d "$backup_date" +%Y)
|
||||
backup_month_int=$((10#$month))
|
||||
current_month_int=$((10#$month))
|
||||
current_year_int=$((10#$backup_year))
|
||||
|
||||
if [ $backup_month_int -lt $current_month_int ]; then
|
||||
months_since=$(( (current_month_int - backup_month_int) + 12 ))
|
||||
else
|
||||
months_since=$(( current_month_int - backup_month_int ))
|
||||
fi
|
||||
|
||||
if [ $backup_year_int -lt $current_year_int ]; then
|
||||
months_since=$(( months_since + 12 ))
|
||||
fi
|
||||
|
||||
# Determine backup type
|
||||
is_daily=0
|
||||
is_weekly=0
|
||||
is_monthly=0
|
||||
|
||||
if [ $age -lt 7 ]; then
|
||||
is_daily=1
|
||||
fi
|
||||
|
||||
if [ $weekday -eq 0 ]; then
|
||||
is_weekly=1
|
||||
fi
|
||||
|
||||
if [ $day -eq 01 ]; then
|
||||
is_monthly=1
|
||||
fi
|
||||
|
||||
# Keep daily backups (less than 7 days old)
|
||||
if [ $is_daily -eq 1 ]; then
|
||||
log "Backup $filename: Keeping (daily backup, age: $age days)"
|
||||
((daily_count++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Keep Sunday backups for the last month (31 days)
|
||||
if [ $is_weekly -eq 1 ] && [ $age -lt 31 ]; then
|
||||
log "Backup $filename: Keeping (weekly Sunday backup, age: $age days)"
|
||||
((weekly_count++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Keep 1st-of-month backups for the last 3 months
|
||||
if [ $is_monthly -eq 1 ] && [ $months_since -lt 3 ]; then
|
||||
log "Backup $filename: Keeping (monthly 1st-of-month backup, age: $age days)"
|
||||
((monthly_count++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Delete backup that doesn't meet retention criteria
|
||||
log "Deleting backup: $filename (age: $age days, days_since: $months_since, type: none)"
|
||||
sudo rm "$backup_file"
|
||||
done
|
||||
else
|
||||
log "No backup files found in $UBUNTU_DEST"
|
||||
fi
|
||||
|
||||
log "All backups completed."
|
||||
31
BackupScript/content.exclude
Normal file
31
BackupScript/content.exclude
Normal file
@@ -0,0 +1,31 @@
|
||||
# Windows cruft
|
||||
$RECYCLE.BIN
|
||||
'$RECYCLE.BIN'
|
||||
System Volume Information
|
||||
'System Volume Information'
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
*.tmp
|
||||
temp/
|
||||
Temp/
|
||||
Temp.*
|
||||
tmp/
|
||||
*.log
|
||||
|
||||
# Syncthing internal state
|
||||
.stfolder
|
||||
*.stfolder
|
||||
.syncthing
|
||||
*.stignore
|
||||
.index.*
|
||||
.clones
|
||||
*.idx*
|
||||
|
||||
# Skip the old Windows DockerContainers on /mnt/Content
|
||||
DockerContainers
|
||||
|
||||
# Android / user cache style (if present under Data/Test)
|
||||
*/.cache/*
|
||||
*/temp/*
|
||||
*/cache/*
|
||||
*/DCIM/.thumbnails/*
|
||||
4
BackupScript/cronjob
Normal file
4
BackupScript/cronjob
Normal file
@@ -0,0 +1,4 @@
|
||||
# Cron job for daily backups at 3:00 AM
|
||||
# Copy this file to /etc/cron.d/backup or create an entry in /etc/crontab
|
||||
|
||||
0 3 * * * root /opt/backup.sh >> /var/log/backup.log 2>&1
|
||||
124
BackupScript/example.sh
Normal file
124
BackupScript/example.sh
Normal file
@@ -0,0 +1,124 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_NAME=$(basename "$0")
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $SCRIPT_NAME: $*" >&2
|
||||
}
|
||||
|
||||
cd / || exit 1
|
||||
|
||||
#UBUNTU_DEST="/mnt/Backup/Ubuntu"
|
||||
UBUNTU_DEST="/mnt/Backup/Ubuntu/Backups"
|
||||
CONTENT_SRC="/mnt/Content"
|
||||
CONTENT_DEST="/mnt/Backup"
|
||||
|
||||
# helper wrapper: run rsync with low CPU and low I/O priority
|
||||
run_rsync() {
|
||||
local label="$1"
|
||||
shift
|
||||
log "Starting low‑priority rsync ($label)"
|
||||
nice -n 19 ionice -c 3 rsync -avHAX --delete "$@"
|
||||
}
|
||||
|
||||
# Helper function: Run a command with low CPU and I/O priority
|
||||
run_low_priority() {
|
||||
local label="$1"
|
||||
shift
|
||||
log "Starting low-priority $label..."
|
||||
nice -n 19 ionice -c 3 "$@"
|
||||
}
|
||||
|
||||
# Function to manage backup lifecycle - keep dailies for the past week, keep weeklies for the past month, keep montlies for 6 months.
|
||||
manage_backups() {
|
||||
#local dir="/mnt/Backup/Ubuntu/Backups" - use UBUNTU_DEST
|
||||
log "Managing backups in $dir..."
|
||||
|
||||
# Create the directory if it doesn't exist
|
||||
[ ! -d "$UBUNTU_DEST" ] && sudo mkdir -p "$UBUNTU_DEST"
|
||||
|
||||
# List all .tgz files and sort them by modification time (oldest first)
|
||||
for backup_file in $(sudo ls -1 $UBUNTU_DEST/*.tgz | sort --key=1.2 --reverse); do
|
||||
filename=$(basename "$backup_file")
|
||||
timestamp=$(echo "$filename" | cut -d'-' -f3- | cut -d '.' -f1)
|
||||
|
||||
# Calculate age in days
|
||||
mtime=$(stat -c %Y "$backup_file")
|
||||
age=$(( ($(date +%s) - $mtime) / (3600 * 24) ))
|
||||
|
||||
# Extract day, month, and weekday from timestamp
|
||||
day=$(date -d "@$timestamp" +%d)
|
||||
month=$(date -d "@$timestamp" +%m)
|
||||
weekday=$(date -d "@$timestamp" +%w)
|
||||
|
||||
# Check if backup is less than a week old (daily retention)
|
||||
if [ $age -lt 7 ]; then
|
||||
log "Backup $filename is less than a week old. Keeping."
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check for weekly backups on Sundays within the last month
|
||||
if [ $weekday == '0' ]; then
|
||||
current_month=$(date +%m)
|
||||
months_since=$(( (current_month - $month) % 12 ))
|
||||
if [ $months_since -lt 1 ]; then
|
||||
log "Backup $filename is a weekly backup on Sunday within the last month. Keeping."
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check for monthly backups on first day of the month within six months
|
||||
if [ $day == '01' ]; then
|
||||
current_month=$(date +%m)
|
||||
months_since=$(( (current_month - $month) % 12 ))
|
||||
if [ $months_since -lt 6 ]; then
|
||||
log "Backup $filename is a monthly backup on the first day within six months. Keeping."
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Delete backups older than retention criteria
|
||||
log "Deleting backup: $filename"
|
||||
sudo rm "$backup_file"
|
||||
done
|
||||
}
|
||||
|
||||
# --- 1. Backup / → /mnt/Backup/Ubuntu (fast, configs + Docker) ---
|
||||
|
||||
log "Starting tar backup of / → $UBUNTU_DEST (excluding /mnt and pseudo‑dirs)"
|
||||
|
||||
sudo mkdir -p "$UBUNTU_DEST"
|
||||
|
||||
# no longer using rsync to backup main filesystem as single file backup using compression works better with remote sync - does not interfere with syncthing state files and folders at the destination
|
||||
#run_rsync "Ubuntu configs & Docker" \
|
||||
# --exclude-from=/etc/backup-excludes/ubuntu.exclude \
|
||||
# / "$UBUNTU_DEST"/
|
||||
|
||||
# Create a compressed tar backup with timestamp in the filename
|
||||
timestamp=$(date +%Y%m%d-%H%M%S)
|
||||
tar_output="$UBUNTU_DEST/backup-full-$timestamp.tgz"
|
||||
|
||||
log "Starting backup of /..."
|
||||
run_low_priority "Ubuntu full backup" \
|
||||
tar -czf "$tar_output" --exclude-from=/etc/backup-excludes/ubuntu.exclude /
|
||||
|
||||
|
||||
log "Finished / → $UBUNTU_DEST backup."
|
||||
|
||||
# --- 2. Backup /mnt/Content → /mnt/Backup (large media‑heavy sync) ---
|
||||
|
||||
if [ -d "$CONTENT_SRC" ]; then
|
||||
log "Starting rsync of $CONTENT_SRC → $CONTENT_DEST (with excludes)"
|
||||
|
||||
# using --delete-before to ensure space is freed at destination for new and updated files
|
||||
run_rsync "Content -> Backup (media)" \
|
||||
--delete-before
|
||||
--exclude-from=/etc/backup-excludes/content.exclude \
|
||||
"$CONTENT_SRC"/ "$CONTENT_DEST"/
|
||||
|
||||
log "Finished $CONTENT_SRC → $CONTENT_DEST backup."
|
||||
else
|
||||
log "WARNING: $CONTENT_SRC not found or not a directory; skipping."
|
||||
fi
|
||||
|
||||
log "All backups completed."
|
||||
12
BackupScript/ubuntu.exclude
Normal file
12
BackupScript/ubuntu.exclude
Normal file
@@ -0,0 +1,12 @@
|
||||
# Don't back up /mnt (Content, Backup, etc.)
|
||||
/mnt
|
||||
|
||||
# OS pseudo‑dirs
|
||||
/dev
|
||||
/proc
|
||||
/run
|
||||
/sys
|
||||
/tmp
|
||||
/var/cache/apt/archives/
|
||||
/var/run
|
||||
/var/tmp
|
||||
Reference in New Issue
Block a user