HOST_NAME=$(hostname -s) DAY=$(date +%u) # MO == 1 BACKUP_MASTER_CHECK="/var/docker/vol/proxy" NAS_HOSTNAME=synology NAS_IP=10.4.2.20 NAS_SCRIPT=/volume1/nfs/Server/bin/backup/main.sh WEEKLY_DAY=1 # SO->MO PASSPHRASE=1234567890 EXT=tgz.gpg LIST_EXT=txt DOCKER="/usr/bin/docker compose -f /var/docker/run/docker-compose.yml --ansi never" LOG_RECIPIENT=hb242@pm.me INFO=INFO WARN=WARN ERROR=ERROR DEBUG=DEBUG SUCCESS="*** success" INF="---" WRN=":::" ERR="###" DBG="===" dump_mariadb() { log $DEBUG "$DBG dump_mariadb( $@ )" local dump_dir=$1 local mysqlconnect="--user=root --password=snafu" log $DEBUG "$DBG > \$DOCKER exec mysql /usr/bin/mariadb --silent --user=xxx --password=xxx -e \"show databases\" -B" for db in $($DOCKER exec mysql /usr/bin/mariadb --silent $mysqlconnect -e "show databases" -B 2>>$LOG_FILE | grep -v information_schema 2>>$LOG_FILE | grep -v performance_schema 2>>$LOG_FILE | sed -e 's/\r//' 2>>$LOG_FILE); do log $DEBUG "$DBG > \$DOCKER exec mysql /usr/bin/mariadb-dump --hex-blob --user=xxx --password=xxx --databases $db" $DOCKER exec mysql /usr/bin/mariadb-dump --hex-blob $mysqlconnect --databases $db >$dump_dir/$db.sql 2>>$LOG_FILE is_error $? false && return 1 done return 0 } dump_postgresql() { log $DEBUG "$DBG dump_postgresql( $@ )" local dump_dir=$1 # users and roles log $DEBUG "$DBG > \$DOCKER exec -u=postgres postgres pg_dumpall --globals-only" $DOCKER exec -u=postgres postgres pg_dumpall --globals-only > $dump_dir/_globals.sql 2>>$LOG_FILE log $DEBUG "$DBG \$DOCKER exec -u=postgres postgres psql -t -c \"select datname from pg_database where not datistemplate\"" for db in $($DOCKER exec -u=postgres postgres psql -t -c "select datname from pg_database where not datistemplate" 2>>$LOG_FILE); do log $DEBUG "$DBG > \$DOCKER exec -u=postgres postgres pg_dump --create $db" $DOCKER exec -u=postgres postgres pg_dump --create $db > $dump_dir/$db.sql 2>>$LOG_FILE is_error $? false && return 1 done return 0 } start_docker() { log $DEBUG "$DBG start_docker()" log $DEBUG "$DBG > \$DOCKER up -d" $DOCKER up -d >>$LOG_FILE 2>&1 return $? } stop_docker() { log $DEBUG "$DBG stop_docker()" log $DEBUG "$DBG > \$DOCKER down" $DOCKER down >>$LOG_FILE 2>&1 return $? } # Script per ssh auf Server $1 im Hintergrund starten # # ssh start in background # -> https://stackoverflow.com/questions/29142/getting-ssh-to-execute-a-command-in-the-background-on-target-machine#comment107117479_29172 # # $1 - Server # $@ - Script incl. Parameter # run_script_remote() { local server=$1 shift ssh $server "sudo sh -c 'nohup $@ &>/dev/null /dev/null } # Reduziertes Log für automatische Läufe # # Das Log wird am Ende des Laufs als Mail verschickt. Die Meldungen # werden zusätzlich im Standard-Log protokolliert. # Dieses log wird nur geschrieben, wenn $MAIL_LOG_FILE definiert ist. # # $1 - Typ, z.B. INFO, WARN, ERROR, etc. # $2 ff. - die Meldung, die ins Log geschreiben wird # mail_log() { local type=$1 shift if [ "$MAIL_LOG_FILE" ]; then echo "$(date '+%d.%m.%Y %H:%M:%S') - $type $@" >> $MAIL_LOG_FILE fi log "$type" "$@" } # Reduziertes Log senden # send_mail_log() { local subject="$@" log $DEBUG "$DBG send_mail_log()" log $DEBUG "$DBG > $HERE/mail.sh $LOG_RECIPIENT \"[$HOST_NAME] $subject\" $MAIL_LOG_FILE" $HERE/mail.sh $LOG_RECIPIENT "[$HOST_NAME] Nightly Backup" $MAIL_LOG_FILE >>$LOG_FILE 2>&1 } # check if $1 is error (!=0) and write result to log # # $1 - return code to check # $2 - if set don't log success # # returns 0 if error, 1 otherwise! # is_error() { local rc=$1 local mlog=$2 if [ $rc -eq 0 ]; then if [ -z "$mlog" ]; then mail_log $INFO "$SUCCESS" fi return 1 else mail_log $ERROR "$ERR an error occured see $LOG_FILE" return 0 fi } # check return code of piped programs # # $1, ff. - return codes # # returns 0 for success, 1 for one or more errors # check_pipe_rc() { local result=0 local count=1 local result_str for rc in $@; do if [ $rc -ne 0 ] ; then result=1 result_str="${result_str}#${count} " fi let "count += 1" done if [ $result -ne 0 ]; then log $ERROR "$ERR in pipe $result_str" fi return $result } # Dateien vom Server zur NAS verschieben # # $1 - Verzeichnis auf Server # $2 - Ziel-Verzeichnis auf der NAS # # returns return code of mv # copy_to_nas() { log $DEBUG "$DBG copy_to_nas( $@ )" local source="$1" local target="$2" delete_dir "$target" if [ $? -ne 0 ]; then log $ERROR "$ERR directory \"$target\" could not be emptied" return 1 fi check_directory "$target" is_error $? true && return 1 log $DEBUG "$DBG > mv \"$source\"/* \"$target\"" mv "$source"/* "$target" >>$LOG_FILE 2>&1 return $? } # Ist $1 in der Liste der restlichen args $2, $3 ... # # zB. if is_in "foo" "foo" "bar" "foo bar" ; then ... # Leere Liste liefert immer false # is_in() { local search="$1" shift 1 local e for e in "$@"; do [[ "$e" == "$search" ]] && return 0; done return 1 } # Alle Verzechnisnamen in einem Verzeichnis holen. # # $1 - Verzeichnis in dem gesucht wird als absoluter Pfad # $2 ff. - auszuschliessende Verzeichnisse (kann leer bleiben) # iterate_dirs_in() { local dirname local root=$1 shift 1 for dir in $(find "$root" -maxdepth 1 -type d \! -wholename "$root"); do dirname=$(basename $dir) if ! is_in "$dirname" "$@"; then echo "$dirname" fi done } # create directory if it not exists # # $1 - dir to check # # returns 0 for success, 1 otherwise # check_directory() { log $DEBUG "$DBG check_directory( $@ )" local dir=$1 if [ -d "$1" ]; then return 0 else log $DEBUG "$DBG > mkdir -p \"$1\"" mkdir -p "$1" >>$LOG_FILE 2>&1 && return 0 log $ERROR "$ERR could not create directory $1!" return 1 fi } # Verzeichnis löschen # # Im Fehlerfall wird rc=1 zurückgegeben # # $1 - zu löschendes Verzeichnis # # returns 0 for success, 1 otherwise # delete_dir() { log $DEBUG "$DBG delete_dir( $@ )" local dir=$1 log $DEBUG "$DBG > rm -rf \"$dir\"/*" rm -rf "$dir"/* >>$LOG_FILE 2>&1 && return 0 # return error, so caller can decide what to do return 1 } # Slashes "/" im Pfadnamen durch Dashes "-" ersetzen, so dass der String als Dateiname verwendet werden kann. # # $1 - path backup_file_prefix() { echo "$1" | sed -E 's/^\///' | sed -E 's/\//-/g' } # Verzeichnise unter $1 in .tgz.gpg-Dateien sichern. # # $1 - Verzeichnis, das gesichert wird: alle Verzeichnisse darunter werden einzeln als .tgz.gpg-Datei gesichert # $2 - Zielverzeichnis für .tgz.gpg-Dateien # $3 ff. - Verzeichnisse, die nicht gesichert werden sollen (kann leer bleiben) # # returns 0 for success, 1 otherwise (Fehlerprotokollierung muss aufrufende fn erledigen) # backup_all_dirs() { log $DEBUG "$DBG backup_all_dirs( $@ )" local root=$1 local target=$2 shift 2 local prefix=$(backup_file_prefix $root) local tar_file local dir for dir in $(iterate_dirs_in "$root" "$@"); do tar_file="$target/$prefix-$dir.$EXT" backup_dir "$root" "$tar_file" "$dir" if [ $? -ne 0 ]; then return 1; fi create_list "$tar_file" "$dir" if [ $? -ne 0 ]; then return 1; fi test_integrity "$tar_file" if [ $? -ne 0 ]; then return 1; fi done return 0 } # Einzelnes Verzeichnis sichern. # # $1 - Pfad um Verzeichnis, das gesichert werden soll # $2 - Zielpfad und -name für das Archiv # $2 - zu sicherndes Verzeichnis # # returns 0 for success, 1 otherwise # backup_dir() { log $DEBUG "$DBG backup_dir( $@ )" local root=$1 local archive=$2 local dir=$3 log $DEBUG "$DBG > tar -c -z --numeric-owner --one-file-system --exclude \"@eaDir\" -C \"$root\" \"$dir\" | gpg --batch -c --passphrase xxxx -o \"$archive\"" tar -c -z --numeric-owner --one-file-system --exclude "@eaDir" -C "$root" "$dir" 2>>$LOG_FILE | gpg --batch -c --passphrase $PASSPHRASE -o "$archive" 2>>$LOG_FILE check_pipe_rc ${PIPESTATUS[@]} return $? } ## # auto backup handling # # hostname-scripte setzen vars und definieren Funktionen backup_daily1, 2, usw. und backup_weekly1, 2, usw. # in den Funktionen werden die vars für den einzelnen backup gesetzt (target, excludes, usw.) # hier wird dann dass hostname-script gesourced (s.o.) und in einer Schleife werden die # entsprechend en backup_*-Funktionen aufgerufen # # returns 0 for success, 1 otherwise # do_auto_backup() { log $DEBUG "$DBG do_auto_backup( $@ )" local func=$1 local count=1 local rc if ! $func$count 2>/dev/null ; then # Seiteneffekt: $func$count wird ausgeführt mail_log $ERROR "$ERR keine Backup-Function $func$count() für $HOST_NAME." exit 1 # kein Mail-Benachrichtigung! fi set_log_info mail_log $INFO "$INF prepare backup" before_backup is_error $? && return 1 delete_dir "$BACKUP_TARGET" mail_log $INFO "$INF check backup directory" check_directory "$BACKUP_TARGET" is_error $? && return 1 until ! $func$count 2>/dev/null # Seiteneffekt: $func$count wird ausgeführt (für nächste Schleife) do mail_log $INFO "$INF backing up directories in $BACKUP_SOURCE" backup_all_dirs "$BACKUP_SOURCE" "$BACKUP_TARGET" "${BACKUP_EXCLUDE_DIRS[@]}" is_error $? && return 1 let "count += 1" done } set_log_info() { local targ if [ -z "$NAS_TARGET" ]; then targ=$BACKUP_TARGET else targ=$NAS_TARGET fi mail_log $INFO ">>> " mail_log $INFO ">>> backup to : $targ" mail_log $INFO ">>> log file : $LOG_FILE" mail_log $INFO ">>> " } backup_daily() { # "master"-Server startet Backup auf NAS if [ -d $BACKUP_MASTER_CHECK ]; then log $DEBUG "$DBG starting daily backup on $NAS_HOSTNAME" run_script_remote $NAS_IP "$NAS_SCRIPT -d" fi mail_log $INFO ">>>>>>>>>> daily auto backup $VER for $HOST_NAME <<<<<<<<<<" do_auto_backup backup_daily # after_backup() muss immer laufen, damit ggf. docker wieder gestartet werden kann mail_log $INFO "$INF finishing backup" after_backup is_error $? mail_log $INFO ">>>>>>>>>> end of auto backup" send_mail_log "Daily Backup" # ggf. weekly backup starten (z.ZT. nur auf NAS) if [ $HAS_WEEKLY -a $WEEKLY_DAY -eq $DAY ]; then log $DEBUG "$DBG starting weekly backup" nohup $THIS_SCRIPT -w &>>$LOG_FILE >>>>>>>>> weekly auto backup $VER for $HOST_NAME <<<<<<<<<<" do_auto_backup backup_weekly mail_log $INFO "$INF finishing backup" after_backup is_error $? mail_log $INFO ">>>>>>>>>> end of auto backup" send_mail_log "Weekly Backup" } # Inhalt des Archivs in eine Textdatei ausgeben. # # $1 - Archiv # # returns 0 for success, 1 otherwise # create_list() { log $DEBUG "$DBG create_list( $@ )" local archive=$1 log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -tzv | tee \"$archive.$LIST_EXT\"" gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -tzv 2>>$LOG_FILE | tee "$archive.$LIST_EXT" >/dev/null 2>>$LOG_FILE check_pipe_rc ${PIPESTATUS[@]} return $? } # Integrität des Archivs testen, indem es entschlüsselt, entpackt # und der Inhalt auf /dev/null ausgegeben wird. # # $1 - Archiv # # returns 0 for success, 1 otherwise # test_integrity() { log $DEBUG "$DBG test_integrity( $@ )" local archive=$1 log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -xzO" gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -xzO >/dev/null 2>>$LOG_FILE check_pipe_rc ${PIPESTATUS[@]} return $? } # Restore aus Archiv # # Wird nach pwd ausgegeben! # # $1 - Archiv, ggf. mit Pfad # $2 ff. - zurückzusichernde Dateien/Verzeichnisse # # returns 0 for success, 1 otherwise # restore() { log $DEBUG "$DBG restore( $@ )" local archive=$1 shift 1 log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -xzv $@" gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -xzv $@ >/dev/null 2>>$LOG_FILE check_pipe_rc ${PIPESTATUS[@]} return $? } # Liste der Archives auf stdout ausgeben # # Zeigt alle Archive incl. Verzeichnis (ohne $1) an # # z.B. # $1 = /volume1/nfs/backups # ... # /daily/synology/backup.3/volume1-nfs-Server.tgz.gpg # /weekly/synology/volume1-media-books.tgz.gpg # # $1 - Verzeichnis unter dem alle Backups liegen # # returns 0 for success, 1 otherwise # list_archives() { log $DEBUG "$DBG list_archives( $@ )" local target=$1 local targ=${target//\//\\\/} # slashes escapen log $DEBUG "$DBG > find \"$target\" -type f -name \"*.$EXT\" | grep \"${HOST_NAME}/\" | sed -E \"s/^$targ//\"" find "$target" -type f -name "*.$EXT" 2>>$LOG_FILE | grep "${HOST_NAME}/" 2>>$LOG_FILE | sed -E "s/^$targ//" 2>>$LOG_FILE check_pipe_rc ${PIPESTATUS[@]} return $? } # Inhalt des Archivs $2 auf stdout ausgeben # # $1 - Verzeichnis unter dem alle Backups liegen # $2 - Archiv-Datei mit Pfad ab $1 # # returns 0 for success, 1 otherwise # list_archive_content() { log $DEBUG "$DBG list_archive_content( $@ )" local root=$1 local archive=$2 log $DEBUG "$DBG > cat \"$root/${archive/#\//}.$LIST_EXT\"" cat "$root/${archive/#\//}.$LIST_EXT" 2>>$LOG_FILE } # --- export --- # exfat-formatierte USB-HDD auf Linux-Server ermitteln # get_usb_device() { local fstype=exfat local lsblk="lsblk -o KNAME,TYPE,SIZE,MODEL,FSTYPE" local device if [ "$($lsblk 2>>$LOG_FILE | grep $fstype 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then echo "ERROR could not query devices" return 1 fi device=$($lsblk 2>>$LOG_FILE | grep $fstype 2>>$LOG_FILE | sed -E 's/^(\S*).*$/\1/' 2>>$LOG_FILE) if [ -z "$device" ]; then echo "ERROR no exfat USB device found" return 1 fi echo $device return 0 } # USB-HDD auf Linux-Server mounten # # $1 - device (ohne /dev/) # mount_usb_device() { local device=$1 local mount_point=/var/mnt/backup check_directory $mount_point if [ $? -ne 0 ]; then echo "ERROR mount point does't exist" return 1 fi mount /dev/$device $mount_point >>$LOG_FILE 2>&1 if [ $? -ne 0 ]; then echo "ERROR mounting $device" return 1 fi echo $mount_point return 0 } # USB-HDD auf Linux-Server unmounten # # $1 - mount point # umount_usb_device() { local mount_point=$1 umount $mount_point >>$LOG_FILE 2>&1 if [ $? -ne 0 ]; then echo "ERROR unmounting $mount_point" return 1 fi return 0 } # USB-HDD suchen und Backup-Typ ermitteln # # Der Typ wird über eine Datei auf der USB-HDD festgelegt: # /.backup#.tag -> z.B. /.backup#int1.tag => Typ=int1 # # $1 - mount point # check_backup_device() { log $DEBUG "$DBG check_backup_drive( $@ )" local mount_point=$1 local tagP1=".backup#" local tagP2=".tag" log $DEBUG "$DBG > ls $mount_point/${tagP1}*${tagP2} | wc -l" if [ "$(ls $mount_point/${tagP1}*${tagP2} 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then mail_log $ERROR "$ERR no tag found on device at $mount_point" return 1 fi log $DEBUG "$DBG > ls $mount_point/${tagP1}*${tagP2} | sed -E \"s/.*$tagP1(.*)$tagP2/\1/\"" local tag=$(ls $mount_point/${tagP1}*${tagP2} 2>>$LOG_FILE | sed -E "s/.*$tagP1(.*)$tagP2/\1/" 2>>$LOG_FILE) if [ $? -ne 0 ]; then mail_log $ERROR "$ERR no tag found on device at $mount_point" return 1 fi echo $tag return 0 } # Datei-Export auf Synology-NAS ausführen # # TODO mount, check, unmount muss raus -> fn macht synology + linux => if host == synology... # fn's ausser unmount liefern string|null => if -z error # do_export() { log $DEBUG "$DBG do_export ()" mail_log $INFO "$INF checking USB device" # get device log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -enum | grep -v devices | wc -l" if [ "$(/usr/syno/bin/synousbdisk -enum 2>>$LOG_FILE | grep -v devices 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then mail_log $ERROR "$ERR no USB device or too many found @$HOST_NAME." return 1 fi log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -enum | grep -v devices" local device=$(/usr/syno/bin/synousbdisk -enum 2>>$LOG_FILE | grep -v devices 2>>$LOG_FILE) # check if exfat log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -info $device | grep -i \"file system\" | sed -E \"s/.*:\s*(.*)\s*/\1/\"" local fs=$(/usr/syno/bin/synousbdisk -info $device 2>>$LOG_FILE | grep -i "file system" 2>>$LOG_FILE | sed -E "s/.*:\s*(.*)\s*/\1/" 2>>$LOG_FILE) if [ "$fs" != "exfat" ]; then mail_log $ERROR "$ERR device $device not exfat formatted" return 1 fi # get mount point log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -info $device | grep -i \"mount path\" | sed -E \"s/.*:\s*(.*)\s*/\1/\"" local mount_path=$(/usr/syno/bin/synousbdisk -info $device 2>>$LOG_FILE | grep -i "mount path" 2>>$LOG_FILE | sed -E "s/.*:\s*(.*)\s*/\1/" 2>>$LOG_FILE) if [ $? -ne 0 ] || [ ! -d $mount_path ]; then mail_log $ERROR "$ERR mount point \"$mount_path\" does not exist." return 1 fi local backup_type=$(check_backup_device $mount_path) if [ -z $backup_type ]; then return 1 fi mail_log $INFO "$SUCCESS" local source_path case $backup_type in int1|int2|1|2) # daily + weekly backups -> Auslagerung 1+2, Backup 1+2 source_path="/volume1/nfs/backups" ;; serien) # Serien -> Auslagerung source_path="/volume1/media/serien" ;; video) # Video -> Auslagerung source_path="/volume1/media/video" ;; ?*) echo "ERROR unknown backup type $backup_type" return 1 ;; esac mail_log $INFO ">>> " mail_log $INFO ">>> export type : $backup_type" mail_log $INFO ">>> export from : $source_path" mail_log $INFO ">>> log file : $LOG_FILE" mail_log $INFO ">>> " mail_log $INFO "$INF exporting $source_path -> $mount_path" log $DEBUG "$DBG rsync -rti --one-file-system --delete --stats --exclude=@eaDir $source_path $mount_path" rsync -rti --one-file-system --delete --stats --exclude=@eaDir $source_path $mount_path >>$LOG_FILE 2>&1 is_error $? mail_log $INFO "$INF unmounting USB device" log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -umount $device" /usr/syno/bin/synousbdisk -umount $device >>$LOG_FILE 2>&1 is_error $? } # Export Backups bzw. Medien-Verzeichnisse auf USB-HDD # # Der Export wird nur auf der NAS ausgeführt # (und wird ggf. dort remote gestartet) # export_backups() { if [ "$HOST_NAME" != "$NAS_HOSTNAME" ]; then run_script_remote $NAS_IP "$NAS_SCRIPT -e" exit fi mail_log $INFO ">>>>>>>>>> export $VER <<<<<<<<<<" do_export mail_log $INFO ">>>>>>>>>> end of export" send_mail_log "Export" }