diff --git a/main.inc.sh b/main.inc.sh index 1523208..2c59333 100755 --- a/main.inc.sh +++ b/main.inc.sh @@ -14,6 +14,7 @@ EXT=tgz.gpg LIST_EXT=txt DOCKER="/usr/bin/docker compose -f /var/docker/run/docker-compose.yml --ansi never" +SYNO_USB=/usr/syno/bin/synousbdisk LOG_RECIPIENT=hb242@pm.me INFO=INFO @@ -26,70 +27,7 @@ WRN=":::" ERR="###" DBG="===" -dump_mariadb() { - log $DEBUG "$DBG dump_mariadb( $@ )" - local dump_dir=$1 - local mysqlconnect="--user=root --password=snafu" - - log $DEBUG "$DBG > \$DOCKER exec mysql /usr/bin/mariadb --silent --user=xxx --password=xxx -e \"show databases\" -B" - for db in $($DOCKER exec mysql /usr/bin/mariadb --silent $mysqlconnect -e "show databases" -B 2>>$LOG_FILE | grep -v information_schema 2>>$LOG_FILE | grep -v performance_schema 2>>$LOG_FILE | sed -e 's/\r//' 2>>$LOG_FILE); do - log $DEBUG "$DBG > \$DOCKER exec mysql /usr/bin/mariadb-dump --hex-blob --user=xxx --password=xxx --databases $db" - $DOCKER exec mysql /usr/bin/mariadb-dump --hex-blob $mysqlconnect --databases $db >$dump_dir/$db.sql 2>>$LOG_FILE - is_error $? false && return 1 - done - return 0 -} - -dump_postgresql() { - log $DEBUG "$DBG dump_postgresql( $@ )" - local dump_dir=$1 - - # users and roles - log $DEBUG "$DBG > \$DOCKER exec -u=postgres postgres pg_dumpall --globals-only" - $DOCKER exec -u=postgres postgres pg_dumpall --globals-only > $dump_dir/_globals.sql 2>>$LOG_FILE - - log $DEBUG "$DBG \$DOCKER exec -u=postgres postgres psql -t -c \"select datname from pg_database where not datistemplate\"" - for db in $($DOCKER exec -u=postgres postgres psql -t -c "select datname from pg_database where not datistemplate" 2>>$LOG_FILE); do - log $DEBUG "$DBG > \$DOCKER exec -u=postgres postgres pg_dump --create $db" - $DOCKER exec -u=postgres postgres pg_dump --create $db > $dump_dir/$db.sql 2>>$LOG_FILE - is_error $? false && return 1 - done - return 0 -} - -start_docker() { - log $DEBUG "$DBG start_docker()" - log $DEBUG "$DBG > \$DOCKER up -d" - $DOCKER up -d >>$LOG_FILE 2>&1 - return $? -} - -stop_docker() { - log $DEBUG "$DBG stop_docker()" - log $DEBUG "$DBG > \$DOCKER down" - $DOCKER down >>$LOG_FILE 2>&1 - return $? -} - -# Script per ssh auf Server $1 im Hintergrund starten -# -# ssh start in background -# -> https://stackoverflow.com/questions/29142/getting-ssh-to-execute-a-command-in-the-background-on-target-machine#comment107117479_29172 -# -# $1 - Server -# $@ - Script incl. Parameter -# -run_script_remote() { - local server=$1 - shift - ssh $server "sudo sh -c 'nohup $@ &>/dev/null $HERE/mail.sh $LOG_RECIPIENT \"[$HOST_NAME] $subject\" $MAIL_LOG_FILE" - $HERE/mail.sh $LOG_RECIPIENT "[$HOST_NAME] Nightly Backup" $MAIL_LOG_FILE >>$LOG_FILE 2>&1 + $HERE/mail.sh $LOG_RECIPIENT "[$HOST_NAME] $subject" $MAIL_LOG_FILE >>$LOG_FILE 2>&1 } # Alte Logs löschen @@ -137,13 +75,82 @@ prune_logs() { log $DEBUG "$DBG prune_logs()" # Dateien älter als 14 Tage löschen log $DEBUG "$DBG > find \"$LOG_FILE_PATH\" -type f -mtime +14 -delete" - find "$LOG_FILE_PATH" -type f -mtime +14 -delete &>>$LOG_FILE + find "$LOG_FILE_PATH" -type f -mtime +14 -delete >>$LOG_FILE 2>&1 if [ $? -ne 0 ]; then # Fehler nur protokollieren, kein Grund abzubrechen mail_log $ERROR "$ERR error deleting old log files." fi } +# ----- db dump ----- + +dump_mariadb() { + log $DEBUG "$DBG dump_mariadb( $@ )" + local dump_dir=$1 + local mysqlconnect="--user=root --password=snafu" + + log $DEBUG "$DBG > \$DOCKER exec mysql /usr/bin/mariadb --silent --user=xxx --password=xxx -e \"show databases\" -B" + for db in $($DOCKER exec mysql /usr/bin/mariadb --silent $mysqlconnect -e "show databases" -B 2>>$LOG_FILE | grep -v information_schema 2>>$LOG_FILE | grep -v performance_schema 2>>$LOG_FILE | sed -e 's/\r//' 2>>$LOG_FILE); do + log $DEBUG "$DBG > \$DOCKER exec mysql /usr/bin/mariadb-dump --hex-blob --user=xxx --password=xxx --databases $db" + $DOCKER exec mysql /usr/bin/mariadb-dump --hex-blob $mysqlconnect --databases $db >$dump_dir/$db.sql 2>>$LOG_FILE + is_error $? false && return 1 + done + return 0 +} + +dump_postgresql() { + log $DEBUG "$DBG dump_postgresql( $@ )" + local dump_dir=$1 + + # users and roles + log $DEBUG "$DBG > \$DOCKER exec -u=postgres postgres pg_dumpall --globals-only" + $DOCKER exec -u=postgres postgres pg_dumpall --globals-only > $dump_dir/_globals.sql 2>>$LOG_FILE + + log $DEBUG "$DBG \$DOCKER exec -u=postgres postgres psql -t -c \"select datname from pg_database where not datistemplate\"" + for db in $($DOCKER exec -u=postgres postgres psql -t -c "select datname from pg_database where not datistemplate" 2>>$LOG_FILE); do + log $DEBUG "$DBG > \$DOCKER exec -u=postgres postgres pg_dump --create $db" + $DOCKER exec -u=postgres postgres pg_dump --create $db > $dump_dir/$db.sql 2>>$LOG_FILE + is_error $? false && return 1 + done + return 0 +} + +# ----- helper functions ----- + +start_docker() { + log $DEBUG "$DBG start_docker()" + log $DEBUG "$DBG > \$DOCKER up -d" + $DOCKER up -d >>$LOG_FILE 2>&1 + return $? +} + +stop_docker() { + log $DEBUG "$DBG stop_docker()" + log $DEBUG "$DBG > \$DOCKER down" + $DOCKER down >>$LOG_FILE 2>&1 + return $? +} + +# Script per ssh auf Server $1 im Hintergrund starten +# +# ssh start in background +# -> https://stackoverflow.com/questions/29142/getting-ssh-to-execute-a-command-in-the-background-on-target-machine#comment107117479_29172 +# +# $1 - Server +# $@ - Script incl. Parameter +# +run_script_remote() { + local server=$1 + shift + ssh $server "sudo sh -c 'nohup $@ &>/dev/null >$LOG_FILE >$LOG_FILE 2>&1 gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -tzv | tee \"$archive.$LIST_EXT\"" - gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -tzv 2>>$LOG_FILE | tee "$archive.$LIST_EXT" >/dev/null 2>>$LOG_FILE - check_pipe_rc ${PIPESTATUS[@]} - return $? -} +# ----- integrity ----- # Integrität des Archivs testen, indem es entschlüsselt, entpackt # und der Inhalt auf /dev/null ausgegeben wird. @@ -462,6 +458,8 @@ test_integrity() { return $? } +# ----- restore ----- + # Restore aus Archiv # # Wird nach pwd ausgegeben! @@ -481,6 +479,23 @@ restore() { return $? } +# ----- archive content ------ + +# Inhalt des Archivs in eine Textdatei ausgeben. +# +# $1 - Archiv +# +# returns 0 for success, 1 otherwise +# +create_list() { + log $DEBUG "$DBG create_list( $@ )" + local archive=$1 + log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -tzv | tee \"$archive.$LIST_EXT\"" + gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -tzv 2>>$LOG_FILE | tee "$archive.$LIST_EXT" >/dev/null 2>>$LOG_FILE + check_pipe_rc ${PIPESTATUS[@]} + return $? +} + # Liste der Archives auf stdout ausgeben # # Zeigt alle Archive incl. Verzeichnis (ohne $1) an @@ -520,22 +535,61 @@ list_archive_content() { cat "$root/${archive/#\//}.$LIST_EXT" 2>>$LOG_FILE } -# --- export --- +# ----- export ----- + +# Synology-NAS (0) or not (1 - linux)? +# +is_synology() { + if [ -f $SYNO_USB ]; then + return 0 + else + return 1 + fi +} + +# Find exfat formatted device +# +# returns device name +# +get_usb_device() { + if is_synology; then + echo $(get_usb_device_syno) + else + echo $(get_usb_device_linux) + fi +} + +# USB-HDD auf Synology-NAS ermitteln +# +get_usb_device_syno() { + log $DEBUG "$DBG get_usb_device_syno()" + log $DEBUG "$DBG > $SYNO_USB -enum | grep -v devices | wc -l" + if [ "$($SYNO_USB -enum 2>>$LOG_FILE | grep -v devices 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then + mail_log $ERROR "$ERR no USB device or too many found @$HOST_NAME." + return 1 + fi + log $DEBUG "$DBG > $SYNO_USB -enum | grep -v devices" + echo $($SYNO_USB -enum 2>>$LOG_FILE | grep -v devices 2>>$LOG_FILE) + return 0 +} # exfat-formatierte USB-HDD auf Linux-Server ermitteln # -get_usb_device() { +get_usb_device_linux() { + log $DEBUG "$DBG get_usb_device_linux()" local fstype=exfat local lsblk="lsblk -o KNAME,TYPE,SIZE,MODEL,FSTYPE" local device + log $DEBUG "$DBG $lsblk | grep $fstype | wc -l" if [ "$($lsblk 2>>$LOG_FILE | grep $fstype 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then - echo "ERROR could not query devices" + mail_log $ERROR "$ERR could not query devices." return 1 fi + log $DEBUG "$DBG $lsblk | grep $fstype | sed -E 's/^(\S*).*$/\1/'" device=$($lsblk 2>>$LOG_FILE | grep $fstype 2>>$LOG_FILE | sed -E 's/^(\S*).*$/\1/' 2>>$LOG_FILE) if [ -z "$device" ]; then - echo "ERROR no exfat USB device found" + mail_log $ERROR "$ERR no exfat USB device found." return 1 fi @@ -543,42 +597,104 @@ get_usb_device() { return 0 } -# USB-HDD auf Linux-Server mounten +# Mount UsB-HDD # -# $1 - device (ohne /dev/) +# returns mount point # mount_usb_device() { + if is_synology; then + echo $(mount_usb_device_syno $1) + else + echo $(mount_usb_device_linux $1) + fi +} + +# Check if auto mounted device is exfat and return mount point +# (fn for consitence with linux fn) +# +# $1 - device name +# returns mount point +# +mount_usb_device_syno() { + log $DEBUG "$DBG mount_usb_device_syno( $@ )" local device=$1 - local mount_point=/var/mnt/backup - check_directory $mount_point - if [ $? -ne 0 ]; then - echo "ERROR mount point does't exist" + log $DEBUG "$DBG > $SYNO_USB -info $device | grep -i \"file system\" | sed -E \"s/.*:\s*(.*)\s*/\1/\"" + local fs=$($SYNO_USB -info $device 2>>$LOG_FILE | grep -i "file system" 2>>$LOG_FILE | sed -E "s/.*:\s*(.*)\s*/\1/" 2>>$LOG_FILE) + if [ "$fs" != "exfat" ]; then + mail_log $ERROR "$ERR device $device not exfat formatted." return 1 fi - mount /dev/$device $mount_point >>$LOG_FILE 2>&1 - if [ $? -ne 0 ]; then - echo "ERROR mounting $device" + # get mount point + log $DEBUG "$DBG > $SYNO_USB -info $device | grep -i \"mount path\" | sed -E \"s/.*:\s*(.*)\s*/\1/\"" + local mount_point=$($SYNO_USB -info $device 2>>$LOG_FILE | grep -i "mount path" 2>>$LOG_FILE | sed -E "s/.*:\s*(.*)\s*/\1/" 2>>$LOG_FILE) + if [ $? -ne 0 ] || [ ! -d $mount_point ]; then + mail_log $ERROR "$ERR mount point \"$mount_point\" does not exist." return 1 fi echo $mount_point return 0 } -# USB-HDD auf Linux-Server unmounten +# USB-HDD auf Linux-Server mounten # -# $1 - mount point +# $1 - device (ohne /dev/) +# return moint point # -umount_usb_device() { - local mount_point=$1 - - umount $mount_point >>$LOG_FILE 2>&1 +mount_usb_device_linux() { + log $DEBUG "$DBG mount_usb_device_linux( $@ )" + local device=$1 + local mount_point=/var/mnt/backup + check_directory $mount_point if [ $? -ne 0 ]; then - echo "ERROR unmounting $mount_point" + mail_log $ERROR "$ERR mount point \"$mount_point\" does not exist." return 1 fi + log $DEBUG "$DBG mount /dev/$device $mount_point" + mount /dev/$device $mount_point >>$LOG_FILE 2>&1 + if [ $? -ne 0 ]; then + mail_log $ERROR "$ERR mounting /dev/$device failed." + return 1 + fi + echo $mount_point return 0 } +# unmount USB-HDD +# +umount_usb_device() { + if is_synology; then + umount_usb_device_syno $1 + return $? + else + umount_usb_device_linux $1 + return $? + fi +} + +# USB-HDD auf Synology-NAS unmounten +# +# $1 - device name +# +umount_usb_device_syno() { + local device=$1 + log $DEBUG "$DBG umount_usb_device_syno( $@ )" + log $DEBUG "$DBG > $SYNO_USB -umount $device" + $SYNO_USB -umount $device >>$LOG_FILE 2>&1 + return $? +} + +# USB-HDD auf Linux-Server unmounten +# +# $1 - device (ohne /dev/) +# +umount_usb_device_linux() { + local device=/dev/$1 + log $DEBUG "$DBG umount_usb_device_linux( $@ )" + log $DEBUG "$DBG > umount $device" + umount $device >>$LOG_FILE 2>&1 + return $? +} + # USB-HDD suchen und Backup-Typ ermitteln # # Der Typ wird über eine Datei auf der USB-HDD festgelegt: @@ -594,14 +710,14 @@ check_backup_device() { log $DEBUG "$DBG > ls $mount_point/${tagP1}*${tagP2} | wc -l" if [ "$(ls $mount_point/${tagP1}*${tagP2} 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then - mail_log $ERROR "$ERR no tag found on device at $mount_point" + mail_log $ERROR "$ERR no tag found on device at $mount_point." return 1 fi log $DEBUG "$DBG > ls $mount_point/${tagP1}*${tagP2} | sed -E \"s/.*$tagP1(.*)$tagP2/\1/\"" local tag=$(ls $mount_point/${tagP1}*${tagP2} 2>>$LOG_FILE | sed -E "s/.*$tagP1(.*)$tagP2/\1/" 2>>$LOG_FILE) if [ $? -ne 0 ]; then - mail_log $ERROR "$ERR no tag found on device at $mount_point" + mail_log $ERROR "$ERR no tag found on device at $mount_point." return 1 fi echo $tag @@ -615,34 +731,22 @@ check_backup_device() { # do_export() { log $DEBUG "$DBG do_export ()" - mail_log $INFO "$INF checking USB device" + mail_log $INFO "$INF checking USB device." # get device - log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -enum | grep -v devices | wc -l" - if [ "$(/usr/syno/bin/synousbdisk -enum 2>>$LOG_FILE | grep -v devices 2>>$LOG_FILE | wc -l 2>>$LOG_FILE)" != "1" ]; then - mail_log $ERROR "$ERR no USB device or too many found @$HOST_NAME." - return 1 - fi - log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -enum | grep -v devices" - local device=$(/usr/syno/bin/synousbdisk -enum 2>>$LOG_FILE | grep -v devices 2>>$LOG_FILE) - - # check if exfat - log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -info $device | grep -i \"file system\" | sed -E \"s/.*:\s*(.*)\s*/\1/\"" - local fs=$(/usr/syno/bin/synousbdisk -info $device 2>>$LOG_FILE | grep -i "file system" 2>>$LOG_FILE | sed -E "s/.*:\s*(.*)\s*/\1/" 2>>$LOG_FILE) - if [ "$fs" != "exfat" ]; then - mail_log $ERROR "$ERR device $device not exfat formatted" + local device=$(get_usb_device) + if [ -z $device ]; then return 1 fi - # get mount point - log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -info $device | grep -i \"mount path\" | sed -E \"s/.*:\s*(.*)\s*/\1/\"" - local mount_path=$(/usr/syno/bin/synousbdisk -info $device 2>>$LOG_FILE | grep -i "mount path" 2>>$LOG_FILE | sed -E "s/.*:\s*(.*)\s*/\1/" 2>>$LOG_FILE) - if [ $? -ne 0 ] || [ ! -d $mount_path ]; then - mail_log $ERROR "$ERR mount point \"$mount_path\" does not exist." + # check if exfat | mount & get mount point + local mount_point=$(mount_usb_device $device) + if [ -z $mount_point ]; then return 1 fi - - local backup_type=$(check_backup_device $mount_path) + + # find export type + local backup_type=$(check_backup_device $mount_point) if [ -z $backup_type ]; then return 1 fi @@ -652,15 +756,27 @@ do_export() { case $backup_type in int1|int2|1|2) # daily + weekly backups -> Auslagerung 1+2, Backup 1+2 - source_path="/volume1/nfs/backups" + if is_synology; then + source_path="/volume1/nfs/backups" + else + source_path="/var/srv/nas/backups" + fi ;; serien) # Serien -> Auslagerung - source_path="/volume1/media/serien" + if is_synology; then + source_path="/volume1/media/serien" + else + source_path="/var/srv/media/serien" + fi ;; video) # Video -> Auslagerung - source_path="/volume1/media/video" + if is_synology; then + source_path="/volume1/media/video" + else + source_path="/var/srv/media/video" + fi ;; ?*) echo "ERROR unknown backup type $backup_type" @@ -674,15 +790,14 @@ do_export() { mail_log $INFO ">>> log file : $LOG_FILE" mail_log $INFO ">>> " - mail_log $INFO "$INF exporting $source_path -> $mount_path" - log $DEBUG "$DBG rsync -rti --one-file-system --delete --stats --exclude=@eaDir $source_path $mount_path" - rsync -rti --one-file-system --delete --stats --exclude=@eaDir $source_path $mount_path >>$LOG_FILE 2>&1 + mail_log $INFO "$INF exporting $source_path -> $mount_point" + log $DEBUG "$DBG rsync -rti --one-file-system --delete --stats --exclude=@eaDir $source_path $mount_point" + rsync -rti --one-file-system --delete --stats --exclude=@eaDir $source_path $mount_point >>$LOG_FILE 2>&1 is_error $? - mail_log $INFO "$INF unmounting USB device" - log $DEBUG "$DBG > /usr/syno/bin/synousbdisk -umount $device" - /usr/syno/bin/synousbdisk -umount $device >>$LOG_FILE 2>&1 - is_error $? + mail_log $INFO "$INF unmounting USB device $device." + umount_usb_device $device + is_error $? } diff --git a/main.sh b/main.sh index 89be191..4464958 100755 --- a/main.sh +++ b/main.sh @@ -3,7 +3,7 @@ # entrypoint for all actions # -VER="2.0.1" +VER="2.0.4" show_help() { echo "usage: $0 options"