logging

TODO.md

@@ -1,6 +1,12 @@
 
 ### TODOs
 
+DC-Dump
+- new for postgres
+  > server exec postgres su - postgres -c pg_dumpall >pg.dump
+- +mariadb
+- hooks für weitere
+
 nextcloud:
 - (x) vol/nextcloud/data/data/hb zeigt auf NAS-share 
 -> alles auf Server
@@ -26,12 +32,17 @@ nas-backup weekly 1
 	(x) TODO: s. nas-backup daily
 
 logging
-- TODO: wo speichern?
-- TODO: ggf. lokal und auf NAS kopieren
-- TODO: full für alles (taugt nur für auto backup, also ggf. noch ein log)
-- TODO: Übersichtliches für die Mail -> Error-Handling!
-- TODO: wo und wie sammeln für Mail
-- TODO: Mail aus py-script übernehmen
+- für auto-fns mail() 
+- mail() loggt in extra Datei und ruft log(), für std log
+- Name std log in mail() ausgeben
+- mail log lokal anlegen (wo?)
+- am Ende von auto mail log in mail und file löschen -> py-script
+- alle Programme müssen stdout/stderr auf std log umleiten
+- für jeden Lauf neue log datei (+date)
+- Name der log Datei bei !auto auf stderr ausgeben (damit stdout ggf. geparst werden kann, alt.: param --machine)
+- log file in nfs/backups/log/$HOST_NAME-$(date '+%Y-%m-%d_%H%M').log
+- bei jedem Lauf alte logs löschen (7d?) -> wie?
+- evtl. logs lokal ablegen und am Ende auf NAS kopieren
 
 backup server+nas extern
 - rsync nas-backup 2x HDD intern (weekly) (5TB?)

hamiller.sh

@@ -10,19 +10,18 @@ backup_daily1() {
     BACKUP_EXCLUDE_DIRS=()
 }
 
-
-copy_to_nas() {
-    # ins NAS-Verzeichnis verschieben
-    delete_dir "$NAS_TARGET"
-    check_dir "$NAS_TARGET"
-    sudo mv "$BACKUP_TARGET/*" "$NAS_TARGET"
-}
-
 before_backup() {
-    run_docker down
+    stop_docker
+    return $?
 }
 
 after_backup() {
-    run_docker up -d
-    copy_to_nas
+    start_docker
+    local dock=$?
+    copy_to_nas "$BACKUP_TARGET" "$NAS_TARGET"
+    local copy=$?
+    if [ $dock -eq 0 ] && [ $copy -eq 0 ]; then
+        return 0
+    fi
+    return 1
 }

mail.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Mail via GMX-Relay versenden (Docker-Image "hb42/smtp")
+#
+# Klappt nur direkt im Container, daher ggf. via ssh aufrufen (Annahme: remote user == local user)
+#
+# $1 = recipient
+# $2 = Mail-Betreff (ggf. in "")
+# $3 || STDIN = Mail-Body
+#
+# Verwendung:
+# > echo "sinnloser Text - äöüÄÖÜß -"| mail.sh hb242@pm.me "noch'n test"
+# > mail.sh hb242@pm.me "log whatever" ../log/some.log
+#
+
+DOCKER=/usr/bin/docker
+MAILCONTAINER=smtp
+MAILSERVER_IP=10.4.2.30
+SSH=""
+
+which ip >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+    # Linux
+    myip=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')
+else
+    # macOS
+    myip=$(ipconfig getifaddr en0) # FIXME: nicht sauber wg. explizitem IF!
+fi
+
+if [ "$myip" != "$MAILSERVER_IP" ]; then
+    SSH="ssh $MAILSERVER_IP "
+fi
+
+TO=$1
+SUBJECT=$2
+shift 2
+if [ -z "$TO" ] || [ -z "$SUBJECT" ]; then
+    echo "ERROR: missing parameter "
+    echo
+    echo "Usage: $0 recipient subject [file || STDIN]"
+    echo
+    exit
+fi
+
+# Mail body aus Datei oder STDIN holen
+CONTENT=$(cat ${1:--})
+
+# construct mail header + content and pipe to sendmail
+cat <<EOMB | $SSH $DOCKER exec -i $MAILCONTAINER /usr/sbin/sendmail -i -t
+FROM: hb42@gmx.de
+TO: ${TO}
+DATE: $(date -R)
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+SUBJECT: $SUBJECT
+
+$CONTENT
+EOMB

main.inc.sh

@@ -1,3 +1,5 @@
+VER="2.0.0-beta-4"
+
 HOST_NAME=$(hostname)
 DAY=$(date +%u)
 
@@ -5,12 +7,37 @@ PASSPHRASE=1234567890
 EXT=tgz.gpg
 LIST_EXT=txt
 
+INFO=INFO
+ERROR=ERROR
+DEBUG=DEBUG
+SUCCESS="*** success"
+INF="---"
+ERR="###"
+DBG="==="
+
 # docker compose ausführen
 #
 # $1 ff. - compose parameter, z.B. up -d
 #
 run_docker() {
-    /usr/bin/docker compose -f /var/docker/run/docker-compose.yml --ansi never $@
+    log $DEBUG "$DBG run_docker $@"
+    
+    #DEBUG ohne docker stop/start
+    return 0
+    
+    log $DEBUG "$DBG > /usr/bin/docker compose -f /var/docker/run/docker-compose.yml --ansi never $@"
+    /usr/bin/docker compose -f /var/docker/run/docker-compose.yml --ansi never $@ >>$LOG_FILE 2>&1
+    return $?
+}
+
+start_docker() {
+    log $DEBUG "$DBG start_docker"
+    return $(run_docker up -d)
+}
+
+stop_docker() {
+    log $DEBUG "$DBG stop_docker"
+    return $(run_docker down)
 }
 
 # check if function $1 exists
@@ -30,12 +57,82 @@ isFunction() { declare -Ff "$1" ; }
 #
 log() {
     local type=$1
-    if [ "$type" == "ERROR" ]; then ERROR=0; fi
+    #if [ "$type" == "ERROR" ]; then ERROR=0; fi
     shift 1
-    echo "$(date '+%d.%m.%Y %H:%M:%S') - $type $@" | sudo tee -a $LOG_FILE
+    echo "$(date '+%d.%m.%Y %H:%M:%S') - $type $@" | tee -a $LOG_FILE >/dev/null
 
 } 
 
+mail_log() {
+    local type=$1
+    shift
+    if [ "$MAIL_LOG_FILE" ]; then
+        echo "$(date '+%d.%m.%Y %H:%M:%S') - $type $@" >> $MAIL_LOG_FILE
+    fi
+    log "$type" "$@"
+}
+
+# check if $1 is error (!=0) and write result to log
+#
+# $1 - return code to check
+# $2 - if set don't log success
+#
+# returns 0 if error, 1 otherwise!
+#
+is_error() {
+    local rc=$1
+    local mlog=$2
+    if [ $rc -eq 0 ]; then
+        if [ "$mlog" ]; then 
+            mail_log $INFO "$SUCCESS"
+        fi
+        return 1
+    else
+        mail_log $ERROR "$ERR an error occured see $LOG_FILE"
+        return 0
+    fi
+}
+
+# check return code of piped programs
+#
+# $1, ff. - return codes
+#
+check_pipe_rc() {
+    local result=0
+    local count=1
+    local result_str
+    for rc in $@; do
+        if [ $rc -ne 0 ] ; then
+            result=1
+            result_str="${result_str}#${count} "
+        fi
+        let "count += 1"
+    done
+    if [ $result -ne 0 ]; then
+        log $ERROR "$ERR in pipe $result_str"
+    fi
+    return $result    
+}
+
+# Dateien vom Server zur NAS verschieben
+#
+# $1 - Verzeichnis auf Server
+# $2 - Ziel-Verzeichnis auf der NAS
+#
+copy_to_nas() {
+    log $DEBUG "$DBG copy_to_nas $@"
+    local source="$1"
+    local target="$2"
+    delete_dir "$target"
+    if [ $(check_directory "$target") -gt 0 ]; then
+        log $ERROR "$ERR directory \"$target\" not found"
+        return 1
+    fi
+    log $DEBUG "$DBG > mv \"$source\"/* \"$target\""
+    mv "$source"/* "$target" >>$LOG_FILE 2>&1
+    return $?
+}
+
 # Ist $1 in der Liste der restlichen args $2, $3 ...
 #
 # zB. if is_in "foo" "foo" "bar" "foo bar" ; then ...
@@ -58,7 +155,7 @@ iterate_dirs_in() {
     local dirname
     local root=$1
     shift 1
-    for dir in $(sudo find "$root" -maxdepth 1 -type d \! -wholename "$root"); do
+    for dir in $(find "$root" -maxdepth 1 -type d \! -wholename "$root"); do
         dirname=$(basename $dir)
         if ! is_in "$dirname" "$@"; then 
             echo "$dirname" 
@@ -71,15 +168,15 @@ iterate_dirs_in() {
 # $1 - dir to check
 #
 check_directory() {
+    log $DEBUG "$DBG check_directory $@"
     local dir=$1
     if [ -d "$1" ]; then 
         return 0
     else
-        mkdir -p "$1" && return 0
-        sudo mkdir -p "$1" && return 0
-        log ERROR "### could not create directory $1!"
-        # no chance to recover
-        exit 1
+        log $DEBUG "$DBG > mkdir -p \"$1\""
+        mkdir -p "$1" >>$LOG_FILE 2>&1 && return 0
+        log $ERROR "$ERR could not create directory $1!"
+        return 1
     fi
 }
 
@@ -90,9 +187,10 @@ check_directory() {
 # $1 - zu löschendes Verzeichnis
 #
 delete_dir() {
+    log $DEBUG "$DBG delete_dir $@"
     local dir=$1
-    sudo rm -rf $dir/* && return 0
-    log ERROR "### could not delete directory $1!"
+    log $DEBUG "$DBG > rm -rf \"$dir\"/*"
+    rm -rf "$dir"/* >>$LOG_FILE 2>&1 && return 0
     # return error, so caller can decide what to do
     return 1
 }
@@ -104,13 +202,14 @@ backup_file_prefix() {
     echo "$1" | sed -E 's/^\///' | sed -E 's/\//-/g'
 }
 
-# Verzeichnise in .tgz.gpg-Dateien sichern.
+# Verzeichnise unter $1 in .tgz.gpg-Dateien sichern.
 #
 # $1 - Verzeichnis, das gesichert wird: alle Verzeichnisse darunter werden einzeln als .tgz.gpg-Datei gesichert
 # $2 - Zielverzeichnis für .tgz.gpg-Dateien
 # $3 ff. - Verzeichnisse, die nicht gesichert werden sollen (kann leer bleiben)
 #
 backup_all_dirs() {
+    log $DEBUG "$DBG backup_all_dirs $@"
     local root=$1
     local target=$2
     shift 2
@@ -120,10 +219,19 @@ backup_all_dirs() {
 
     for dir in $(iterate_dirs_in "$root" "$@"); do
         tar_file="$target/$prefix-$dir.$EXT"
+        mail_log $INFO "$INF directory: $root/$dir"
         backup_dir "$root" "$tar_file" "$dir"
+        is_error $? false && return 1
+
+        #mail_log $INFO "$INF list directory"
         create_list "$tar_file" "$dir"
+        is_error $? false && return 1
+
+        #mail_log $INFO "$INF test integrity of $tar_file"
         test_integrity "$tar_file"
+        is_error $? false && return 1
     done
+    return 0
 }
 
 # Einzelnes Verzeichnis sichern.
@@ -133,17 +241,14 @@ backup_all_dirs() {
 # $2 - zu sicherndes Verzeichnis
 #
 backup_dir() {
+    log $DEBUG "$DBG backup_dir $@"
     local root=$1
     local archive=$2
     local dir=$3
-    log INFO "--- Backup $root/$dir -> $archive"
-    sudo tar -c -z --numeric-owner --one-file-system --exclude "@eaDir" -C "$root" "$dir" | sudo gpg --batch -c --passphrase $PASSPHRASE -o "$archive"
-    local rc=$?
-    if [ "$rc" == "0" ] ; then
-        log INFO "*** success"
-    else
-        log ERROR "### see full log"
-    fi
+    log $DEBUG "$DBG > tar -c -z --numeric-owner --one-file-system --exclude \"@eaDir\" -C \"$root\" \"$dir\" | gpg --batch -c --passphrase xxxx -o \"$archive\""
+    tar -c -z --numeric-owner --one-file-system --exclude "@eaDir" -C "$root" "$dir" 2>>$LOG_FILE  | gpg --batch -c --passphrase $PASSPHRASE -o "$archive" 2>>$LOG_FILE 
+    check_pipe_rc ${PIPESTATUS[@]}
+    return $?
 }
 
 ##
@@ -155,32 +260,64 @@ backup_dir() {
 # entsprechend en backup_*-Funktionen aufgerufen
 #
 do_auto_backup() {
+    log $DEBUG "$DBG do_auto_backup $@"
     local func=$1
     local count=1
+    local rc
     
-    if ! $func§count 2>/dev/null ; then  # Seiteneffekt: $func$count wird ausgeführt
-        echo "ERROR: keine Backup-Functions für $HOST_NAME."
+    if ! $func$count 2>/dev/null ; then  # Seiteneffekt: $func$count wird ausgeführt
+        mail_log $ERROR "$ERR keine Backup-Function $func$count() für $HOST_NAME."
         exit 1
     fi
 
+    mail_log $INFO "$INF prepare backup"
     before_backup
+    is_error $? && return 1
+
     delete_dir "$BACKUP_TARGET" 
+
+    mail_log $INFO "$INF check backup directory"
     check_directory "$BACKUP_TARGET"
-    # ab hier std logging (Datei ist in $BACKUP_TARGET)
+    is_error $? && return 1
 
-    do 
-        backup_all_dirs "$BACKUP_SOURCE" "$BACKUP_TARGET" "${BACKUP_EXCLUDE_DIRS[@]}" 
-        let "count += 1"
     until ! $func$count 2>/dev/null # Seiteneffekt: $func$count wird ausgeführt (für nächste Schleife)
-    
+    do 
+        mail_log $INFO "$INF backing up directories in $BACKUP_SOURCE"
+        backup_all_dirs "$BACKUP_SOURCE" "$BACKUP_TARGET" "${BACKUP_EXCLUDE_DIRS[@]}" 
+        rc=$?
+        # error logging in backup_all_dirs
+        if [ $rc -gt 0 ]; then
+            return 1
+        fi
+
+        let "count += 1"
+    done
+    mail_log $INFO "$INF finishing backup"
     after_backup
+    is_error $? && return 1
 }
 
 backup_daily() {
+    mail_log $INFO ">>>>>>>>>> daily auto backup $VER for $HOST_NAME <<<<<<<<<<"
     do_auto_backup backup_daily
+    mail_log $INFO ">>>>>>>>>> end of auto backup"
+    # TODO mail log
 }
 backup_weekly() {
+    mail_log $INFO ">>>>>>>>>> weekly auto backup $VER for $HOST_NAME <<<<<<<<<<"
     do_auto_backuo backup_weekly
+    mail_log $INFO ">>>>>>>>>> end of auto backup"
+    # TODO mail log
+}
+
+export_backups() {
+    # TODO
+    echo
+    # check host (only nathan|hamiller?)
+    # check USB drive
+    # del USB drive (?)
+    # rsync USB -> 1|2, video, serien
+    # mail log
 }
 
 # Inhalt des Archivs in eine Textdatei ausgeben.
@@ -188,15 +325,12 @@ backup_weekly() {
 # $1 - Archiv
 #
 create_list() {
+    log $DEBUG "$DBG create_list $@"
     local archive=$1
-    log INFO "--- List $archive"
-    sudo gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" | sudo tar -tzv | sudo tee "$archive.$LIST_EXT" >/dev/null
-    local rc=$?
-    if [ "$rc" == "0" ] ; then
-        log INFO "*** success"
-    else
-        log ERROR "### see full log"
-    fi
+    log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -tzv | tee \"$archive.$LIST_EXT\""
+    gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -tzv 2>>$LOG_FILE | tee "$archive.$LIST_EXT" >/dev/null 2>>$LOG_FILE
+    check_pipe_rc ${PIPESTATUS[@]}
+    return $?
 }
 
 # Integrität des Archivs testen, indem es entschlüsselt, entpackt 
@@ -205,15 +339,12 @@ create_list() {
 # $1 - Archiv
 #
 test_integrity() {
+    log $DEBUG "$DBG test_integrity $@"
     local archive=$1
-    log INFO "--- Check integrity of $archive"
-    sudo gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" | sudo tar -xzO >/dev/null
-    local rc=$?
-    if [ "$rc" == "0" ] ; then
-        log INFO "*** success"
-    else
-        log ERROR "### see full log"
-    fi
+    log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -xzO"
+    gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -xzO >/dev/null 2>>$LOG_FILE
+    check_pipe_rc ${PIPESTATUS[@]}
+    return $?
 }
 
 # Restore aus Archiv
@@ -224,15 +355,13 @@ test_integrity() {
 # $2 ff. - zurückzusichernde Dateien/Verzeichnisse
 #
 restore() {
+    log $DEBUG "$DBG restore $@"
     local archive=$1
     shift 1
-    sudo gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" | sudo tar -xzv $@ >/dev/null
-    local rc=$?
-    if [ "$rc" == "0" ] ; then
-        log INFO "*** success"
-    else
-        log ERROR "### see full log"
-    fi
+    log $DEBUG "$DBG > gpg --batch --decrypt --passphrase xxxx \"$archive\" | tar -xzv $@"
+    gpg --batch --decrypt --passphrase $PASSPHRASE "$archive" 2>>$LOG_FILE | tar -xzv $@ >/dev/null 2>>$LOG_FILE
+    check_pipe_rc ${PIPESTATUS[@]}
+    return $?
 }
 
 # Liste der Archives auf stdout ausgeben
@@ -248,10 +377,13 @@ restore() {
 # $1 - Verzeichnis unter dem alle Backups liegen
 #
 list_archives() {
+    log $DEBUG "$DBG list_archives $@"
     local target=$1
     local targ=${target//\//\\\/} # slashes escapen
-    echo "#$targ#"
-    sudo find "$target" -type f -name "*.$EXT" | grep "${HOST_NAME}/" | sed -E "s/^$targ//"
+    log $DEBUG "$DBG > find \"$target\" -type f -name \"*.$EXT\" | grep \"${HOST_NAME}/\" | sed -E \"s/^$targ//\""
+    find "$target" -type f -name "*.$EXT" 2>>$LOG_FILE | grep "${HOST_NAME}/" 2>>$LOG_FILE | sed -E "s/^$targ//" 2>>$LOG_FILE
+    check_pipe_rc ${PIPESTATUS[@]}
+    return $?
 }
 
 # Inhalt des Archivs $2 auf stdout ausgeben
@@ -260,8 +392,9 @@ list_archives() {
 # $2 - Archiv-Datei mit Pfad ab $1
 #
 list_archive_content() {
+    log $DEBUG "$DBG list_archive_content $@"
     local root=$1
     local archive=$2
-
-    sudo cat "$root/${archive/#\//}.$LIST_EXT"
+    log $DEBUG "$DBG > cat \"$root/${archive/#\//}.$LIST_EXT\""
+    cat "$root/${archive/#\//}.$LIST_EXT" 2>>$LOG_FILE
 }

main.sh

@@ -18,6 +18,17 @@ show_help() {
     echo " "
 }
 
+#
+# Script muss unter root bzw. sudo laufen, sonst gibt es jede Menge Probleme
+# mit Zugriff auf die diversen Verzeichnisse.
+# 
+if [ "$(id -u)" -ne 0 ]; then
+    echo "**"
+    echo "** ERROR: script has to be run as superuser!"
+    echo "**"
+    exit 100
+fi
+
 HERE=$(dirname "$0")
 
 if [ -f $HERE/main.inc.sh ]; then
@@ -28,7 +39,7 @@ else
     echo "**        Script wird abgebrochen"
     echo "**"
     show_help
-    exit 1
+    exit 101
 fi
 
 if [ -f $HERE/$HOST_NAME.sh ]; then
@@ -39,10 +50,11 @@ else
     echo "**        Script wird abgebrochen."
     echo "**"
     show_help
-    exit 2
+    exit 102
 fi
 
-LOG_FILE="$NAS_ROOT/log/$LOG_FILE-$(date '+%Y-%m-%d_%H%M').log"
+log_file_name="$NAS_ROOT/log/$HOST_NAME-$(date '+%Y-%m-%d_%H%M')"
+LOG_FILE="${log_file_name}.log"
 
 file=
 archive=
@@ -54,15 +66,21 @@ while :; do
             exit 0
             ;;
         -d|--daily) 
-            LOG_FILE=$BACKUP_TARGET/backup.log
+            MAIL_LOG_FILE="${log_file_name}.auto.log"
+          ##  copy_to_nas
             backup_daily 
             break 
             ;;
         -w|--weekly) 
-            LOG_FILE=$BACKUP_TARGET/backup.log
+            MAIL_LOG_FILE="${log_file_name}.auto.log"
             backup_weekly
             break 
             ;;
+        -e|--export) 
+            MAIL_LOG_FILE="${log_file_name}.auto.log"
+            export_backups
+            break 
+            ;;
         --list)
             list_archives "$NAS_ROOT/daily/$HOST_NAME"
             if [ -d "$NAS_ROOT/weekly/$HOST_NAME" ]; then
@@ -88,22 +106,6 @@ while :; do
             fi
             break
             ;;
-
-    # --- example -> https://mywiki.wooledge.org/BashFAQ/035
-        -f|--file)       # Takes an option argument; ensure it has been specified.
-            if [ "$2" ]; then
-                file=$2
-                shift
-            else
-                die 'ERROR: "--file" requires a non-empty option argument.'
-            fi
-            ;;
-        --file=?*)
-            file=${1#*=} # Delete everything up to "=" and assign the remainder.
-            ;;
-        --file=)         # Handle the case of an empty --file=
-            die 'ERROR: "--file" requires a non-empty option argument.'
-            ;;
         --)              # End of all options.
             shift
             break

nathan.sh

@@ -19,17 +19,19 @@ backup_daily3() {
     BACKUP_EXCLUDE_DIRS=()
 }
 
-copy_to_nas() {
-    delete_dir "$NAS_TARGET"
-    check_dir "$NAS_TARGET"
-    sudo mv "$BACKUP_TARGET/*" "$NAS_TARGET"
-}
 
 before_backup() {
-    run_docker down
+    stop_docker
+    return $?
 }
 
 after_backup() {
-    run_docker up -d
-    copy_to_nas
+    start_docker
+    local dock=$?
+    copy_to_nas "$BACKUP_TARGET" "$NAS_TARGET"
+    local copy=$?
+    if [ $dock -eq 0 ] && [ $copy -eq 0 ]; then
+        return 0
+    fi
+    return 1
 }

synology.sh

@@ -21,8 +21,10 @@ backup_weekly1() {
 
 before_backup() {
     # nothing to do
+    echo
 }
 
 after_backup() {
     # nothing to do
+    echo
 }

test.sh

@@ -33,9 +33,24 @@ list2() {
     #for i in "${a[@]}"; do echo "$i"; done
 }
 
-func=list3
+fun0() {
+    return 0
+}
 
-if ! $func 2>/dev/null ; then echo "success"; else echo "error"; fi
-echo "$EXCL"
+if [ "$(id -u)" -ne 0 ]; then
+    echo "**"
+    echo "** ERROR: script has to be run as superuser!"
+    echo "**"
+    exit 100
+fi
 
-exit
+fun0
+rc=$?
+
+if [ $rc -eq 0 ]; then
+    echo "$rc -eq 0"
+fi
+
+#echo "result=$rc0"
+
+exit