Backup exclusions

bin/v-backup-user

@@ -62,11 +62,12 @@ send_mail="$VESTA/web/inc/mail-wrapper.php"
 la=$(cat /proc/loadavg | cut -f 1 -d ' ' | cut -f 1 -d '.')
 i=0
 while [ "$la" -ge "$BACKUP_LA_LIMIT" ]; do
-    echo "$(date "+%F %T") Load Average $la"
-    echo
-    sleep 60
-    if [ "$i" -ge "15" ]; then
-        echo "LoadAverage $i is above threshold" | $send_mail -s "$subj" $email
+    echo "$(date "+%F %T") LoadAverage $la is above threshold. Sleeping..."
+    sleep 120
+    if [ "$i" -ge "5" ]; then
+        mail_top=$(top -b| head -n 30)
+        mail_text="LoadAverage $i is above threshold\n\n$mail_top\n"
+        echo -e "$mail_text" | $send_mail -s "$subj" $email
         echo "Error: LA is too high"
         sed -i "/ $user /d" $VESTA/data/queue/backup.pipe
         log_event "$E_LA" "$EVENT"
@@ -106,10 +107,10 @@ if [ -e "$USER_DATA/history.log" ]; then
     cp -r $USER_DATA/history.log $tmpdir/vesta/
 fi
 
-if [ -e "$USER_DATA/backup.excludes" ]; then
-    echo -e "$(date "+%F %T") backup.excludes"
-    msg="$msg\n$(date "+%F %T") backup.excludes"
-    cp -r $USER_DATA/backup.excludes $tmpdir/vesta/
+if [ -e "$USER_DATA/backup-excludes.conf" ]; then
+    echo -e "$(date "+%F %T") backup-excludes.conf"
+    msg="$msg\n$(date "+%F %T") backup-excludes.conf"
+    cp -r $USER_DATA/backup-excludes.conf $tmpdir/vesta/
 fi
 
 # Backup PAM
@@ -123,14 +124,9 @@ echo
 msg="$msg\n"
 
 # Parsing excludes
-OLD_IFS="$IFS"
-IFS=$'\n'
-if [ -e "$USER_DATA/backup.excludes" ]; then
-    for exclude in $(cat $USER_DATA/backup.excludes); do
-        eval ${exclude%%=*}=${exclude#*=}
-    done
+if [ -e "$USER_DATA/backup-excludes.conf" ]; then
+    source $USER_DATA/backup-excludes.conf
 fi
-IFS="$OLD_IFS"
 
 # WEB domains
 if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
@@ -138,12 +134,15 @@ if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
     msg="$msg\n-- WEB --"
     mkdir $tmpdir/web/
 
-    # Parsing unsuspeneded domains
+    # Parsing web domain exclusions
     conf="$USER_DATA/web.conf"
     for domain in $(search_objects 'web' 'SUSPENDED' "*" 'DOMAIN'); do
-        check_exl=$(echo "$WEB"|grep -w $domain)
+        check_exl=$(echo -e "${WEB//,/\n}" |grep "^$domain$")
         if [ -z "$check_exl" ]; then
             web_list="$web_list $domain"
+        else
+            echo "$(date "+%F %T") excluding $domain"
+            msg="$msg\n$(date "+%F %T") excluding $domain"
         fi
     done
     web_list=$(echo "$web_list" | sed -e "s/  */\ /g" -e "s/^ //")
@@ -207,9 +206,26 @@ if [ ! -z "$WEB_SYSTEM" ] && [ "$WEB" != '*' ]; then
         fi
 
         # Packing data folders
+        touch $tmpdir/web/$domain/domain_data.tar
         cd $HOMEDIR/$user/web/$domain
-        domain_direcotries=$(ls |grep -v logs)
-        tar -cpf $tmpdir/web/$domain/domain_data.tar $domain_direcotries
+        set -f
+        fargs+=(-not)
+        fargs+=(-path)
+        fargs+=("./logs*")
+        check_exlusion=$(echo -e "${WEB//,/\n}" | grep "^$domain:")
+        if [ ! -z "$check_exlusion" ]; then
+            xdirs="$(echo -e "${check_exlusion//:/\n}" |grep -v $domain)"
+            for xpath in $xdirs; do
+                xpath="$(echo $xpath | sed -e 's/\/*$//' -e 's/^\/*//')"
+                fargs+=(-not)
+                fargs+=(-path)
+                fargs+=("./$xpath*")
+                echo "$(date "+%F %T") excluding directory $xpath"
+                msg="$msg\n$(date "+%F %T") excluding directory $xpath"
+            done
+        fi
+        find . ${fargs[@]} |grep -v "^./$" | grep -v "^.$" |\
+            xargs tar -rpf $tmpdir/web/$domain/domain_data.tar
         gzip -$BACKUP_GZIP $tmpdir/web/$domain/domain_data.tar
     done
 
@@ -230,11 +246,14 @@ if [ ! -z "$DNS_SYSTEM" ] && [ "$DNS" != '*' ]; then
     msg="$msg\n-- DNS --"
     mkdir $tmpdir/dns/
 
-    # Parsing unsuspeneded domains
+    # Parsing dns domain exclusions
     for domain in $(search_objects 'dns' 'SUSPENDED' "*" 'DOMAIN'); do
-        check_exl=$(echo "$DNS"|grep -w $domain)
+        check_exl=$(echo -e "${DNS//,/\n}" |grep "^$domain$")
         if [ -z "$check_exl" ]; then
             dns_list="$dns_list $domain"
+        else
+            echo "$(date "+%F %T") excluding $domain"
+            msg="$msg\n$(date "+%F %T") excluding $domain"
         fi
     done
     dns_list=$(echo "$dns_list" | sed -e "s/  */\ /g" -e "s/^ //")
@@ -256,7 +275,9 @@ if [ ! -z "$DNS_SYSTEM" ] && [ "$DNS" != '*' ]; then
 
         # Backingup dns recods
         cp $USER_DATA/dns/$domain.conf vesta/$domain.conf
-        cp $HOMEDIR/$user/conf/dns/$domain.db conf/$domain.db
+        if [ "$DNS_SYSTEM" != 'remote' ]; then
+            cp $HOMEDIR/$user/conf/dns/$domain.db conf/$domain.db
+        fi
     done
 
     if [ "$i" -eq 1 ]; then
@@ -276,12 +297,15 @@ if [ ! -z "$MAIL_SYSTEM" ] && [ "$MAIL" != '*' ]; then
     msg="$msg\n-- MAIL --"
     mkdir $tmpdir/mail/
 
-    # Parsing unsuspeneded domains
+    # Parsing mail domain exclusions
     conf="$USER_DATA/mail.conf"
     for domain in $(search_objects 'mail' 'SUSPENDED' "*" 'DOMAIN'); do
-        check_exl=$(echo "$MAIL"|grep -w $domain)
+        check_exl=$(echo -e "${MAIL//,/\n}" |grep "^$domain$")
         if [ -z "$check_exl" ]; then
             mail_list="$mail_list $domain"
+        else
+            echo "$(date "+%F %T") excluding $domain"
+            msg="$msg\n$(date "+%F %T") excluding $domain"
         fi
     done
     mail_list=$(echo "$mail_list" | sed -e "s/  */\ /g" -e "s/^ //")
@@ -311,9 +335,19 @@ if [ ! -z "$MAIL_SYSTEM" ] && [ "$MAIL" != '*' ]; then
 
         # Packing mailboxes
         cd $HOMEDIR/$user/mail/$domain_idn
-        accounts=$(ls)
-        if [ ! -z "$accounts" ] && [[ "$MAIL_SYSTEM" =~ exim ]]; then
-            tar -cpf $tmpdir/mail/$domain/accounts.tar  $accounts
+        for account in $(ls); do
+            exclusion=$(echo -e "${MAIL//,/\n}" |grep "$domain:")
+            exclusion=$(echo -e "${exclusion//:/\n}" |grep "^$account$")
+            if [ -z "$exclusion" ] && [[ "$MAIL_SYSTEM" =~ exim ]]; then
+                echo "$(date "+%F %T") $account"
+                touch $tmpdir/mail/$domain/accounts.tar
+                tar -rpf $tmpdir/mail/$domain/accounts.tar $account
+            else
+                echo "$(date "+%F %T") excluding account $account"
+                msg="$msg\n$(date "+%F %T") excluding account $account"
+            fi
+        done
+        if [ -e "$tmpdir/mail/$domain/accounts.tar" ]; then
             gzip -$BACKUP_GZIP $tmpdir/mail/$domain/accounts.tar
         fi
     done
@@ -336,11 +370,14 @@ if [ ! -z "$DB_SYSTEM" ] && [ "$DB" != '*' ]; then
     msg="$msg\n-- DB --"
     mkdir $tmpdir/db/
 
-    # Parsing unsuspeneded domains
+    # Parsing database exclusions
     for database in $(search_objects 'db' 'SUSPENDED' "*" 'DB'); do
-        check_exl=$(echo "$DB"|grep -w $database)
+        check_exl=$(echo -e "${DB//,/\n}" |grep "^$database$")
         if [ -z "$check_exl" ]; then
             db_list="$db_list $database"
+        else
+            echo "$(date "+%F %T") excluding $database"
+            msg="$msg\n$(date "+%F %T") excluding $database"
         fi
     done
     db_list=$(echo "$db_list" | sed -e "s/  */\ /g" -e "s/^ //")
@@ -410,38 +447,52 @@ if [ ! -z "$CRON_SYSTEM" ] && [ "$CRON" != '*' ]; then
 fi
 
 # User Directories
-echo "-- User Dir --"
-msg="$msg\n-- User Dir --"
-mkdir $tmpdir/user_dir
-user_dir=$(ls $HOMEDIR/$user | \
-    grep -v conf | \
-    grep -v web | \
-    grep -v dns | \
-    grep -v mail | \
-    grep -v tmp)
-i=0
-for udir in $user_dir; do
-    udir_list="$udir_list $udir"
-    ((i ++))
-    echo -e "$(date "+%F %T") $udir"
-    msg="$msg\n$(date "+%F %T") $udir"
-    cp -pr $HOMEDIR/$user/$udir $tmpdir/user_dir/
-    cd $tmpdir/user_dir/
-    tar -czpf $udir.tar.gz $udir
-    rm -rf $udir
-done
-udir_list=$(echo "$udir_list" | sed -e "s/  */\ /g" -e "s/^ //")
+if [ "$USER" != '*' ]; then
+    echo "-- User Dir --"
+    msg="$msg\n-- User Dir --"
+    mkdir $tmpdir/user_dir
+    cd $HOMEDIR/$user
+
+    # Default excludes
+    set -f
+    fargs=''
+
+    # Parsing directory exlusion list
+    exlusion_list=$(echo -e "${USER//,/\n}")
+    for xpath in $exlusion_list; do
+        fargs+=(-not)
+        fargs+=(-path)
+        fargs+=("./$xpath*")
+        echo "$(date "+%F %T") excluding directory $xpath"
+        msg="$msg\n$(date "+%F %T") excluding directory $xpath"
+    done
 
-if [ "$i" -eq 1 ]; then
-    echo -e "$(date "+%F %T") $i user directory"
-    msg="$msg\n$(date "+%F %T") $i directory"
-else
-    echo -e "$(date "+%F %T") $i directories"
-    msg="$msg\n$(date "+%F %T") $i directories"
-fi
-echo
-msg="$msg\n"
+    for udir in $(ls |egrep -v "conf|web|dns|mail"); do
+        check_exl=$(echo -e "${USER//,/\n}" |grep "^$udir$")
+        if [ -z "$check_exl" ]; then
+            ((i ++))
+            udir_list="$udir_list $udir"
+            echo -e "$(date "+%F %T") adding directory $udir"
+            msg="$msg\n$(date "+%F %T") adding directory $udir"
+            touch $tmpdir/user_dir/$udir.tar
+            find ./$udir ${fargs[@]} |grep -v "^./$" |grep -v "^.$" |\
+                grep -v "./$udir$" |\
+                xargs tar -rpf $tmpdir/user_dir/$udir.tar
+            gzip -$BACKUP_GZIP $tmpdir/user_dir/$udir.tar
+        fi
+    done
+    udir_list=$(echo "$udir_list" | sed -e "s/  */\ /g" -e "s/^ //")
 
+    if [ "$i" -eq 1 ]; then
+        echo -e "$(date "+%F %T") $i user directory"
+        msg="$msg\n$(date "+%F %T") $i directory"
+    else
+        echo -e "$(date "+%F %T") $i directories"
+        msg="$msg\n$(date "+%F %T") $i directories"
+    fi
+    echo
+    msg="$msg\n"
+fi
 
 # Get backup size
 size="$(du -shm $tmpdir | cut -f 1)"

bin/v-list-user-backup-exclusions

@@ -0,0 +1,94 @@
+#!/bin/bash
+# info: list backup exclusions
+# options: USER [FORMAT]
+#
+# The function for obtaining the backup exclusion list
+
+
+#----------------------------------------------------------#
+#                    Variable&Function                     #
+#----------------------------------------------------------#
+
+# Argument defenition
+user=$1
+format=${2-shell}
+USER=''
+
+# Includes
+source $VESTA/func/main.sh
+
+# Json function
+json_list_backup_xld() {
+    set -f
+    echo '{'
+    fields_count=$(echo "$fields" | wc -w)
+    i=1
+    source $USER_DATA/backup-excludes.conf
+    for field in $fields; do
+        eval value=$field
+        j=1
+        echo -e "\t\"${field//$/}\": {"
+        for exlcude in ${value//,/ }; do
+            exlcude_obj=$(echo $exlcude |cut -f 1 -d:)
+            exclude_param=$(echo $exlcude |sed -e "s/$exlcude_obj://")
+            if [ "$exlcude_obj" = "$exclude_param" ]; then
+                exclude_param=''
+            fi
+            if [ $j -lt "$(echo ${value//,/ } |wc -w)" ]; then
+                echo -e "\t\t\"$exlcude_obj\": \"$exclude_param\","
+            else
+                echo -e "\t\t\"$exlcude_obj\": \"$exclude_param\""
+            fi
+            (( ++j))
+        done
+        if [ $i -lt $fields_count ]; then
+            echo -e "\t},"
+        else
+            echo -e "\t}"
+        fi
+        (( ++i))
+    done
+    echo '}'
+}
+
+# Shell function
+shell_list_backup_xld() {
+    source $USER_DATA/backup-excludes.conf
+    for field in $fields; do
+        eval value=$field
+        echo -e "${field//$/}: $value"
+    done
+}
+
+
+#----------------------------------------------------------#
+#                    Verifications                         #
+#----------------------------------------------------------#
+
+check_args '1' "$#" 'USER [FORMAT]'
+validate_format 'user'
+is_object_valid 'user' 'USER' "$user"
+
+
+#----------------------------------------------------------#
+#                       Action                             #
+#----------------------------------------------------------#
+
+# Defining fileds to select
+conf=$VESTA/data/ips/*
+fields="\$WEB \$DNS \$MAIL \$DB \$CRON \$USER"
+
+# Listing backup exclusions
+case $format in
+    json)   json_list_backup_xld ;;
+    plain)  nohead=1; shell_list_backup_xld ;;
+    shell)  shell_list_backup_xld;;
+    *)      check_args '1' '0' '[FORMAT]'
+esac
+
+
+#----------------------------------------------------------#
+#                       Vesta                              #
+#----------------------------------------------------------#
+
+exit

bin/v-update-user-backup-exclusions

@@ -0,0 +1,82 @@
+#!/bin/bash
+# info: update backup exclusion list
+# options: USER FILE
+#
+# The function for updating backup exclusion list
+
+
+#----------------------------------------------------------#
+#                    Variable&Function                     #
+#----------------------------------------------------------#
+
+# Argument defenition
+user=$1
+vfile=$2
+
+# Includes
+source $VESTA/func/main.sh
+source $VESTA/conf/vesta.conf
+
+is_file_available() {
+    if [ ! -e "$vfile" ]; then
+        echo "Error: file $vfile doesn't exist"
+        log_event "$E_NOTEXIST" "$EVENT"
+        exit $E_NOTEXIST
+    fi
+}
+
+is_file_valid() {
+    exclude="[!$#&;()\]"
+    vcontent=$(cat $vfile)
+    if [[ "$vcontent" =~ $exclude ]]; then
+        echo "Error: invalid characters in the exlusion list"
+        log_event "$E_INVALID" "$EVENT"
+        exit $E_INVALID
+    fi
+}
+
+
+#----------------------------------------------------------#
+#                    Verifications                         #
+#----------------------------------------------------------#
+
+check_args '2' "$#" 'USER FILE'
+validate_format 'user' 'vfile'
+is_object_valid 'user' 'USER' "$user"
+is_file_available
+is_file_valid
+
+
+#----------------------------------------------------------#
+#                       Action                             #
+#----------------------------------------------------------#
+
+# Flush variables
+WEB=''
+DNS=''
+MAIL=''
+DB=''
+CRON=''
+USER=''
+
+# Source exclusion list
+source $vfile
+
+# Updating exlusion list
+echo "WEB='$WEB'" > $USER_DATA/backup-excludes.conf
+echo "DNS='$DNS'" >> $USER_DATA/backup-excludes.conf
+echo "MAIL='$MAIL'" >> $USER_DATA/backup-excludes.conf
+echo "DB='$DB'" >> $USER_DATA/backup-excludes.conf
+echo "CRON='$DB'" >> $USER_DATA/backup-excludes.conf
+echo "USER='$USER'" >> $USER_DATA/backup-excludes.conf
+
+
+#----------------------------------------------------------#
+#                       Vesta                              #
+#----------------------------------------------------------#
+
+# Logging
+log_history "updated exlusion list"
+log_event "$OK" "$EVENT"
+
+exit