Rclone support (#2928)

* Add support rclone

* Fix naming

* Fix 2 small bugs

* Update v-list-backup-host

* Changes to web ui

* Update web/templates/pages/edit_server.html

Co-authored-by: Alec Rust <me@alecrust.com>

Co-authored-by: Alec Rust <me@alecrust.com>

bin/v-add-backup-host

@@ -157,9 +157,9 @@ EOF
 #----------------------------------------------------------#
 
 if [ "$type" != 'local' ];then
-    check_args '4' "$#" "TYPE HOST USERNAME PASSWORD [PATH] [PORT]"
+    check_args '2' "$#" "TYPE HOST USERNAME PASSWORD [PATH] [PORT]"
     is_format_valid 'host' 'path' 'port'
-    is_type_valid  'sftp,ftp,b2' "$type"
+    is_type_valid  'sftp,ftp,b2,rclone' "$type"
     is_username_format_valid "$user" "username"
     privatekey="no"
     if [ -f "$raw_password" ]; then
@@ -178,7 +178,7 @@ if [ "$type" != 'local' ];then
         which expect >/dev/null 2>&1
         check_result $? "expect command not found"  "$E_NOTEXIST"
     fi
-    if [ "$type" != 'b2' ]; then
+    if [ "$type" != 'b2' ] && [ "$type" != 'rclone' ]; then
         if ! (is_ip_format_valid "$host" >/dev/null); then
             host "$host" >/dev/null 2>&1
             check_result $? "host connection failed" "$E_CONNECT"
@@ -283,8 +283,26 @@ if [ "$type" = 'b2' ]; then
     fi
 fi
 
+if [ "$type" = 'rclone' ]; then
+    curl -s https://rclone.org/install.sh | bash /dev/null > /dev/null 2>&1
+    # Verify account exists
+    if [ ! -z "$(cat /root/.config/rclone/rclone.conf | grep "\[$host\]")" ]; then 
+        echo "test" > /tmp/hestia-backup.txt
+        # Try to upload a single file
+        if [ -z "$path" ]; then
+            rclone copy /tmp/hestia-backup $host:/hestia-backup.txt
+            rclone delete $host:/hestia-backup.txt
+        else
+            rclone copy /tmp/hestia-backup $host:$path/hestia-backup.txt
+            rclone delete $host:$path/hestia-backup.txt
+        fi
+    else
+        check_result "$E_CONNECT" "Rclone config does not exits"
+    fi
+fi
+
 # Adding backup host
-if [ $type != 'local' ] && [ $type != 'b2' ]; then
+if [ $type == 'ftp' ] || [ $type = 'sftp' ] ; then
     new_timestamp
     str="HOST='$host'\nUSERNAME='$user'\nPASSWORD='$password'\nPRIVATEKEY='$privatekey'"
     str="$str\nBPATH='$path'\nPORT='$port'\nTIME='$time'\nDATE='$date'"
@@ -296,6 +314,11 @@ elif [ $type == 'b2' ]; then
     str="$str\nTIME='$time'\nDATE='$date'"
     echo -e "$str" > $HESTIA/conf/$type.backup.conf
     chmod 660 $HESTIA/conf/$type.backup.conf
+elif [ $type == "rclone" ]; then
+    new_timestamp
+    str="HOST='$host'\nBPATH='$path'"
+    str="$str\nTIME='$time'\nDATE='$date'"
+    echo -e "$str" > $HESTIA/conf/$type.backup.conf
 fi
 
 #----------------------------------------------------------#

bin/v-backup-user

@@ -706,6 +706,7 @@ for backup_type in $(echo -e "${BACKUP_SYSTEM//,/\\n}"); do
         sftp)  sftp_backup ;;
         google) google_backup ;;
         b2) b2_backup ;;
+        rclone) rclone_backup ;;
     esac
 done
 

bin/v-delete-user-backup

@@ -57,6 +57,9 @@ fi
 if [[ "$TYPE" =~ "b2" ]]; then
     b2_delete "$user" "$backup"
 fi
+if [[ "$TYPE" =~ "rclone" ]]; then
+    rclone_delete "$backup"
+fi
 if [[ "$TYPE" =~ "local" ]]; then
     rm -f "$backup_folder/$2"
 fi

bin/v-download-backup

@@ -74,6 +74,14 @@ if [ ! -e "$BACKUP/$backup" ]; then
         ftp_download "$backup"
         downloaded='yes'
     fi
+    if [[ "$BACKUP_SYSTEM" =~ "rclone" ]] && [ -z "$downloaded" ]; then
+        rclone_download "$backup"
+        downloaded='yes'
+    fi
+    if [[ "$BACKUP_SYSTEM" =~ "b2" ]] && [ -z "$downloaded" ]; then
+        b2_download "$backup"
+        downloaded='yes'
+    fi
     if [ -z "$downloaded" ]; then
         subj="Download of $backup failed for $user"
         $BIN/v-add-user-notification $user "$subj" "<b>Unable to retrieve backup file from remote server.</b><br><b>Error:</b> $backup file doesn't exist in '${BACKUP}' directory."

bin/v-list-backup-host

@@ -25,7 +25,7 @@ source_conf "$HESTIA/conf/hestia.conf"
 # JSON list function
 json_list() {
     case $type in 
-    'sftp' | 'ftp')
+    'sftp' | 'ftp' | 'rclone')
     echo '{'
     echo '    "'$type'": {
         "HOST": "'$HOST'",
@@ -55,7 +55,7 @@ json_list() {
 # SHELL list function
 shell_list() {
     case $type in
-    'sftp' | 'ftp')
+    'sftp' | 'ftp' | 'rclone')
     echo "HOST:           $HOST"
     echo "USERNAME:       $USERNAME"
     echo "PORT:           $PORT"
@@ -77,7 +77,7 @@ shell_list() {
 # PLAIN list function
 plain_list() {
     case $type in
-    'sftp' | 'ftp')
+    'sftp' | 'ftp' | 'rclone')
     echo -e "$HOST\t$USERNAME\t$PORT\t$type\t$BPATH\t$TIME\t$DATE"
     ;;
     'b2')
@@ -89,7 +89,7 @@ plain_list() {
 # CSV list function
 csv_list() {
     case $type in
-    'sftp' | 'ftp')
+    'sftp' | 'ftp' | 'rclone')
     echo "HOST,USERNAME,PORT,TYPE,PATH,TIME,DATE"
     echo "$HOST,$USERNAME,$PORT,$type,$BPATH,$TIME,$DATE"
     ;;
@@ -113,7 +113,7 @@ is_type_format_valid() {
 #----------------------------------------------------------#
 
 check_args '1' "$#" 'TYPE [FORMAT]'
-is_type_format_valid "$type"
+is_common_format_valid $type "type" 
 
 #----------------------------------------------------------#
 #                       Action                             #

bin/v-restore-user

@@ -85,6 +85,10 @@ if [ ! -e "$BACKUP/$backup" ]; then
         b2_download "$backup"
         downloaded='yes'
     fi
+    if [[ "$BACKUP_SYSTEM" =~ "rclone" ]] && [ -z "$downloaded" ]; then
+         rclone_download "$backup"
+         downloaded='yes'
+    fi
     if [ -z "$downloaded" ]; then
         check_result "$E_NOTEXIST" "backup file $backup doesn't exist in '${BACKUP}' folder"
     fi

func/backup.sh

@@ -556,3 +556,67 @@ b2_delete(){
 
     b2 delete-file-version $1/$2 > /dev/null 2>&1
 }
+
+rclone_backup(){
+    # Define rclone config
+    source_conf "$HESTIA/conf/rclone.backup.conf"
+    echo -e "$(date "+%F %T") Upload With Rclone: $user.$backup_new_date.tar"
+    if [ "$localbackup" != 'yes' ]; then
+        cd $tmpdir
+        tar -cf $BACKUP/$user.$backup_new_date.tar .
+    fi
+    cd $BACKUP/
+    
+    if [ -z "$BPATH" ]; then
+        rclone copy -v $user.$backup_new_date.tar $HOST
+        if [ "$?" -ne 0 ]; then
+            check_result "$E_CONNECT" "Unable to upload backup"
+        fi
+        
+        backup_list=$(rclone lsf $HOST | cut -d' ' -f1 |grep "^$user\.");
+        backups_count=$(echo "$backup_list" |wc -l)
+        backups_rm_number=$((backups_count - BACKUPS))
+        if [ "$backups_count" -ge "$BACKUPS" ]; then
+        for backup in $(echo "$backup_list" |head -n $backups_rm_number); do
+            echo "Delete file: $backup"
+            rclone deletefile $HOST:/$backup
+        done    
+        fi
+    else
+        rclone copy -v $user.$backup_new_date.tar $HOST:$BPATH
+        if [ "$?" -ne 0 ]; then
+            check_result "$E_CONNECT" "Unable to upload backup"
+        fi
+        
+        backup_list=$(rclone lsf $HOST:$BPATH | cut -d' ' -f1 |grep "^$user\.");
+        backups_count=$(echo "$backup_list" |wc -l)
+        backups_rm_number=$((backups_count - BACKUPS))
+        if [ "$backups_count" -ge "$BACKUPS" ]; then
+        for backup in $(echo "$backup_list" |head -n $backups_rm_number); do
+            echo "Delete file: $backup"
+            rclone deletefile $HOST:$BPATH/$backup
+        done    
+        fi
+    fi
+}
+
+rclone_delete(){
+    # Defining rclone settings
+    source_conf "$HESTIA/conf/rclone.backup.conf"
+    if [ -z "$BPATH" ]; then
+        rclone deletefile $HOST:/$1
+    else
+        rclone deletefile $HOST:$BPATH/$1
+    fi
+}
+
+rclone_download() {
+    # Defining rclone b2 settings
+    source_conf "$HESTIA/conf/rclone.backup.conf"
+    cd $BACKUP
+    if [ -z "$BPATH" ]; then
+        rclone copy -v $HOST:/$1 ./
+    else
+        rclone copy -v $HOST:$BPATH/$1 ./
+    fi
+}

web/edit/server/index.php

@@ -219,9 +219,9 @@ foreach ($backup_types as $backup_type) {
         if (in_array($backup_type, array('ftp','sftp'))) {
             $v_backup_host = $v_remote_backup[$backup_type]['HOST'];
             $v_backup_type = $v_remote_backup[$backup_type]['TYPE'];
-            $v_backup_username = $v_remote_backup[$backup_type]['USERNAME'];
+            $v_backup_username = $v_remote_backup[$backup_type]['USERNAME'] ?? '';
             $v_backup_password = "";
-            $v_backup_port = $v_remote_backup[$backup_type]['PORT'];
+            $v_backup_port = $v_remote_backup[$backup_type]['PORT'] ?? '';
             $v_backup_bpath = $v_remote_backup[$backup_type]['BPATH'];
             $v_backup_remote_adv = "yes";
         } elseif (in_array($backup_type, array('b2'))) {
@@ -230,6 +230,11 @@ foreach ($backup_types as $backup_type) {
             $v_backup_application_id = $v_remote_backup[$backup_type]['B2_KEY_ID'];
             $v_backup_application_key = '';
             $v_backup_remote_adv = "yes";
+        } elseif (in_array($backup_type, array('rclone'))) {
+            $v_backup_type = $v_remote_backup[$backup_type]['TYPE'];
+            $v_rclone_host = $v_remote_backup[$backup_type]['HOST'];
+            $v_rclone_path = $v_remote_backup[$backup_type]['BPATH'];
+            $v_backup_remote_adv = "yes";
         }
     }
 }
@@ -266,8 +271,11 @@ if (empty($v_backup_application_key)) {
 if (empty($v_backup_remote_adv)) {
     $v_backup_remote_adv = '';
 }
-if (empty($v_backup_remote_adv)) {
-    $v_backup_remote_adv = '';
+if (empty($v_rclone_host)) {
+    $v_rclone_host = '';
+}
+if (empty($v_rclone_path)) {
+    $v_rclone_path = '';
 }
 
 // List ssl certificate info
@@ -812,6 +820,17 @@ if (!empty($_POST['save'])) {
                 $v_backup_remote_adv = 'yes';
             }
         }
+        if ($v_rclone_host == '' && !empty($_POST['v_rclone_host']) && $_POST['v_backup_type'] =="rclone" ) {
+            $v_rclone_host = quoteshellarg($_POST['v_rclone_host']);
+            $v_backup_type = quoteshellarg($_POST['v_backup_type']);
+            $v_rclone_path = quoteshellarg($_POST['v_rclone_path']);
+            exec(HESTIA_CMD."v-add-backup-host ". $v_backup_type ." ". $v_rclone_host ." '' '' ".$v_rclone_path, $output, $return_var);
+            check_return_code($return_var, $output);
+            unset($output);
+            $v_backup_new = 'yes';
+            $v_backup_adv = 'yes';
+            $v_backup_remote_adv = 'yes';
+        }
     }
 
     // Change remote backup host type

web/js/pages/edit_server.js

@@ -2,9 +2,15 @@ $('#backup_type').change(function (){
    if(this.value == 'b2'){
        $('#backup_bucket').show();
        $('#backup_sftp').hide();
+       $('#backup_rclone').hide(); 
+   }else if(this.value == 'rclone'){
+      $('#backup_bucket').hide();
+      $('#backup_sftp').hide();
+      $('#backup_rclone').show(); 
    }else{
-       $('#backup_bucket').hide();
-       $('#backup_sftp').show();
+      $('#backup_bucket').hide();
+      $('#backup_sftp').show();
+      $('#backup_rclone').hide(); 
    }
 });
 

web/templates/pages/edit_server.html

@@ -661,6 +661,7 @@
 															<option value='ftp'><?=_('ftp'); ?></option>
 															<option value='sftp' <?php if((!empty($v_backup_type)) && (trim($v_backup_type,"'")  == 'sftp' )) echo 'selected="selected"'; ?>><?=_('sftp'); ?></option>
 															<option value="b2" <?php if((!empty($v_backup_type)) && (trim($v_backup_type,"'")  == 'b2' )) echo 'selected="selected"'; ?>><?=_('Backblaze'); ?>
+															<option value="rclone" <?php if((!empty($v_backup_type)) && (trim($v_backup_type,"'")  == 'rclone' )) echo 'selected="selected"'; ?>><?=_('Rclone'); ?>
 														</select>
 													</td>
 												</tr>
@@ -726,6 +727,29 @@
 														</table>
 													</td>
 												</tr>
+												<tr style="display: <?php if ((empty($v_backup_type)) || !in_array(trim($v_backup_type, "'"),array('rclone'))){ echo 'none';} else {echo 'block';} ?>" id="backup_rclone">
+													<td>
+														<table>
+															<tr>
+																<td class="vst-text">
+																	<label for="v_rclone_host" class="form-label"><?=_('Host');?></label>
+																	<input type="text" class="form-control" name="v_rclone_host" id="v_rclone_host" value="<?=trim($v_rclone_host, "'")?>">
+																</td>
+															</tr>
+															<tr>
+																<td class="vst-text">
+																	<?=_('Directory');?>
+																</td>
+															</tr>
+															<tr>
+																<td>
+																	<input type="text" size="20" class="vst-input" name="v_rclone_path" value="<?=trim($v_rclone_path, "'")?>">
+																	<br><br>
+																</td>
+															</tr>
+														</table>
+													</td>	
+												</tr>
 											</table>
 										</td>
 									</tr>