[cli] add support for using linux ipset lists

bin/v-add-firewall-ipset

@@ -0,0 +1,171 @@
+#!/bin/bash
+# info: add firewall ipset
+# options: NAME [SOURCE] [IPVERSION] [AUTOUPDATE] [FORCE]
+#
+# The function adds new ipset to system firewall
+
+#----------------------------------------------------------#
+#                    Variable&Function                     #
+#----------------------------------------------------------#
+
+ip_name=${1}
+data_source=${2}
+ip_version=${3-v4}
+autoupdate=${4-yes}
+force=${5-no}
+
+# Includes
+source $HESTIA/func/main.sh
+source $HESTIA/conf/hestia.conf
+
+
+#----------------------------------------------------------#
+#                    Verifications                         #
+#----------------------------------------------------------#
+
+check_args '1' "$#" 'NAME [SOURCE] [IPVERSION] [AUTOUPDATE] [FORCE]'
+is_format_valid 'ip_name'
+is_boolean_format_valid "$autoupdate" 'bool (yes/no)'
+is_boolean_format_valid "$force" 'bool (yes/no)'
+is_system_enabled "$FIREWALL_SYSTEM" 'FIREWALL_SYSTEM'
+
+ipset_hstobject='../../data/firewall/ipset'
+
+IPSET_BIN="$(which ipset)"
+IPSET_PATH="$HESTIA/data/firewall/ipset"
+
+if [ -z "$data_source" ]; then
+    if [ ! -f "${IPSET_PATH}.conf" ] || [[ ! $(grep "LISTNAME='$ip_name'" "${IPSET_PATH}.conf") ]]; then
+        check_args '2' "$#" 'NAME SOURCE [IPVERSION] [AUTOUPDATE] [FORCE]'
+    fi
+
+    data_source="$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ip_name" '$SOURCE')"
+    ip_version="$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ip_name" '$IP_VERSION')"
+else
+    is_object_new "$ipset_hstobject" 'LISTNAME' "$ip_name"
+fi
+
+if [ "$ip_version" != "v4" ] && [ "$ip_version" != "v6" ]; then
+    check_result $E_INVALID "invalid ip version, valid: (v4|v6)"
+fi
+
+if ! echo "$data_source" | egrep -q '^(https?|script|file):'; then
+    check_result $E_INVALID "invalid ipset source, valid: (http[s]://|script:|file:)"
+fi
+
+
+IPSET_FILE="${ip_name}.${ip_version}"
+IPSET_MIN_SIZE=10
+
+# Perform verification if read-only mode is enabled
+check_hestia_demo_mode
+
+# Install ipset package if missing
+if [ -z "$IPSET_BIN" ]; then
+    apt-get --quiet --yes install ipset > /dev/null
+    check_result $? "Installing ipset package"
+
+    IPSET_BIN="$(which ipset)"
+    check_result $? "ipset binary not found"
+fi
+
+
+#----------------------------------------------------------#
+#                       Action                             #
+#----------------------------------------------------------#
+
+mkdir -p "$IPSET_PATH"
+
+# Generate ip lists file if missing or when forced
+if [ ! -f "${IPSET_PATH}/${IPSET_FILE}.iplist" ] || [ "$force" = "yes" ]; then
+
+    iplist_tempfile=$(mktemp)
+
+    if [[ "$data_source" =~ ^https?:// ]]; then
+
+        wget --tries=3 --timeout=15 --read-timeout=15 --waitretry=3 --no-dns-cache --quiet "$data_source" -O "$iplist_tempfile"
+        check_result $? "Downloading ip list"
+
+        # Advanced: execute script with the same basename for aditional pre-processing
+        # ex: 
+        if [ -x "${IPSET_PATH}/${IPSET_FILE}.sh" ]; then 
+            setpriv --clear-groups --reuid nobody --regid nogroup -- ${IPSET_PATH}/${IPSET_FILE}.sh "$ip_name" "$iplist_tempfile"
+        fi
+
+    elif [[ "$data_source" =~ ^script:/ ]]; then
+
+        # Generate the ip list file trough a external script
+        # ex: compiling a ip list from multiple sources on demand
+
+        if [ -x "${data_source#script:}" ]; then
+
+            setpriv --clear-groups --reuid nobody --regid nogroup -- ${data_source#script:} "$ip_name" > "$iplist_tempfile"
+            check_result $? "Running custom ip list update script"
+
+        fi
+
+    elif [[ "$data_source" =~ ^file:/ ]]; then
+
+        # Use a external ip-list file managed by other apps
+        # ex: Using a ip list that is continously updated
+
+        [ -f "${data_source#file:}" ] && cp -f "${data_source#file:}" "$iplist_tempfile"
+
+    fi
+
+    # Validate iplist file size
+    iplist_size=$(sed -r -e '/^#|^$/d' "$iplist_tempfile" | wc -l)
+    [[ "$iplist_size" -le $IPSET_MIN_SIZE ]] && check_result $E_INVALID "iplist file too small (<${IPSET_MIN_SIZE}), ignoring"
+    mv -f "$iplist_tempfile" "${IPSET_PATH}/${IPSET_FILE}.iplist"
+
+fi
+
+# Load ipset in kernel
+inet_ver="inet"
+[ "$ip_version" == "v6" ] && inet_ver="inet6"
+
+$IPSET_BIN create "$ip_name" -exist hash:net family $inet_ver
+$IPSET_BIN create "${ip_name}-tmp" -exist hash:net family $inet_ver
+$IPSET_BIN flush "${ip_name}-tmp"
+
+sed -rn -e '/^#|^$/d'  -e "s/^(.*)/add ${ip_name}-tmp \\1/p" "${IPSET_PATH}/${IPSET_FILE}.iplist" | $IPSET_BIN -quiet restore
+check_result $? "Populating ipset table"
+
+$IPSET_BIN swap "${ip_name}-tmp" "${ip_name}"
+$IPSET_BIN --quiet destroy "${ip_name}-tmp"
+
+
+# Generating timestamp
+time_n_date=$(date +'%T %F')
+time=$(echo "$time_n_date" |cut -f 1 -d \ )
+date=$(echo "$time_n_date" |cut -f 2 -d \ )
+
+if [ ! -f "${IPSET_PATH}.conf" ] || [ -z "$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ip_name" '$LISTNAME')" ]; then
+
+    # Concatenating rule
+    str="LISTNAME='$ip_name' IP_VERSION='$ip_version' SOURCE='$data_source'"
+    str="$str AUTOUPDATE='$autoupdate' SUSPENDED='no'"
+    str="$str TIME='$time' DATE='$date'"
+    echo "$str" >> $HESTIA/data/firewall/ipset.conf
+
+elif [ "$force" = "yes" ]; then
+
+    # update iplist last regen time
+    update_object_value "$ipset_hstobject" 'LISTNAME' "$ip_name" '$TIME' "$time"
+    update_object_value "$ipset_hstobject" 'LISTNAME' "$ip_name" '$DATE' "$date"
+
+fi
+
+# Changing permissions
+chmod 660 $HESTIA/data/firewall/ipset.conf
+chmod 660 "${IPSET_PATH}/${IPSET_FILE}.iplist"
+
+
+#----------------------------------------------------------#
+#                       Hestia                             #
+#----------------------------------------------------------#
+
+# Logging
+log_event "$OK" "$ARGUMENTS"
+
+exit

bin/v-delete-firewall-ipset

@@ -0,0 +1,72 @@
+#!/bin/bash
+# info: delete firewall ipset
+# options: NAME
+#
+# The function removes ipset from system and from hestia
+
+
+#----------------------------------------------------------#
+#                    Variable&Function                     #
+#----------------------------------------------------------#
+
+ip_name=${1}
+
+# Includes
+source $HESTIA/func/main.sh
+source $HESTIA/conf/hestia.conf
+
+
+#----------------------------------------------------------#
+#                    Verifications                         #
+#----------------------------------------------------------#
+
+ipset_hstobject='../../data/firewall/ipset'
+
+check_args '1' "$#" 'NAME'
+is_format_valid 'ip_name'
+is_system_enabled "$FIREWALL_SYSTEM" 'FIREWALL_SYSTEM'
+is_object_valid "$ipset_hstobject" 'LISTNAME' "$ip_name"
+
+ip_version="$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ip_name" '$IP_VERSION')"
+
+IPSET_BIN="$(which ipset)"
+IPSET_PATH="$HESTIA/data/firewall/ipset"
+IPSET_FILE="${ip_name}.${ip_version}"
+
+# Perform verification if read-only mode is enabled
+check_hestia_demo_mode
+
+# Install ipset package if missing
+if [ -z "$IPSET_BIN" ]; then
+    apt-get --quiet --yes install ipset > /dev/null
+    check_result $? "Installing ipset package"
+
+    IPSET_BIN="$(which ipset)"
+    check_result $? "ipset binary not found"
+fi
+
+
+#----------------------------------------------------------#
+#                       Action                             #
+#----------------------------------------------------------#
+
+if $IPSET_BIN --quiet list "${ip_name}-tmp"; then
+    $IPSET_BIN --quiet destroy "${ip_name}-tmp"
+fi
+
+if $IPSET_BIN --quiet list "${ip_name}"; then
+    $IPSET_BIN --quiet destroy "${ip_name}"
+    check_result $? "ipset ${ip_name} still used by iptables. Cannot remove"
+fi
+
+sed -i "/LISTNAME='$ip_name'/d" "${IPSET_PATH}.conf"
+rm -f "${IPSET_PATH}/${IPSET_FILE}.iplist"
+
+#----------------------------------------------------------#
+#                       Hestia                             #
+#----------------------------------------------------------#
+
+# Logging
+log_event "$OK" "$ARGUMENTS"
+
+exit

bin/v-list-firewall-ipset

@@ -0,0 +1,102 @@
+#!/bin/bash
+# info: List firewall ipset
+# options: [FORMAT]
+#
+# The function prints defined ipset lists
+
+
+#----------------------------------------------------------#
+#                    Variable&Function                     #
+#----------------------------------------------------------#
+
+# Argument definition
+format=${1-shell}
+
+# Includes
+source $HESTIA/func/main.sh
+source $HESTIA/conf/hestia.conf
+
+
+#----------------------------------------------------------#
+#                    Verifications                         #
+#----------------------------------------------------------#
+
+is_system_enabled "$FIREWALL_SYSTEM" 'FIREWALL_SYSTEM'
+
+
+#----------------------------------------------------------#
+#                       Action                             #
+#----------------------------------------------------------#
+
+# JSON list function
+json_list() {
+    IFS=$'\n'
+    i=1
+    objects=$(grep LISTNAME $HESTIA/data/firewall/ipset.conf |wc -l)
+    echo "{"
+    while read str; do
+        [[ -z "$str" ]] && continue;
+        parse_object_kv_list "$str"
+        echo -n '    "'$LISTNAME'": {
+        "IP_VERSION": "'$IP_VERSION'",
+        "AUTOUPDATE": "'$AUTOUPDATE'",
+        "SUSPENDED": "'$SUSPENDED'",
+        "SOURCE": "'$SOURCE'",
+        "TIME": "'$TIME'",
+        "DATE": "'$DATE'"
+    }'
+    [[ "$i" -lt "$objects" ]] && echo ',' || echo
+    ((i++))
+    done < <(cat $HESTIA/data/firewall/ipset.conf)
+    echo '}'
+}
+
+# SHELL list function
+shell_list() {
+    IFS=$'\n'
+    echo "LISTNAME^IP_VERSION^AUTOUPDATE^SUSPENDED^SOURCE^TIME^DATE"
+    echo "----^------^-----^----^--^----^----"
+    while read str; do
+        [[ -z "$str" ]] && continue;
+        parse_object_kv_list "$str"
+        echo "$LISTNAME^$IP_VERSION^$AUTOUPDATE^$SUSPENDED^$SOURCE^$TIME^$DATE"
+    done < <(cat $HESTIA/data/firewall/ipset.conf)
+}
+
+# PLAIN list function
+plain_list() {
+    IFS=$'\n'
+    while read str; do
+        [[ -z "$str" ]] && continue;
+        parse_object_kv_list "$str"
+        echo -ne "$LISTNAME\t$IP_VERSION\t$AUTOUPDATE\t$SUSPENDED\t$SOURCE\t"
+        echo -e "$TIME\t$DATE"
+    done < <(cat $HESTIA/data/firewall/ipset.conf)
+}
+
+# CSV list function
+csv_list() {
+    IFS=$'\n'
+    echo "LISTNAME,IP_VERSION,AUTOUPDATE,SUSPENDED,SOURCE,SUSPENDED,TIME,DATE"
+    while read str; do
+        [[ -z "$str" ]] && continue;
+        parse_object_kv_list "$str"
+        echo -n "$LISTNAME,$IP_VERSION,$AUTOUPDATE,$SUSPENDED,\"$SOURCE\","
+        echo "$TIME,$DATE"
+    done < <(cat $HESTIA/data/firewall/ipset.conf)
+}
+
+# Listing data
+case $format in
+    json)   json_list ;;
+    plain)  plain_list ;;
+    csv)    csv_list ;;
+    shell)  shell_list |column -t -s '^' ;;
+esac
+
+
+#----------------------------------------------------------#
+#                       Hestia                             #
+#----------------------------------------------------------#
+
+exit

bin/v-update-firewall-ipset

@@ -0,0 +1,62 @@
+#!/bin/bash
+# info: update firewall ipset
+# options: [REFRESH]
+#
+# The function creates ipset lists and updates the lists if they are expired or ondemand
+
+
+#----------------------------------------------------------#
+#                    Variable&Function                     #
+#----------------------------------------------------------#
+
+# Force refresh
+force=${1-no}
+
+# Includes
+source $HESTIA/func/main.sh
+source $HESTIA/conf/hestia.conf
+
+
+#----------------------------------------------------------#
+#                    Verifications                         #
+#----------------------------------------------------------#
+
+is_boolean_format_valid "$force" 'bool (yes/no)'
+is_system_enabled "$FIREWALL_SYSTEM" 'FIREWALL_SYSTEM'
+
+ipset_hstobject='../../data/firewall/ipset'
+
+for ipset_name in $(search_objects "$ipset_hstobject" 'SUSPENDED' 'no' 'LISTNAME'); do
+
+    ipset_time="$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ipset_name" '$TIME')"
+    ipset_date="$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ipset_name" '$DATE')"
+    ipset_au="$(get_object_value "$ipset_hstobject" 'LISTNAME' "$ipset_name" '$AUTOUPDATE')"
+
+    if [ "$ipset_au" = 'no' ] ; then
+        # load existing ip list files in the kernel but don't auto update them
+        $BIN/v-add-firewall-ipset "$ipset_name"
+        continue
+    fi
+
+    last_updated_ts=$(date -d "$ipset_date $ipset_time" +%s)
+    now=$(date +%s)
+    hours_since_update=$(( (now - last_updated_ts) / 60 ))
+
+    if [[ "$hours_since_update" -lt 70 ]] && [ "$force" = 'no' ]; then
+        # load existing ip list files in the kernel but don't auto update them
+        $BIN/v-add-firewall-ipset "$ipset_name"
+        continue
+    fi
+
+    $BIN/v-add-firewall-ipset "$ipset_name" '' '' '' 'yes'
+done
+
+
+#----------------------------------------------------------#
+#                       Hestia                             #
+#----------------------------------------------------------#
+
+# Logging
+log_event "$OK" "$ARGUMENTS"
+
+exit