extern/zsync
1
0
Fork 1
mirror of http://gitlab.bashclub.org/bashclub/zsync.git synced 2024-11-27 05:33:08 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

SSH performance tuning

Browse Source
This commit is contained in:
Administrator 2024-07-13 16:54:01 +00:00
parent e3ce1dd8be
commit 63e3fdfddc
1 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
bashclub-zsync/usr/bin/bashclub-zsync
View File

@ -21,6 +21,7 @@ sed=$(which sed)
zfs_auto_snapshot=$(which zfs-auto-snapshot) zfs_auto_snapshot=$(which zfs-auto-snapshot)
checkzfs=$(which checkzfs) checkzfs=$(which checkzfs)
mv=$(which mv) mv=$(which mv)
sshopts=-o\ ControlMaster=auto\ -o\ ControlPath=~/.ssh/bashclub-zsync-%r@%h-%p\ -o\ ControlPersist=15
rc=0 rc=0
debug= debug=
@ -154,7 +155,7 @@ else
fi fi
local_os_id=$($grep -E "^ID=" /etc/os-release | $cut -d'=' -f2) local_os_id=$($grep -E "^ID=" /etc/os-release | $cut -d'=' -f2)
remote_os_id=$($ssh $source $zsync_sshport "grep -E \"^ID=\" /etc/os-release | cut -d'=' -f2") remote_os_id=$($ssh $sshopts $source $zsync_sshport "grep -E \"^ID=\" /etc/os-release | cut -d'=' -f2")
if [[ $local_os_id == "freebsd" ]]; then if [[ $local_os_id == "freebsd" ]]; then
local_aes=$($grep -o AES /var/run/dmesg.boot | $uniq) local_aes=$($grep -o AES /var/run/dmesg.boot | $uniq)
@ -163,9 +164,9 @@ else
fi fi
if [[ $remote_os_id == "freebsd" ]]; then if [[ $remote_os_id == "freebsd" ]]; then
remote_aes=$($ssh $source $zsync_sshport "grep -o AES /var/run/dmesg.boot | uniq") remote_aes=$($ssh $sshopts $source $zsync_sshport "grep -o AES /var/run/dmesg.boot | uniq")
else else
remote_aes=$($ssh $source $zsync_sshport "grep -m1 -o aes /proc/cpuinfo | uniq") remote_aes=$($ssh $sshopts $source $zsync_sshport "grep -m1 -o aes /proc/cpuinfo | uniq")
fi fi
if [[ $local_aes == "aes" ]] && [[ $remote_aes == "aes" ]]; then if [[ $local_aes == "aes" ]] && [[ $remote_aes == "aes" ]]; then
@ -181,7 +182,7 @@ exclude_list="[INFO] Excluded datasets:\n"
checkzfs_headline="[INFO] checkzfs filter:" checkzfs_headline="[INFO] checkzfs filter:"
checkzfs_filter= checkzfs_filter=
IFS=$'\n' IFS=$'\n'
for zvol in $($ssh $sshcipher $zsync_sshport $source "zfs get -H -o name,value,source -t filesystem,volume $tag"); do for zvol in $($ssh $sshopts $sshcipher $zsync_sshport $source "zfs get -H -o name,value,source -t filesystem,volume $tag"); do
name=$(echo $zvol | $cut -f1) name=$(echo $zvol | $cut -f1)
if [[ "$(echo $zvol | $cut -f2)" == "subvols" ]]; then if [[ "$(echo $zvol | $cut -f2)" == "subvols" ]]; then
if [[ $(echo $zvol | $grep -vE "local|received") ]]; then if [[ $(echo $zvol | $grep -vE "local|received") ]]; then
@ -222,7 +223,7 @@ fi
if [ $zfs_auto_snapshot_keep -gt 1 ]; then if [ $zfs_auto_snapshot_keep -gt 1 ]; then
if [[ $debug == "-v" ]]; then log "[DEBUG] Running zfs-auto-snapshot"; fi if [[ $debug == "-v" ]]; then log "[DEBUG] Running zfs-auto-snapshot"; fi
$ssh $sshcipher $zsync_sshport $source "which zfs-auto-snapshot > /dev/null || exit 0 ; zfs-auto-snapshot --quiet --syslog --label=$zfs_auto_snapshot_label --keep=$zfs_auto_snapshot_keep //" $ssh $sshopts $sshcipher $zsync_sshport $source "which zfs-auto-snapshot > /dev/null || exit 0 ; zfs-auto-snapshot --quiet --syslog --label=$zfs_auto_snapshot_label --keep=$zfs_auto_snapshot_keep //"
if [[ $snapshot_filter == "" ]]; then if [[ $snapshot_filter == "" ]]; then
snapshot_filter="$zfs_auto_snapshot_label" snapshot_filter="$zfs_auto_snapshot_label"
else else
@ -232,7 +233,7 @@ fi
for name in "${syncvols[@]}"; do for name in "${syncvols[@]}"; do
log "[INFO] Replicate $name" log "[INFO] Replicate $name"
fstype=$($ssh $sshcipher $zsync_sshport $source zfs get -H -o value type $name) fstype=$($ssh $sshopts $sshcipher $zsync_sshport $source zfs get -H -o value type $name)
if [[ $fstype == "filesystem" ]]; then if [[ $fstype == "filesystem" ]]; then
mp=-xmountpoint mp=-xmountpoint
cm=-ocanmount=noauto cm=-ocanmount=noauto
@ -241,7 +242,7 @@ for name in "${syncvols[@]}"; do
cm= cm=
fi fi
if [[ $($ssh $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name 2>/dev/null | grep -E \"@.*($snapshot_filter)\" | wc -l | tr -d ' '") -gt 0 ]]; then if [[ $($ssh $sshopts $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name 2>/dev/null | grep -E \"@.*($snapshot_filter)\" | wc -l | tr -d ' '") -gt 0 ]]; then
IFS=$' ' IFS=$' '
if ! $zfs list -H $target/$name > /dev/null 2>&1 ; then if ! $zfs list -H $target/$name > /dev/null 2>&1 ; then
if [[ $debug == "-v" ]]; then log "[DEBUG] $target/$name does not exist"; fi if [[ $debug == "-v" ]]; then log "[DEBUG] $target/$name does not exist"; fi
@ -257,9 +258,9 @@ for name in "${syncvols[@]}"; do
if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start initial replication"; fi if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start initial replication"; fi
IFS=$'\n' IFS=$'\n'
for snap in $($ssh $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name | grep -E \"@.*($snapshot_filter)\" | tail -1"); do for snap in $($ssh $sshopts $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name | grep -E \"@.*($snapshot_filter)\" | tail -1"); do
log "[INFO] Start initial replication: $snap => $target/$(echo $name | $cut -d'/' -f1)" log "[INFO] Start initial replication: $snap => $target/$(echo $name | $cut -d'/' -f1)"
$ssh $sshcipher $zsync_sshport $source "zfs send -w -p $debug $snap" | $zfs receive $mp $cm -x $tag -x com.sun:auto-snapshot $debug -dF $target/$(echo $name | $cut -d'/' -f1) $ssh $sshopts $sshcipher $zsync_sshport $source "zfs send -w -p $debug $snap" | $zfs receive $mp $cm -x $tag -x com.sun:auto-snapshot $debug -dF $target/$(echo $name | $cut -d'/' -f1)
if [ $? -gt $rc ]; then rc=1; log "[ERROR] initial replication to $target/$name failed."; fi if [ $? -gt $rc ]; then rc=1; log "[ERROR] initial replication to $target/$name failed."; fi
done done
fi fi
@ -267,15 +268,15 @@ for name in "${syncvols[@]}"; do
if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start incremental replication"; fi if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Start incremental replication"; fi
guid=$($zfs list -H -o guid -s creation -t snapshot $target/$name | $tail -1) guid=$($zfs list -H -o guid -s creation -t snapshot $target/$name | $tail -1)
if [[ $guid != "" ]]; then if [[ $guid != "" ]]; then
last=$($ssh $sshcipher $zsync_sshport $source "zfs list -H -o name,guid -t snapshot $name | grep $guid | tail -1 | cut -f1") last=$($ssh $sshopts $sshcipher $zsync_sshport $source "zfs list -H -o name,guid -t snapshot $name | grep $guid | tail -1 | cut -f1")
if [[ $last != "" ]]; then if [[ $last != "" ]]; then
IFS=$'\n' IFS=$'\n'
if [[ $fstype == "filesystem" ]] && [[ $($zfs get -H -o value canmount $target/$name) != "noauto" ]]; then if [[ $fstype == "filesystem" ]] && [[ $($zfs get -H -o value canmount $target/$name) != "noauto" ]]; then
$zfs set canmount=noauto $target/$name $zfs set canmount=noauto $target/$name
fi fi
for snap in $($ssh $sshcipher $zsync_sshport $source "zfs list -H -o name,guid -s creation -t snapshot $name | grep -E \"@.*($snapshot_filter)\" | grep --after-context=200 $guid | grep -v $guid | cut -f1"); do for snap in $($ssh $sshopts $sshcipher $zsync_sshport $source "zfs list -H -o name,guid -s creation -t snapshot $name | grep -E \"@.*($snapshot_filter)\" | grep --after-context=200 $guid | grep -v $guid | cut -f1"); do
log "[INFO] Replicating delta of $last => $snap to $target/$name" log "[INFO] Replicating delta of $last => $snap to $target/$name"
$ssh $sshcipher $zsync_sshport $source "zfs send -w $debug -i $last $snap" | zfs receive -x $tag -x com.sun:auto-snapshot -F $debug $target/$name $ssh $sshopts $sshcipher $zsync_sshport $source "zfs send -w $debug -i $last $snap" | zfs receive -x $tag -x com.sun:auto-snapshot -F $debug $target/$name
if [ $? -gt $rc ]; then rc=1; log "[ERROR] incremental replication to $target/$name failed."; fi if [ $? -gt $rc ]; then rc=1; log "[ERROR] incremental replication to $target/$name failed."; fi
last=$snap last=$snap
done done
@ -293,7 +294,7 @@ for name in "${syncvols[@]}"; do
IFS=$'\n' IFS=$'\n'
for interval in $filter ; do for interval in $filter ; do
if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Checking interval $interval"; fi if [[ $debug == "-v" ]]; then log "[DEBUG] $name - Checking interval $interval"; fi
guid=$($ssh $sshcipher $zsync_sshport $source "zfs list -H -o guid,name -S creation -t snapshot $name | grep -E \"@.*$interval\" | cut -f1 | tail -1") guid=$($ssh $sshopts $sshcipher $zsync_sshport $source "zfs list -H -o guid,name -S creation -t snapshot $name | grep -E \"@.*$interval\" | cut -f1 | tail -1")
if [[ "$(echo -e "$guid" | sed 's/\n//g')" != "" ]]; then if [[ "$(echo -e "$guid" | sed 's/\n//g')" != "" ]]; then
snaps_to_delete=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $grep --after-context=200 $guid | $grep -v $guid | $cut -f1) snaps_to_delete=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $grep --after-context=200 $guid | $grep -v $guid | $cut -f1)
snap_count=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $wc -l | $tr -d ' ') snap_count=$($zfs list -H -o name,guid -S creation -t snapshot $target/$name | $grep -E "@.*$interval" | $wc -l | $tr -d ' ')