README.md

@@ -2,10 +2,10 @@
 
 ZFS replication script by Thorsten Spille <thorsten@spille-edv.de>
 - replicates ZFS filesystems/volumes with user parameter bashclub:zsync (or custom name) configured
-- creates optional snapshot before replication (required zfs-auto-znapshot)
+- creates optional snapshot before replication (via zfs-auto-snapshot or included remote-snapshot-manager)
 - parameter setting uses zfs hierarchy on source
 - mirrored replication with existing snapshots (filtered by snapshot_filter)
-- pull/local replication only
+- pull replication only
 - auto creates full path on target pool, enforce com.sun:auto-snapshot=false, inherits mountpoint and sets canmount=noauto
 - raw replication
 - tested on Proxmox VE 7.x/8.x
@@ -25,7 +25,10 @@ apt install bashclub-zsync
 ~~~
 wget -q --no-cache -O /usr/bin/bashclub-zsync https://gitlab.bashclub.org/bashclub/zsync/-/raw/main/bashclub-zsync/usr/bin/bashclub-zsync
 chmod +x /usr/bin/bashclub-zsync
-bashclub-zsync
+
+# optional: download remote-snapshot-manager
+wget -q --no-cache -O /usr/bin/remote-snapshot-manager https://gitlab.bashclub.org/bashclub/zsync/-/raw/main/bashclub-zsync/usr/bin/remote-snapshot-manager
+chmod +x /usr/bin/remote-snapshot-manager
 ~~~
 
 ## Documentation
@@ -33,7 +36,8 @@ bashclub-zsync
 
 
 ## Configuration
-After first execution adjust the default config file `/etc/bashclub/zsync.conf`:
+After first execution bashclub-zsync will create `/etc/bashclub/zsync.conf` if no configuration parameter is given and file not exists.
+The Debian installation package provides `/etc/bashclub/zsync.conf.example` with the default values.
 
 ~~~
 # replication target path on local machine 
@@ -60,6 +64,9 @@ zfs_auto_snapshot_keep=0
 # make snapshot via zfs-auto-snapshot before replication
 zfs_auto_snapshot_label="backup"
 
+# select snapshot engine: "zas" or "internal"
+zfs_auto_snapshot_engine="zas"
+
 # disable checkzfs with value > 0
 checkzfs_disabled=0
 
@@ -92,6 +99,17 @@ File: /etc/cron.hourly/bashclub-zsync
 /usr/bin/bashclub-zsync -c /etc/bashclub/zsync.conf > /var/log/bashclub-zsync/zsync.log
 ~~~
 
+# Roadmap
+
+Following features are on the wishlist:
+- Local replication without SSH connections
+- Removable device support (Autostart on connect, remove when finished)
+- E-Mail notifications
+- Internal verification logic for checkzfs replacement
+- Parallel replication (multi-threaded, per dataset)
+- Resume replication after failure caused by changes on source
+
+
 # Author
 
 ### Thorsten Spille

bashclub-zsync/DEBIAN/postinst

@@ -11,6 +11,8 @@ fi
 
 chown root:root /usr/bin/bashclub-zsync
 chmod 755 /usr/bin/bashclub-zsync
+chown root:root /usr/bin/remote-snapshot-manager
+chmod 755 /usr/bin/remote-snapshot-manager
 
 if [ -f /etc/logrotate.d/bashclub_zsync ]; then
   rm -f /etc/logrotate.d/bashclub_zsync

bashclub-zsync/etc/bashclub/zsync.conf.example

@@ -22,6 +22,9 @@ zfs_auto_snapshot_keep=0
 # make snapshot via zfs-auto-snapshot before replication
 zfs_auto_snapshot_label="backup"
 
+# select the zfs auto snapshot engine (zfs-auto-snapshot = "zas" or remote-snapshot-manager = "internal")
+zfs_auto_snapshot_engine="zas"
+
 # disable checkzfs with value > 0
 checkzfs_disabled=0
 

bashclub-zsync/usr/bin/bashclub-zsync

@@ -3,7 +3,7 @@
 # bashclub zfs replication script
 # Author: (C) 2024 Thorsten Spille <thorsten@spille-edv.de>
 
-__version__=v1.0.15
+__version__=v1.0.16
 
 PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 
@@ -21,6 +21,7 @@ sed=$(which sed)
 zfs_auto_snapshot=$(which zfs-auto-snapshot)
 checkzfs=$(which checkzfs)
 mv=$(which mv)
+rsm=$(which remote-snapshot-manager)
 
 rc=0
 debug=
@@ -53,6 +54,9 @@ zfs_auto_snapshot_keep=0
 # make snapshot via zfs-auto-snapshot before replication
 zfs_auto_snapshot_label="backup"
 
+# select snapshot engine: "zas" or "internal"
+zfs_auto_snapshot_engine="zas"
+
 # disable checkzfs with value > 0
 checkzfs_disabled=0
 
@@ -131,6 +135,7 @@ snapshot_filter="$snapshot_filter"
 min_keep=$min_keep
 zfs_auto_snapshot_keep=$zfs_auto_snapshot_keep
 zfs_auto_snapshot_label=$zfs_auto_snapshot_label
+zfs_auto_snapshot_engine=$zfs_auto_snapshot_engine
 checkzfs_disabled=$checkzfs_disabled
 checkzfs_local=$checkzfs_local
 checkzfs_prefix=$checkzfs_prefix
@@ -147,10 +152,10 @@ if [[ $source == "" ]]; then
     log "[INFO] source is empty, switching to local mode."
     ssh=
     zsync_sshport=
-    log "[INFO] Configuration:\n\ttarget=$target\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n\ŧzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\n\tzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n\tcheckzfs_disabled=$checkzfs_disabled\n\tcheckzfs_local=$checkzfs_local\n\tcheckzfs_prefix=$checkzfs_prefix\n\tcheckzfs_max_age=$checkzfs_max_age\n\tcheckzfs_max_snapshot_count=$checkzfs_max_snapshot_count\n\tcheckzfs_spool=$checkzfs_spool\n\tcheckzfs_spool_maxage=$checkzfs_spool_maxage\n"
+    log "[INFO] Configuration:\n\ttarget=$target\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n\ŧzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\n\tzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n\tzfs_auto_snapshot_engine=$zfs_auto_snapshot_engine\n\tcheckzfs_disabled=$checkzfs_disabled\n\tcheckzfs_local=$checkzfs_local\n\tcheckzfs_prefix=$checkzfs_prefix\n\tcheckzfs_max_age=$checkzfs_max_age\n\tcheckzfs_max_snapshot_count=$checkzfs_max_snapshot_count\n\tcheckzfs_spool=$checkzfs_spool\n\tcheckzfs_spool_maxage=$checkzfs_spool_maxage\n"
 else
     zsync_sshport=-p$sshport
-    log "[INFO] Configuration:\n\ttarget=$target\n\tsource=$source\n\tsshport=$sshport\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n\tzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\n\tzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n\tcheckzfs_disabled=$checkzfs_disabled\n\tcheckzfs_local=$checkzfs_local\n\tcheckzfs_prefix=$checkzfs_prefix\n\tcheckzfs_max_age=$checkzfs_max_age\n\tcheckzfs_max_snapshot_count=$checkzfs_max_snapshot_count\n\tcheckzfs_spool=$checkzfs_spool\n\tcheckzfs_spool_maxage=$checkzfs_spool_maxage\n"
+    log "[INFO] Configuration:\n\ttarget=$target\n\tsource=$source\n\tsshport=$sshport\n\ttag=$tag\n\tsnapshot_filter=$snapshot_filter\n\tmin_keep=$min_keep\n\tzfs_auto_snapshot_keep=$zfs_auto_snapshot_keep\n\tzfs_auto_snapshot_label=$zfs_auto_snapshot_label\n\tzfs_auto_snapshot_engine=$zfs_auto_snapshot_engine\n\tcheckzfs_disabled=$checkzfs_disabled\n\tcheckzfs_local=$checkzfs_local\n\tcheckzfs_prefix=$checkzfs_prefix\n\tcheckzfs_max_age=$checkzfs_max_age\n\tcheckzfs_max_snapshot_count=$checkzfs_max_snapshot_count\n\tcheckzfs_spool=$checkzfs_spool\n\tcheckzfs_spool_maxage=$checkzfs_spool_maxage\n"
 fi
 
 controlmaster=-oControlMaster=auto
@@ -225,8 +230,11 @@ else
 fi
 
 if [ $zfs_auto_snapshot_keep -gt 1 ]; then
-    if [[ $debug == "-v" ]]; then log "[DEBUG] Running zfs-auto-snapshot"; fi
-    $ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "which zfs-auto-snapshot > /dev/null || exit 0 ; zfs-auto-snapshot --quiet --syslog --label=$zfs_auto_snapshot_label --keep=$zfs_auto_snapshot_keep //"
+    if [[ $zfs_auto_snapshot_engine == "zas" ]]; then
+        if [[ $debug == "-v" ]]; then log "[DEBUG] Running zfs-auto-snapshot"; fi
+        $ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "which zfs-auto-snapshot > /dev/null || exit 0 ; zfs-auto-snapshot --quiet --syslog --label=$zfs_auto_snapshot_label --keep=$zfs_auto_snapshot_keep //"
+    fi
+
     if [[ $snapshot_filter == "" ]]; then
         snapshot_filter="$zfs_auto_snapshot_label"
     else
@@ -245,6 +253,13 @@ for name in "${syncvols[@]}"; do
         cm=
     fi
 
+    if [ $zfs_auto_snapshot_keep -gt 1 ]; then
+        if [[ $zfs_auto_snapshot_engine == "internal" ]]; then
+            if [[ $debug == "-v" ]]; then log "[DEBUG] Running remote-snapshot-manager"; fi
+            $rsm $debug -d $name -k $zfs_auto_snapshot_keep -l $zfs_auto_snapshot_label -h $source -p $sshport
+        fi
+    fi
+
     if [[ $($ssh $controlmaster $controlpath $controlpersist $sshcipher $zsync_sshport $source "zfs list -H -t snapshot -o name -S creation $name 2>/dev/null | grep -E \"@.*($snapshot_filter)\" | wc -l | tr -d ' '") -gt 0 ]]; then
         IFS=$' '
         if ! $zfs list -H $target/$name > /dev/null 2>&1 ; then

bashclub-zsync/usr/bin/bashclub-zsync.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# 
+# bashclub zfs replication script
+# Author: (C) 2025 Thorsten Spille <thorsten@spille-edv.de>
+
+from argparse import Namespace
+import os
+import subprocess
+import sys
+import configparser
+import shutil
+from enum import Enum
+from datetime import datetime
+
+__version__ = "2.0-001"
+
+class Severity (Enum):
+    DEBUG = 7
+    INFO = 6
+    NOTICE = 5
+    WARN = 4
+    ERROR = 3
+    CRITICAL = 2
+    ALERT = 1
+    EMERGENCY = 0
+
+__loglevel: Severity = Severity.INFO
+
+def log(message: str, severity: Severity) -> None:
+    if severity.value <= __loglevel.value:
+        print(f"{datetime.now().strftime('%Y-%m-%d %T %Z')} [{severity.name}] {message}")
+
+
+def usage(exit_code=0):
+    print(f"""
+    bashclub-zsync.py Version {__version__}
+    -------------------------------------------------------------------------------
+    Usage: bashclub-zsync.py [-h] [-d] [-c CONFIG]
+        Creates a mirrored replication of configured ZFS filesystems/volumes
+
+        -c CONFIG    Configuration file for this script
+        -d           Debug mode
+    -------------------------------------------------------------------------------
+    (C) 2024 by Spille IT Solutions for bashclub (github.com/bashclub)
+    Author: Thorsten Spille <thorsten@spille-edv.de>
+    -------------------------------------------------------------------------------
+    """)
+    sys.exit(exit_code)
+
+def load_config(config_path) -> configparser.ConfigParser:
+    config: dict  = {}
+    if os.path.exists(config_path):
+        log(f"Reading configuration {config_path}",Severity.INFO)
+        with open(config_path) as file:
+            for line in file:
+                if line[0] != '#' and line[0] != '\n':
+                    key, value= line.partition("=")[::2]
+                    value = value.removesuffix('\n')
+                    config[key] = value
+    else:
+        log(f"Config file {config_path} not found.", Severity.CRITICAL)
+        usage(1)
+    return config
+
+def execute(command: str, failed_when: bool = False, debug: bool = False) -> str:
+    if debug:
+        log(f"Executing command: {command}", Severity.DEBUG)
+    try:
+        result: subprocess.CompletedProcess[str] = subprocess.run(command, shell=True, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        return result.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        log(f"Command failed: {e}", Severity.ERROR)
+        if failed_when:
+          usage(e.returncode)
+
+def check_command_availability(commands) -> dict:
+    cmds: dict = {}
+    for cmd in commands:
+        log(f"Check command '{cmd}'...", Severity.DEBUG)
+        try:
+            cmdpath: str = shutil.which(cmd)
+            log(f"Command '{cmd}' is located at '{cmdpath}'", Severity.DEBUG)
+            cmds[cmd] = cmdpath
+        except:
+            log(f"Required command '{cmd}' not found.", Severity.ERROR)
+            usage(1)
+    
+    return cmds
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="ZFS replication script in Python")
+    parser.add_argument("-c", "--config", type=str, default="/etc/bashclub/zsync.conf", help="Path to config file")
+    parser.add_argument("-d", "--debug", action="store_true", help="Enable debug mode")
+    args: Namespace = parser.parse_args()
+
+    config_path = args.config
+    debug = args.debug
+    local_mode: bool = False
+
+    config: configparser.ConfigParser = load_config(config_path)
+
+    # Extract configuration variables
+    target: str = config.get("target", "pool/dataset")
+    source: str = config.get("source", "user@host")
+    if source == "":
+        local_mode = True
+    sshport: int = config.get("sshport", int(22))
+    tag: str = config.get("tag", "bashclub:zsync")
+    snapshot_filter: str = config.get("snapshot_filter", "hourly|daily|weekly|monthly")
+    min_keep: int = config.get("min_keep", 3)
+
+    cmds: dict = check_command_availability(["zfs", "ssh", "scp", "checkzfs" ])
+
+    log(f"source: {source}, target: {target}, sshport: {sshport}, tag: {tag}, snapshot_filter: {snapshot_filter}, min_keep: {min_keep}", Severity.INFO)
+
+    # Add logic for replication using ZFS and SSH commands
+
+if __name__ == "__main__":
+    main()

bashclub-zsync/usr/bin/remote-snapshot-manager

@@ -0,0 +1,134 @@
+#!/bin/bash
+#
+# ZFS Snapshot Management Script
+# Author: (C) 2024 Thorsten Spille <thorsten@spille-edv.de>
+
+__version__=v1.0.0
+
+# Function to display usage information
+usage() {
+    echo "Usage: $0 -d <dataset> -k <keep> -l <label> [-f <dateformat>] [-v] [-h <host>] [-p <port>]"
+    echo "  -d: Name of the ZFS dataset"
+    echo "  -k: Number of snapshots to keep"
+    echo "  -l: Label for the snapshot"
+    echo "  -f: (Optional) Date format to append to the snapshot name (default: '%Y-%m-%d-%H%M')"
+    echo "  -v: (Optional) Enable verbose/debug logging"
+    echo "  -h: (Optional) Remote host for SSH management"
+    echo "  -p: (Optional) SSH port (default: 22)"
+    exit 1
+}
+
+# Logging function
+function log() {
+    echo -e "$(date +'%b %d %T') $1"
+}
+
+# Debug logging function
+function debug() {
+    if [[ "$verbose" == true ]]; then
+        echo -e "$(date +'%b %d %T') [DEBUG] $1"
+    fi
+}
+
+# Default date format
+default_dateformat="%Y-%m-%d-%H%M"
+verbose=false
+ssh_host=""
+ssh_port=22
+
+# Parse command-line arguments
+while getopts "d:k:l:f:vh:p:" opt; do
+    case "$opt" in
+        d) dataset="$OPTARG" ;;
+        k) keep="$OPTARG" ;;
+        l) label="$OPTARG" ;;
+        f) dateformat="$OPTARG" ;;
+        v) verbose=true ;;
+        h) ssh_host="$OPTARG" ;;
+        p) ssh_port="$OPTARG" ;;
+        *) usage ;;
+    esac
+done
+
+# Validate required parameters
+if [[ -z "$dataset" || -z "$keep" || -z "$label" ]]; then
+    usage
+fi
+
+# Set dateformat to default if not provided
+if [[ -z "$dateformat" ]]; then
+    dateformat="$default_dateformat"
+fi
+
+# Validate "keep" parameter is a number
+if ! [[ "$keep" =~ ^[0-9]+$ ]]; then
+    log "Error: 'keep' must be a positive integer."
+    exit 1
+fi
+
+# Base command for ZFS operations
+zfs_cmd="zfs"
+if [[ -n "$ssh_host" ]]; then
+    # Test SSH connection before proceeding
+    debug "Testing SSH connection to $ssh_host on port $ssh_port"
+    ssh -q -o PasswordAuthentication=no -p $ssh_port $ssh_host exit
+    if [[ $? -ne 0 ]]; then
+        log "Error: Unable to connect to $ssh_host on port $ssh_port. Ensure SSH access is configured correctly."
+        exit 1
+    fi
+    zfs_cmd="ssh -p $ssh_port $ssh_host zfs"
+    debug "Using remote host: $ssh_host (port $ssh_port)"
+fi
+
+debug "Dataset: $dataset"
+debug "Keep: $keep"
+debug "Label: $label"
+debug "Date format: $dateformat"
+debug "Verbose mode: $verbose"
+
+# Generate the snapshot name
+snapshot_name="${dataset}@${label}-$(date -u +"$dateformat")"
+debug "Generated snapshot name: $snapshot_name"
+
+# Create the ZFS snapshot
+log "Creating snapshot: $snapshot_name"
+$zfs_cmd snapshot "$snapshot_name"
+if [[ $? -ne 0 ]]; then
+    log "Error: Failed to create snapshot."
+    exit 1
+fi
+
+# List existing snapshots for the dataset
+debug "Listing existing snapshots for dataset: $dataset"
+existing_snapshots=$($zfs_cmd list -t snapshot -o name -s creation "$dataset" | grep "^${dataset}@${label}-")
+debug "Existing snapshots:\n$existing_snapshots"
+
+# Count the total number of snapshots matching the label
+snapshot_count=$(echo "$existing_snapshots" | wc -l)
+debug "Snapshot count: $snapshot_count"
+
+# Check if pruning is necessary
+if [[ "$snapshot_count" -gt "$keep" ]]; then
+    # Calculate the number of snapshots to delete
+    delete_count=$((snapshot_count - keep))
+    debug "Number of snapshots to delete: $delete_count"
+
+    # Get the snapshots to delete (oldest first)
+    snapshots_to_delete=$(echo "$existing_snapshots" | head -n "$delete_count")
+    debug "Snapshots to delete:\n$snapshots_to_delete"
+
+    # Delete the old snapshots
+    log "Deleting $delete_count old snapshot(s):"
+    for snapshot in $snapshots_to_delete; do
+        log "Deleting snapshot: $snapshot"
+        $zfs_cmd destroy "$snapshot"
+        if [[ $? -ne 0 ]]; then
+            log "Error: Failed to delete snapshot $snapshot."
+        fi
+    done
+else
+    log "No snapshots to delete. Total snapshots ($snapshot_count) <= keep ($keep)."
+fi
+
+log "Snapshot management completed."
+exit 0

build-package

@@ -21,6 +21,7 @@ chmod 755 $package/DEBIAN/p*
 
 # remove bashclub-zsync-config from package
 rm $package/usr/bin/bashclub-zsync-config
+rm $package/usr/bin/bashclub-zsync.py
 
 # add license and changelog
 mkdir -p ./$package/usr/share/doc/$package/copyright