diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..5443da9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docs/themes/docdock"] + path = docs/themes/docdock + url = git@github.com:zrepl/hugo-theme-docdock.git diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..364fdec --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +public/ diff --git a/docs/archetypes/default.md b/docs/archetypes/default.md new file mode 100644 index 0000000..f5a9e45 --- /dev/null +++ b/docs/archetypes/default.md @@ -0,0 +1,6 @@ +--- +title: "{{ replace .TranslationBaseName "-" " " | title }}" +date: {{ .Date }} +draft: true +--- + diff --git a/docs/config.toml b/docs/config.toml new file mode 100644 index 0000000..d459d44 --- /dev/null +++ b/docs/config.toml @@ -0,0 +1,17 @@ +baseURL = "https://zrepl.github.io" +languageCode = "en-us" +title = "zrepl - ZFS replication" +theme = "docdock" + +[[menu.shortcuts]] +pre = "

More

" +name = " GitHub repo" +identifier = "ds" +url = "https://github.com/zrepl/zrepl" +weight = 10 + +[[menu.shortcuts]] +name = " Primary Author on Twitter" +url = "https://twitter.com/problame" +weight = 11 + diff --git a/docs/content/_header.md b/docs/content/_header.md new file mode 100644 index 0000000..3951719 --- /dev/null +++ b/docs/content/_header.md @@ -0,0 +1 @@ +zrepl Documentation diff --git a/docs/content/_index.md b/docs/content/_index.md new file mode 100644 index 0000000..85c3d9c --- /dev/null +++ b/docs/content/_index.md @@ -0,0 +1,39 @@ ++++ +title = "zrepl - ZFS replication" ++++ + +# zrepl - ZFS replication + +`zrepl` is a tool for replicating ZFS filesystems. + +{{% panel theme="danger" header="Important" %}} +`zrepl` as well as this documentation is still under active +development. Some of the features below are not implemented yet. Use & test at your own risk ;) +{{% / panel %}} + +## Main Features + +* filesystem replication + * local & over network (SSH) + * push & pull mode + * snapshots & bookmarks support + * feature-negotiation for + * resumable `send & receive` + * compressed `send & receive` + * raw encrypted `send & receive` (as soon as it is available) + * access control checks when pulling datasets + * flexible mappings for filesystems +* automatic snapshot creation + * periodic interval +* automatic snapshot pruning + * *retention grid* TODO link explain + +## Contributing + +We are happy about contributors, both for the `zrepl` codebase and theses docs. +Feel free to open a ticket or even submit a pull request ;) + +* [zrepl GitHub repository](https://github.com/zrepl/zrepl) +* these docs: + * were started on 2017-08-08 and are not committed yet. + * please use GitHub flavored markdown and open an issue in the `zrepl` repo diff --git a/docs/content/configuration/_index.md b/docs/content/configuration/_index.md new file mode 100644 index 0000000..791bcd1 --- /dev/null +++ b/docs/content/configuration/_index.md @@ -0,0 +1,6 @@ ++++ +title = "Configuration" +alwaysopen = true ++++ + +{{% children description="true" %}} diff --git a/docs/content/configuration/example_pull_backup.md b/docs/content/configuration/example_pull_backup.md new file mode 100644 index 0000000..7283f6e --- /dev/null +++ b/docs/content/configuration/example_pull_backup.md @@ -0,0 +1,9 @@ ++++ +title = "Example: Pull Backup" +description = "Example configuration for a typical pull-backup scenario, e.g. server to server" ++++ + +Example configuration for a typical pull-backup scenario, e.g. server to server + +{{% alert theme="warning"%}}TBD{{% /alert %}} + diff --git a/docs/content/configuration/example_push_backup.md b/docs/content/configuration/example_push_backup.md new file mode 100644 index 0000000..ffb70ef --- /dev/null +++ b/docs/content/configuration/example_push_backup.md @@ -0,0 +1,8 @@ ++++ +title = "Example: Push Backup" +description = "Example configuration for a typical push-backup scenario, e.g. laptop to NAS" ++++ + +Example configuration for a typical push-backup scenario, e.g. laptop to NAS + +{{% alert theme="warning"%}}TBD{{% /alert %}} diff --git a/docs/content/configuration/overview.md b/docs/content/configuration/overview.md new file mode 100644 index 0000000..b8bd402 --- /dev/null +++ b/docs/content/configuration/overview.md @@ -0,0 +1,144 @@ ++++ +title = "Overview" +weight = 100 +description = "Configuration format, SSH authentication, etc." ++++ + +{{% panel header="Recommendation" %}} +Keep the [sample configuration file](https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/zrepl.yml) open on the side while reading this document! +{{% / panel %}} + +All configuration is managed in a single YAML file.
+It is structured by sections roughly corresponding to `zrepl` subcommands: + +```yaml +# REPLICATION +# Remote zrepl instances where pull and push jobs connect to +remotes: + name_of_remote: #... +# Push jobs (replication from local to remote) +pushs: + name_of_push_job: #... + name_of_other_push_job: #... +# pull jobs (replication from remote to local & local to local) +pulls: + name_of_pull_job: #... +# mapping incoming pushs to local datasets +sinks: + client_identity: #... +# access control for remote pull jobs +pull_acls: + client_identity: #... + +# SNAPSHOT MANAGEMENT +# Automatic snapshotting of filesystems +autosnap: + name_of_autosnap_job: #... +# Automatic pruning of snapshots based on creation date +prune: + name_of_prune_job: #... +``` + +When using `zrepl(8)`, a *subcommand* is passed the *job name* as a positional argument: + +```yaml +autosnap: # subcommand + db: # job name + prefix: zrepl_ + interval: 10m + dataset_filter: { + "tank/db<": ok + } +``` +```bash +$ zrepl autosnap --config zrepl.yml db +``` + +Run `zrepl --help` for a list of subcommands and options. + +## Mapping & Filter Syntax + +For various job types, a filesystem `mapping` or `filter` needs to be +specified. Both have in common that they return a result for a given filesystem +path (in the ZFS filesystem hierarchy). + +A mapping returns a *target filesystem*, a filter returns a *filter result*. +The matching syntax & rules are the same for mappings and filters. + +#### Matching Rules + +Given a mapping / filter with a list of patterns and results, which result depends on the following rules: + +* More specific path patterns win over less specific ones +* Direct mappings win over *subtree wildcards* (`<` at end of pattern) + +{{% panel %}} +The **subtree wildcard** `<` means "*the dataset left of `<` and all its children*". +{{% / panel %}} + +##### Example + +``` +# Rule number and its pattern +1: tank< +2: tank/foo/bar +3: tank/foo< + +# Which rule applies to given path? +tank/foo/bar/loo => 3 +tank/bar => 1 +tank/foo/bar => 2 +zroot => NO MATCH +tank/var/log => 1 +``` + +#### Mappings + +The example below shows a pull job that would pull remote datasets to the given local paths.
+If there exists no mapping (`NO MATCH`), the filesystem will not be pulled. + +```yaml +pull: + app01.example.com: # client identity + from: app01.example.com + mapping: { + "tank/var/db<": "zroot/backups/app01/tank/var/db", + "tank/var/www<": "zroot/backups/app01/tank/var/www", + "tank/var/log": "zroot/logbackup/app01", + } +``` + +``` +tank/var/db` => zroot/backups/app01/tank/var/db +tank/var/db/a/child => zroot/backups/app01/tank/var/db/a/child +... +tank/var/www => zroot/backups/app01/tank/var/www +tank/var/www/a/child => zroot/backups/app01/tank/var/www/a/child +... +tank/var/log => zroot/logbackup/app01 +tank/var/log/foo => NOT MAPPED +``` + +#### Filters + +Valid filter results: `ok` or `omit`. + +The example below shows a pull ACL that allows access to the user homes but not +to the rest of the system's datasets. + +``` +# Example for filter syntax +pull_acls: + backups.example.com: # client identity + filter: { + "<": omit, + "tank/usr/home<": ok, + } +``` + +## Next up + +* [Automating snapshot creation & pruning]({{< ref "configuration/snapshots.md" >}}) + +* [Replicating filesystems]({{< ref "configuration/replication.md" >}}) + diff --git a/docs/content/configuration/replication.md b/docs/content/configuration/replication.md new file mode 100644 index 0000000..81d6998 --- /dev/null +++ b/docs/content/configuration/replication.md @@ -0,0 +1,40 @@ ++++ +title = "Filesystem Replication" +description = "Replicating filesystems with existing bookmarks & snapshots" +weight = 300 ++++ + +{{% alert theme="warning"%}}Under Construction{{% /alert %}} + +### Remotes + +The `remotes` section specifies remote `zrepl` instances from which to pull from / push backups to: + +```yaml +remotes: + offsite_backups: + transport: + ssh: + host: 192.168.122.6 + user: root + port: 22 + identity_file: /etc/zrepl/identities/offsite_backups +``` + +#### SSH Transport + +The SSH transport connects to the remote server using the SSH binary in +`$PATH` and the parameters specified in the `zrepl` config file. + +However, instead of a traditional interactive SSH session, `zrepl` expects +another instance of `zrepl` on the other side of the connection; You may be +familiar with this concept from [git shell](https://git-scm.com/docs/git-shell) +or [Borg Backup](https://borgbackup.readthedocs.io/en/stable/deployment.html). + +Check the examples for instructions on how to set this up on your machines! + +{{% panel %}} +The environment variables of the underlying SSH process are cleared. `$SSH_AUTH_SOCK` will not be available. We suggest creating a separate, unencrypted SSH key. +{{% / panel %}} + + diff --git a/docs/content/configuration/snapshots.md b/docs/content/configuration/snapshots.md new file mode 100644 index 0000000..0981cae --- /dev/null +++ b/docs/content/configuration/snapshots.md @@ -0,0 +1,7 @@ ++++ +title = "Snapshot Management" +description = "Automated snapshot creation & pruning" +weight = 200 ++++ + +{{% alert theme="warning"%}}TBD{{% /alert %}} diff --git a/docs/content/impl/_index.md b/docs/content/impl/_index.md new file mode 100644 index 0000000..973ba71 --- /dev/null +++ b/docs/content/impl/_index.md @@ -0,0 +1,44 @@ ++++ +title = "Implementation Overview" ++++ + +The following design aspects may convince you that `zrepl` is superior to a hacked-together shell script solution. + +## Language + +`zrepl` is written in Go, a real programming language with type safety, +reasonable performance, testing infrastructure and an (opinionated) idea of +software engineering. + +* key parts & algorithms of `zrepl` are covered by unit tests +* zrepl is noticably faster than comparable shell scripts + + +## RPC protocol + +While it is tempting to just issue a few `ssh remote 'zfs send ...' | zfs recv`, this has a number of drawbacks: + +* The snapshot streams need to be compatible. +* Communication is still unidirectional. Thus, you will most likely + * either not take advantage of features such as *compressed send & recv* + * or issue additional `ssh` commands in advance to figure out what features are supported on the other side. +* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain. + +`zrepl` takes a different approach: + +* Define an RPC protocol. +* Establish an encrypted, authenticated, bidirectional communication channel... +* ... with `zrepl` running at both ends of it. + + This has several obvious benefits: + +* No blank root shell access is given to the other side. +* Instead, access control lists (ACLs) are used to grant permissions to *authenticated* peers. +* The transport mechanism is decoupled from the remaining logic, keeping it extensible (e.g. TCP+TLS) + +{{% panel %}} +Currently, the bidirectional communication channel is multiplexed on top of a single SSH connection. +Local replication is of course handled efficiently via simple method calls +See TODO for details. +{{% / panel %}} + diff --git a/docs/publish.sh b/docs/publish.sh new file mode 100755 index 0000000..46369dc --- /dev/null +++ b/docs/publish.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -eo pipefail + +GHPAGESREPO="git@github.com:zrepl/zrepl.github.io.git" + +exit_msg() { + echo "error, exiting..." +} +trap exit_msg EXIT + +echo -n "PRESS ENTER to confirm you commited the docs changes to the zrepl repo" +read + +cd public +echo "verify we're in the GitHub pages repo..." +git remote get-url origin | grep -E "^${GHPAGESREPO}\$" +echo "cleaning GitHub pages repo" +git clean -dn +cd .. +echo "building site" +hugo +cd public +echo "adding and commiting all changes in GitHub pages repo" +git add -A +git commit -m "hugo render from publish.sh - `date -u`" +git push origin master + diff --git a/docs/themes/docdock b/docs/themes/docdock new file mode 160000 index 0000000..2b29738 --- /dev/null +++ b/docs/themes/docdock @@ -0,0 +1 @@ +Subproject commit 2b29738da2e8efe45a567937ada0fcdf9beaddd5