From 6f7b8ca1afe881777489118309c498ccd5885bc7 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Nov 2017 12:48:41 +0100 Subject: [PATCH] docs: adjust jobs documentation to rst + use extlinks extension --- docs/conf.py | 9 +- docs/configuration/jobs.rst | 191 ++++++++++++++--------- docs/configuration/map_filter_syntax.rst | 3 + docs/configuration/prune.rst | 2 + docs/configuration/transports.rst | 4 + docs/global.rst.inc | 3 + docs/tutorial.rst | 2 +- 7 files changed, 140 insertions(+), 74 deletions(-) create mode 100644 docs/global.rst.inc diff --git a/docs/conf.py b/docs/conf.py index 21d2e1a..8f34692 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,7 +32,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.todo', - 'sphinx.ext.githubpages'] + 'sphinx.ext.githubpages', + 'sphinx.ext.extlinks'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -155,4 +156,10 @@ texinfo_documents = [ ] +# -- Options for the extlinks extension ----------------------------------- +# http://www.sphinx-doc.org/en/stable/ext/extlinks.html +extlinks = { + 'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #'), + 'sampleconf':('https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/%s', 'cmd/sampleconf') + } diff --git a/docs/configuration/jobs.rst b/docs/configuration/jobs.rst index 9c968ca..96203c8 100644 --- a/docs/configuration/jobs.rst +++ b/docs/configuration/jobs.rst @@ -1,119 +1,163 @@ +.. include:: ../global.rst.inc + +.. |patient| replace:: :ref:`patient ` +.. |serve-transport| replace:: :ref:`serve transport` +.. |connect-transport| replace:: :ref:`connect transport` +.. |mapping| replace:: :ref:`mapping ` +.. |filter| replace:: :ref:`filter ` +.. |prune| replace:: :ref:`prune ` + Job Types ========= -A *job* is the unit of activity tracked by the zrepl daemon and configured in the [configuration file]({{< relref "install/_index.md#configuration-files" >}}). - -Every job has a unique `name`, a `type` and type-dependent fields which are documented on this page. - -Check out the [tutorial]({{< relref "tutorial/_index.md" >}}) and {{< sampleconflink >}} for examples on how job types are actually used. +A *job* is the unit of activity tracked by the zrepl daemon and configured in the |mainconfig|. +Every job has a unique ``name``, a ``type`` and type-dependent fields which are documented on this page. +Check out the :ref:`tutorial` and :sampleconf:`/` for examples on how job types are actually used. .. ATTENTION:: Currently, zrepl does not replicate filesystem properties. Whe receiving a filesystem, it is never mounted (`-u` flag) and `mountpoint=none` is set. - This is temporary and being worked on {{< zrepl-issue 24 >}}. + This is temporary and being worked on :issue:`24`. .. _job-source: Source Job ---------- -========== ======= ===================== -Parameter Default Description / Example -========== ======= ===================== -========== ======= ===================== +Example: :sampleconf:`pullbackup/productionhost.yml`. -:: +.. list-table:: + :widths: 20 80 + :header-rows: 1 - |-----|-------|-------| - |`type`||`source`| - |`name`||unique name of the job| - |`serve`||{{< zrepl-transport "serve transport" >}} specification| - |`datasets`||{{< zrepl-filter >}} for datasets to expose to client| - |`snapshot_prefix`||prefix for ZFS snapshots taken by this job| - |`interval`||snapshotting interval| - |`prune`||{{< zrepl-prune >}} policy for datasets in `datasets` with prefix `snapshot_prefix`| + * - Parameter + - Comment + * - ``type`` + - = ``source`` + * - ``name`` + - unique name of the job + * - ``serve`` + - |serve-transport| + * - ``datasets`` + - |filter| for datasets to expose to client + * - ``snapshot_prefix`` + - prefix for ZFS snapshots taken by this job + * - ``interval`` + - snapshotting interval + * - ``prune`` + - |prune| policy for datasets in ``datasets`` with prefix ``snapshot_prefix`` -* Snapshotting Task (every `interval`, {{% zrepl-job-patient %}}) - 1. A snapshot of filesystems matched by `datasets` is taken every `interval` with prefix `snapshot_prefix`. - 1. The `prune` policy is triggered on datasets matched by `datasets` with snapshots matched by `snapshot_prefix`. -* Serve Task - * Wait for connections from pull job using `serve` -A source job is the counterpart to a [pull job]({{< relref "#pull" >}}). +- Snapshotting Task (every ``interval``, |patient|) + + - A snapshot of filesystems matched by ``datasets`` is taken every ``interval`` with prefix ``snapshot_prefix``. + - The ``prune`` policy is triggered on datasets matched by ``datasets`` with snapshots matched by ``snapshot_prefix``. + +- Serve Task + + - Wait for connections from pull job using ``serve``. + +A source job is the counterpart to a :ref:`job-pull`. Note that the prune policy determines the maximum replication lag: a pull job may stop replication due to link failure, misconfiguration or administrative action. The source prune policy will eventually destroy the last common snapshot between source and pull job, requiring full replication. -Make sure you read the [prune policy documentation]({{< relref "configuration/prune.md" >}}). +Make sure you read the |prune| policy documentation. -Example: {{< sampleconflink "pullbackup/productionhost.yml" >}} .. _job-pull: Pull Job -------- -:: +Example: :sampleconf:`pullbackup/backuphost.yml` - |Parameter|Default|Description / Example| - |-----|-------|-------| - |`type`||`pull`| - |`name`||unique name of the job| - |`connect`||{{< zrepl-transport "connect transport" >}} specification| - |`interval`||Interval between pull attempts| - |`mapping`||{{< zrepl-mapping >}} for remote to local filesystems| - |`initial_repl_policy`|`most_recent`|initial replication policy| - |`snapshot_prefix`||prefix filter used for replication & pruning| - |`prune`||{{< zrepl-prune >}} policy for local filesystems reachable by `mapping`| +.. list-table:: + :widths: 20 80 + :header-rows: 1 - - Main Task (every `interval`, {{% zrepl-job-patient %}}) - #. A connection to the remote source job is established using the strategy in `connect` - #. `mapping` maps filesystems presented by the remote side to local *target filesystems* - #. Those remote filesystems with a local *target filesystem* are replicated - #. Only snapshots with prefix `snapshot_prefix` are replicated. - #. If possible, incremental replication takes place. - #. If the local target filesystem does not exist, `initial_repl_policy` is used. - #. On conflicts, an error is logged but replication of other filesystems with mapping continues. - #. The `prune` policy is triggered for all *target filesystems* + * - Parameter + - Comment + * - ``type`` + - = ``pull`` + * - ``name`` + - unqiue name of the job + * - ``connect`` + - |connect-transport| + * - ``interval`` + - Interval between pull attempts + * - ``mapping`` + - |mapping| for remote to local filesystems + * - ``initial_repl_policy`` + - default = ``most_recent``, initial replication policy + * - ``snapshot_prefix`` + - prefix snapshots must match to be considered for replication & pruning + * - ``prune`` + - |prune| policy for local filesystems reachable by ``mapping`` -A pull job is the counterpart to a [source job]({{< relref "#source" >}}). +* Main Task (every ``interval``, |patient|) + + #. A connection to the remote source job is established using the strategy in ``connect`` + #. ``mapping`` maps filesystems presented by the remote side to local *target filesystems* + #. Those remote filesystems with a local *target filesystem* are replicated + + #. Only snapshots with prefix ``snapshot_prefix`` are replicated. + #. If possible, incremental replication takes place. + #. If the local target filesystem does not exist, ``initial_repl_policy`` is used. + #. On conflicts, an error is logged but replication of other filesystems with mapping continues. + + #. The ``prune`` policy is triggered for all *target filesystems* + +A pull job is the counterpart to a :ref:`job-source`. -Example: {{< sampleconflink "pullbackup/backuphost.yml" >}} .. _job-local: Local Job --------- -:: +Example: :sampleconf:`localbackup/host1.yml` - |Parameter|Default|Description / Example| - |-----|-------|-------| - |`type`||`local`| - |`name`||unique name of the job| - |`mapping`||{{}} from source to target filesystem (both local)| - |`snapshot_prefix`||prefix for ZFS snapshots taken by this job| - |`interval`|snapshotting & replication interval| - |`initial_repl_policy`|`most_recent`|initial replication policy| - |`prune_lhs`||pruning policy on left-hand-side (source)| - |`prune_rhs`||pruning policy on right-hand-side (target)| +.. list-table:: + :widths: 20 80 + :header-rows: 1 - * Main Task (every `interval`, {{% zrepl-job-patient %}}) - 1. Evaluate `mapping` for local filesystems, those with a *target filesystem* are called *mapped filesystems*. - 1. Snapshot *mapped filesystems* with `snapshot_prefix`. - 1. Replicate *mapped filesystems* to their respective *target filesystems*: - 1. Only snapshots with prefix `snapshot_prefix` are replicated. - 1. If possible, incremental replication takes place. - 1. If the *target filesystem* does not exist, `initial_repl_policy` is used. - 1. On conflicts, an error is logged but replication of other *mapped filesystems* continues. - 1. The `prune_lhs` policy is triggered for all *mapped filesystems* - 1. The `prune_rhs` policy is triggered for all *target filesystems* + * - Parameter + - Comment + * - ``type`` + - = ``local`` + * - ``name`` + - unqiue name of the job + * - ``mapping`` + - |mapping| from source to target filesystem (both local) + * - ``snapshot_prefix`` + - prefix for ZFS snapshots taken by this job + * - ``interval`` + - snapshotting & replication interval + * - ``initial_repl_policy`` + - default = ``most_recent``, initial replication policy + * - ``prune_lhs`` + - pruning policy on left-hand-side (source) + * - ``prune_rhs`` + - pruning policy on right-hand-side (target) + +* Main Task (every ``interval``, |patient|) + + #. Evaluate ``mapping`` for local filesystems, those with a *target filesystem* are called *mapped filesystems*. + #. Snapshot *mapped filesystems* with ``snapshot_prefix``. + #. Replicate *mapped filesystems* to their respective *target filesystems*: + + #. Only snapshots with prefix ``snapshot_prefix`` are replicated. + #. If possible, incremental replication takes place. + #. If the *target filesystem* does not exist, ``initial_repl_policy`` is used. + #. On conflicts, an error is logged but replication of other *mapped filesystems* continues. + + #. The ``prune_lhs`` policy is triggered for all *mapped filesystems* + #. The ``prune_rhs`` policy is triggered for all *target filesystems* A local job is combination of source & pull job executed on the same machine. -Example: {{< sampleconflink "localbackup/host1.yml" >}} - Terminology ----------- @@ -122,8 +166,11 @@ task A job consists of one or more tasks and a task consists of one or more steps. Some tasks may be periodic while others wait for an event to occur. + patient task + .. _job-term-patient: + A patient task is supposed to execute some task every `interval`. We call the start of the task an *invocation*. diff --git a/docs/configuration/map_filter_syntax.rst b/docs/configuration/map_filter_syntax.rst index 40fa819..5d152b9 100644 --- a/docs/configuration/map_filter_syntax.rst +++ b/docs/configuration/map_filter_syntax.rst @@ -37,6 +37,7 @@ Example zroot => NO MATCH tank/var/log => 1 +.. _pattern-mapping: Mappings -------- @@ -78,6 +79,8 @@ Results in the following mappings zroot/poudriere/ports/2017Q3 => NOT MAPPED zroot/poudriere/ports/HEAD => NOT MAPPED +.. _pattern-filter: + Filters ------- diff --git a/docs/configuration/prune.rst b/docs/configuration/prune.rst index 27d1359..daf10f1 100644 --- a/docs/configuration/prune.rst +++ b/docs/configuration/prune.rst @@ -1,3 +1,5 @@ +.. _prune: + Snapshot Pruning ================ diff --git a/docs/configuration/transports.rst b/docs/configuration/transports.rst index cc9f936..a3fca9a 100644 --- a/docs/configuration/transports.rst +++ b/docs/configuration/transports.rst @@ -17,6 +17,8 @@ The way the `ssh+stdinserver` transport works is inspired by [git shell](https:/ It is implemented in the Go package `github.com/zrepl/zrepl/sshbytestream`. The config excerpts are taken from the [tutorial]({{< relref "tutorial/_index.md" >}}) which you should complete before reading further. +.. _transport-ssh+stdinserver-serve: + `serve` ~~~~~~~ @@ -64,6 +66,8 @@ To recap, this is of how client authentication works with the `ssh+stdinserver` * The admin of the host with the serving zrepl daemon controls the `authorized_keys` file. * Thus, the administrator controls the mapping `PUBKEY -> CLIENT_IDENTITY`. +.. _transport-ssh+stdinserver-connect: + `connect` ~~~~~~~~~ diff --git a/docs/global.rst.inc b/docs/global.rst.inc new file mode 100644 index 0000000..bbe9204 --- /dev/null +++ b/docs/global.rst.inc @@ -0,0 +1,3 @@ +.. global list of substitutions + +.. |mainconfig| replace:: :ref:`main configuration file ` diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c9d13b4..0683971 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1,4 +1,4 @@ -.. |mainconfig| replace:: :ref:`main configuration file ` +.. include:: global.rst.inc .. _tutorial: