From 393fc10a6901f7ee86182fa0a26244cf81f9abb4 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 7 Sep 2020 01:20:57 +0200 Subject: [PATCH] [#285] support setting zfs send / recv flags in the config (send: -wLcepbS, recv: -ox) Co-authored-by: Christian Schwarz Signed-off-by: InsanePrawn closes #285 closes #276 closes #24 --- config/config.go | 23 +- config/config_recv_test.go | 134 +++++++ config/config_send_test.go | 88 ++++- daemon/job/build_jobs_sendrecvoptions.go | 20 +- docs/configuration/overview.rst | 16 + docs/configuration/sendrecvoptions.rst | 181 ++++++++- docs/index.rst | 6 +- endpoint/endpoint.go | 76 +++- platformtest/tests/generated_cases.go | 1 + .../tests/recvForceIntoEncryptedErr.go | 12 +- platformtest/tests/recvRollback.go | 12 +- platformtest/tests/replication.go | 191 +++++++++- .../tests/resumableRecvAndTokenHandling.go | 10 +- platformtest/tests/sendArgsValidation.go | 30 +- zfs/encryption.go | 2 +- zfs/placeholder.go | 5 +- zfs/property/property.go | 33 ++ zfs/propertysource_enumer.go | 82 ++++ zfs/versions.go | 2 +- zfs/zfs.go | 355 ++++++++++++------ zfs/zfs_test.go | 154 +++++++- 21 files changed, 1229 insertions(+), 204 deletions(-) create mode 100644 config/config_recv_test.go create mode 100644 zfs/property/property.go create mode 100644 zfs/propertysource_enumer.go diff --git a/config/config.go b/config/config.go index 9bbf1b3..58a4ce5 100644 --- a/config/config.go +++ b/config/config.go @@ -12,6 +12,8 @@ import ( "github.com/pkg/errors" "github.com/zrepl/yaml-config" + + zfsprop "github.com/zrepl/zrepl/zfs/property" ) type Config struct { @@ -77,15 +79,23 @@ type SnapJob struct { } type SendOptions struct { - Encrypted bool `yaml:"encrypted,optional,default=false"` + Encrypted bool `yaml:"encrypted,optional,default=false"` + Raw bool `yaml:"raw,optional,default=false"` + SendProperties bool `yaml:"send_properties,optional,default=false"` + BackupProperties bool `yaml:"backup_properties,optional,default=false"` + LargeBlocks bool `yaml:"large_blocks,optional,default=false"` + Compressed bool `yaml:"compressed,optional,default=false"` + EmbeddedData bool `yaml:"embbeded_data,optional,default=false"` + Saved bool `yaml:"saved,optional,default=false"` } type RecvOptions struct { // Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it // Encrypted bool `yaml:"may_encrypted"` - // Future: // Reencrypt bool `yaml:"reencrypt"` + + Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"` } type Replication struct { @@ -97,6 +107,15 @@ type ReplicationOptionsProtection struct { Incremental string `yaml:"incremental,optional,default=guarantee_resumability"` } +func (l *RecvOptions) SetDefault() { + *l = RecvOptions{Properties: &PropertyRecvOptions{}} +} + +type PropertyRecvOptions struct { + Inherit []zfsprop.Property `yaml:"inherit,optional"` + Override map[zfsprop.Property]string `yaml:"override,optional"` +} + type PushJob struct { ActiveJob `yaml:",inline"` Snapshotting SnapshottingEnum `yaml:"snapshotting"` diff --git a/config/config_recv_test.go b/config/config_recv_test.go new file mode 100644 index 0000000..d793546 --- /dev/null +++ b/config/config_recv_test.go @@ -0,0 +1,134 @@ +package config + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + zfsprop "github.com/zrepl/zrepl/zfs/property" +) + +func TestRecvOptions(t *testing.T) { + tmpl := ` +jobs: +- name: foo + type: pull + connect: + type: local + listener_name: foo + client_identity: bar + root_fs: "zreplplatformtest" + %s + interval: manual + pruning: + keep_sender: + - type: last_n + count: 10 + keep_receiver: + - type: last_n + count: 10 + +` + + recv_properties_empty := ` + recv: + properties: +` + + recv_inherit_empty := ` + recv: + properties: + inherit: +` + + recv_inherit := ` + recv: + properties: + inherit: + - testprop +` + + recv_override_empty := ` + recv: + properties: + override: +` + + recv_override := ` + recv: + properties: + override: + testprop2: "test123" +` + + recv_override_and_inherit := ` + recv: + properties: + inherit: + - testprop + override: + testprop2: "test123" +` + + recv_empty := ` + recv: {} +` + + recv_not_specified := ` +` + + fill := func(s string) string { return fmt.Sprintf(tmpl, s) } + + t.Run("recv_inherit_empty", func(t *testing.T) { + c := testValidConfig(t, fill(recv_inherit_empty)) + assert.NotNil(t, c) + }) + + t.Run("recv_inherit", func(t *testing.T) { + c := testValidConfig(t, fill(recv_inherit)) + inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit + assert.NotEmpty(t, inherit) + assert.Contains(t, inherit, zfsprop.Property("testprop")) + }) + + t.Run("recv_override_empty", func(t *testing.T) { + c := testValidConfig(t, fill(recv_override_empty)) + assert.NotNil(t, c) + }) + + t.Run("recv_override", func(t *testing.T) { + c := testValidConfig(t, fill(recv_override)) + override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override + require.Len(t, override, 1) + require.Equal(t, "test123", override["testprop2"]) + + }) + + t.Run("recv_override_and_inherit", func(t *testing.T) { + c := testValidConfig(t, fill(recv_override_and_inherit)) + inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit + override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override + assert.NotEmpty(t, inherit) + assert.Contains(t, inherit, zfsprop.Property("testprop")) + assert.NotEmpty(t, override) + assert.Equal(t, "test123", override["testprop2"]) + }) + + t.Run("recv_properties_empty", func(t *testing.T) { + c := testValidConfig(t, fill(recv_properties_empty)) + assert.NotNil(t, c) + }) + + t.Run("recv_empty", func(t *testing.T) { + c := testValidConfig(t, fill(recv_empty)) + assert.NotNil(t, c) + }) + + t.Run("send_not_specified", func(t *testing.T) { + c := testValidConfig(t, fill(recv_not_specified)) + assert.NotNil(t, c) + }) + +} diff --git a/config/config_send_test.go b/config/config_send_test.go index b708789..69e9a21 100644 --- a/config/config_send_test.go +++ b/config/config_send_test.go @@ -38,7 +38,41 @@ jobs: encrypted: true ` - encrypted_unspecified := ` + raw_true := ` + send: + raw: true + +` + + raw_false := ` + send: + raw: false + +` + + raw_and_encrypted := ` + send: + encrypted: true + raw: true +` + + properties_and_encrypted := ` + send: + encrypted: true + send_properties: true +` + + properties_true := ` + send: + send_properties: true +` + + properties_false := ` + send: + send_properties: false +` + + send_empty := ` send: {} ` @@ -46,28 +80,66 @@ jobs: ` fill := func(s string) string { return fmt.Sprintf(tmpl, s) } - var c *Config t.Run("encrypted_false", func(t *testing.T) { - c = testValidConfig(t, fill(encrypted_false)) + c := testValidConfig(t, fill(encrypted_false)) encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted assert.Equal(t, false, encrypted) }) t.Run("encrypted_true", func(t *testing.T) { - c = testValidConfig(t, fill(encrypted_true)) + c := testValidConfig(t, fill(encrypted_true)) encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted assert.Equal(t, true, encrypted) }) - t.Run("encrypted_unspecified", func(t *testing.T) { - c = testValidConfig(t, fill(encrypted_unspecified)) + t.Run("send_empty", func(t *testing.T) { + c := testValidConfig(t, fill(send_empty)) encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted assert.Equal(t, false, encrypted) }) + t.Run("properties_and_encrypted", func(t *testing.T) { + c := testValidConfig(t, fill(properties_and_encrypted)) + encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted + properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties + assert.Equal(t, true, encrypted) + assert.Equal(t, true, properties) + }) + + t.Run("properties_false", func(t *testing.T) { + c := testValidConfig(t, fill(properties_false)) + properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties + assert.Equal(t, false, properties) + }) + + t.Run("properties_true", func(t *testing.T) { + c := testValidConfig(t, fill(properties_true)) + properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties + assert.Equal(t, true, properties) + }) + + t.Run("raw_true", func(t *testing.T) { + c := testValidConfig(t, fill(raw_true)) + raw := c.Jobs[0].Ret.(*PushJob).Send.Raw + assert.Equal(t, true, raw) + }) + + t.Run("raw_false", func(t *testing.T) { + c := testValidConfig(t, fill(raw_false)) + raw := c.Jobs[0].Ret.(*PushJob).Send.Raw + assert.Equal(t, false, raw) + }) + + t.Run("raw_and_encrypted", func(t *testing.T) { + c := testValidConfig(t, fill(raw_and_encrypted)) + raw := c.Jobs[0].Ret.(*PushJob).Send.Raw + encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted + assert.Equal(t, true, raw) + assert.Equal(t, true, encrypted) + }) + t.Run("send_not_specified", func(t *testing.T) { - c, err := testConfig(t, fill(send_not_specified)) - assert.NoError(t, err) + c := testValidConfig(t, fill(send_not_specified)) assert.NotNil(t, c) }) diff --git a/daemon/job/build_jobs_sendrecvoptions.go b/daemon/job/build_jobs_sendrecvoptions.go index f5c76a4..13c8c7f 100644 --- a/daemon/job/build_jobs_sendrecvoptions.go +++ b/daemon/job/build_jobs_sendrecvoptions.go @@ -21,11 +21,19 @@ func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.Sen if err != nil { return nil, errors.Wrap(err, "cannot build filesystem filter") } - + sendOpts := in.GetSendOptions() return &endpoint.SenderConfig{ - FSF: fsf, - Encrypt: &nodefault.Bool{B: in.GetSendOptions().Encrypted}, - JobID: jobID, + FSF: fsf, + JobID: jobID, + + Encrypt: &nodefault.Bool{B: sendOpts.Encrypted}, + SendRaw: sendOpts.Raw, + SendProperties: sendOpts.SendProperties, + SendBackupProperties: sendOpts.BackupProperties, + SendLargeBlocks: sendOpts.LargeBlocks, + SendCompressed: sendOpts.Compressed, + SendEmbeddedData: sendOpts.EmbeddedData, + SendSaved: sendOpts.Saved, }, nil } @@ -44,10 +52,14 @@ func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoi return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver } + recvOpts := in.GetRecvOptions() rc = endpoint.ReceiverConfig{ JobID: jobID, RootWithoutClientComponent: rootFs, AppendClientIdentity: in.GetAppendClientIdentity(), + + InheritProperties: recvOpts.Properties.Inherit, + OverrideProperties: recvOpts.Properties.Override, } if err := rc.Validate(); err != nil { return rc, errors.Wrap(err, "cannot build receiver config") diff --git a/docs/configuration/overview.rst b/docs/configuration/overview.rst index 74bb0e3..2230cc0 100644 --- a/docs/configuration/overview.rst +++ b/docs/configuration/overview.rst @@ -148,6 +148,22 @@ Incremental sends require that ``@from`` be present on the receiving side when r Incremental sends can also use a ZFS bookmark as *from* on the sending side (``zfs send -i #bm_from fs@to``), where ``#bm_from`` was created using ``zfs bookmark fs@from fs#bm_from``. The receiving side must always have the actual snapshot ``@from``, regardless of whether the sending side uses ``@from`` or a bookmark of it. +.. _zfs-background-knowledge-plain-vs-raw-sends: + +**Plain and raw sends** +By default, ``zfs send`` sends the most generic, backwards-compatible data stream format (so-called 'plain send'). +If the sent uses newer features, e.g. compression or encryption, ``zfs send`` has to un-do these operations on the fly to produce the plain send stream. +If the receiver uses newer features (e.g. compression or encryption inherited from the parent FS), it applies the necessary transformations again on the fly during ``zfs recv``. + +Flags such as ``-e``, ``-c`` and ``-L`` tell ZFS to produce a send stream that is closer to how the data is stored on disk. +Sending with those flags removes computational overhead from sender and receiver. +However, the receiver will not apply certain transformations, e.g., it will not compress with the receive-side ``compression`` algorithm. + +The ``-w`` (``--raw``) flag produces a send stream that is as *raw* as possible. +For unencrypted datasets, its current effect is the same as ``-Lce``. + +Encrypted datasets can only be sent plain (unencrypted) or raw (encrypted) using the ``-w`` flag. + **Resumable Send & Recv** The ``-s`` flag for ``zfs recv`` tells zfs to save the partially received send stream in case it is interrupted. To resume the replication, the receiving side filesystem's ``receive_resume_token`` must be passed to a new ``zfs send -t | zfs recv`` command. diff --git a/docs/configuration/sendrecvoptions.rst b/docs/configuration/sendrecvoptions.rst index 3e75d72..9e865de 100644 --- a/docs/configuration/sendrecvoptions.rst +++ b/docs/configuration/sendrecvoptions.rst @@ -9,22 +9,62 @@ Send & Recv Options Send Options ~~~~~~~~~~~~ +:ref:`Source` and :ref:`push` jobs have an optional ``send`` configuration section. + :: jobs: - type: push filesystems: ... send: - encrypted: true + # flags from the table below go here ... -:ref:`Source` and :ref:`push` jobs have an optional ``send`` configuration section. +The following table specifies the list of (boolean) options. +Flags with an entry in the ``zfs send`` column map directly to the zfs send CLI flags. +zrepl does not perform feature checks for these flags. +If you enable a flag that is not supported by the installed version of ZFS, the zfs error will show up at runtime in the logs and zrepl status. +See the `upstream man page `_ (``man zfs-send``) for their semantics. -``encryption`` option ---------------------- +.. list-table:: + :widths: 20 10 70 + :header-rows: 1 -The ``encryption`` variable controls whether the matched filesystems are sent as `OpenZFS native encryption `_ raw sends. -More specifically, if ``encryption=true``, zrepl + * - ``send.`` + - ``zfs send`` + - Comment + * - ``encrypted`` + - + - Specific to zrepl, :ref:`see below `. + * - ``raw`` + - ``-w`` + - Use ``encrypted`` to only allow encrypted sends. + * - ``send_properties`` + - ``-p`` + - **Be careful**, read the :ref:`note on property replication below `. + * - ``backup_properties`` + - ``-b`` + - **Be careful**, read the :ref:`note on property replication below `. + * - ``large_blocks`` + - ``-L`` + - **Potential data loss on OpenZFS < 2.0**, see :ref:`warning below `. + * - ``compressed`` + - ``-c`` + - + * - ``embbeded_data`` + - ``-e`` + - + * - ``saved`` + - ``-S`` + - + +.. _job-send-options-encrypted: + +``encrypted`` +------------- + +The ``encrypted`` option controls whether the matched filesystems are sent as `OpenZFS native encryption `_ raw sends. +More specifically, if ``encrypted=true``, zrepl * checks for any of the filesystems matched by ``filesystems`` whether the ZFS ``encryption`` property indicates that the filesystem is actually encrypted with ZFS native encryption and * invokes the ``zfs send`` subcommand with the ``-w`` option (raw sends) and @@ -32,14 +72,137 @@ More specifically, if ``encryption=true``, zrepl Filesystems matched by ``filesystems`` that are not encrypted are not sent and will cause error log messages. -If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys. +If ``encrypted=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys. + +.. NOTE:: + + Use ``encrypted`` instead of ``raw`` to make your intent clear that zrepl must only replicate filesystems that are actually encrypted by OpenZFS native encryption. + It is meant as a safeguard to prevent unintended sends of unencrypted filesystems in raw mode. + +.. _job-send-options-properties: + +``properties`` +-------------- +Sends the dataset properties along with snapshots. +Please be careful with this option and read the :ref:`note on property replication below `. + +.. _job-send-options-backup-properties: + +``backup_properties`` +--------------------- + +When properties are modified on a filesystem that was received from a send stream with ``send.properties=true``, ZFS archives the original received value internally. +This also applies to :ref:`inheriting or overriding properties during zfs receive `. + +When sending those received filesystems another hop, the ``backup_properties`` flag instructs ZFS to send the original property values rather than the current locally set values. + +This is useful for replicating properties across multiple levels of backup machines. +**Example:** +Suppose we want to flow snapshots from Machine A to B, then from B to C. +A will enable the :ref:`properties send option `. +B will want to override :ref:`critical properties such as mountpoint or canmount `. +But the job that replicates from B to C should be sending the original property values received from A. +Thus, B sets the ``backup_properties`` option. + +Please be careful with this option and read the :ref:`note on property replication below `. + +.. _job-send-options-large-blocks: + +``large_blocks`` +---------------- + +This flag should not be changed after initial replication. +Prior to `OpenZFS commit 7bcb7f08 `_ +it was possible to change this setting which resulted in **data loss on the receiver**. +The commit in question is included in OpenZFS 2.0 and works around the problem by prohibiting receives of incremental streams with a flipped setting. + +.. WARNING:: + + This bug has **not been fixed in the OpenZFS 0.8 releases** which means that changing this flag after initial replication might cause **data loss** on the receiver. .. _job-recv-options: Recv Options ~~~~~~~~~~~~ -:ref:`Sink` and :ref:`pull` jobs have an optional ``recv`` configuration section. -However, there are currently no variables to configure there. +:ref:`Sink` and :ref:`pull` jobs have an optional ``recv`` configuration section: + +:: + + jobs: + - type: pull + recv: + properties: + inherit: + - "mountpoint" + override: { + "org.openzfs.systemd:ignore": "on" + } + ... + +.. _job-recv-options--inherit-and-override: + +``properties`` +-------------- + +``override`` maps directly to the `zfs recv -o flag `_. +Property name-value pairs specified in this map will apply to all received filesystems, regardless of whether the send stream contains properties or not. + +``inherit`` maps directly to the `zfs recv -x flag `_. +Property names specified in this list will be inherited from the receiving side's parent filesystem (e.g. ``root_fs``). + +With both options, the sending side's property value is still stored on the receiver, but the local override or inherit is the one that takes effect. +You can send the original properties from the first receiver to another receiver using :ref:`send.backup_properties`. +.. _job-note-property-replication: + +A Note on Property Replication +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a send stream contains properties, as per ``send.properties`` or ``send.backup_properties``, +the default ZFS behavior is to use those properties on the receiving side, verbatim. + +In many use cases for zrepl, this can have devastating consequences. +For example, when backing up a filesystem that has ``mountpoint=/`` to a storage server, +that storage server's root filesystem will be shadowed by the received file system on some platforms. +Also, many scripts and tools use ZFS user properties for configuration and do not check the property source (``local`` vs. ``received``). +If they are installed on the receiving side as well as the sending side, property replication could have unintended effects. + +**zrepl currently does not provide any automatic safe-guards for property replication:** + +* Make sure to read the entire man page on zfs recv (`man zfs recv `_) before enabling this feature. +* Use ``recv.properties.override`` whenever possible, e.g. for ``mountpoint=none`` or ``canmount=off``. +* Use ``recv.properties.inherit`` if that makes more sense to you. + +Below is an **non-exhaustive list of problematic properties**. +Please open a pull request if you find a property that is missing from this list. +(Both with regards to core ZFS tools and other software in the broader ecosystem.) + +Mount behaviour +--------------- + +* ``mountpoint`` +* ``canmount`` +* ``overlay`` + +Systemd +------- + +With systemd, you should also consider the properties processed by the `zfs-mount-generator `_ . + +Most notably: + +* ``org.openzfs.systemd:ignore`` +* ``org.openzfs.systemd:wanted-by`` +* ``org.openzfs.systemd:required-by`` + +Encryption +---------- + +If the sender filesystems are encrypted but the sender does :ref:`plain sends ` +and property replication is enabled, the receiver must :ref:`inherit the following properties`: + +* ``keylocation`` +* ``keyformat`` +* ``encryption`` diff --git a/docs/index.rst b/docs/index.rst index 73783ac..2383f1d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -61,7 +61,11 @@ Main Features * [x] Automatic ZFS holds during send & receive * [x] Automatic bookmark \& hold management for guaranteed incremental send & recv * [x] Encrypted raw send & receive to untrusted receivers (OpenZFS native encryption) - * [ ] Compressed send & receive + * [x] Properties send & receive + * [x] Compressed send & receive + * [x] Large blocks send & receive + * [x] Embedded data send & receive + * [x] Resume state send & receive * **Automatic snapshot management** diff --git a/endpoint/endpoint.go b/endpoint/endpoint.go index dbd1eee..ec41f64 100644 --- a/endpoint/endpoint.go +++ b/endpoint/endpoint.go @@ -20,12 +20,21 @@ import ( "github.com/zrepl/zrepl/util/nodefault" "github.com/zrepl/zrepl/util/semaphore" "github.com/zrepl/zrepl/zfs" + zfsprop "github.com/zrepl/zrepl/zfs/property" ) type SenderConfig struct { - FSF zfs.DatasetFilter - Encrypt *nodefault.Bool - JobID JobID + FSF zfs.DatasetFilter + JobID JobID + + Encrypt *nodefault.Bool + SendRaw bool + SendProperties bool + SendBackupProperties bool + SendLargeBlocks bool + SendCompressed bool + SendEmbeddedData bool + SendSaved bool } func (c *SenderConfig) Validate() error { @@ -44,8 +53,8 @@ type Sender struct { pdu.UnsafeReplicationServer // prefer compilation errors over default 'method X not implemented' impl FSFilter zfs.DatasetFilter - encrypt *nodefault.Bool jobId JobID + config SenderConfig } func NewSender(conf SenderConfig) *Sender { @@ -54,8 +63,8 @@ func NewSender(conf SenderConfig) *Sender { } return &Sender{ FSFilter: conf.FSF, - encrypt: conf.Encrypt, jobId: conf.JobID, + config: conf, } } @@ -155,12 +164,12 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea // use s.encrypt setting // ok, fallthrough outer case pdu.Tri_False: - if s.encrypt.B { + if s.config.Encrypt.B { return nil, nil, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested") } // fallthrough outer case pdu.Tri_True: - if !s.encrypt.B { + if !s.config.Encrypt.B { return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested") } // fallthrough outer @@ -169,11 +178,20 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea } sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{ - FS: r.Filesystem, - From: uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend - To: uncheckedSendArgsFromPDU(r.GetTo()), // validated by zfs.ZFSSendDry / zfs.ZFSSend - Encrypted: s.encrypt, - ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success + FS: r.Filesystem, + From: uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend + To: uncheckedSendArgsFromPDU(r.GetTo()), // validated by zfs.ZFSSendDry / zfs.ZFSSend + ZFSSendFlags: zfs.ZFSSendFlags{ + ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success + Encrypted: s.config.Encrypt, + Properties: s.config.SendProperties, + BackupProperties: s.config.SendBackupProperties, + Raw: s.config.SendRaw, + LargeBlocks: s.config.SendLargeBlocks, + Compressed: s.config.SendCompressed, + EmbeddedData: s.config.SendEmbeddedData, + Saved: s.config.SendSaved, + }, } sendArgs, err := sendArgsUnvalidated.Validate(ctx) @@ -425,19 +443,49 @@ type FSMap interface { // FIXME unused AsFilter() FSFilter } +// NOTE: when adding members to this struct, remember +// to add them to `ReceiverConfig.copyIn()` type ReceiverConfig struct { JobID JobID RootWithoutClientComponent *zfs.DatasetPath // TODO use AppendClientIdentity bool + + InheritProperties []zfsprop.Property + OverrideProperties map[zfsprop.Property]string } func (c *ReceiverConfig) copyIn() { c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy() + + pInherit := make([]zfsprop.Property, len(c.InheritProperties)) + copy(pInherit, c.InheritProperties) + c.InheritProperties = pInherit + + pOverride := make(map[zfsprop.Property]string, len(c.OverrideProperties)) + for key, value := range c.OverrideProperties { + pOverride[key] = value + } + c.OverrideProperties = pOverride } func (c *ReceiverConfig) Validate() error { c.JobID.MustValidate() + + for _, prop := range c.InheritProperties { + err := prop.Validate() + if err != nil { + return errors.Wrapf(err, "inherit property %q", prop) + } + } + + for prop := range c.OverrideProperties { + err := prop.Validate() + if err != nil { + return errors.Wrapf(err, "override property %q", prop) + } + } + if c.RootWithoutClientComponent.Length() <= 0 { return errors.New("RootWithoutClientComponent must not be an empty dataset path") } @@ -727,6 +775,10 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io. return nil, errors.Wrap(err, "cannot get placeholder state") } log.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).Debug("placeholder state") + + recvOpts.InheritProperties = s.conf.InheritProperties + recvOpts.OverrideProperties = s.conf.OverrideProperties + if ph.FSExists && ph.IsPlaceholder { recvOpts.RollbackAndForceRecv = true clearPlaceholderProperty = true diff --git a/platformtest/tests/generated_cases.go b/platformtest/tests/generated_cases.go index da1cc74..9a94c15 100644 --- a/platformtest/tests/generated_cases.go +++ b/platformtest/tests/generated_cases.go @@ -24,6 +24,7 @@ var Cases = []Case{BatchDestroy, ReplicationIsResumableFullSend__both_GuaranteeResumability, ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication, ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication, + ReplicationPropertyReplicationWorks, ReplicationReceiverErrorWhileStillSending, ReplicationStepCompletedLostBehavior__GuaranteeIncrementalReplication, ReplicationStepCompletedLostBehavior__GuaranteeResumability, diff --git a/platformtest/tests/recvForceIntoEncryptedErr.go b/platformtest/tests/recvForceIntoEncryptedErr.go index f9a9e7d..53b8a6a 100644 --- a/platformtest/tests/recvForceIntoEncryptedErr.go +++ b/platformtest/tests/recvForceIntoEncryptedErr.go @@ -32,11 +32,13 @@ func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) { sfsSnap1 := sendArgVersion(ctx, sfs, "@1") sendArgs, err := zfs.ZFSSendArgsUnvalidated{ - FS: sfs, - Encrypted: &nodefault.Bool{B: false}, - From: nil, - To: &sfsSnap1, - ResumeToken: "", + FS: sfs, + From: nil, + To: &sfsSnap1, + ZFSSendFlags: zfs.ZFSSendFlags{ + Encrypted: &nodefault.Bool{B: false}, + ResumeToken: "", + }, }.Validate(ctx) require.NoError(ctx, err) diff --git a/platformtest/tests/recvRollback.go b/platformtest/tests/recvRollback.go index dbc1a4f..e75f5ff 100644 --- a/platformtest/tests/recvRollback.go +++ b/platformtest/tests/recvRollback.go @@ -28,11 +28,13 @@ func ReceiveForceRollbackWorksUnencrypted(ctx *platformtest.Context) { sfsSnap1 := sendArgVersion(ctx, sfs, "@1") sendArgs, err := zfs.ZFSSendArgsUnvalidated{ - FS: sfs, - Encrypted: &nodefault.Bool{B: false}, - From: nil, - To: &sfsSnap1, - ResumeToken: "", + FS: sfs, + From: nil, + To: &sfsSnap1, + ZFSSendFlags: zfs.ZFSSendFlags{ + Encrypted: &nodefault.Bool{B: false}, + ResumeToken: "", + }, }.Validate(ctx) require.NoError(ctx, err) diff --git a/platformtest/tests/replication.go b/platformtest/tests/replication.go index 1c3d01e..ee08c2a 100644 --- a/platformtest/tests/replication.go +++ b/platformtest/tests/replication.go @@ -21,6 +21,7 @@ import ( "github.com/zrepl/zrepl/util/limitio" "github.com/zrepl/zrepl/util/nodefault" "github.com/zrepl/zrepl/zfs" + zfsprop "github.com/zrepl/zrepl/zfs/property" ) // mimics the replication invocations of an active-side job @@ -30,13 +31,15 @@ import ( // of a new sender and receiver instance and one blocking invocation // of the replication engine without encryption type replicationInvocation struct { - sjid, rjid endpoint.JobID - sfs string - sfilter *filters.DatasetMapFilter - rfsRoot string - interceptSender func(e *endpoint.Sender) logic.Sender - interceptReceiver func(e *endpoint.Receiver) logic.Receiver - guarantee *pdu.ReplicationConfigProtection + sjid, rjid endpoint.JobID + sfs string + sfilter *filters.DatasetMapFilter + rfsRoot string + interceptSender func(e *endpoint.Sender) logic.Sender + interceptReceiver func(e *endpoint.Receiver) logic.Receiver + guarantee *pdu.ReplicationConfigProtection + senderConfigHook func(*endpoint.SenderConfig) + receiverConfigHook func(*endpoint.ReceiverConfig) } func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report { @@ -56,16 +59,30 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report { err := i.sfilter.Add(i.sfs, "ok") require.NoError(ctx, err) } - sender := i.interceptSender(endpoint.NewSender(endpoint.SenderConfig{ + + senderConfig := endpoint.SenderConfig{ FSF: i.sfilter.AsFilter(), Encrypt: &nodefault.Bool{B: false}, JobID: i.sjid, - })) - receiver := i.interceptReceiver(endpoint.NewReceiver(endpoint.ReceiverConfig{ + } + if i.senderConfigHook != nil { + i.senderConfigHook(&senderConfig) + } + + receiverConfig := endpoint.ReceiverConfig{ JobID: i.rjid, AppendClientIdentity: false, RootWithoutClientComponent: mustDatasetPath(i.rfsRoot), - })) + } + if i.receiverConfigHook != nil { + i.receiverConfigHook(&receiverConfig) + } + + require.Equal(ctx, senderConfig.JobID, i.sjid) + require.Equal(ctx, receiverConfig.JobID, i.rjid) + + sender := i.interceptSender(endpoint.NewSender(senderConfig)) + receiver := i.interceptReceiver(endpoint.NewReceiver(receiverConfig)) plannerPolicy := logic.PlannerPolicy{ EncryptedSend: logic.TriFromBool(false), ReplicationConfig: &pdu.ReplicationConfig{ @@ -901,7 +918,7 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest. } require.Contains(ctx, fsByName, fsA) - require.Contains(ctx, fsByName, fsAA) + require.Contains(ctx, fsByName, fsAChild) require.Contains(ctx, fsByName, fsAA) checkFS := func(fs string, expectErrMsg string) { @@ -916,3 +933,153 @@ func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest. checkFS(fsAChild, "parent(s) failed during initial replication") checkFS(fsAA, mockRecvErr.Error()) // fsAA is not treated as a child of fsA } + +func ReplicationPropertyReplicationWorks(ctx *platformtest.Context) { + + platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, ` + CREATEROOT + + "sender" + + "sender/a" + + "sender/a@1" + + "sender/a/child" + + "sender/a/child@1" + + "receiver" + `) + + sjid := endpoint.MustMakeJobID("sender-job") + rjid := endpoint.MustMakeJobID("receiver-job") + + fsA := ctx.RootDataset + "/sender/a" + fsAChild := ctx.RootDataset + "/sender/a/child" + + sfilter := filters.NewDatasetMapFilter(2, true) + mustAddToSFilter := func(fs string) { + err := sfilter.Add(fs, "ok") + require.NoError(ctx, err) + } + mustAddToSFilter(fsA) + mustAddToSFilter(fsAChild) + rfsRoot := ctx.RootDataset + "/receiver" + + type testPropExpectation struct { + Exists bool + Source zfs.PropertySource + ExpectSpecialValue string + } + type testProp struct { + Name string + SetOnSender map[string]bool + ExpectReceiver map[string]testPropExpectation + } + testProps := []testProp{ + { + Name: "zrepl:ignored", + SetOnSender: map[string]bool{fsA: true, fsAChild: true}, + ExpectReceiver: map[string]testPropExpectation{ + fsA: { + Exists: false, + }, + fsAChild: { + Exists: false, + }, + }, + }, + { + Name: "zrepl:replicate", + SetOnSender: map[string]bool{fsA: true, fsAChild: true}, + ExpectReceiver: map[string]testPropExpectation{ + fsA: {Exists: true, Source: zfs.SourceReceived}, + fsAChild: {Exists: true, Source: zfs.SourceReceived}, + }, + }, + { + Name: "zrepl:overridden", + SetOnSender: map[string]bool{fsA: false, fsAChild: true}, + ExpectReceiver: map[string]testPropExpectation{ + fsA: {Exists: true, Source: zfs.SourceLocal, ExpectSpecialValue: "overridden value"}, + fsAChild: {Exists: true, Source: zfs.SourceLocal, ExpectSpecialValue: "overridden value"}, + }, + }, + } + + for _, prop := range testProps { + for fs := range prop.SetOnSender { + err := zfs.ZFSSet(ctx, mustDatasetPath(fs), map[string]string{prop.Name: prop.Name}) + require.NoError(ctx, err) + } + } + + rep := replicationInvocation{ + sjid: sjid, + rjid: rjid, + sfilter: sfilter, + rfsRoot: rfsRoot, + guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing), + receiverConfigHook: func(c *endpoint.ReceiverConfig) { + c.InheritProperties = []zfsprop.Property{"zrepl:ignored"} + c.OverrideProperties = map[zfsprop.Property]string{ + "zrepl:overridden": "overridden value", + } + }, + senderConfigHook: func(c *endpoint.SenderConfig) { + c.SendProperties = true // TODO: do another tier with SendBackupProperties + }, + } + + r := rep.Do(ctx) + ctx.Logf("\n%s", pretty.Sprint(r)) + + require.Len(ctx, r.Attempts, 1) + attempt := r.Attempts[0] + require.Nil(ctx, attempt.PlanError) + require.Len(ctx, attempt.Filesystems, 2) + + fsByName := make(map[string]*report.FilesystemReport, len(attempt.Filesystems)) + for _, fs := range attempt.Filesystems { + fsByName[fs.Info.Name] = fs + } + + require.Contains(ctx, fsByName, fsA) + require.Contains(ctx, fsByName, fsAChild) + require.Len(ctx, fsByName, 2) + + requireFSReportSucceeded := func(fs string) *zfs.DatasetPath { + rep := fsByName[fs] + require.Len(ctx, rep.Steps, 1) + require.Nil(ctx, rep.PlanError) + require.Nil(ctx, rep.StepError) + require.Len(ctx, rep.Steps, 1) + require.Equal(ctx, 1, rep.CurrentStep) + return mustDatasetPath(path.Join(rfsRoot, fs)) + } + + rfsA := requireFSReportSucceeded(fsA) + rfsAChild := requireFSReportSucceeded(fsAChild) + + rfsmap := map[string]*zfs.DatasetPath{ + fsA: rfsA, + fsAChild: rfsAChild, + } + + for fs, rfs := range rfsmap { + for _, tp := range testProps { + r_, err := zfs.ZFSGet(ctx, rfs, []string{tp.Name}) + require.NoError(ctx, err) + r := r_.GetDetails(tp.Name) + + expect := tp.ExpectReceiver[fs] + + if !expect.Exists { + require.Equal(ctx, zfs.SourceNone, r.Source) + } else { + require.Equal(ctx, expect.Source, r.Source) + + if expect.ExpectSpecialValue != "" { + require.Equal(ctx, expect.ExpectSpecialValue, r.Value) + } else { + require.Equal(ctx, tp.Name, r.Value) + } + } + } + } +} diff --git a/platformtest/tests/resumableRecvAndTokenHandling.go b/platformtest/tests/resumableRecvAndTokenHandling.go index 95d1738..f173eda 100644 --- a/platformtest/tests/resumableRecvAndTokenHandling.go +++ b/platformtest/tests/resumableRecvAndTokenHandling.go @@ -27,10 +27,12 @@ func ResumableRecvAndTokenHandling(ctx *platformtest.Context) { src := makeDummyDataSnapshots(ctx, sendFS) s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgsUnvalidated{ - FS: sendFS, - To: src.snapA, - Encrypted: &nodefault.Bool{B: false}, - ResumeToken: "", + FS: sendFS, + To: src.snapA, + ZFSSendFlags: zfs.ZFSSendFlags{ + Encrypted: &nodefault.Bool{B: false}, + ResumeToken: "", + }, }, zfs.RecvOptions{ RollbackAndForceRecv: false, // doesnt' exist yet SavePartialRecvState: true, diff --git a/platformtest/tests/sendArgsValidation.go b/platformtest/tests/sendArgsValidation.go index 3f91092..c1e96e1 100644 --- a/platformtest/tests/sendArgsValidation.go +++ b/platformtest/tests/sendArgsValidation.go @@ -43,8 +43,10 @@ func sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx *plat RelName: "@a snap", GUID: props.Guid, }, - Encrypted: &nodefault.Bool{B: true}, - ResumeToken: "", + ZFSSendFlags: zfs.ZFSSendFlags{ + Encrypted: &nodefault.Bool{B: true}, + ResumeToken: "", + }, }.Validate(ctx) var stream *zfs.SendStream @@ -96,18 +98,18 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest. src := makeDummyDataSnapshots(ctx, sendFS) unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgsUnvalidated{ - FS: sendFS, - To: src.snapA, - Encrypted: &nodefault.Bool{B: false}, // ! + FS: sendFS, + To: src.snapA, + ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: false}}, // ! }, zfs.RecvOptions{ RollbackAndForceRecv: false, SavePartialRecvState: true, }) encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgsUnvalidated{ - FS: sendFS, - To: src.snapA, - Encrypted: &nodefault.Bool{B: true}, // ! + FS: sendFS, + To: src.snapA, + ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: true}}, // ! }, zfs.RecvOptions{ RollbackAndForceRecv: false, SavePartialRecvState: true, @@ -173,9 +175,9 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest src2 := makeDummyDataSnapshots(ctx, sendFS2) rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgsUnvalidated{ - FS: sendFS1, - To: src1.snapA, - Encrypted: &nodefault.Bool{B: false}, + FS: sendFS1, + To: src1.snapA, + ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: false}}, }, zfs.RecvOptions{ RollbackAndForceRecv: false, SavePartialRecvState: true, @@ -189,8 +191,10 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest RelName: src2.snapA.RelName, GUID: src2.snapA.GUID, }, - Encrypted: &nodefault.Bool{B: false}, - ResumeToken: rs.recvErrDecoded.ResumeTokenRaw, + ZFSSendFlags: zfs.ZFSSendFlags{ + Encrypted: &nodefault.Bool{B: false}, + ResumeToken: rs.recvErrDecoded.ResumeTokenRaw, + }, } _, err = maliciousSend.Validate(ctx) require.Error(ctx, err) diff --git a/zfs/encryption.go b/zfs/encryption.go index 8da07fc..5e4228e 100644 --- a/zfs/encryption.go +++ b/zfs/encryption.go @@ -51,7 +51,7 @@ func ZFSGetEncryptionEnabled(ctx context.Context, fs string) (enabled bool, err return false, err } - props, err := zfsGet(ctx, fs, []string{"encryption"}, sourceAny) + props, err := zfsGet(ctx, fs, []string{"encryption"}, SourceAny) if err != nil { return false, errors.Wrap(err, "cannot get `encryption` property") } diff --git a/zfs/placeholder.go b/zfs/placeholder.go index b43c855..818f30c 100644 --- a/zfs/placeholder.go +++ b/zfs/placeholder.go @@ -67,7 +67,7 @@ type FilesystemPlaceholderState struct { func ZFSGetFilesystemPlaceholderState(ctx context.Context, p *DatasetPath) (state *FilesystemPlaceholderState, err error) { state = &FilesystemPlaceholderState{FS: p.ToString()} state.FS = p.ToString() - props, err := zfsGet(ctx, p.ToString(), []string{PlaceholderPropertyName}, sourceLocal) + props, err := zfsGet(ctx, p.ToString(), []string{PlaceholderPropertyName}, SourceLocal) var _ error = (*DatasetDoesNotExist)(nil) // weak assertion on zfsGet's interface if _, ok := err.(*DatasetDoesNotExist); ok { return state, nil @@ -110,12 +110,11 @@ func ZFSCreatePlaceholderFilesystem(ctx context.Context, fs *DatasetPath, parent } func ZFSSetPlaceholder(ctx context.Context, p *DatasetPath, isPlaceholder bool) error { - props := NewZFSProperties() prop := placeholderPropertyOff if isPlaceholder { prop = placeholderPropertyOn } - props.Set(PlaceholderPropertyName, prop) + props := map[string]string{PlaceholderPropertyName: prop} return zfsSet(ctx, p.ToString(), props) } diff --git a/zfs/property/property.go b/zfs/property/property.go new file mode 100644 index 0000000..405a3b4 --- /dev/null +++ b/zfs/property/property.go @@ -0,0 +1,33 @@ +package property + +import ( + "fmt" + "regexp" +) + +type Property string + +// Check property name conforms to zfsprops(8), section "User Properties" +// Keep regex and error message in sync! +var ( + propertyValidNameChars = regexp.MustCompile(`^[0-9a-zA-Z-_\.:]+$`) + propertyValidNameCharsErr = fmt.Errorf("property name must only contain alphanumeric chars and any in %q", "-_.:") +) + +func (p Property) Validate() error { + const PROPERTYNAMEMAXLEN int = 256 + + if len(p) < 1 { + return fmt.Errorf("property name cannot be empty") + } + if len(p) > PROPERTYNAMEMAXLEN { + return fmt.Errorf("property name longer than %d characters", PROPERTYNAMEMAXLEN) + } + if p[0] == '-' { + return fmt.Errorf("property name cannot start with '-'") + } + if !propertyValidNameChars.MatchString(string(p)) { + return propertyValidNameCharsErr + } + return nil +} diff --git a/zfs/propertysource_enumer.go b/zfs/propertysource_enumer.go new file mode 100644 index 0000000..41db54f --- /dev/null +++ b/zfs/propertysource_enumer.go @@ -0,0 +1,82 @@ +// Code generated by "enumer -type=PropertySource -trimprefix=Source"; DO NOT EDIT. + +// +package zfs + +import ( + "fmt" +) + +const ( + _PropertySourceName_0 = "LocalDefault" + _PropertySourceName_1 = "Inherited" + _PropertySourceName_2 = "None" + _PropertySourceName_3 = "Temporary" + _PropertySourceName_4 = "Received" + _PropertySourceName_5 = "Any" +) + +var ( + _PropertySourceIndex_0 = [...]uint8{0, 5, 12} + _PropertySourceIndex_1 = [...]uint8{0, 9} + _PropertySourceIndex_2 = [...]uint8{0, 4} + _PropertySourceIndex_3 = [...]uint8{0, 9} + _PropertySourceIndex_4 = [...]uint8{0, 8} + _PropertySourceIndex_5 = [...]uint8{0, 3} +) + +func (i PropertySource) String() string { + switch { + case 1 <= i && i <= 2: + i -= 1 + return _PropertySourceName_0[_PropertySourceIndex_0[i]:_PropertySourceIndex_0[i+1]] + case i == 4: + return _PropertySourceName_1 + case i == 8: + return _PropertySourceName_2 + case i == 16: + return _PropertySourceName_3 + case i == 32: + return _PropertySourceName_4 + case i == 18446744073709551615: + return _PropertySourceName_5 + default: + return fmt.Sprintf("PropertySource(%d)", i) + } +} + +var _PropertySourceValues = []PropertySource{1, 2, 4, 8, 16, 32, 18446744073709551615} + +var _PropertySourceNameToValueMap = map[string]PropertySource{ + _PropertySourceName_0[0:5]: 1, + _PropertySourceName_0[5:12]: 2, + _PropertySourceName_1[0:9]: 4, + _PropertySourceName_2[0:4]: 8, + _PropertySourceName_3[0:9]: 16, + _PropertySourceName_4[0:8]: 32, + _PropertySourceName_5[0:3]: 18446744073709551615, +} + +// PropertySourceString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func PropertySourceString(s string) (PropertySource, error) { + if val, ok := _PropertySourceNameToValueMap[s]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to PropertySource values", s) +} + +// PropertySourceValues returns all values of the enum +func PropertySourceValues() []PropertySource { + return _PropertySourceValues +} + +// IsAPropertySource returns "true" if the value is listed in the enum definition. "false" otherwise +func (i PropertySource) IsAPropertySource() bool { + for _, v := range _PropertySourceValues { + if i == v { + return true + } + } + return false +} diff --git a/zfs/versions.go b/zfs/versions.go index 1eb415d..c7f43ec 100644 --- a/zfs/versions.go +++ b/zfs/versions.go @@ -267,7 +267,7 @@ func ZFSListFilesystemVersions(ctx context.Context, fs *DatasetPath, options Lis } func ZFSGetFilesystemVersion(ctx context.Context, ds string) (v FilesystemVersion, _ error) { - props, err := zfsGet(ctx, ds, []string{"createtxg", "guid", "creation", "userrefs"}, sourceAny) + props, err := zfsGet(ctx, ds, []string{"createtxg", "guid", "creation", "userrefs"}, SourceAny) if err != nil { return v, err } diff --git a/zfs/zfs.go b/zfs/zfs.go index d8c441e..c26bcd1 100644 --- a/zfs/zfs.go +++ b/zfs/zfs.go @@ -22,6 +22,7 @@ import ( "github.com/zrepl/zrepl/util/circlog" "github.com/zrepl/zrepl/util/envconst" "github.com/zrepl/zrepl/util/nodefault" + zfsprop "github.com/zrepl/zrepl/zfs/property" "github.com/zrepl/zrepl/zfs/zfscmd" ) @@ -326,45 +327,6 @@ func absVersion(fs string, v *ZFSSendArgVersion) (full string, err error) { return fmt.Sprintf("%s%s", fs, v.RelName), nil } -// tok is allowed to be nil -// a must already be validated -// -// SECURITY SENSITIVE because Raw must be handled correctly -func (a ZFSSendArgsUnvalidated) buildCommonSendArgs() ([]string, error) { - - args := make([]string, 0, 3) - // ResumeToken takes precedence, we assume that it has been validated to reflect - // what is described by the other fields in ZFSSendArgs - if a.ResumeToken != "" { - args = append(args, "-t", a.ResumeToken) - return args, nil - } - - if a.Encrypted.B { - args = append(args, "-w") - } - - toV, err := absVersion(a.FS, a.To) - if err != nil { - return nil, err - } - - fromV := "" - if a.From != nil { - fromV, err = absVersion(a.FS, a.From) - if err != nil { - return nil, err - } - } - - if fromV == "" { // Initial - args = append(args, toV) - } else { - args = append(args, "-i", fromV, toV) - } - return args, nil -} - func pipeWithCapacityHint(capacity int) (r, w *os.File, err error) { if capacity <= 0 { panic(fmt.Sprintf("capacity must be positive %v", capacity)) @@ -566,12 +528,9 @@ func (v ZFSSendArgVersion) MustBeBookmark() { // When updating this struct, check Validate and ValidateCorrespondsToResumeToken (POTENTIALLY SECURITY SENSITIVE) type ZFSSendArgsUnvalidated struct { - FS string - From, To *ZFSSendArgVersion // From may be nil - Encrypted *nodefault.Bool - - // Preferred if not empty - ResumeToken string // if not nil, must match what is specified in From, To (covered by ValidateCorrespondsToResumeToken) + FS string + From, To *ZFSSendArgVersion // From may be nil + ZFSSendFlags } type ZFSSendArgsValidated struct { @@ -580,6 +539,20 @@ type ZFSSendArgsValidated struct { ToVersion FilesystemVersion } +type ZFSSendFlags struct { + Encrypted *nodefault.Bool + Properties bool + BackupProperties bool + Raw bool + LargeBlocks bool + Compressed bool + EmbeddedData bool + Saved bool + + // Preferred if not empty + ResumeToken string // if not nil, must match what is specified in From, To (covered by ValidateCorrespondsToResumeToken) +} + type zfsSendArgsValidationContext struct { encEnabled *nodefault.Bool } @@ -638,8 +611,8 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali // fallthrough } - if err := a.Encrypted.ValidateNoDefault(); err != nil { - return v, newGenericValidationError(a, errors.Wrap(err, "`Raw` invalid")) + if err := a.ZFSSendFlags.Validate(); err != nil { + return v, newGenericValidationError(a, errors.Wrap(err, "send flags invalid")) } valCtx := &zfsSendArgsValidationContext{} @@ -655,10 +628,8 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali errors.Errorf("encrypted send requested, but filesystem %q is not encrypted", a.FS)) } - if a.ResumeToken != "" { - if err := a.validateCorrespondsToResumeToken(ctx, valCtx); err != nil { - return v, newValidationError(a, ZFSSendArgsResumeTokenMismatch, err) - } + if err := a.validateEncryptionFlagsCorrespondToResumeToken(ctx, valCtx); err != nil { + return v, newValidationError(a, ZFSSendArgsResumeTokenMismatch, err) } return ZFSSendArgsValidated{ @@ -668,6 +639,89 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali }, nil } +func (f ZFSSendFlags) Validate() error { + if err := f.Encrypted.ValidateNoDefault(); err != nil { + return errors.Wrap(err, "flag `Encrypted` invalid") + } + return nil +} + +// If ResumeToken is empty, builds a command line with the flags specified. +// If ResumeToken is not empty, build a command line with just `-t {{.ResumeToken}}`. +// +// SECURITY SENSITIVE it is the caller's responsibility to ensure that a.Encrypted semantics +// hold for the file system that will be sent with the send flags returned by this function +func (a ZFSSendFlags) buildSendFlagsUnchecked() []string { + + args := make([]string, 0) + + // ResumeToken takes precedence, we assume that it has been validated + // to reflect what is described by the other fields. + if a.ResumeToken != "" { + args = append(args, "-t", a.ResumeToken) + return args + } + + if a.Encrypted.B || a.Raw { + args = append(args, "-w") + } + + if a.Properties { + args = append(args, "-p") + } + + if a.BackupProperties { + args = append(args, "-b") + } + + if a.LargeBlocks { + args = append(args, "-L") + } + + if a.Compressed { + args = append(args, "-c") + } + + if a.EmbeddedData { + args = append(args, "-e") + } + + if a.Saved { + args = append(args, "-S") + } + + return args +} + +func (a ZFSSendArgsValidated) buildSendCommandLine() ([]string, error) { + + flags := a.buildSendFlagsUnchecked() + + if a.ZFSSendFlags.ResumeToken != "" { + return flags, nil + } + + toV, err := absVersion(a.FS, a.To) + if err != nil { + return nil, err + } + + fromV := "" + if a.From != nil { + fromV, err = absVersion(a.FS, a.From) + if err != nil { + return nil, err + } + } + + if fromV == "" { // Initial + flags = append(flags, toV) + } else { + flags = append(flags, "-i", fromV, toV) + } + return flags, nil +} + type ZFSSendArgsResumeTokenMismatchError struct { What ZFSSendArgsResumeTokenMismatchErrorCode Err error @@ -692,10 +746,17 @@ func (c ZFSSendArgsResumeTokenMismatchErrorCode) fmt(format string, args ...inte } } -// This is SECURITY SENSITIVE and requires exhaustive checking of both side's values -// An attacker requesting a Send with a crafted ResumeToken may encode different parameters in the resume token than expected: +// Validate that the encryption settings specified in `a` correspond to the encryption settings encoded in the resume token. +// +// This is SECURITY SENSITIVE: +// It is possible for an attacker to craft arbitrary resume tokens. +// Those malicious resume tokens could encode different parameters in the resume token than expected: // for example, they may specify another file system (e.g. the filesystem with secret data) or request unencrypted send instead of encrypted raw send. -func (a ZFSSendArgsUnvalidated) validateCorrespondsToResumeToken(ctx context.Context, valCtx *zfsSendArgsValidationContext) error { +// +// Note that we don't check correspondence of all other send flags because +// a) the resume token does not capture all send flags (e.g. send -p is implemented in libzfs and thus not represented in the resume token) +// b) it would force us to either reject resume tokens with unknown flags. +func (a ZFSSendArgsUnvalidated) validateEncryptionFlagsCorrespondToResumeToken(ctx context.Context, valCtx *zfsSendArgsValidationContext) error { if a.ResumeToken == "" { return nil // nothing to do @@ -720,6 +781,7 @@ func (a ZFSSendArgsUnvalidated) validateCorrespondsToResumeToken(ctx context.Con "filesystem in resume token field `toname` = %q does not match expected value %q", tokenFS.ToString(), a.FS) } + // If From is set, it must match. if (a.From != nil) != t.HasFromGUID { // existence must be same if t.HasFromGUID { return gen.fmt("resume token not expected to be incremental, but `fromguid` = %v", t.FromGUID) @@ -774,18 +836,20 @@ func ZFSSend(ctx context.Context, sendArgs ZFSSendArgsValidated) (*SendStream, e args = append(args, "send") // pre-validation of sendArgs for plain ErrEncryptedSendNotSupported error - // TODO go1.13: push this down to sendArgs.Validate - if encryptedSendValid := sendArgs.Encrypted.ValidateNoDefault(); encryptedSendValid == nil && sendArgs.Encrypted.B { - supported, err := EncryptionCLISupported(ctx) + // we tie BackupProperties (send -b) and SendRaw (-w, same as with Encrypted) to this + // since these were released together. + if sendArgs.Encrypted.B || sendArgs.Raw || sendArgs.BackupProperties { + encryptionSupported, err := EncryptionCLISupported(ctx) if err != nil { return nil, errors.Wrap(err, "cannot determine CLI native encryption support") } - if !supported { + + if !encryptionSupported { return nil, ErrEncryptedSendNotSupported } } - sargs, err := sendArgs.buildCommonSendArgs() + sargs, err := sendArgs.buildSendCommandLine() if err != nil { return nil, err } @@ -959,7 +1023,7 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI args := make([]string, 0) args = append(args, "send", "-n", "-v", "-P") - sargs, err := sendArgs.buildCommonSendArgs() + sargs, err := sendArgs.buildSendCommandLine() if err != nil { return nil, err } @@ -977,14 +1041,6 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI return &si, nil } -type RecvOptions struct { - // Rollback to the oldest snapshot, destroy it, then perform `recv -F`. - // Note that this doesn't change property values, i.e. an existing local property value will be kept. - RollbackAndForceRecv bool - // Set -s flag used for resumable send & recv - SavePartialRecvState bool -} - type ErrRecvResumeNotSupported struct { FS string CheckErr error @@ -1001,6 +1057,39 @@ func (e *ErrRecvResumeNotSupported) Error() string { return buf.String() } +type RecvOptions struct { + // Rollback to the oldest snapshot, destroy it, then perform `recv -F`. + // Note that this doesn't change property values, i.e. an existing local property value will be kept. + RollbackAndForceRecv bool + // Set -s flag used for resumable send & recv + SavePartialRecvState bool + + InheritProperties []zfsprop.Property + OverrideProperties map[zfsprop.Property]string +} + +func (opts RecvOptions) buildRecvFlags() []string { + args := make([]string, 0) + if opts.RollbackAndForceRecv { + args = append(args, "-F") + } + if opts.SavePartialRecvState { + args = append(args, "-s") + } + if opts.InheritProperties != nil { + for _, prop := range opts.InheritProperties { + args = append(args, "-x", string(prop)) + } + } + if opts.OverrideProperties != nil { + for prop, value := range opts.OverrideProperties { + args = append(args, "-o", fmt.Sprintf("%s=%s", prop, value)) + } + } + + return args +} + const RecvStderrBufSiz = 1 << 15 func ZFSRecv(ctx context.Context, fs string, v *ZFSSendArgVersion, stream io.ReadCloser, opts RecvOptions) (err error) { @@ -1049,17 +1138,14 @@ func ZFSRecv(ctx context.Context, fs string, v *ZFSSendArgVersion, stream io.Rea } } - args := make([]string, 0) - args = append(args, "recv") - if opts.RollbackAndForceRecv { - args = append(args, "-F") - } if opts.SavePartialRecvState { if supported, err := ResumeRecvSupported(ctx, fsdp); err != nil || !supported { return &ErrRecvResumeNotSupported{FS: fs, CheckErr: err} } - args = append(args, "-s") } + + args := []string{"recv"} + args = append(args, opts.buildRecvFlags()...) args = append(args, v.FullPath(fs)) ctx, cancelCmd := context.WithCancel(ctx) @@ -1235,43 +1321,38 @@ func ZFSRecvClearResumeToken(ctx context.Context, fs string) (err error) { return nil } +type PropertyValue struct { + Value string + Source PropertySource +} + type ZFSProperties struct { - m map[string]string + m map[string]PropertyValue } func NewZFSProperties() *ZFSProperties { - return &ZFSProperties{make(map[string]string, 4)} -} - -func (p *ZFSProperties) Set(key, val string) { - p.m[key] = val + return &ZFSProperties{make(map[string]PropertyValue, 4)} } func (p *ZFSProperties) Get(key string) string { + return p.m[key].Value +} + +func (p *ZFSProperties) GetDetails(key string) PropertyValue { return p.m[key] } -func (p *ZFSProperties) appendArgs(args *[]string) (err error) { - for prop, val := range p.m { +func zfsSet(ctx context.Context, path string, props map[string]string) error { + args := make([]string, 0) + args = append(args, "set") + + for prop, val := range props { if strings.Contains(prop, "=") { return errors.New("prop contains rune '=' which is the delimiter between property name and value") } - *args = append(*args, fmt.Sprintf("%s=%s", prop, val)) + args = append(args, fmt.Sprintf("%s=%s", prop, val)) } - return nil -} -func ZFSSet(ctx context.Context, fs *DatasetPath, props *ZFSProperties) (err error) { - return zfsSet(ctx, fs.ToString(), props) -} - -func zfsSet(ctx context.Context, path string, props *ZFSProperties) (err error) { - args := make([]string, 0) - args = append(args, "set") - err = props.appendArgs(&args) - if err != nil { - return err - } args = append(args, path) cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, args...) @@ -1283,11 +1364,15 @@ func zfsSet(ctx context.Context, path string, props *ZFSProperties) (err error) } } - return + return err +} + +func ZFSSet(ctx context.Context, fs *DatasetPath, props map[string]string) error { + return zfsSet(ctx, fs.ToString(), props) } func ZFSGet(ctx context.Context, fs *DatasetPath, props []string) (*ZFSProperties, error) { - return zfsGet(ctx, fs.ToString(), props, sourceAny) + return zfsGet(ctx, fs.ToString(), props, SourceAny) } // The returned error includes requested filesystem and version as quoted strings in its error message @@ -1307,7 +1392,7 @@ func ZFSGetGUID(ctx context.Context, fs string, version string) (g uint64, err e return 0, errors.New("version does not start with @ or #") } path := fmt.Sprintf("%s%s", fs, version) - props, err := zfsGet(ctx, path, []string{"guid"}, sourceAny) // always local + props, err := zfsGet(ctx, path, []string{"guid"}, SourceAny) // always local if err != nil { return 0, err } @@ -1323,7 +1408,7 @@ func ZFSGetMountpoint(ctx context.Context, fs string) (*GetMountpointOutput, err if err := EntityNamecheck(fs, EntityTypeFilesystem); err != nil { return nil, err } - props, err := zfsGet(ctx, fs, []string{"mountpoint", "mounted"}, sourceAny) + props, err := zfsGet(ctx, fs, []string{"mountpoint", "mounted"}, SourceAny) if err != nil { return nil, err } @@ -1340,7 +1425,7 @@ func ZFSGetMountpoint(ctx context.Context, fs string) (*GetMountpointOutput, err } func ZFSGetRawAnySource(ctx context.Context, path string, props []string) (*ZFSProperties, error) { - return zfsGet(ctx, path, props, sourceAny) + return zfsGet(ctx, path, props, SourceAny) } var zfsGetDatasetDoesNotExistRegexp = regexp.MustCompile(`^cannot open '([^)]+)': (dataset does not exist|no such pool or dataset)`) // verified in platformtest @@ -1360,46 +1445,68 @@ func tryDatasetDoesNotExist(expectPath string, stderr []byte) *DatasetDoesNotExi return nil } -type zfsPropertySource uint +//go:generate enumer -type=PropertySource -trimprefix=Source +type PropertySource uint const ( - sourceLocal zfsPropertySource = 1 << iota - sourceDefault - sourceInherited - sourceNone - sourceTemporary - sourceReceived + SourceLocal PropertySource = 1 << iota + SourceDefault + SourceInherited + SourceNone + SourceTemporary + SourceReceived - sourceAny zfsPropertySource = ^zfsPropertySource(0) + SourceAny PropertySource = ^PropertySource(0) ) -func (s zfsPropertySource) zfsGetSourceFieldPrefixes() []string { +var propertySourceParseLUT = map[string]PropertySource{ + "local": SourceLocal, + "default": SourceDefault, + "inherited": SourceInherited, + "-": SourceNone, + "temporary": SourceTemporary, + "received": SourceReceived, +} + +func parsePropertySource(s string) (PropertySource, error) { + fields := strings.Fields(s) + if len(fields) > 0 { + v, ok := propertySourceParseLUT[fields[0]] + if ok { + return v, nil + } + // fallthrough + } + return 0, fmt.Errorf("unknown property source %q", s) +} + +func (s PropertySource) zfsGetSourceFieldPrefixes() []string { prefixes := make([]string, 0, 7) - if s&sourceLocal != 0 { + if s&SourceLocal != 0 { prefixes = append(prefixes, "local") } - if s&sourceDefault != 0 { + if s&SourceDefault != 0 { prefixes = append(prefixes, "default") } - if s&sourceInherited != 0 { + if s&SourceInherited != 0 { prefixes = append(prefixes, "inherited") } - if s&sourceNone != 0 { + if s&SourceNone != 0 { prefixes = append(prefixes, "-") } - if s&sourceTemporary != 0 { + if s&SourceTemporary != 0 { prefixes = append(prefixes, "temporary") } - if s&sourceReceived != 0 { + if s&SourceReceived != 0 { prefixes = append(prefixes, "received") } - if s == sourceAny { + if s == SourceAny { prefixes = append(prefixes, "") } return prefixes } -func zfsGet(ctx context.Context, path string, props []string, allowedSources zfsPropertySource) (*ZFSProperties, error) { +func zfsGet(ctx context.Context, path string, props []string, allowedSources PropertySource) (*ZFSProperties, error) { args := []string{"get", "-Hp", "-o", "property,value,source", strings.Join(props, ","), path} cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, args...) stdout, err := cmd.Output() @@ -1425,7 +1532,7 @@ func zfsGet(ctx context.Context, path string, props []string, allowedSources zfs return nil, fmt.Errorf("zfs get did not return the number of expected property values") } res := &ZFSProperties{ - make(map[string]string, len(lines)), + make(map[string]PropertyValue, len(lines)), } allowedPrefixes := allowedSources.zfsGetSourceFieldPrefixes() for _, line := range lines[:len(lines)-1] { @@ -1436,8 +1543,16 @@ func zfsGet(ctx context.Context, path string, props []string, allowedSources zfs return nil, fmt.Errorf("zfs get did not return property,value,source tuples") } for _, p := range allowedPrefixes { + // prefix-match so that SourceAny (= "") works if strings.HasPrefix(fields[2], p) { - res.m[fields[0]] = fields[1] + source, err := parsePropertySource(fields[2]) + if err != nil { + return nil, errors.Wrap(err, "parse property source") + } + res.m[fields[0]] = PropertyValue{ + Value: fields[1], + Source: source, + } break } } diff --git a/zfs/zfs_test.go b/zfs/zfs_test.go index 2f781de..c4fe562 100644 --- a/zfs/zfs_test.go +++ b/zfs/zfs_test.go @@ -5,6 +5,9 @@ import ( "strings" "testing" + "github.com/zrepl/zrepl/util/nodefault" + zfsprop "github.com/zrepl/zrepl/zfs/property" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,20 +44,20 @@ func TestDatasetPathTrimNPrefixComps(t *testing.T) { func TestZFSPropertySource(t *testing.T) { tcs := []struct { - in zfsPropertySource + in PropertySource exp []string }{ { - in: sourceAny, + in: SourceAny, // although empty prefix matches any source exp: []string{"local", "default", "inherited", "-", "temporary", "received", ""}, }, { - in: sourceTemporary, + in: SourceTemporary, exp: []string{"temporary"}, }, { - in: sourceLocal | sourceInherited, + in: SourceLocal | SourceInherited, exp: []string{"local", "inherited"}, }, } @@ -69,6 +72,21 @@ func TestZFSPropertySource(t *testing.T) { for _, tc := range tcs { + t.Logf("TEST CASE %v", tc) + + // give the parsing code some coverage + for _, e := range tc.exp { + if e == "" { + continue // "" is the prefix that matches SourceAny + } + s, err := parsePropertySource(e) + assert.NoError(t, err) + t.Logf("s: %x %s", s, s) + t.Logf("in: %x %s", tc.in, tc.in) + assert.True(t, s&tc.in != 0) + } + + // prefix matching res := tc.in.zfsGetSourceFieldPrefixes() resSet := toSet(res) expSet := toSet(tc.exp) @@ -304,3 +322,131 @@ func TestTryRecvDestroyOrOverwriteEncryptedErr(t *testing.T) { require.NotNil(t, err) assert.EqualError(t, err, strings.TrimSpace(msg)) } + +func TestZFSSendArgsBuildSendFlags(t *testing.T) { + + type args = ZFSSendFlags + type SendTest struct { + conf args + exactMatch bool + flagsInclude []string + flagsExclude []string + } + + withEncrypted := func(justFlags ZFSSendFlags) ZFSSendFlags { + if justFlags.Encrypted == nil { + justFlags.Encrypted = &nodefault.Bool{B: false} + } + return justFlags + } + + var sendTests = map[string]SendTest{ + "Empty Args": { + conf: withEncrypted(args{}), + flagsInclude: []string{}, + flagsExclude: []string{"-w", "-p", "-b"}, + }, + "Raw": { + conf: withEncrypted(args{Raw: true}), + flagsInclude: []string{"-w"}, + flagsExclude: []string{}, + }, + "Encrypted": { + conf: withEncrypted(args{Encrypted: &nodefault.Bool{B: true}}), + flagsInclude: []string{"-w"}, + flagsExclude: []string{}, + }, + "Unencrypted_And_Raw": { + conf: withEncrypted(args{Encrypted: &nodefault.Bool{B: false}, Raw: true}), + flagsInclude: []string{"-w"}, + flagsExclude: []string{}, + }, + "Encrypted_And_Raw": { + conf: withEncrypted(args{Encrypted: &nodefault.Bool{B: true}, Raw: true}), + flagsInclude: []string{"-w"}, + flagsExclude: []string{}, + }, + "Send properties": { + conf: withEncrypted(args{Properties: true}), + flagsInclude: []string{"-p"}, + flagsExclude: []string{}, + }, + "Send backup properties": { + conf: withEncrypted(args{BackupProperties: true}), + flagsInclude: []string{"-b"}, + flagsExclude: []string{}, + }, + "Send -b and -p": { + conf: withEncrypted(args{Properties: true, BackupProperties: true}), + flagsInclude: []string{"-p", "-b"}, + flagsExclude: []string{}, + }, + "Send resume state": { + conf: withEncrypted(args{Saved: true}), + flagsInclude: []string{"-S"}, + }, + "Resume token wins if not empty": { + conf: withEncrypted(args{ResumeToken: "$theresumetoken$", Compressed: true}), + flagsInclude: []string{"-t", "$theresumetoken$"}, + exactMatch: true, + }, + } + + for testName, test := range sendTests { + t.Run(testName, func(t *testing.T) { + flags := test.conf.buildSendFlagsUnchecked() + assert.GreaterOrEqual(t, len(flags), len(test.flagsInclude)) + assert.Subset(t, flags, test.flagsInclude) + if test.exactMatch { + assert.Equal(t, flags, test.flagsInclude) + } + for flag := range flags { + assert.NotContains(t, test.flagsExclude, flag) + } + }) + } +} + +func TestZFSCommonRecvArgsBuild(t *testing.T) { + type RecvTest struct { + conf RecvOptions + flagsInclude []string + flagsExclude []string + } + var recvTests = map[string]RecvTest{ + "Empty Args": { + conf: RecvOptions{}, + flagsInclude: []string{}, + flagsExclude: []string{"-x", "-o", "-F", "-s"}, + }, + "ForceRollback": { + conf: RecvOptions{RollbackAndForceRecv: true}, + flagsInclude: []string{"-F"}, + flagsExclude: []string{"-x", "-o", "-s"}, + }, + "PartialSend": { + conf: RecvOptions{SavePartialRecvState: true}, + flagsInclude: []string{"-s"}, + flagsExclude: []string{"-x", "-o", "-F"}, + }, + "Override properties": { + conf: RecvOptions{OverrideProperties: map[zfsprop.Property]string{zfsprop.Property("abc"): "123"}}, + flagsInclude: []string{"-o", "abc=123"}, + flagsExclude: []string{"-x", "-F", "-s"}, + }, + "Exclude/inherit properties": { + conf: RecvOptions{InheritProperties: []zfsprop.Property{"abc", "123"}}, + flagsInclude: []string{"-x", "abc", "123"}, flagsExclude: []string{"-o", "-F", "-s"}, + }, + } + + for testName, test := range recvTests { + t.Run(testName, func(t *testing.T) { + flags := test.conf.buildRecvFlags() + assert.Subset(t, flags, test.flagsInclude) + for flag := range flags { + assert.NotContains(t, test.flagsExclude, flag) + } + }) + } +}