rework resume token validation to allow resuming from raw sends of unencrypted datasets

Before this change, resuming from an unencrypted dataset with
send.raw=true specified wouldn't work with zrepl due to overly
restrictive resume token checking.

An initial PR to fix this was made in https://github.com/zrepl/zrepl/pull/503
but it didn't address the core of the problem.
The core of the problem was that zrepl assumed that if a resume token
contained `rawok=true, compressok=true`, the resulting send would be
encrypted. But if the sender dataset was unencrypted, such a resume would
actually result in an unencrypted send.
Which could be totally legitimate but zrepl failed to recognize that.

BACKGROUND
==========

The following snippets of OpenZFS code are insightful regarding how the
various ${X}ok values in the resume token are handled:

- 6c3c5fcfbe/module/zfs/dmu_send.c (L1947-L2012)
- 6c3c5fcfbe/module/zfs/dmu_recv.c (L877-L891)
- https://github.com/openzfs/zfs/blob/6c3c5fc/lib/libzfs/libzfs_sendrecv.c#L1663-L1672

Basically, some zfs send flags make the DMU send code set some DMU send
stream featureflags, although it's not a pure mapping, i.e, which DMU
send stream flags are used depends somewhat on the dataset (e.g., is it
encrypted or not, or, does it use zstd or not).

Then, the receiver looks at some (but not all) feature flags and maps
them to ${X}ok dataset zap attributes.

These are funnelled back to the sender 1:1 through the resume_token.

And the sender turns them into lzc flags.

As an example, let's look at zfs send --raw.
if the sender requests a raw send on an unencrypted dataset, the send
stream (and hence the resume token) will not have the raw stream
featureflag set, and hence the resume token will not have the rawok
field set. Instead, it will have compressok, embedok, and depending
on whether large blocks are present in the dataset, largeblockok set.

WHAT'S ZREPL'S ROLE IN THIS?
============================

zrepl provides a virtual encrypted sendflag that is like `raw`,
but further ensures that we only send encrypted datasets.

For any other resume token stuff, it shoudn't do any checking,
because it's a futile effort to keep up with ZFS send/recv features
that are orthogonal to encryption.

CHANGES MADE IN THIS COMMIT
===========================

- Rip out a bunch of needless checking that zrepl would do during
  planning. These checks were there to give better error messages,
  but actually, the error messages created by the endpoint.Sender.Send
  RPC upon send args validation failure are good enough.
- Add platformtests to validate all combinations of
  (Unencrypted/Encrypted FS) x (send.encrypted = true | false) x (send.raw = true | false)
  for cases both non-resuming and resuming send.

Additional manual testing done:
1. With zrepl 0.5, setup with unencrypted dataset, send.raw=true specified, no send.encrypted specified.
2. Observe that regular non-resuming send works, but resuming doesn't work.
3. Upgrade zrepl to this change.
4. Observe that both regular and resuming send works.

closes https://github.com/zrepl/zrepl/pull/613

zfs/zfs.go

@@ -632,6 +632,14 @@ func (e ZFSSendArgsValidationError) Error() string {
 	return e.Msg.Error()
 }
 
+type zfsSendArgsSkipValidationKeyType struct{}
+
+var zfsSendArgsSkipValidationKey = zfsSendArgsSkipValidationKeyType{}
+
+func ZFSSendArgsSkipValidation(ctx context.Context) context.Context {
+	return context.WithValue(ctx, zfsSendArgsSkipValidationKey, true)
+}
+
 // - Recursively call Validate on each field.
 // - Make sure that if ResumeToken != "", it reflects the same operation as the other parameters would.
 //
@@ -659,6 +667,16 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali
 		// fallthrough
 	}
 
+	validated := ZFSSendArgsValidated{
+		ZFSSendArgsUnvalidated: a,
+		FromVersion:            fromVersion,
+		ToVersion:              toVersion,
+	}
+
+	if ctx.Value(zfsSendArgsSkipValidationKey) != nil {
+		return validated, nil
+	}
+
 	if err := a.ZFSSendFlags.Validate(); err != nil {
 		return v, newGenericValidationError(a, errors.Wrap(err, "send flags invalid"))
 	}
@@ -673,18 +691,19 @@ func (a ZFSSendArgsUnvalidated) Validate(ctx context.Context) (v ZFSSendArgsVali
 
 	if a.Encrypted.B && !fsEncrypted {
 		return v, newValidationError(a, ZFSSendArgsEncryptedSendRequestedButFSUnencrypted,
-			errors.Errorf("encrypted send requested, but filesystem %q is not encrypted", a.FS))
+			errors.Errorf("encrypted send mandated by policy, but filesystem %q is not encrypted", a.FS))
+	}
+
+	if a.Raw && fsEncrypted && !a.Encrypted.B {
+		return v, newValidationError(a, ZFSSendArgsGenericValidationError,
+			errors.Errorf("policy mandates raw+unencrypted sends, but filesystem %q is encrypted", a.FS))
 	}
 
 	if err := a.validateEncryptionFlagsCorrespondToResumeToken(ctx, valCtx); err != nil {
 		return v, newValidationError(a, ZFSSendArgsResumeTokenMismatch, err)
 	}
 
-	return ZFSSendArgsValidated{
-		ZFSSendArgsUnvalidated: a,
-		FromVersion:            fromVersion,
-		ToVersion:              toVersion,
-	}, nil
+	return validated, nil
 }
 
 func (f ZFSSendFlags) Validate() error {
@@ -852,21 +871,34 @@ func (a ZFSSendArgsUnvalidated) validateEncryptionFlagsCorrespondToResumeToken(c
 		return gen.fmt("resume token `toguid` != expected: %v != %v", t.ToGUID, a.To.GUID)
 	}
 
-	if a.Encrypted.B {
-		if !(t.RawOK && t.CompressOK) {
-			return ZFSSendArgsResumeTokenMismatchEncryptionNotSet.fmt(
-				"resume token must have `rawok` and `compressok` = true but got %v %v", t.RawOK, t.CompressOK)
-		}
-		// fallthrough
-	} else {
-		if t.RawOK || t.CompressOK {
-			return ZFSSendArgsResumeTokenMismatchEncryptionSet.fmt(
-				"resume token must not have `rawok` or `compressok` set but got %v %v", t.RawOK, t.CompressOK)
-		}
-		// fallthrough
+	// ensure resume stream will be encrypted/unencrypted as specified in policy
+	if err := valCtx.encEnabled.ValidateNoDefault(); err != nil {
+		panic(valCtx)
+	}
+	wouldSendEncryptedIfFilesystemIsEncrypted := t.RawOK
+	filesystemIsEncrypted := valCtx.encEnabled.B
+	resumeWillBeEncryptedSend := filesystemIsEncrypted && wouldSendEncryptedIfFilesystemIsEncrypted
+	if a.Encrypted.B {
+		if resumeWillBeEncryptedSend {
+			return nil // encrypted send in policy, and that's what's going to happen
+		} else {
+			if !filesystemIsEncrypted {
+				// NB: a.Encrypted.B && !valCtx.encEnabled.B
+				//     is handled in the caller, because it doesn't concern the resume token (different kind of error)
+				panic("caller should have already raised an error")
+			}
+			// XXX we have no test coverage for this case. We'd need to forge a resume token for that.
+			return ZFSSendArgsResumeTokenMismatchEncryptionNotSet.fmt(
+				"resume token does not have rawok=true which would result in an unencrypted send, but policy mandates encrypted sends only")
+		}
+	} else {
+		if resumeWillBeEncryptedSend {
+			return ZFSSendArgsResumeTokenMismatchEncryptionSet.fmt(
+				"resume token has rawok=true which would result in encrypted send, but policy mandates unencrypted sends only")
+		} else {
+			return nil // unencrypted send in policy, and that's what's going to happen
+		}
 	}
-
-	return nil
 }
 
 var zfsSendStderrCaptureMaxSize = envconst.Int("ZREPL_ZFS_SEND_STDERR_MAX_CAPTURE_SIZE", 1<<15)