Improve handling of custom values in plugin examples (#12409)

# Description
Requested by @ayax79. This makes the custom value behavior more correct,
by calling the methods on the plugin to handle the custom values in
examples rather than the methods on the custom values themselves. This
helps for handle-type custom values (like what he's doing with
dataframes).

- Equality checking in `PluginTest::test_examples()` changed to use
`PluginInterface::custom_value_partial_cmp()`
- Base value rendering for `PluginSignature` changed to use
`Plugin::custom_value_to_base_value()`
- Had to be moved closer to `serve_plugin` for this reason, so the test
for writing signatures containing custom values was removed
- That behavior should still be tested to some degree, since if custom
values are not handled, signatures will fail to parse, so all of the
other tests won't work.

# User-Facing Changes

- `Record::sort_cols()` method added to share functionality required by
`PartialCmp`, and it might also be slightly faster
- Otherwise, everything should mostly be the same but better. Plugins
that don't implement special handling for custom values will still work
the same way, because the default implementation is just a pass-through
to the `CustomValue` methods.

# Tests + Formatting
- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

crates/nu-plugin/src/plugin/command.rs

@@ -1,5 +1,6 @@
 use nu_protocol::{
-    Example, LabeledError, PipelineData, PluginExample, PluginSignature, Signature, Value,
+    Example, IntoSpanned, LabeledError, PipelineData, PluginExample, PluginSignature, ShellError,
+    Signature, Value,
 };
 
 use crate::{EngineInterface, EvaluatedCall, Plugin};
@@ -359,3 +360,36 @@ pub fn create_plugin_signature(command: &(impl PluginCommand + ?Sized)) -> Plugi
             .collect(),
     )
 }
+
+/// Render examples to their base value so they can be sent in the response to `Signature`.
+pub(crate) fn render_examples(
+    plugin: &impl Plugin,
+    engine: &EngineInterface,
+    examples: &mut [PluginExample],
+) -> Result<(), ShellError> {
+    for example in examples {
+        if let Some(ref mut value) = example.result {
+            value.recurse_mut(&mut |value| {
+                let span = value.span();
+                match value {
+                    Value::Custom { .. } => {
+                        let value_taken = std::mem::replace(value, Value::nothing(span));
+                        let Value::Custom { val, .. } = value_taken else {
+                            unreachable!()
+                        };
+                        *value =
+                            plugin.custom_value_to_base_value(engine, val.into_spanned(span))?;
+                        Ok::<_, ShellError>(())
+                    }
+                    // Collect LazyRecord before proceeding
+                    Value::LazyRecord { ref val, .. } => {
+                        *value = val.collect()?;
+                        Ok(())
+                    }
+                    _ => Ok(()),
+                }
+            })?;
+        }
+    }
+    Ok(())
+}

crates/nu-plugin/src/plugin/interface/engine.rs

@@ -403,16 +403,8 @@ impl EngineInterface {
     /// Any custom values in the examples will be rendered using `to_base_value()`.
     pub(crate) fn write_signature(
         &self,
-        mut signature: Vec<PluginSignature>,
+        signature: Vec<PluginSignature>,
     ) -> Result<(), ShellError> {
-        // Render any custom values in the examples to plain values so that the engine doesn't
-        // have to keep custom values around just to render the help pages.
-        for sig in signature.iter_mut() {
-            for value in sig.examples.iter_mut().flat_map(|e| e.result.as_mut()) {
-                PluginCustomValue::render_to_base_value_in(value)?;
-            }
-        }
-
         let response = PluginCallResponse::Signature(signature);
         self.write(PluginOutput::CallResponse(self.context()?, response))?;
         self.flush()

crates/nu-plugin/src/plugin/interface/engine/tests.rs

@@ -11,7 +11,7 @@ use crate::{
 };
 use nu_protocol::{
     engine::Closure, Config, CustomValue, IntoInterruptiblePipelineData, LabeledError,
-    PipelineData, PluginExample, PluginSignature, ShellError, Signature, Span, Spanned, Value,
+    PipelineData, PluginSignature, ShellError, Span, Spanned, Value,
 };
 use std::{
     collections::HashMap,
@@ -782,49 +782,6 @@ fn interface_write_signature() -> Result<(), ShellError> {
     Ok(())
 }
 
-#[test]
-fn interface_write_signature_custom_value() -> Result<(), ShellError> {
-    let test = TestCase::new();
-    let interface = test.engine().interface_for_context(38);
-    let signatures = vec![PluginSignature::new(
-        Signature::build("test command"),
-        vec![PluginExample {
-            example: "test command".into(),
-            description: "a test".into(),
-            result: Some(Value::test_custom_value(Box::new(
-                expected_test_custom_value(),
-            ))),
-        }],
-    )];
-    interface.write_signature(signatures.clone())?;
-
-    let written = test.next_written().expect("nothing written");
-
-    match written {
-        PluginOutput::CallResponse(id, response) => {
-            assert_eq!(38, id, "id");
-            match response {
-                PluginCallResponse::Signature(sigs) => {
-                    assert_eq!(1, sigs.len(), "sigs.len");
-
-                    let sig = &sigs[0];
-                    assert_eq!(1, sig.examples.len(), "sig.examples.len");
-
-                    assert_eq!(
-                        Some(Value::test_int(expected_test_custom_value().0 as i64)),
-                        sig.examples[0].result,
-                    );
-                }
-                _ => panic!("unexpected response: {response:?}"),
-            }
-        }
-        _ => panic!("unexpected message written: {written:?}"),
-    }
-
-    assert!(!test.has_unconsumed_write());
-    Ok(())
-}
-
 #[test]
 fn interface_write_engine_call_registers_subscription() -> Result<(), ShellError> {
     let mut manager = TestCase::new().engine();

crates/nu-plugin/src/plugin/mod.rs

@@ -33,8 +33,8 @@ use std::os::unix::process::CommandExt;
 #[cfg(windows)]
 use std::os::windows::process::CommandExt;
 
-use self::gc::PluginGc;
 pub use self::interface::{PluginRead, PluginWrite};
+use self::{command::render_examples, gc::PluginGc};
 
 mod command;
 mod context;
@@ -651,7 +651,12 @@ where
                     let sigs = commands
                         .values()
                         .map(|command| create_plugin_signature(command.deref()))
-                        .collect();
+                        .map(|mut sig| {
+                            render_examples(plugin, &engine, &mut sig.examples)?;
+                            Ok(sig)
+                        })
+                        .collect::<Result<Vec<_>, ShellError>>()
+                        .try_to_report(&engine)?;
                     engine.write_signature(sigs).try_to_report(&engine)?;
                 }
                 // Run the plugin on a background thread, handling any input or output streams

crates/nu-plugin/src/plugin/source.rs

@@ -40,7 +40,10 @@ impl PluginSource {
 
     /// Try to upgrade the persistent reference, and return an error referencing `span` as the
     /// object that referenced it otherwise
-    pub(crate) fn persistent(&self, span: Option<Span>) -> Result<Arc<dyn GetPlugin>, ShellError> {
+    ///
+    /// This is not a public API.
+    #[doc(hidden)]
+    pub fn persistent(&self, span: Option<Span>) -> Result<Arc<dyn GetPlugin>, ShellError> {
         self.persistent
             .upgrade()
             .ok_or_else(|| ShellError::GenericError {

crates/nu-plugin/src/protocol/plugin_custom_value.rs

@@ -224,7 +224,7 @@ impl PluginCustomValue {
 
     /// Serialize a custom value into a [`PluginCustomValue`]. This should only be done on the
     /// plugin side.
-    pub(crate) fn serialize_from_custom_value(
+    pub fn serialize_from_custom_value(
         custom_value: &dyn CustomValue,
         span: Span,
     ) -> Result<PluginCustomValue, ShellError> {
@@ -240,7 +240,7 @@ impl PluginCustomValue {
 
     /// Deserialize a [`PluginCustomValue`] into a `Box<dyn CustomValue>`. This should only be done
     /// on the plugin side.
-    pub(crate) fn deserialize_to_custom_value(
+    pub fn deserialize_to_custom_value(
         &self,
         span: Span,
     ) -> Result<Box<dyn CustomValue>, ShellError> {
@@ -272,7 +272,7 @@ impl PluginCustomValue {
     ///
     /// This method will collapse `LazyRecord` in-place as necessary to make the guarantee,
     /// since `LazyRecord` could return something different the next time it is called.
-    pub(crate) fn verify_source(
+    pub fn verify_source(
         value: Spanned<&dyn CustomValue>,
         source: &PluginSource,
     ) -> Result<(), ShellError> {
@@ -356,7 +356,7 @@ impl PluginCustomValue {
     }
 
     /// Render any custom values in the `Value` using `to_base_value()`
-    pub(crate) fn render_to_base_value_in(value: &mut Value) -> Result<(), ShellError> {
+    pub fn render_to_base_value_in(value: &mut Value) -> Result<(), ShellError> {
         value.recurse_mut(&mut |value| {
             let span = value.span();
             match value {