Merge pull request #76 from nushell/from_json

Add 'from json'

Cargo.lock

@@ -171,12 +171,39 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
+[[package]]
+name = "dirs-next"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+dependencies = [
+ "cfg-if",
+ "dirs-sys-next",
+]
+
+[[package]]
+name = "dirs-sys-next"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
 [[package]]
 name = "doc-comment"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
+[[package]]
+name = "dunce"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541"
+
 [[package]]
 name = "either"
 version = "1.6.1"
@@ -192,7 +219,9 @@ dependencies = [
  "nu-cli",
  "nu-command",
  "nu-engine",
+ "nu-json",
  "nu-parser",
+ "nu-path",
  "nu-protocol",
  "nu-table",
  "pretty_assertions",
@@ -256,6 +285,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -268,6 +303,16 @@ version = "0.2.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
+dependencies = [
+ "serde",
+ "serde_test",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.5"
@@ -393,8 +438,10 @@ version = "0.1.0"
 dependencies = [
  "glob",
  "nu-engine",
+ "nu-json",
  "nu-protocol",
  "nu-table",
+ "thiserror",
 ]
 
 [[package]]
@@ -405,6 +452,19 @@ dependencies = [
  "nu-protocol",
 ]
 
+[[package]]
+name = "nu-json"
+version = "0.37.1"
+dependencies = [
+ "lazy_static",
+ "linked-hash-map",
+ "nu-path",
+ "num-traits",
+ "regex",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "nu-parser"
 version = "0.1.0"
@@ -414,11 +474,20 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "nu-path"
+version = "0.37.1"
+dependencies = [
+ "dirs-next",
+ "dunce",
+]
+
 [[package]]
 name = "nu-protocol"
 version = "0.1.0"
 dependencies = [
  "miette",
+ "serde",
  "thiserror",
 ]
 
@@ -623,6 +692,16 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "redox_users"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
+dependencies = [
+ "getrandom",
+ "redox_syscall",
+]
+
 [[package]]
 name = "reedline"
 version = "0.2.0"
@@ -674,6 +753,12 @@ version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
 
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -700,6 +785,26 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_test"
+version = "1.0.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d82178225dbdeae2d5d190e8649287db6a3a32c6d24da22ae3146325aa353e4c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "signal-hook"
 version = "0.3.10"

Cargo.toml

@@ -13,7 +13,9 @@ reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" }
 nu-cli = { path="./crates/nu-cli" }
 nu-command = { path="./crates/nu-command" }
 nu-engine = { path="./crates/nu-engine" }
+nu-json = { path="./crates/nu-json" }
 nu-parser = { path="./crates/nu-parser" }
+nu-path = { path="./crates/nu-path" }
 nu-protocol = { path = "./crates/nu-protocol" }
 nu-table = { path = "./crates/nu-table" }
 miette = "3.0.0"

crates/nu-command/Cargo.toml

@@ -6,9 +6,11 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-protocol = { path = "../nu-protocol" }
 nu-engine = { path = "../nu-engine" }
+nu-json = { path = "../nu-json" }
+nu-protocol = { path = "../nu-protocol" }
 nu-table = { path = "../nu-table" }
 
 # Potential dependencies for extras
-glob = "0.3.0"
+glob = "0.3.0"
+thiserror = "1.0.29"

crates/nu-command/src/default_context.rs

@@ -6,8 +6,8 @@ use nu_protocol::{
 };
 
 use crate::{
-    Alias, Benchmark, BuildString, Def, Do, Each, External, For, Git, GitCheckout, If, Length, Let,
+    Alias, Benchmark, BuildString, Def, Do, Each, External, For, From, FromJson, Git, GitCheckout,
-    LetEnv, Lines, ListGitBranches, Ls, Module, Table, Use, Where,
+    If, Length, Let, LetEnv, Lines, ListGitBranches, Ls, Module, Table, Use, Where,
 };
 
 pub fn create_default_context() -> Rc<RefCell<EngineState>> {
@@ -20,41 +20,26 @@ pub fn create_default_context() -> Rc<RefCell<EngineState>> {
             Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition");
         working_set.add_decl(sig.predeclare());
 
-        working_set.add_decl(Box::new(If));
-
-        working_set.add_decl(Box::new(Let));
-
-        working_set.add_decl(Box::new(LetEnv));
-
         working_set.add_decl(Box::new(Alias));
-
-        working_set.add_decl(Box::new(BuildString));
-
-        working_set.add_decl(Box::new(Def));
-
-        working_set.add_decl(Box::new(For));
-
-        working_set.add_decl(Box::new(Each));
-
-        working_set.add_decl(Box::new(Where));
-
-        working_set.add_decl(Box::new(Do));
-
         working_set.add_decl(Box::new(Benchmark));
-
+        working_set.add_decl(Box::new(BuildString));
-        working_set.add_decl(Box::new(Length));
+        working_set.add_decl(Box::new(Def));
-
+        working_set.add_decl(Box::new(Do));
-        working_set.add_decl(Box::new(Ls));
+        working_set.add_decl(Box::new(Each));
-
-        working_set.add_decl(Box::new(Module));
-
-        working_set.add_decl(Box::new(Use));
-
-        working_set.add_decl(Box::new(Table));
-
         working_set.add_decl(Box::new(External));
-
+        working_set.add_decl(Box::new(For));
+        working_set.add_decl(Box::new(From));
+        working_set.add_decl(Box::new(FromJson));
+        working_set.add_decl(Box::new(If));
+        working_set.add_decl(Box::new(Length));
+        working_set.add_decl(Box::new(Let));
+        working_set.add_decl(Box::new(LetEnv));
         working_set.add_decl(Box::new(Lines));
+        working_set.add_decl(Box::new(Ls));
+        working_set.add_decl(Box::new(Module));
+        working_set.add_decl(Box::new(Table));
+        working_set.add_decl(Box::new(Use));
+        working_set.add_decl(Box::new(Where));
 
         // This is a WIP proof of concept
         working_set.add_decl(Box::new(ListGitBranches));

crates/nu-command/src/formats/from/command.rs

@@ -0,0 +1,28 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EvaluationContext};
+use nu_protocol::{ShellError, Signature, Value};
+
+pub struct From;
+
+impl Command for From {
+    fn name(&self) -> &str {
+        "from"
+    }
+
+    fn usage(&self) -> &str {
+        "Parse a string or binary data into structured data"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("from")
+    }
+
+    fn run(
+        &self,
+        _context: &EvaluationContext,
+        _call: &Call,
+        _input: Value,
+    ) -> Result<nu_protocol::Value, ShellError> {
+        Ok(Value::nothing())
+    }
+}

crates/nu-command/src/formats/from/json.rs

@@ -0,0 +1,111 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EvaluationContext};
+use nu_protocol::{IntoValueStream, ShellError, Signature, Span, Value};
+
+pub struct FromJson;
+
+impl Command for FromJson {
+    fn name(&self) -> &str {
+        "from json"
+    }
+
+    fn usage(&self) -> &str {
+        "Convert from json to structured data"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("from json").switch(
+            "objects",
+            "treat each line as a separate value",
+            Some('o'),
+        )
+    }
+
+    fn run(
+        &self,
+        _context: &EvaluationContext,
+        call: &Call,
+        input: Value,
+    ) -> Result<nu_protocol::Value, ShellError> {
+        let span = input.span();
+        let mut string_input = input.into_string();
+        string_input.push('\n');
+
+        // TODO: turn this into a structured underline of the nu_json error
+        if call.has_flag("objects") {
+            #[allow(clippy::needless_collect)]
+            let lines: Vec<String> = string_input.lines().map(|x| x.to_string()).collect();
+            Ok(Value::Stream {
+                stream: lines
+                    .into_iter()
+                    .map(move |mut x| {
+                        x.push('\n');
+                        match convert_string_to_value(x, span) {
+                            Ok(v) => v,
+                            Err(error) => Value::Error { error },
+                        }
+                    })
+                    .into_value_stream(),
+                span,
+            })
+        } else {
+            convert_string_to_value(string_input, span)
+        }
+    }
+}
+
+fn convert_nujson_to_value(value: &nu_json::Value, span: Span) -> Value {
+    match value {
+        nu_json::Value::Array(array) => {
+            let v: Vec<Value> = array
+                .iter()
+                .map(|x| convert_nujson_to_value(x, span))
+                .collect();
+
+            Value::List { vals: v, span }
+        }
+        nu_json::Value::Bool(b) => Value::Bool { val: *b, span },
+        nu_json::Value::F64(f) => Value::Float { val: *f, span },
+        nu_json::Value::I64(i) => Value::Int { val: *i, span },
+        nu_json::Value::Null => Value::Nothing { span },
+        nu_json::Value::Object(k) => {
+            let mut cols = vec![];
+            let mut vals = vec![];
+
+            for item in k {
+                cols.push(item.0.clone());
+                vals.push(convert_nujson_to_value(item.1, span));
+            }
+
+            Value::Record { cols, vals, span }
+        }
+        nu_json::Value::U64(u) => {
+            if *u > i64::MAX as u64 {
+                Value::Error {
+                    error: ShellError::CantConvert("i64 sized integer".into(), span),
+                }
+            } else {
+                Value::Int {
+                    val: *u as i64,
+                    span,
+                }
+            }
+        }
+        nu_json::Value::String(s) => Value::String {
+            val: s.clone(),
+            span,
+        },
+    }
+}
+
+fn convert_string_to_value(string_input: String, span: Span) -> Result<Value, ShellError> {
+    let result: Result<nu_json::Value, nu_json::Error> = nu_json::from_str(&string_input);
+    match result {
+        Ok(value) => Ok(convert_nujson_to_value(&value, span)),
+
+        Err(_x) => Err(ShellError::CantConvert(
+            "structured data from json".into(),
+            span,
+        )),
+    }
+}

crates/nu-command/src/formats/from/mod.rs

@@ -0,0 +1,5 @@
+mod command;
+mod json;
+
+pub use command::From;
+pub use json::FromJson;

crates/nu-command/src/formats/mod.rs

@@ -0,0 +1,3 @@
+mod from;
+
+pub use from::*;

crates/nu-command/src/lib.rs

@@ -4,6 +4,7 @@ mod env;
 mod experimental;
 mod filesystem;
 mod filters;
+mod formats;
 mod strings;
 mod system;
 mod viewers;
@@ -14,6 +15,7 @@ pub use env::*;
 pub use experimental::*;
 pub use filesystem::*;
 pub use filters::*;
+pub use formats::*;
 pub use strings::*;
 pub use system::*;
 pub use viewers::*;

crates/nu-json/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+authors = ["The Nu Project Contributors", "Christian Zangl <laktak@cdak.net>"]
+description = "Fork of serde-hjson"
+edition = "2018"
+license = "MIT"
+name = "nu-json"
+version = "0.37.1"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+preserve_order = ["linked-hash-map", "linked-hash-map/serde_impl"]
+default = ["preserve_order"]
+
+[dependencies]
+serde = "1.0"
+num-traits = "0.2.14"
+regex = "^1.0"
+lazy_static = "1"
+linked-hash-map = { version="0.5", optional=true }
+
+[dev-dependencies]
+nu-path = { version = "0.37.1", path="../nu-path" }
+serde_json = "1.0.39"

crates/nu-json/LICENSE

@@ -0,0 +1,29 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 The Rust Project Developers
+Copyright (c) 2016 Christian Zangl
+Copyright (c) 2020 The Nu Project Contributors
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

crates/nu-json/src/builder.rs

@@ -0,0 +1,115 @@
+use serde::ser;
+
+use crate::value::{self, Map, Value};
+
+/// This structure provides a simple interface for constructing a JSON array.
+pub struct ArrayBuilder {
+    array: Vec<Value>,
+}
+
+impl Default for ArrayBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ArrayBuilder {
+    /// Construct an `ObjectBuilder`.
+    pub fn new() -> ArrayBuilder {
+        ArrayBuilder { array: Vec::new() }
+    }
+
+    /// Return the constructed `Value`.
+    pub fn unwrap(self) -> Value {
+        Value::Array(self.array)
+    }
+
+    /// Insert a value into the array.
+    pub fn push<T: ser::Serialize>(mut self, v: T) -> ArrayBuilder {
+        self.array
+            .push(value::to_value(&v).expect("failed to serialize"));
+        self
+    }
+
+    /// Creates and passes an `ArrayBuilder` into a closure, then inserts the resulting array into
+    /// this array.
+    pub fn push_array<F>(mut self, f: F) -> ArrayBuilder
+    where
+        F: FnOnce(ArrayBuilder) -> ArrayBuilder,
+    {
+        let builder = ArrayBuilder::new();
+        self.array.push(f(builder).unwrap());
+        self
+    }
+
+    /// Creates and passes an `ArrayBuilder` into a closure, then inserts the resulting object into
+    /// this array.
+    pub fn push_object<F>(mut self, f: F) -> ArrayBuilder
+    where
+        F: FnOnce(ObjectBuilder) -> ObjectBuilder,
+    {
+        let builder = ObjectBuilder::new();
+        self.array.push(f(builder).unwrap());
+        self
+    }
+}
+
+/// This structure provides a simple interface for constructing a JSON object.
+pub struct ObjectBuilder {
+    object: Map<String, Value>,
+}
+
+impl Default for ObjectBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ObjectBuilder {
+    /// Construct an `ObjectBuilder`.
+    pub fn new() -> ObjectBuilder {
+        ObjectBuilder { object: Map::new() }
+    }
+
+    /// Return the constructed `Value`.
+    pub fn unwrap(self) -> Value {
+        Value::Object(self.object)
+    }
+
+    /// Insert a key-value pair into the object.
+    pub fn insert<S, V>(mut self, key: S, value: V) -> ObjectBuilder
+    where
+        S: Into<String>,
+        V: ser::Serialize,
+    {
+        self.object.insert(
+            key.into(),
+            value::to_value(&value).expect("failed to serialize"),
+        );
+        self
+    }
+
+    /// Creates and passes an `ObjectBuilder` into a closure, then inserts the resulting array into
+    /// this object.
+    pub fn insert_array<S, F>(mut self, key: S, f: F) -> ObjectBuilder
+    where
+        S: Into<String>,
+        F: FnOnce(ArrayBuilder) -> ArrayBuilder,
+    {
+        let builder = ArrayBuilder::new();
+        self.object.insert(key.into(), f(builder).unwrap());
+        self
+    }
+
+    /// Creates and passes an `ObjectBuilder` into a closure, then inserts the resulting object into
+    /// this object.
+    pub fn insert_object<S, F>(mut self, key: S, f: F) -> ObjectBuilder
+    where
+        S: Into<String>,
+        F: FnOnce(ObjectBuilder) -> ObjectBuilder,
+    {
+        let builder = ObjectBuilder::new();
+        self.object.insert(key.into(), f(builder).unwrap());
+        self
+    }
+}

crates/nu-json/src/de.rs

@@ -0,0 +1,833 @@
+//! Hjson Deserialization
+//!
+//! This module provides for Hjson deserialization with the type `Deserializer`.
+
+use std::char;
+use std::io;
+use std::marker::PhantomData;
+use std::str;
+
+use serde::de;
+
+use super::error::{Error, ErrorCode, Result};
+use super::util::StringReader;
+use super::util::{Number, ParseNumber};
+
+enum State {
+    Normal,
+    Root,
+    Keyname,
+}
+
+/// A structure that deserializes Hjson into Rust values.
+pub struct Deserializer<Iter: Iterator<Item = u8>> {
+    rdr: StringReader<Iter>,
+    str_buf: Vec<u8>,
+    state: State,
+}
+
+// macro_rules! try_or_invalid {
+//     ($self_:expr, $e:expr) => {
+//         match $e {
+//             Some(v) => v,
+//             None => { return Err($self_.error(ErrorCode::InvalidNumber)); }
+//         }
+//     }
+// }
+
+impl<Iter> Deserializer<Iter>
+where
+    Iter: Iterator<Item = u8>,
+{
+    /// Creates the Hjson parser from an `std::iter::Iterator`.
+    #[inline]
+    pub fn new(rdr: Iter) -> Deserializer<Iter> {
+        Deserializer {
+            rdr: StringReader::new(rdr),
+            str_buf: Vec::with_capacity(128),
+            state: State::Normal,
+        }
+    }
+
+    /// Creates the Hjson parser from an `std::iter::Iterator`.
+    #[inline]
+    pub fn new_for_root(rdr: Iter) -> Deserializer<Iter> {
+        let mut res = Deserializer::new(rdr);
+        res.state = State::Root;
+        res
+    }
+
+    /// The `Deserializer::end` method should be called after a value has been fully deserialized.
+    /// This allows the `Deserializer` to validate that the input stream is at the end or that it
+    /// only has trailing whitespace.
+    #[inline]
+    pub fn end(&mut self) -> Result<()> {
+        self.rdr.parse_whitespace()?;
+        if self.rdr.eof()? {
+            Ok(())
+        } else {
+            Err(self.rdr.error(ErrorCode::TrailingCharacters))
+        }
+    }
+
+    fn is_punctuator_char(&mut self, ch: u8) -> bool {
+        matches!(ch, b'{' | b'}' | b'[' | b']' | b',' | b':')
+    }
+
+    fn parse_keyname<'de, V>(&mut self, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        // quotes for keys are optional in Hjson
+        // unless they include {}[],: or whitespace.
+        // assume whitespace was already eaten
+
+        self.str_buf.clear();
+
+        let mut space: Option<usize> = None;
+        loop {
+            let ch = self.rdr.next_char_or_null()?;
+
+            if ch == b':' {
+                if self.str_buf.is_empty() {
+                    return Err(self.rdr.error(ErrorCode::Custom(
+                        "Found ':' but no key name (for an empty key name use quotes)".to_string(),
+                    )));
+                } else if space.is_some()
+                    && space.expect("Internal error: json parsing") != self.str_buf.len()
+                {
+                    return Err(self.rdr.error(ErrorCode::Custom(
+                        "Found whitespace in your key name (use quotes to include)".to_string(),
+                    )));
+                }
+                self.rdr.uneat_char(ch);
+                let s = str::from_utf8(&self.str_buf).expect("Internal error: json parsing");
+                return visitor.visit_str(s);
+            } else if ch <= b' ' {
+                if ch == 0 {
+                    return Err(self.rdr.error(ErrorCode::EofWhileParsingObject));
+                } else if space.is_none() {
+                    space = Some(self.str_buf.len());
+                }
+            } else if self.is_punctuator_char(ch) {
+                return Err(self.rdr.error(ErrorCode::Custom("Found a punctuator where a key name was expected (check your syntax or use quotes if the key name includes {}[],: or whitespace)".to_string())));
+            } else {
+                self.str_buf.push(ch);
+            }
+        }
+    }
+
+    fn parse_value<'de, V>(&mut self, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        self.rdr.parse_whitespace()?;
+
+        if self.rdr.eof()? {
+            return Err(self.rdr.error(ErrorCode::EofWhileParsingValue));
+        }
+
+        match self.state {
+            State::Keyname => {
+                self.state = State::Normal;
+                return self.parse_keyname(visitor);
+            }
+            State::Root => {
+                self.state = State::Normal;
+                return self.visit_map(true, visitor);
+            }
+            _ => {}
+        }
+
+        match self.rdr.peek_or_null()? {
+            /*
+            b'-' => {
+                self.rdr.eat_char();
+                self.parse_integer(false, visitor)
+            }
+            b'0' ... b'9' => {
+                self.parse_integer(true, visitor)
+            }
+            */
+            b'"' => {
+                self.rdr.eat_char();
+                self.parse_string()?;
+                let s = str::from_utf8(&self.str_buf).expect("Internal error: json parsing");
+                visitor.visit_str(s)
+            }
+            b'[' => {
+                self.rdr.eat_char();
+                let ret = visitor.visit_seq(SeqVisitor::new(self))?;
+                self.rdr.parse_whitespace()?;
+                match self.rdr.next_char()? {
+                    Some(b']') => Ok(ret),
+                    Some(_) => Err(self.rdr.error(ErrorCode::TrailingCharacters)),
+                    None => Err(self.rdr.error(ErrorCode::EofWhileParsingList)),
+                }
+            }
+            b'{' => {
+                self.rdr.eat_char();
+                self.visit_map(false, visitor)
+            }
+            b'\x00' => Err(self.rdr.error(ErrorCode::ExpectedSomeValue)),
+            _ => self.parse_tfnns(visitor),
+        }
+    }
+
+    fn visit_map<'de, V>(&mut self, root: bool, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        let ret = visitor.visit_map(MapVisitor::new(self, root))?;
+        self.rdr.parse_whitespace()?;
+        match self.rdr.next_char()? {
+            Some(b'}') => {
+                if !root {
+                    Ok(ret)
+                } else {
+                    Err(self.rdr.error(ErrorCode::TrailingCharacters))
+                } // todo
+            }
+            Some(_) => Err(self.rdr.error(ErrorCode::TrailingCharacters)),
+            None => {
+                if root {
+                    Ok(ret)
+                } else {
+                    Err(self.rdr.error(ErrorCode::EofWhileParsingObject))
+                }
+            }
+        }
+    }
+
+    fn parse_ident(&mut self, ident: &[u8]) -> Result<()> {
+        for c in ident {
+            if Some(*c) != self.rdr.next_char()? {
+                return Err(self.rdr.error(ErrorCode::ExpectedSomeIdent));
+            }
+        }
+
+        Ok(())
+    }
+
+    fn parse_tfnns<'de, V>(&mut self, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        // Hjson strings can be quoteless
+        // returns string, true, false, or null.
+        self.str_buf.clear();
+
+        let first = self.rdr.peek()?.expect("Internal error: json parsing");
+
+        if self.is_punctuator_char(first) {
+            return Err(self.rdr.error(ErrorCode::PunctuatorInQlString));
+        }
+
+        loop {
+            let ch = self.rdr.next_char_or_null()?;
+
+            let is_eol = ch == b'\r' || ch == b'\n' || ch == b'\x00';
+            let is_comment = ch == b'#'
+                || if ch == b'/' {
+                    let next = self.rdr.peek_or_null()?;
+                    next == b'/' || next == b'*'
+                } else {
+                    false
+                };
+            if is_eol || is_comment || ch == b',' || ch == b'}' || ch == b']' {
+                let chf = self.str_buf[0];
+                match chf {
+                    b'f' => {
+                        if str::from_utf8(&self.str_buf)
+                            .expect("Internal error: json parsing")
+                            .trim()
+                            == "false"
+                        {
+                            self.rdr.uneat_char(ch);
+                            return visitor.visit_bool(false);
+                        }
+                    }
+                    b'n' => {
+                        if str::from_utf8(&self.str_buf)
+                            .expect("Internal error: json parsing")
+                            .trim()
+                            == "null"
+                        {
+                            self.rdr.uneat_char(ch);
+                            return visitor.visit_unit();
+                        }
+                    }
+                    b't' => {
+                        if str::from_utf8(&self.str_buf)
+                            .expect("Internal error: json parsing")
+                            .trim()
+                            == "true"
+                        {
+                            self.rdr.uneat_char(ch);
+                            return visitor.visit_bool(true);
+                        }
+                    }
+                    _ => {
+                        if chf == b'-' || (b'0'..=b'9').contains(&chf) {
+                            let mut pn = ParseNumber::new(self.str_buf.iter().copied());
+                            match pn.parse(false) {
+                                Ok(Number::F64(v)) => {
+                                    self.rdr.uneat_char(ch);
+                                    return visitor.visit_f64(v);
+                                }
+                                Ok(Number::U64(v)) => {
+                                    self.rdr.uneat_char(ch);
+                                    return visitor.visit_u64(v);
+                                }
+                                Ok(Number::I64(v)) => {
+                                    self.rdr.uneat_char(ch);
+                                    return visitor.visit_i64(v);
+                                }
+                                Err(_) => {} // not a number, continue
+                            }
+                        }
+                    }
+                }
+                if is_eol {
+                    // remove any whitespace at the end (ignored in quoteless strings)
+                    return visitor.visit_str(
+                        str::from_utf8(&self.str_buf)
+                            .expect("Internal error: json parsing")
+                            .trim(),
+                    );
+                }
+            }
+            self.str_buf.push(ch);
+
+            if self.str_buf == b"'''" {
+                return self.parse_ml_string(visitor);
+            }
+        }
+    }
+
+    fn decode_hex_escape(&mut self) -> Result<u16> {
+        let mut i = 0;
+        let mut n = 0u16;
+        while i < 4 && !self.rdr.eof()? {
+            n = match self.rdr.next_char_or_null()? {
+                c @ b'0'..=b'9' => n * 16_u16 + ((c as u16) - (b'0' as u16)),
+                b'a' | b'A' => n * 16_u16 + 10_u16,
+                b'b' | b'B' => n * 16_u16 + 11_u16,
+                b'c' | b'C' => n * 16_u16 + 12_u16,
+                b'd' | b'D' => n * 16_u16 + 13_u16,
+                b'e' | b'E' => n * 16_u16 + 14_u16,
+                b'f' | b'F' => n * 16_u16 + 15_u16,
+                _ => {
+                    return Err(self.rdr.error(ErrorCode::InvalidEscape));
+                }
+            };
+
+            i += 1;
+        }
+
+        // Error out if we didn't parse 4 digits.
+        if i != 4 {
+            return Err(self.rdr.error(ErrorCode::InvalidEscape));
+        }
+
+        Ok(n)
+    }
+
+    fn ml_skip_white(&mut self) -> Result<bool> {
+        match self.rdr.peek_or_null()? {
+            b' ' | b'\t' | b'\r' => {
+                self.rdr.eat_char();
+                Ok(true)
+            }
+            _ => Ok(false),
+        }
+    }
+
+    fn ml_skip_indent(&mut self, indent: usize) -> Result<()> {
+        let mut skip = indent;
+        while self.ml_skip_white()? && skip > 0 {
+            skip -= 1;
+        }
+        Ok(())
+    }
+
+    fn parse_ml_string<'de, V>(&mut self, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        self.str_buf.clear();
+
+        // Parse a multiline string value.
+        let mut triple = 0;
+
+        // we are at ''' +1 - get indent
+        let (_, col) = self.rdr.pos();
+        let indent = col - 4;
+
+        // skip white/to (newline)
+        while self.ml_skip_white()? {}
+        if self.rdr.peek_or_null()? == b'\n' {
+            self.rdr.eat_char();
+            self.ml_skip_indent(indent)?;
+        }
+
+        // When parsing multiline string values, we must look for ' characters.
+        loop {
+            if self.rdr.eof()? {
+                return Err(self.rdr.error(ErrorCode::EofWhileParsingString));
+            } // todo error("Bad multiline string");
+            let ch = self.rdr.next_char_or_null()?;
+
+            if ch == b'\'' {
+                triple += 1;
+                if triple == 3 {
+                    if self.str_buf.last() == Some(&b'\n') {
+                        self.str_buf.pop();
+                    }
+                    let res = str::from_utf8(&self.str_buf).expect("Internal error: json parsing");
+                    //todo if (self.str_buf.slice(-1) === '\n') self.str_buf=self.str_buf.slice(0, -1); // remove last EOL
+                    return visitor.visit_str(res);
+                } else {
+                    continue;
+                }
+            }
+
+            while triple > 0 {
+                self.str_buf.push(b'\'');
+                triple -= 1;
+            }
+
+            if ch != b'\r' {
+                self.str_buf.push(ch);
+            }
+            if ch == b'\n' {
+                self.ml_skip_indent(indent)?;
+            }
+        }
+    }
+
+    fn parse_string(&mut self) -> Result<()> {
+        self.str_buf.clear();
+
+        loop {
+            let ch = match self.rdr.next_char()? {
+                Some(ch) => ch,
+                None => {
+                    return Err(self.rdr.error(ErrorCode::EofWhileParsingString));
+                }
+            };
+
+            match ch {
+                b'"' => {
+                    return Ok(());
+                }
+                b'\\' => {
+                    let ch = match self.rdr.next_char()? {
+                        Some(ch) => ch,
+                        None => {
+                            return Err(self.rdr.error(ErrorCode::EofWhileParsingString));
+                        }
+                    };
+
+                    match ch {
+                        b'"' => self.str_buf.push(b'"'),
+                        b'\\' => self.str_buf.push(b'\\'),
+                        b'/' => self.str_buf.push(b'/'),
+                        b'b' => self.str_buf.push(b'\x08'),
+                        b'f' => self.str_buf.push(b'\x0c'),
+                        b'n' => self.str_buf.push(b'\n'),
+                        b'r' => self.str_buf.push(b'\r'),
+                        b't' => self.str_buf.push(b'\t'),
+                        b'u' => {
+                            let c = match self.decode_hex_escape()? {
+                                0xDC00..=0xDFFF => {
+                                    return Err(self
+                                        .rdr
+                                        .error(ErrorCode::LoneLeadingSurrogateInHexEscape));
+                                }
+
+                                // Non-BMP characters are encoded as a sequence of
+                                // two hex escapes, representing UTF-16 surrogates.
+                                n1 @ 0xD800..=0xDBFF => {
+                                    match (self.rdr.next_char()?, self.rdr.next_char()?) {
+                                        (Some(b'\\'), Some(b'u')) => (),
+                                        _ => {
+                                            return Err(self
+                                                .rdr
+                                                .error(ErrorCode::UnexpectedEndOfHexEscape));
+                                        }
+                                    }
+
+                                    let n2 = self.decode_hex_escape()?;
+
+                                    if !(0xDC00..=0xDFFF).contains(&n2) {
+                                        return Err(self
+                                            .rdr
+                                            .error(ErrorCode::LoneLeadingSurrogateInHexEscape));
+                                    }
+
+                                    let n = (((n1 - 0xD800) as u32) << 10 | (n2 - 0xDC00) as u32)
+                                        + 0x1_0000;
+
+                                    match char::from_u32(n as u32) {
+                                        Some(c) => c,
+                                        None => {
+                                            return Err(self
+                                                .rdr
+                                                .error(ErrorCode::InvalidUnicodeCodePoint));
+                                        }
+                                    }
+                                }
+
+                                n => match char::from_u32(n as u32) {
+                                    Some(c) => c,
+                                    None => {
+                                        return Err(self
+                                            .rdr
+                                            .error(ErrorCode::InvalidUnicodeCodePoint));
+                                    }
+                                },
+                            };
+
+                            self.str_buf.extend(c.encode_utf8(&mut [0; 4]).as_bytes());
+                        }
+                        _ => {
+                            return Err(self.rdr.error(ErrorCode::InvalidEscape));
+                        }
+                    }
+                }
+                ch => {
+                    self.str_buf.push(ch);
+                }
+            }
+        }
+    }
+
+    fn parse_object_colon(&mut self) -> Result<()> {
+        self.rdr.parse_whitespace()?;
+
+        match self.rdr.next_char()? {
+            Some(b':') => Ok(()),
+            Some(_) => Err(self.rdr.error(ErrorCode::ExpectedColon)),
+            None => Err(self.rdr.error(ErrorCode::EofWhileParsingObject)),
+        }
+    }
+}
+
+impl<'de, 'a, Iter> de::Deserializer<'de> for &'a mut Deserializer<Iter>
+where
+    Iter: Iterator<Item = u8>,
+{
+    type Error = Error;
+
+    #[inline]
+    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        if let State::Root = self.state {}
+
+        self.parse_value(visitor)
+    }
+
+    /// Parses a `null` as a None, and any other values as a `Some(...)`.
+    #[inline]
+    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        self.rdr.parse_whitespace()?;
+
+        match self.rdr.peek_or_null()? {
+            b'n' => {
+                self.rdr.eat_char();
+                self.parse_ident(b"ull")?;
+                visitor.visit_none()
+            }
+            _ => visitor.visit_some(self),
+        }
+    }
+
+    /// Parses a newtype struct as the underlying value.
+    #[inline]
+    fn deserialize_newtype_struct<V>(self, _name: &str, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        visitor.visit_newtype_struct(self)
+    }
+
+    serde::forward_to_deserialize_any! {
+        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
+        bytes byte_buf unit unit_struct seq tuple map
+        tuple_struct struct enum identifier ignored_any
+    }
+}
+
+struct SeqVisitor<'a, Iter: 'a + Iterator<Item = u8>> {
+    de: &'a mut Deserializer<Iter>,
+}
+
+impl<'a, Iter: Iterator<Item = u8>> SeqVisitor<'a, Iter> {
+    fn new(de: &'a mut Deserializer<Iter>) -> Self {
+        SeqVisitor { de }
+    }
+}
+
+impl<'de, 'a, Iter> de::SeqAccess<'de> for SeqVisitor<'a, Iter>
+where
+    Iter: Iterator<Item = u8>,
+{
+    type Error = Error;
+
+    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>>
+    where
+        T: de::DeserializeSeed<'de>,
+    {
+        self.de.rdr.parse_whitespace()?;
+
+        match self.de.rdr.peek()? {
+            Some(b']') => {
+                return Ok(None);
+            }
+            Some(_) => {}
+            None => {
+                return Err(self.de.rdr.error(ErrorCode::EofWhileParsingList));
+            }
+        }
+
+        let value = seed.deserialize(&mut *self.de)?;
+
+        // in Hjson the comma is optional and trailing commas are allowed
+        self.de.rdr.parse_whitespace()?;
+        if self.de.rdr.peek()? == Some(b',') {
+            self.de.rdr.eat_char();
+            self.de.rdr.parse_whitespace()?;
+        }
+
+        Ok(Some(value))
+    }
+}
+
+struct MapVisitor<'a, Iter: 'a + Iterator<Item = u8>> {
+    de: &'a mut Deserializer<Iter>,
+    first: bool,
+    root: bool,
+}
+
+impl<'a, Iter: Iterator<Item = u8>> MapVisitor<'a, Iter> {
+    fn new(de: &'a mut Deserializer<Iter>, root: bool) -> Self {
+        MapVisitor {
+            de,
+            first: true,
+            root,
+        }
+    }
+}
+
+impl<'de, 'a, Iter> de::MapAccess<'de> for MapVisitor<'a, Iter>
+where
+    Iter: Iterator<Item = u8>,
+{
+    type Error = Error;
+
+    fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>>
+    where
+        K: de::DeserializeSeed<'de>,
+    {
+        self.de.rdr.parse_whitespace()?;
+
+        if self.first {
+            self.first = false;
+        } else if self.de.rdr.peek()? == Some(b',') {
+            // in Hjson the comma is optional and trailing commas are allowed
+            self.de.rdr.eat_char();
+            self.de.rdr.parse_whitespace()?;
+        }
+
+        match self.de.rdr.peek()? {
+            Some(b'}') => return Ok(None), // handled later for root
+            Some(_) => {}
+            None => {
+                if self.root {
+                    return Ok(None);
+                } else {
+                    return Err(self.de.rdr.error(ErrorCode::EofWhileParsingObject));
+                }
+            }
+        }
+
+        match self.de.rdr.peek()? {
+            Some(ch) => {
+                self.de.state = if ch == b'"' {
+                    State::Normal
+                } else {
+                    State::Keyname
+                };
+                Ok(Some(seed.deserialize(&mut *self.de)?))
+            }
+            None => Err(self.de.rdr.error(ErrorCode::EofWhileParsingValue)),
+        }
+    }
+
+    fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value>
+    where
+        V: de::DeserializeSeed<'de>,
+    {
+        self.de.parse_object_colon()?;
+
+        seed.deserialize(&mut *self.de)
+    }
+}
+
+impl<'de, 'a, Iter> de::VariantAccess<'de> for &'a mut Deserializer<Iter>
+where
+    Iter: Iterator<Item = u8>,
+{
+    type Error = Error;
+
+    fn unit_variant(self) -> Result<()> {
+        de::Deserialize::deserialize(self)
+    }
+
+    fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value>
+    where
+        T: de::DeserializeSeed<'de>,
+    {
+        seed.deserialize(self)
+    }
+
+    fn tuple_variant<V>(self, _len: usize, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        de::Deserializer::deserialize_any(self, visitor)
+    }
+
+    fn struct_variant<V>(self, _fields: &'static [&'static str], visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        de::Deserializer::deserialize_any(self, visitor)
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+/// Iterator that deserializes a stream into multiple Hjson values.
+pub struct StreamDeserializer<T, Iter>
+where
+    Iter: Iterator<Item = u8>,
+    T: de::DeserializeOwned,
+{
+    deser: Deserializer<Iter>,
+    _marker: PhantomData<T>,
+}
+
+impl<T, Iter> StreamDeserializer<T, Iter>
+where
+    Iter: Iterator<Item = u8>,
+    T: de::DeserializeOwned,
+{
+    /// Returns an `Iterator` of decoded Hjson values from an iterator over
+    /// `Iterator<Item=u8>`.
+    pub fn new(iter: Iter) -> StreamDeserializer<T, Iter> {
+        StreamDeserializer {
+            deser: Deserializer::new(iter),
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<T, Iter> Iterator for StreamDeserializer<T, Iter>
+where
+    Iter: Iterator<Item = u8>,
+    T: de::DeserializeOwned,
+{
+    type Item = Result<T>;
+
+    fn next(&mut self) -> Option<Result<T>> {
+        // skip whitespaces, if any
+        // this helps with trailing whitespaces, since whitespaces between
+        // values are handled for us.
+        if let Err(e) = self.deser.rdr.parse_whitespace() {
+            return Some(Err(e));
+        };
+
+        match self.deser.rdr.eof() {
+            Ok(true) => None,
+            Ok(false) => match de::Deserialize::deserialize(&mut self.deser) {
+                Ok(v) => Some(Ok(v)),
+                Err(e) => Some(Err(e)),
+            },
+            Err(e) => Some(Err(e)),
+        }
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+/// Decodes a Hjson value from an iterator over an iterator
+/// `Iterator<Item=u8>`.
+pub fn from_iter<I, T>(iter: I) -> Result<T>
+where
+    I: Iterator<Item = io::Result<u8>>,
+    T: de::DeserializeOwned,
+{
+    let fold: io::Result<Vec<_>> = iter.collect();
+
+    if let Err(e) = fold {
+        return Err(Error::Io(e));
+    }
+
+    let bytes = fold.expect("Internal error: json parsing");
+
+    // deserialize tries first to decode with legacy support (new_for_root)
+    // and then with the standard method if this fails.
+    // todo: add compile switch
+
+    // deserialize and make sure the whole stream has been consumed
+    let mut de = Deserializer::new_for_root(bytes.iter().copied());
+    de::Deserialize::deserialize(&mut de)
+        .and_then(|x| de.end().map(|()| x))
+        .or_else(|_| {
+            let mut de2 = Deserializer::new(bytes.iter().copied());
+            de::Deserialize::deserialize(&mut de2).and_then(|x| de2.end().map(|()| x))
+        })
+
+    /* without legacy support:
+    // deserialize and make sure the whole stream has been consumed
+    let mut de = Deserializer::new(bytes.iter().map(|b| *b));
+    let value = match de::Deserialize::deserialize(&mut de)
+        .and_then(|x| { try!(de.end()); Ok(x) })
+    {
+        Ok(v) => Ok(v),
+        Err(e) => Err(e),
+    };
+    */
+}
+
+/// Decodes a Hjson value from a `std::io::Read`.
+pub fn from_reader<R, T>(rdr: R) -> Result<T>
+where
+    R: io::Read,
+    T: de::DeserializeOwned,
+{
+    from_iter(rdr.bytes())
+}
+
+/// Decodes a Hjson value from a byte slice `&[u8]`.
+pub fn from_slice<T>(v: &[u8]) -> Result<T>
+where
+    T: de::DeserializeOwned,
+{
+    from_iter(v.iter().map(|&byte| Ok(byte)))
+}
+
+/// Decodes a Hjson value from a `&str`.
+pub fn from_str<T>(s: &str) -> Result<T>
+where
+    T: de::DeserializeOwned,
+{
+    from_slice(s.as_bytes())
+}

crates/nu-json/src/error.rs

@@ -0,0 +1,166 @@
+//! JSON Errors
+//!
+//! This module is centered around the `Error` and `ErrorCode` types, which represents all possible
+//! `serde_hjson` errors.
+
+use std::error;
+use std::fmt;
+use std::io;
+use std::result;
+use std::string::FromUtf8Error;
+
+use serde::de;
+use serde::ser;
+
+/// The errors that can arise while parsing a JSON stream.
+#[derive(Clone, PartialEq)]
+pub enum ErrorCode {
+    /// Catchall for syntax error messages
+    Custom(String),
+
+    /// EOF while parsing a list.
+    EofWhileParsingList,
+
+    /// EOF while parsing an object.
+    EofWhileParsingObject,
+
+    /// EOF while parsing a string.
+    EofWhileParsingString,
+
+    /// EOF while parsing a JSON value.
+    EofWhileParsingValue,
+
+    /// Expected this character to be a `':'`.
+    ExpectedColon,
+
+    /// Expected this character to be either a `','` or a `]`.
+    ExpectedListCommaOrEnd,
+
+    /// Expected this character to be either a `','` or a `}`.
+    ExpectedObjectCommaOrEnd,
+
+    /// Expected to parse either a `true`, `false`, or a `null`.
+    ExpectedSomeIdent,
+
+    /// Expected this character to start a JSON value.
+    ExpectedSomeValue,
+
+    /// Invalid hex escape code.
+    InvalidEscape,
+
+    /// Invalid number.
+    InvalidNumber,
+
+    /// Invalid Unicode code point.
+    InvalidUnicodeCodePoint,
+
+    /// Object key is not a string.
+    KeyMustBeAString,
+
+    /// Lone leading surrogate in hex escape.
+    LoneLeadingSurrogateInHexEscape,
+
+    /// JSON has non-whitespace trailing characters after the value.
+    TrailingCharacters,
+
+    /// Unexpected end of hex escape.
+    UnexpectedEndOfHexEscape,
+
+    /// Found a punctuator character when expecting a quoteless string.
+    PunctuatorInQlString,
+}
+
+impl fmt::Debug for ErrorCode {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        //use std::fmt::Debug;
+
+        match *self {
+            ErrorCode::Custom(ref msg) => write!(f, "{}", msg),
+            ErrorCode::EofWhileParsingList => "EOF while parsing a list".fmt(f),
+            ErrorCode::EofWhileParsingObject => "EOF while parsing an object".fmt(f),
+            ErrorCode::EofWhileParsingString => "EOF while parsing a string".fmt(f),
+            ErrorCode::EofWhileParsingValue => "EOF while parsing a value".fmt(f),
+            ErrorCode::ExpectedColon => "expected `:`".fmt(f),
+            ErrorCode::ExpectedListCommaOrEnd => "expected `,` or `]`".fmt(f),
+            ErrorCode::ExpectedObjectCommaOrEnd => "expected `,` or `}`".fmt(f),
+            ErrorCode::ExpectedSomeIdent => "expected ident".fmt(f),
+            ErrorCode::ExpectedSomeValue => "expected value".fmt(f),
+            ErrorCode::InvalidEscape => "invalid escape".fmt(f),
+            ErrorCode::InvalidNumber => "invalid number".fmt(f),
+            ErrorCode::InvalidUnicodeCodePoint => "invalid Unicode code point".fmt(f),
+            ErrorCode::KeyMustBeAString => "key must be a string".fmt(f),
+            ErrorCode::LoneLeadingSurrogateInHexEscape => {
+                "lone leading surrogate in hex escape".fmt(f)
+            }
+            ErrorCode::TrailingCharacters => "trailing characters".fmt(f),
+            ErrorCode::UnexpectedEndOfHexEscape => "unexpected end of hex escape".fmt(f),
+            ErrorCode::PunctuatorInQlString => {
+                "found a punctuator character when expecting a quoteless string".fmt(f)
+            }
+        }
+    }
+}
+
+/// This type represents all possible errors that can occur when serializing or deserializing a
+/// value into JSON.
+#[derive(Debug)]
+pub enum Error {
+    /// The JSON value had some syntactic error.
+    Syntax(ErrorCode, usize, usize),
+
+    /// Some IO error occurred when serializing or deserializing a value.
+    Io(io::Error),
+
+    /// Some UTF8 error occurred while serializing or deserializing a value.
+    FromUtf8(FromUtf8Error),
+}
+
+impl error::Error for Error {
+    fn cause(&self) -> Option<&dyn error::Error> {
+        match *self {
+            Error::Io(ref error) => Some(error),
+            Error::FromUtf8(ref error) => Some(error),
+            _ => None,
+        }
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            Error::Syntax(ref code, line, col) => {
+                write!(fmt, "{:?} at line {} column {}", code, line, col)
+            }
+            Error::Io(ref error) => fmt::Display::fmt(error, fmt),
+            Error::FromUtf8(ref error) => fmt::Display::fmt(error, fmt),
+        }
+    }
+}
+
+impl From<io::Error> for Error {
+    fn from(error: io::Error) -> Error {
+        Error::Io(error)
+    }
+}
+
+impl From<FromUtf8Error> for Error {
+    fn from(error: FromUtf8Error) -> Error {
+        Error::FromUtf8(error)
+    }
+}
+
+impl de::Error for Error {
+    fn custom<T: fmt::Display>(msg: T) -> Error {
+        Error::Syntax(ErrorCode::Custom(msg.to_string()), 0, 0)
+    }
+}
+
+impl ser::Error for Error {
+    /// Raised when there is general error when deserializing a type.
+    fn custom<T: fmt::Display>(msg: T) -> Error {
+        Error::Syntax(ErrorCode::Custom(msg.to_string()), 0, 0)
+    }
+}
+
+/// Helper alias for `Result` objects that return a JSON `Error`.
+pub type Result<T> = result::Result<T, Error>;

crates/nu-json/src/lib.rs

@@ -0,0 +1,13 @@
+pub use self::de::{
+    from_iter, from_reader, from_slice, from_str, Deserializer, StreamDeserializer,
+};
+pub use self::error::{Error, ErrorCode, Result};
+pub use self::ser::{to_string, to_vec, to_writer, Serializer};
+pub use self::value::{from_value, to_value, Map, Value};
+
+pub mod builder;
+pub mod de;
+pub mod error;
+pub mod ser;
+mod util;
+pub mod value;

crates/nu-json/src/util.rs

@@ -0,0 +1,333 @@
+use std::io;
+use std::str;
+
+use super::error::{Error, ErrorCode, Result};
+
+pub struct StringReader<Iter: Iterator<Item = u8>> {
+    iter: Iter,
+    line: usize,
+    col: usize,
+    ch: Vec<u8>,
+}
+
+impl<Iter> StringReader<Iter>
+where
+    Iter: Iterator<Item = u8>,
+{
+    #[inline]
+    pub fn new(iter: Iter) -> Self {
+        StringReader {
+            iter,
+            line: 1,
+            col: 0,
+            ch: Vec::new(),
+        }
+    }
+
+    fn next(&mut self) -> Option<io::Result<u8>> {
+        match self.iter.next() {
+            None => None,
+            Some(b'\n') => {
+                self.line += 1;
+                self.col = 0;
+                Some(Ok(b'\n'))
+            }
+            Some(c) => {
+                self.col += 1;
+                Some(Ok(c))
+            }
+        }
+    }
+
+    pub fn pos(&mut self) -> (usize, usize) {
+        (self.line, self.col)
+    }
+
+    pub fn eof(&mut self) -> Result<bool> {
+        Ok(self.peek()?.is_none())
+    }
+
+    pub fn peek_next(&mut self, idx: usize) -> Result<Option<u8>> {
+        while self.ch.len() <= idx {
+            match self.next() {
+                Some(Err(err)) => return Err(Error::Io(err)),
+                Some(Ok(ch)) => self.ch.push(ch),
+                None => return Ok(None),
+            }
+        }
+        Ok(Some(self.ch[idx]))
+    }
+
+    // pub fn peek_next_or_null(&mut self, idx: usize) -> Result<u8> {
+    //     Ok(try!(self.peek_next(idx)).unwrap_or(b'\x00'))
+    // }
+
+    pub fn peek(&mut self) -> Result<Option<u8>> {
+        self.peek_next(0)
+    }
+
+    pub fn peek_or_null(&mut self) -> Result<u8> {
+        Ok(self.peek()?.unwrap_or(b'\x00'))
+    }
+
+    pub fn eat_char(&mut self) -> u8 {
+        self.ch.remove(0)
+    }
+
+    pub fn uneat_char(&mut self, ch: u8) {
+        self.ch.insert(0, ch);
+    }
+
+    pub fn next_char(&mut self) -> Result<Option<u8>> {
+        match self.ch.first() {
+            Some(&ch) => {
+                self.eat_char();
+                Ok(Some(ch))
+            }
+            None => match self.next() {
+                Some(Err(err)) => Err(Error::Io(err)),
+                Some(Ok(ch)) => Ok(Some(ch)),
+                None => Ok(None),
+            },
+        }
+    }
+
+    pub fn next_char_or_null(&mut self) -> Result<u8> {
+        Ok(self.next_char()?.unwrap_or(b'\x00'))
+    }
+
+    fn eat_line(&mut self) -> Result<()> {
+        loop {
+            match self.peek()? {
+                Some(b'\n') | None => return Ok(()),
+                _ => {}
+            }
+            self.eat_char();
+        }
+    }
+
+    pub fn parse_whitespace(&mut self) -> Result<()> {
+        loop {
+            match self.peek_or_null()? {
+                b' ' | b'\n' | b'\t' | b'\r' => {
+                    self.eat_char();
+                }
+                b'#' => self.eat_line()?,
+                b'/' => {
+                    match self.peek_next(1)? {
+                        Some(b'/') => self.eat_line()?,
+                        Some(b'*') => {
+                            self.eat_char();
+                            self.eat_char();
+                            while !(self.peek()?.unwrap_or(b'*') == b'*'
+                                && self.peek_next(1)?.unwrap_or(b'/') == b'/')
+                            {
+                                self.eat_char();
+                            }
+                            self.eat_char();
+                            self.eat_char();
+                        }
+                        Some(_) => {
+                            self.eat_char();
+                        }
+                        None => return Err(self.error(ErrorCode::TrailingCharacters)), //todo
+                    }
+                }
+                _ => {
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    pub fn error(&mut self, reason: ErrorCode) -> Error {
+        Error::Syntax(reason, self.line, self.col)
+    }
+}
+
+pub enum Number {
+    I64(i64),
+    U64(u64),
+    F64(f64),
+}
+
+pub struct ParseNumber<Iter: Iterator<Item = u8>> {
+    rdr: StringReader<Iter>,
+    result: Vec<u8>,
+}
+
+// macro_rules! try_or_invalid {
+//     ($e:expr) => {
+//         match $e {
+//             Some(v) => v,
+//             None => { return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)); }
+//         }
+//     }
+// }
+
+impl<Iter: Iterator<Item = u8>> ParseNumber<Iter> {
+    #[inline]
+    pub fn new(iter: Iter) -> Self {
+        ParseNumber {
+            rdr: StringReader::new(iter),
+            result: Vec::new(),
+        }
+    }
+
+    pub fn parse(&mut self, stop_at_next: bool) -> Result<Number> {
+        match self.try_parse() {
+            Ok(()) => {
+                self.rdr.parse_whitespace()?;
+
+                let mut ch = self.rdr.next_char_or_null()?;
+
+                if stop_at_next {
+                    let ch2 = self.rdr.peek_or_null()?;
+                    // end scan if we find a punctuator character like ,}] or a comment
+                    if ch == b','
+                        || ch == b'}'
+                        || ch == b']'
+                        || ch == b'#'
+                        || ch == b'/' && (ch2 == b'/' || ch2 == b'*')
+                    {
+                        ch = b'\x00';
+                    }
+                }
+
+                match ch {
+                    b'\x00' => {
+                        let res =
+                            str::from_utf8(&self.result).expect("Internal error: json parsing");
+
+                        let mut is_float = false;
+                        for ch in res.chars() {
+                            if ch == '.' || ch == 'e' || ch == 'E' {
+                                is_float = true;
+                                break;
+                            }
+                        }
+
+                        if is_float {
+                            Ok(Number::F64(
+                                res.parse::<f64>().expect("Internal error: json parsing"),
+                            ))
+                        } else if res.starts_with('-') {
+                            Ok(Number::I64(
+                                res.parse::<i64>().expect("Internal error: json parsing"),
+                            ))
+                        } else {
+                            Ok(Number::U64(
+                                res.parse::<u64>().expect("Internal error: json parsing"),
+                            ))
+                        }
+                    }
+                    _ => Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0)),
+                }
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    fn try_parse(&mut self) -> Result<()> {
+        if self.rdr.peek_or_null()? == b'-' {
+            self.result.push(self.rdr.eat_char());
+        }
+
+        let mut has_value = false;
+
+        if self.rdr.peek_or_null()? == b'0' {
+            self.result.push(self.rdr.eat_char());
+            has_value = true;
+
+            // There can be only one leading '0'.
+            if let b'0'..=b'9' = self.rdr.peek_or_null()? {
+                return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0));
+            }
+        }
+
+        loop {
+            match self.rdr.peek_or_null()? {
+                b'0'..=b'9' => {
+                    self.result.push(self.rdr.eat_char());
+                    has_value = true;
+                }
+                b'.' => {
+                    if !has_value {
+                        return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0));
+                    }
+                    self.rdr.eat_char();
+                    return self.try_decimal();
+                }
+                b'e' | b'E' => {
+                    if !has_value {
+                        return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0));
+                    }
+                    self.rdr.eat_char();
+                    return self.try_exponent();
+                }
+                _ => {
+                    if !has_value {
+                        return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0));
+                    }
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    fn try_decimal(&mut self) -> Result<()> {
+        self.result.push(b'.');
+
+        // Make sure a digit follows the decimal place.
+        match self.rdr.next_char_or_null()? {
+            c @ b'0'..=b'9' => {
+                self.result.push(c);
+            }
+            _ => {
+                return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0));
+            }
+        };
+
+        while let b'0'..=b'9' = self.rdr.peek_or_null()? {
+            self.result.push(self.rdr.eat_char());
+        }
+
+        match self.rdr.peek_or_null()? {
+            b'e' | b'E' => {
+                self.rdr.eat_char();
+                self.try_exponent()
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn try_exponent(&mut self) -> Result<()> {
+        self.result.push(b'e');
+
+        match self.rdr.peek_or_null()? {
+            b'+' => {
+                self.result.push(self.rdr.eat_char());
+            }
+            b'-' => {
+                self.result.push(self.rdr.eat_char());
+            }
+            _ => {}
+        };
+
+        // Make sure a digit follows the exponent place.
+        match self.rdr.next_char_or_null()? {
+            c @ b'0'..=b'9' => {
+                self.result.push(c);
+            }
+            _ => {
+                return Err(Error::Syntax(ErrorCode::InvalidNumber, 0, 0));
+            }
+        };
+
+        while let b'0'..=b'9' = self.rdr.peek_or_null()? {
+            self.result.push(self.rdr.eat_char());
+        }
+
+        Ok(())
+    }
+}

crates/nu-json/tests/main.rs

@@ -0,0 +1,212 @@
+// FIXME: re-enable tests
+/*
+use nu_json::Value;
+use regex::Regex;
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+
+fn txt(text: &str) -> String {
+    let out = String::from_utf8_lossy(text.as_bytes());
+
+    #[cfg(windows)]
+    {
+        out.replace("\r\n", "").replace("\n", "")
+    }
+
+    #[cfg(not(windows))]
+    {
+        out.to_string()
+    }
+}
+
+fn hjson_expectations() -> PathBuf {
+    let assets = nu_test_support::fs::assets().join("nu_json");
+
+    nu_path::canonicalize(assets.clone()).unwrap_or_else(|e| {
+        panic!(
+            "Couldn't canonicalize hjson assets path {}: {:?}",
+            assets.display(),
+            e
+        )
+    })
+}
+
+fn get_test_content(name: &str) -> io::Result<String> {
+    let expectations = hjson_expectations();
+
+    let mut p = format!("{}/{}_test.hjson", expectations.display(), name);
+
+    if !Path::new(&p).exists() {
+        p = format!("{}/{}_test.json", expectations.display(), name);
+    }
+
+    fs::read_to_string(&p)
+}
+
+fn get_result_content(name: &str) -> io::Result<(String, String)> {
+    let expectations = hjson_expectations();
+
+    let p1 = format!("{}/{}_result.json", expectations.display(), name);
+    let p2 = format!("{}/{}_result.hjson", expectations.display(), name);
+
+    Ok((fs::read_to_string(&p1)?, fs::read_to_string(&p2)?))
+}
+
+macro_rules! run_test {
+    // {{ is a workaround for rust stable
+    ($v: ident, $list: expr, $fix: expr) => {{
+        let name = stringify!($v);
+        $list.push(format!("{}_test", name));
+        println!("- running {}", name);
+        let should_fail = name.starts_with("fail");
+        let test_content = get_test_content(name).unwrap();
+        let data: nu_json::Result<Value> = nu_json::from_str(&test_content);
+        assert!(should_fail == data.is_err());
+
+        if !should_fail {
+            let udata = data.unwrap();
+            let (rjson, rhjson) = get_result_content(name).unwrap();
+            let rjson = txt(&rjson);
+            let rhjson = txt(&rhjson);
+            let actual_hjson = nu_json::to_string(&udata).unwrap();
+            let actual_hjson = txt(&actual_hjson);
+            let actual_json = $fix(serde_json::to_string_pretty(&udata).unwrap());
+            let actual_json = txt(&actual_json);
+            if rhjson != actual_hjson {
+                println!(
+                    "{:?}\n---hjson expected\n{}\n---hjson actual\n{}\n---\n",
+                    name, rhjson, actual_hjson
+                );
+            }
+            if rjson != actual_json {
+                println!(
+                    "{:?}\n---json expected\n{}\n---json actual\n{}\n---\n",
+                    name, rjson, actual_json
+                );
+            }
+            assert!(rhjson == actual_hjson && rjson == actual_json);
+        }
+    }};
+}
+
+// add fixes where rust's json differs from javascript
+
+fn std_fix(json: String) -> String {
+    // serde_json serializes integers with a superfluous .0 suffix
+    let re = Regex::new(r"(?m)(?P<d>\d)\.0(?P<s>,?)$").unwrap();
+    re.replace_all(&json, "$d$s").to_string()
+}
+
+fn fix_kan(json: String) -> String {
+    std_fix(json).replace("    -0,", "    0,")
+}
+
+fn fix_pass1(json: String) -> String {
+    std_fix(json)
+        .replace("1.23456789e34", "1.23456789e+34")
+        .replace("2.3456789012e76", "2.3456789012e+76")
+}
+
+#[test]
+fn test_hjson() {
+    let mut done: Vec<String> = Vec::new();
+
+    println!();
+    run_test!(charset, done, std_fix);
+    run_test!(comments, done, std_fix);
+    run_test!(empty, done, std_fix);
+    run_test!(failCharset1, done, std_fix);
+    run_test!(failJSON02, done, std_fix);
+    run_test!(failJSON05, done, std_fix);
+    run_test!(failJSON06, done, std_fix);
+    run_test!(failJSON07, done, std_fix);
+    run_test!(failJSON08, done, std_fix);
+    run_test!(failJSON10, done, std_fix);
+    run_test!(failJSON11, done, std_fix);
+    run_test!(failJSON12, done, std_fix);
+    run_test!(failJSON13, done, std_fix);
+    run_test!(failJSON14, done, std_fix);
+    run_test!(failJSON15, done, std_fix);
+    run_test!(failJSON16, done, std_fix);
+    run_test!(failJSON17, done, std_fix);
+    run_test!(failJSON19, done, std_fix);
+    run_test!(failJSON20, done, std_fix);
+    run_test!(failJSON21, done, std_fix);
+    run_test!(failJSON22, done, std_fix);
+    run_test!(failJSON23, done, std_fix);
+    run_test!(failJSON24, done, std_fix);
+    run_test!(failJSON26, done, std_fix);
+    run_test!(failJSON28, done, std_fix);
+    run_test!(failJSON29, done, std_fix);
+    run_test!(failJSON30, done, std_fix);
+    run_test!(failJSON31, done, std_fix);
+    run_test!(failJSON32, done, std_fix);
+    run_test!(failJSON33, done, std_fix);
+    run_test!(failJSON34, done, std_fix);
+    run_test!(failKey1, done, std_fix);
+    run_test!(failKey2, done, std_fix);
+    run_test!(failKey3, done, std_fix);
+    run_test!(failKey4, done, std_fix);
+    run_test!(failMLStr1, done, std_fix);
+    run_test!(failObj1, done, std_fix);
+    run_test!(failObj2, done, std_fix);
+    run_test!(failObj3, done, std_fix);
+    run_test!(failStr1a, done, std_fix);
+    run_test!(failStr1b, done, std_fix);
+    run_test!(failStr1c, done, std_fix);
+    run_test!(failStr1d, done, std_fix);
+    run_test!(failStr2a, done, std_fix);
+    run_test!(failStr2b, done, std_fix);
+    run_test!(failStr2c, done, std_fix);
+    run_test!(failStr2d, done, std_fix);
+    run_test!(failStr3a, done, std_fix);
+    run_test!(failStr3b, done, std_fix);
+    run_test!(failStr3c, done, std_fix);
+    run_test!(failStr3d, done, std_fix);
+    run_test!(failStr4a, done, std_fix);
+    run_test!(failStr4b, done, std_fix);
+    run_test!(failStr4c, done, std_fix);
+    run_test!(failStr4d, done, std_fix);
+    run_test!(failStr5a, done, std_fix);
+    run_test!(failStr5b, done, std_fix);
+    run_test!(failStr5c, done, std_fix);
+    run_test!(failStr5d, done, std_fix);
+    run_test!(failStr6a, done, std_fix);
+    run_test!(failStr6b, done, std_fix);
+    run_test!(failStr6c, done, std_fix);
+    run_test!(failStr6d, done, std_fix);
+    run_test!(kan, done, fix_kan);
+    run_test!(keys, done, std_fix);
+    run_test!(oa, done, std_fix);
+    run_test!(pass1, done, fix_pass1);
+    run_test!(pass2, done, std_fix);
+    run_test!(pass3, done, std_fix);
+    run_test!(pass4, done, std_fix);
+    run_test!(passSingle, done, std_fix);
+    run_test!(root, done, std_fix);
+    run_test!(stringify1, done, std_fix);
+    run_test!(strings, done, std_fix);
+    run_test!(trail, done, std_fix);
+
+    // check if we include all assets
+    let paths = fs::read_dir(hjson_expectations()).unwrap();
+
+    let all = paths
+        .map(|item| String::from(item.unwrap().path().file_stem().unwrap().to_str().unwrap()))
+        .filter(|x| x.contains("_test"));
+
+    let missing = all
+        .into_iter()
+        .filter(|x| done.iter().find(|y| &x == y) == None)
+        .collect::<Vec<String>>();
+
+    if !missing.is_empty() {
+        for item in missing {
+            println!("missing: {}", item);
+        }
+        panic!();
+    }
+}
+
+*/

crates/nu-path/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+authors = ["The Nu Project Contributors"]
+description = "Path handling library for Nushell"
+edition = "2018"
+license = "MIT"
+name = "nu-path"
+version = "0.37.1"
+
+[dependencies]
+dirs-next = "2.0.0"
+dunce = "1.0.1"
+

crates/nu-path/README.md

@@ -0,0 +1,3 @@
+# nu-path
+
+This crate takes care of path handling in Nushell, such as canonicalization and component expansion, as well as other path-related utilities.

crates/nu-path/src/dots.rs

@@ -0,0 +1,259 @@
+use std::path::{is_separator, Component, Path, PathBuf};
+
+const EXPAND_STR: &str = if cfg!(windows) { r"..\" } else { "../" };
+
+fn handle_dots_push(string: &mut String, count: u8) {
+    if count < 1 {
+        return;
+    }
+
+    if count == 1 {
+        string.push('.');
+        return;
+    }
+
+    for _ in 0..(count - 1) {
+        string.push_str(EXPAND_STR);
+    }
+
+    string.pop(); // remove last '/'
+}
+
+/// Expands any occurence of more than two dots into a sequence of ../ (or ..\ on windows), e.g.,
+/// "..." into "../..", "...." into "../../../", etc.
+pub fn expand_ndots(path: impl AsRef<Path>) -> PathBuf {
+    // Check if path is valid UTF-8 and if not, return it as it is to avoid breaking it via string
+    // conversion.
+    let path_str = match path.as_ref().to_str() {
+        Some(s) => s,
+        None => return path.as_ref().into(),
+    };
+
+    // find if we need to expand any >2 dot paths and early exit if not
+    let mut dots_count = 0u8;
+    let ndots_present = {
+        for chr in path_str.chars() {
+            if chr == '.' {
+                dots_count += 1;
+            } else {
+                if is_separator(chr) && (dots_count > 2) {
+                    // this path component had >2 dots
+                    break;
+                }
+
+                dots_count = 0;
+            }
+        }
+
+        dots_count > 2
+    };
+
+    if !ndots_present {
+        return path.as_ref().into();
+    }
+
+    let mut dots_count = 0u8;
+    let mut expanded = String::new();
+    for chr in path_str.chars() {
+        if chr == '.' {
+            dots_count += 1;
+        } else {
+            if is_separator(chr) {
+                // check for dots expansion only at path component boundaries
+                handle_dots_push(&mut expanded, dots_count);
+                dots_count = 0;
+            } else {
+                // got non-dot within path component => do not expand any dots
+                while dots_count > 0 {
+                    expanded.push('.');
+                    dots_count -= 1;
+                }
+            }
+            expanded.push(chr);
+        }
+    }
+
+    handle_dots_push(&mut expanded, dots_count);
+
+    expanded.into()
+}
+
+/// Expand "." and ".." into nothing and parent directory, respectively.
+pub fn expand_dots(path: impl AsRef<Path>) -> PathBuf {
+    let path = path.as_ref();
+
+    // Early-exit if path does not contain '.' or '..'
+    if !path
+        .components()
+        .any(|c| std::matches!(c, Component::CurDir | Component::ParentDir))
+    {
+        return path.into();
+    }
+
+    let mut result = PathBuf::with_capacity(path.as_os_str().len());
+
+    // Only pop/skip path elements if the previous one was an actual path element
+    let prev_is_normal = |p: &Path| -> bool {
+        p.components()
+            .next_back()
+            .map(|c| std::matches!(c, Component::Normal(_)))
+            .unwrap_or(false)
+    };
+
+    path.components().for_each(|component| match component {
+        Component::ParentDir if prev_is_normal(&result) => {
+            result.pop();
+        }
+        Component::CurDir if prev_is_normal(&result) => {}
+        _ => result.push(component),
+    });
+
+    dunce::simplified(&result).to_path_buf()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn expand_two_dots() {
+        let path = Path::new("/foo/bar/..");
+
+        assert_eq!(
+            PathBuf::from("/foo"), // missing path
+            expand_dots(path)
+        );
+    }
+
+    #[test]
+    fn expand_dots_with_curdir() {
+        let path = Path::new("/foo/./bar/./baz");
+
+        assert_eq!(PathBuf::from("/foo/bar/baz"), expand_dots(path));
+    }
+
+    fn check_ndots_expansion(expected: &str, s: &str) {
+        let expanded = expand_ndots(Path::new(s));
+        assert_eq!(Path::new(expected), &expanded);
+    }
+
+    // common tests
+    #[test]
+    fn string_without_ndots() {
+        check_ndots_expansion("../hola", "../hola");
+    }
+
+    #[test]
+    fn string_with_three_ndots_and_chars() {
+        check_ndots_expansion("a...b", "a...b");
+    }
+
+    #[test]
+    fn string_with_two_ndots_and_chars() {
+        check_ndots_expansion("a..b", "a..b");
+    }
+
+    #[test]
+    fn string_with_one_dot_and_chars() {
+        check_ndots_expansion("a.b", "a.b");
+    }
+
+    #[test]
+    fn expand_dots_double_dots_no_change() {
+        // Can't resolve this as we don't know our parent dir
+        assert_eq!(Path::new(".."), expand_dots(Path::new("..")));
+    }
+
+    #[test]
+    fn expand_dots_single_dot_no_change() {
+        // Can't resolve this as we don't know our current dir
+        assert_eq!(Path::new("."), expand_dots(Path::new(".")));
+    }
+
+    #[test]
+    fn expand_dots_multi_single_dots_no_change() {
+        assert_eq!(Path::new("././."), expand_dots(Path::new("././.")));
+    }
+
+    #[test]
+    fn expand_multi_double_dots_no_change() {
+        assert_eq!(Path::new("../../../"), expand_dots(Path::new("../../../")));
+    }
+
+    #[test]
+    fn expand_dots_no_change_with_dirs() {
+        // Can't resolve this as we don't know our parent dir
+        assert_eq!(
+            Path::new("../../../dir1/dir2/"),
+            expand_dots(Path::new("../../../dir1/dir2"))
+        );
+    }
+
+    #[test]
+    fn expand_dots_simple() {
+        assert_eq!(Path::new("/foo"), expand_dots(Path::new("/foo/bar/..")));
+    }
+
+    #[test]
+    fn expand_dots_complex() {
+        assert_eq!(
+            Path::new("/test"),
+            expand_dots(Path::new("/foo/./bar/../../test/././test2/../"))
+        );
+    }
+
+    #[cfg(windows)]
+    mod windows {
+        use super::*;
+
+        #[test]
+        fn string_with_three_ndots() {
+            check_ndots_expansion(r"..\..", "...");
+        }
+
+        #[test]
+        fn string_with_mixed_ndots_and_chars() {
+            check_ndots_expansion(
+                r"a...b/./c..d/../e.f/..\..\..//.",
+                "a...b/./c..d/../e.f/....//.",
+            );
+        }
+
+        #[test]
+        fn string_with_three_ndots_and_final_slash() {
+            check_ndots_expansion(r"..\../", ".../");
+        }
+
+        #[test]
+        fn string_with_three_ndots_and_garbage() {
+            check_ndots_expansion(r"ls ..\../ garbage.*[", "ls .../ garbage.*[");
+        }
+    }
+
+    #[cfg(not(windows))]
+    mod non_windows {
+        use super::*;
+        #[test]
+        fn string_with_three_ndots() {
+            check_ndots_expansion(r"../..", "...");
+        }
+
+        #[test]
+        fn string_with_mixed_ndots_and_chars() {
+            check_ndots_expansion(
+                "a...b/./c..d/../e.f/../../..//.",
+                "a...b/./c..d/../e.f/....//.",
+            );
+        }
+
+        #[test]
+        fn string_with_three_ndots_and_final_slash() {
+            check_ndots_expansion("../../", ".../");
+        }
+
+        #[test]
+        fn string_with_three_ndots_and_garbage() {
+            check_ndots_expansion("ls ../../ garbage.*[", "ls .../ garbage.*[");
+        }
+    }
+}

crates/nu-path/src/expansions.rs

@@ -0,0 +1,75 @@
+use std::io;
+use std::path::{Path, PathBuf};
+
+use super::dots::{expand_dots, expand_ndots};
+use super::tilde::expand_tilde;
+
+// Join a path relative to another path. Paths starting with tilde are considered as absolute.
+fn join_path_relative<P, Q>(path: P, relative_to: Q) -> PathBuf
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let path = path.as_ref();
+    let relative_to = relative_to.as_ref();
+
+    if path == Path::new(".") {
+        // Joining a Path with '.' appends a '.' at the end, making the prompt
+        // more ugly - so we don't do anything, which should result in an equal
+        // path on all supported systems.
+        relative_to.into()
+    } else if path.starts_with("~") {
+        // do not end up with "/some/path/~"
+        path.into()
+    } else {
+        relative_to.join(path)
+    }
+}
+
+/// Resolve all symbolic links and all components (tilde, ., .., ...+) and return the path in its
+/// absolute form.
+///
+/// Fails under the same conditions as
+/// [std::fs::canonicalize](https://doc.rust-lang.org/std/fs/fn.canonicalize.html).
+pub fn canonicalize(path: impl AsRef<Path>) -> io::Result<PathBuf> {
+    let path = expand_tilde(path);
+    let path = expand_ndots(path);
+
+    dunce::canonicalize(path)
+}
+
+/// Same as canonicalize() but the input path is specified relative to another path
+pub fn canonicalize_with<P, Q>(path: P, relative_to: Q) -> io::Result<PathBuf>
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let path = join_path_relative(path, relative_to);
+
+    canonicalize(path)
+}
+
+/// Resolve only path components (tilde, ., .., ...+), if possible.
+///
+/// The function works in a "best effort" mode: It does not fail but rather returns the unexpanded
+/// version if the expansion is not possible.
+///
+/// Furthermore, unlike canonicalize(), it does not use sys calls (such as readlink).
+///
+/// Does not convert to absolute form nor does it resolve symlinks.
+pub fn expand_path(path: impl AsRef<Path>) -> PathBuf {
+    let path = expand_tilde(path);
+    let path = expand_ndots(path);
+    expand_dots(path)
+}
+
+/// Same as expand_path() but the input path is specified relative to another path
+pub fn expand_path_with<P, Q>(path: P, relative_to: Q) -> PathBuf
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let path = join_path_relative(path, relative_to);
+
+    expand_path(path)
+}

crates/nu-path/src/lib.rs

@@ -0,0 +1,8 @@
+mod dots;
+mod expansions;
+mod tilde;
+mod util;
+
+pub use expansions::{canonicalize, canonicalize_with, expand_path, expand_path_with};
+pub use tilde::expand_tilde;
+pub use util::trim_trailing_slash;

crates/nu-path/src/tilde.rs

@@ -0,0 +1,85 @@
+use std::path::{Path, PathBuf};
+
+fn expand_tilde_with(path: impl AsRef<Path>, home: Option<PathBuf>) -> PathBuf {
+    let path = path.as_ref();
+
+    if !path.starts_with("~") {
+        return path.into();
+    }
+
+    match home {
+        None => path.into(),
+        Some(mut h) => {
+            if h == Path::new("/") {
+                // Corner case: `h` is a root directory;
+                // don't prepend extra `/`, just drop the tilde.
+                path.strip_prefix("~").unwrap_or(path).into()
+            } else {
+                if let Ok(p) = path.strip_prefix("~/") {
+                    h.push(p)
+                }
+                h
+            }
+        }
+    }
+}
+
+/// Expand tilde ("~") into a home directory if it is the first path component
+pub fn expand_tilde(path: impl AsRef<Path>) -> PathBuf {
+    // TODO: Extend this to work with "~user" style of home paths
+    expand_tilde_with(path, dirs_next::home_dir())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn check_expanded(s: &str) {
+        let home = Path::new("/home");
+        let buf = Some(PathBuf::from(home));
+        assert!(expand_tilde_with(Path::new(s), buf).starts_with(&home));
+
+        // Tests the special case in expand_tilde for "/" as home
+        let home = Path::new("/");
+        let buf = Some(PathBuf::from(home));
+        assert!(!expand_tilde_with(Path::new(s), buf).starts_with("//"));
+    }
+
+    fn check_not_expanded(s: &str) {
+        let home = PathBuf::from("/home");
+        let expanded = expand_tilde_with(Path::new(s), Some(home));
+        assert!(expanded == Path::new(s));
+    }
+
+    #[test]
+    fn string_with_tilde() {
+        check_expanded("~");
+    }
+
+    #[test]
+    fn string_with_tilde_forward_slash() {
+        check_expanded("~/test/");
+    }
+
+    #[test]
+    fn string_with_tilde_double_forward_slash() {
+        check_expanded("~//test/");
+    }
+
+    #[test]
+    fn does_not_expand_tilde_if_tilde_is_not_first_character() {
+        check_not_expanded("1~1");
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn string_with_tilde_backslash() {
+        check_expanded("~\\test/test2/test3");
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn string_with_double_tilde_backslash() {
+        check_expanded("~\\\\test\\test2/test3");
+    }
+}

crates/nu-path/src/util.rs

@@ -0,0 +1,4 @@
+/// Trim trailing path separator from a string
+pub fn trim_trailing_slash(s: &str) -> &str {
+    s.trim_end_matches(std::path::is_separator)
+}

crates/nu-path/tests/mod.rs

@@ -0,0 +1 @@
+mod util;

crates/nu-path/tests/util.rs

@@ -0,0 +1,45 @@
+use nu_path::trim_trailing_slash;
+use std::path::MAIN_SEPARATOR;
+
+/// Helper function that joins string literals with '/' or '\', based on the host OS
+fn join_path_sep(pieces: &[&str]) -> String {
+    let sep_string = String::from(MAIN_SEPARATOR);
+    pieces.join(&sep_string)
+}
+
+#[test]
+fn trims_trailing_slash_without_trailing_slash() {
+    let path = join_path_sep(&["some", "path"]);
+
+    let actual = trim_trailing_slash(&path);
+
+    assert_eq!(actual, &path)
+}
+
+#[test]
+fn trims_trailing_slash() {
+    let path = join_path_sep(&["some", "path", ""]);
+
+    let actual = trim_trailing_slash(&path);
+    let expected = join_path_sep(&["some", "path"]);
+
+    assert_eq!(actual, &expected)
+}
+
+#[test]
+fn trims_many_trailing_slashes() {
+    let path = join_path_sep(&["some", "path", "", "", "", ""]);
+
+    let actual = trim_trailing_slash(&path);
+    let expected = join_path_sep(&["some", "path"]);
+
+    assert_eq!(actual, &expected)
+}
+
+#[test]
+fn trims_trailing_slash_empty() {
+    let path = String::from(MAIN_SEPARATOR);
+    let actual = trim_trailing_slash(&path);
+
+    assert_eq!(actual, "")
+}

crates/nu-protocol/Cargo.toml

@@ -8,3 +8,4 @@ edition = "2018"
 [dependencies]
 thiserror = "1.0.29"
 miette = "3.0.0"
+serde = "1.0.130"

crates/nu-protocol/src/ast/operator.rs

@@ -1,8 +1,9 @@
 use crate::Span;
 
+use serde::{Deserialize, Serialize};
 use std::fmt::Display;
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub enum Operator {
     Equal,
     NotEqual,
@@ -49,7 +50,7 @@ impl Display for Operator {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq)]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub enum RangeInclusion {
     Inclusive,
     RightExclusive,

crates/nu-protocol/src/shell_error.rs

@@ -1,9 +1,10 @@
 use miette::Diagnostic;
+use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
 use crate::{ast::Operator, Span, Type};
 
-#[derive(Debug, Clone, Error, Diagnostic)]
+#[derive(Debug, Clone, Error, Diagnostic, Serialize, Deserialize)]
 pub enum ShellError {
     #[error("Type mismatch during operation.")]
     #[diagnostic(code(nu::shell::type_mismatch), url(docsrs))]

crates/nu-protocol/src/span.rs

@@ -1,6 +1,7 @@
 use miette::SourceSpan;
+use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Span {
     pub start: usize,
     pub end: usize,

crates/nu-protocol/src/ty.rs

@@ -1,6 +1,8 @@
+use serde::{Deserialize, Serialize};
+
 use std::fmt::Display;
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum Type {
     Int,
     Float,

crates/nu-protocol/src/value/mod.rs

@@ -4,6 +4,7 @@ mod stream;
 
 pub use range::*;
 pub use row::*;
+use serde::{Deserialize, Serialize};
 pub use stream::*;
 
 use std::fmt::Debug;
@@ -14,7 +15,7 @@ use crate::{span, BlockId, Span, Type};
 use crate::ShellError;
 
 /// Core structured values that pass through the pipeline in engine-q
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum Value {
     Bool {
         val: bool,

crates/nu-protocol/src/value/range.rs

@@ -1,3 +1,4 @@
+use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 
 use crate::{
@@ -5,7 +6,7 @@ use crate::{
     *,
 };
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Range {
     pub from: Value,
     pub incr: Value,

crates/nu-protocol/src/value/stream.rs

@@ -1,6 +1,8 @@
 use crate::*;
 use std::{cell::RefCell, fmt::Debug, rc::Rc};
 
+use serde::{Deserialize, Serialize};
+
 #[derive(Clone)]
 pub struct ValueStream(pub Rc<RefCell<dyn Iterator<Item = Value>>>);
 
@@ -35,6 +37,26 @@ impl Iterator for ValueStream {
     }
 }
 
+impl Serialize for ValueStream {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        // FIXME: implement these
+        todo!()
+    }
+}
+
+impl<'de> Deserialize<'de> for ValueStream {
+    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        // FIXME: implement these
+        todo!()
+    }
+}
+
 pub trait IntoValueStream {
     fn into_value_stream(self) -> ValueStream;
 }

src/tests.rs

@@ -382,3 +382,17 @@ fn module_imports_5() -> TestResult {
         "3",
     )
 }
+
+#[test]
+fn from_json_1() -> TestResult {
+    run_test(r#"('{"name": "Fred"}' | from json).name"#, "Fred")
+}
+
+#[test]
+fn from_json_2() -> TestResult {
+    run_test(
+        r#"('{"name": "Fred"}
+                   {"name": "Sally"}' | from json -o).name.1"#,
+        "Sally",
+    )
+}