added nu-utils crate, fixed issue where externals turn off vt processing (#4857)

* added `nu-utils` crate, fixed issue where externals turn off vt processing

* hopefully make work in non-windows environments

* clippy

crates/nu-utils/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod utils;
+
+pub use utils::enable_vt_processing;

crates/nu-utils/src/main.rs

@@ -0,0 +1,10 @@
+#[cfg(windows)]
+use nu_utils::utils::enable_vt_processing;
+
+fn main() {
+    // reset vt processing, aka ansi because illbehaved externals can break it
+    #[cfg(windows)]
+    {
+        let _ = enable_vt_processing();
+    }
+}

crates/nu-utils/src/utils.rs

@@ -0,0 +1,25 @@
+use std::io::Result;
+
+pub fn enable_vt_processing() -> Result<()> {
+    #[cfg(windows)]
+    {
+        use crossterm_winapi::{ConsoleMode, Handle};
+
+        pub const ENABLE_PROCESSED_OUTPUT: u32 = 0x0001;
+        pub const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
+        // let mask = ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+
+        let console_mode = ConsoleMode::from(Handle::current_out_handle()?);
+        let old_mode = console_mode.mode()?;
+
+        // researching odd ansi behavior in windows terminal repo revealed that
+        // enable_processed_output and enable_virtual_terminal_processing should be used
+        // also, instead of checking old_mode & mask, just set the mode already
+
+        // if old_mode & mask == 0 {
+        console_mode
+            .set_mode(old_mode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING)?;
+        // }
+    }
+    Ok(())
+}