this fixes garbage ansi when externals turn off vt processing (#422)

* this fixes garbage ansi when externals turn off vt processing

* clippy

* changes are only for windows

* type-o

Cargo.lock

@@ -711,6 +711,7 @@ version = "0.1.0"
 dependencies = [
  "assert_cmd",
  "crossterm",
+ "crossterm_winapi",
  "ctrlc",
  "dialoguer",
  "miette",

Cargo.toml

@@ -34,6 +34,7 @@ nu-term-grid = { path = "./crates/nu-term-grid" }
 nu-ansi-term = { path = "./crates/nu-ansi-term" }
 miette = "3.0.0"
 ctrlc = "3.2.1"
+crossterm_winapi = "0.9.0"
 # mimalloc = { version = "*", default-features = false }
 
 [features]

src/main.rs

@@ -1,11 +1,5 @@
-use std::{
-    io::Write,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-};
-
+#[cfg(windows)]
+use crossterm_winapi::{ConsoleMode, Handle};
 use dialoguer::{
     console::{Style, Term},
     theme::ColorfulTheme,
@@ -22,6 +16,13 @@ use nu_protocol::{
     PipelineData, ShellError, Span, Value, CONFIG_VARIABLE_ID,
 };
 use reedline::{Completer, CompletionActionHandler, DefaultPrompt, LineBuffer, Prompt};
+use std::{
+    io::Write,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+};
 
 #[cfg(test)]
 mod tests;
@@ -455,16 +456,56 @@ fn eval_source(
                 let working_set = StateWorkingSet::new(engine_state);
 
                 report_error(&working_set, &err);
+
+                // reset vt processing, aka ansi because illbehaved externals can break it
+                #[cfg(windows)]
+                {
+                    let _ = enable_vt_processing();
+                }
+
                 return false;
             }
+
+            // reset vt processing, aka ansi because illbehaved externals can break it
+            #[cfg(windows)]
+            {
+                let _ = enable_vt_processing();
+            }
         }
         Err(err) => {
             let working_set = StateWorkingSet::new(engine_state);
 
             report_error(&working_set, &err);
+
+            // reset vt processing, aka ansi because illbehaved externals can break it
+            #[cfg(windows)]
+            {
+                let _ = enable_vt_processing();
+            }
             return false;
         }
     }
 
     true
 }
+
+#[cfg(windows)]
+fn enable_vt_processing() -> Result<(), ShellError> {
+    pub const ENABLE_PROCESSED_OUTPUT: u32 = 0x0001;
+    pub const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
+    // let mask = ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+
+    let console_mode = ConsoleMode::from(Handle::current_out_handle()?);
+    let old_mode = console_mode.mode()?;
+
+    // researching odd ansi behavior in windows terminal repo revealed that
+    // enable_processed_output and enable_virtual_terminal_processing should be used
+    // also, instead of checking old_mode & mask, just set the mode already
+
+    // if old_mode & mask == 0 {
+    console_mode
+        .set_mode(old_mode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING)?;
+    // }
+
+    Ok(())
+}