nushell/crates/nu-ansi-term/src/windows.rs
2021-02-22 12:33:34 -06:00

63 lines
2.4 KiB
Rust

/// Enables ANSI code support on Windows 10.
///
/// This uses Windows API calls to alter the properties of the console that
/// the program is running in.
///
/// https://msdn.microsoft.com/en-us/library/windows/desktop/mt638032(v=vs.85).aspx
///
/// Returns a `Result` with the Windows error code if unsuccessful.
#[cfg(windows)]
pub fn enable_ansi_support() -> Result<(), u32> {
// ref: https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#EXAMPLE_OF_ENABLING_VIRTUAL_TERMINAL_PROCESSING @@ https://archive.is/L7wRJ#76%
use std::ffi::OsStr;
use std::iter::once;
use std::os::windows::ffi::OsStrExt;
use std::ptr::null_mut;
use winapi::um::consoleapi::{GetConsoleMode, SetConsoleMode};
use winapi::um::errhandlingapi::GetLastError;
use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING};
use winapi::um::handleapi::INVALID_HANDLE_VALUE;
use winapi::um::winnt::{FILE_SHARE_WRITE, GENERIC_READ, GENERIC_WRITE};
const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
unsafe {
// ref: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
// Using `CreateFileW("CONOUT$", ...)` to retrieve the console handle works correctly even if STDOUT and/or STDERR are redirected
let console_out_name: Vec<u16> =
OsStr::new("CONOUT$").encode_wide().chain(once(0)).collect();
let console_handle = CreateFileW(
console_out_name.as_ptr(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_WRITE,
null_mut(),
OPEN_EXISTING,
0,
null_mut(),
);
if console_handle == INVALID_HANDLE_VALUE {
return Err(GetLastError());
}
// ref: https://docs.microsoft.com/en-us/windows/console/getconsolemode
let mut console_mode: u32 = 0;
if 0 == GetConsoleMode(console_handle, &mut console_mode) {
return Err(GetLastError());
}
// VT processing not already enabled?
if console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 {
// https://docs.microsoft.com/en-us/windows/console/setconsolemode
if 0 == SetConsoleMode(
console_handle,
console_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING,
) {
return Err(GetLastError());
}
}
}
Ok(())
}