extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-12-23 15:39:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve performance of explore - 1 (#13116)

Browse Source 
Could be improved further I guess; but not here;

You can test the speed differences using data from #13088

```nu
open data.db | get profiles | explore
```

address: #13062

________

1. Noticed that search does not work anymore (even on `main` branch).
2. Not sure about resolved merged conflicts, seems fine, but maybe
something was lost.

---------

Co-authored-by: Reilly Wood <reilly.wood@icloud.com>
This commit is contained in:
Maxim Zhiburt 2024-06-13 04:35:04 +03:00 committed by GitHub
parent 634361b2d1
commit 323d5457f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 159 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
crates/nu-explore/src/commands/nu.rs
View File

@ -27,7 +27,7 @@ impl NuCmd {
}
impl ViewCommand for NuCmd {
type View = NuView<'static>;
type View = NuView;
fn name(&self) -> &'static str {
Self::NAME
@ -72,12 +72,12 @@ impl ViewCommand for NuCmd {
}
}
pub enum NuView<'a> {
Records(RecordView<'a>),
pub enum NuView {
Records(RecordView),
Preview(Preview),
}
impl View for NuView<'_> {
impl View for NuView {
fn draw(&mut self, f: &mut Frame, area: Rect, cfg: ViewConfig<'_>, layout: &mut Layout) {
match self {
NuView::Records(v) => v.draw(f, area, cfg, layout),

2
crates/nu-explore/src/commands/table.rs
View File

@ -30,7 +30,7 @@ impl TableCmd {
}
impl ViewCommand for TableCmd {
type View = RecordView<'static>;
type View = RecordView;
fn name(&self) -> &'static str {
Self::NAME

2
crates/nu-explore/src/commands/try.rs
View File

@ -22,7 +22,7 @@ impl TryCmd {
}
impl ViewCommand for TryCmd {
type View = TryView<'static>;
type View = TryView;
fn name(&self) -> &'static str {
Self::NAME

170
crates/nu-explore/src/views/record/mod.rs
View File

@ -23,23 +23,20 @@ use nu_protocol::{
Config, Record, Span, Value,
};
use ratatui::{layout::Rect, widgets::Block};
use std::{borrow::Cow, collections::HashMap};
use std::collections::HashMap;
pub use self::table_widget::Orientation;
#[derive(Debug, Clone)]
pub struct RecordView<'a> {
layer_stack: Vec<RecordLayer<'a>>,
pub struct RecordView {
layer_stack: Vec<RecordLayer>,
mode: UIMode,
orientation: Orientation,
cfg: ExploreConfig,
}
impl<'a> RecordView<'a> {
pub fn new(
columns: impl Into<Cow<'a, [String]>>,
records: impl Into<Cow<'a, [Vec<Value>]>>,
) -> Self {
impl RecordView {
pub fn new(columns: Vec<String>, records: Vec<Vec<Value>>) -> Self {
Self {
layer_stack: vec![RecordLayer::new(columns, records)],
mode: UIMode::View,
@ -64,13 +61,13 @@ impl<'a> RecordView<'a> {
}
// todo: rename to get_layer
pub fn get_layer_last(&self) -> &RecordLayer<'a> {
pub fn get_layer_last(&self) -> &RecordLayer {
self.layer_stack
.last()
.expect("we guarantee that 1 entry is always in a list")
}
pub fn get_layer_last_mut(&mut self) -> &mut RecordLayer<'a> {
pub fn get_layer_last_mut(&mut self) -> &mut RecordLayer {
self.layer_stack
.last_mut()
.expect("we guarantee that 1 entry is always in a list")
@ -134,32 +131,39 @@ impl<'a> RecordView<'a> {
Orientation::Left => (column, row),
};
if layer.records.len() > row && layer.records[row].len() > column {
layer.records[row][column].clone()
} else {
Value::nothing(Span::unknown())
}
if row >= layer.count_rows() || column >= layer.count_columns() {
// actually must never happen; unless cursor works incorrectly
// if being sure about cursor it can be deleted;
return Value::nothing(Span::unknown());
}
/// Create a table widget.
/// WARNING: this is currently really slow on large data sets.
/// It creates a string representation of every cell in the table and looks at every row for lscolorize.
fn create_table_widget(&'a self, cfg: ViewConfig<'a>) -> TableWidget<'a> {
let layer = self.get_layer_last();
let mut data = convert_records_to_string(&layer.records, cfg.nu_config, cfg.style_computer);
lscolorize(&layer.columns, &mut data, cfg.lscolors);
layer.record_values[row][column].clone()
}
let headers = layer.columns.as_ref();
fn create_table_widget<'a>(&'a mut self, cfg: ViewConfig<'a>) -> TableWidget<'a> {
let style = self.cfg.table;
let style_computer = cfg.style_computer;
let Position { row, column } = self.get_window_origin();
let layer = self.get_layer_last_mut();
if layer.record_text.is_none() {
let mut data =
convert_records_to_string(&layer.record_values, cfg.nu_config, cfg.style_computer);
lscolorize(&layer.column_names, &mut data, cfg.lscolors);
layer.record_text = Some(data);
}
let headers = &layer.column_names;
let data = layer.record_text.as_ref().expect("always ok");
TableWidget::new(
headers,
data,
style_computer,
row,
column,
self.cfg.table,
style,
layer.orientation,
)
}
@ -197,7 +201,7 @@ impl<'a> RecordView<'a> {
}
}
impl View for RecordView<'_> {
impl View for RecordView {
fn draw(&mut self, f: &mut Frame, area: Rect, cfg: ViewConfig<'_>, layout: &mut Layout) {
let mut table_layout = TableWidgetState::default();
// TODO: creating the table widget is O(N) where N is the number of cells in the grid.
@ -265,7 +269,7 @@ impl View for RecordView<'_> {
let style_computer = StyleComputer::new(&dummy_engine_state, &dummy_stack, HashMap::new());
let data = convert_records_to_string(
&self.get_layer_last().records,
&self.get_layer_last().record_values,
&nu_protocol::Config::default(),
&style_computer,
);
@ -274,7 +278,7 @@ impl View for RecordView<'_> {
}
fn show_data(&mut self, pos: usize) -> bool {
let data = &self.get_layer_last().records;
let data = &self.get_layer_last().record_values;
let mut i = 0;
for (row, cells) in data.iter().enumerate() {
@ -332,30 +336,35 @@ enum UIMode {
}
#[derive(Debug, Clone)]
pub struct RecordLayer<'a> {
columns: Cow<'a, [String]>,
records: Cow<'a, [Vec<Value>]>,
pub struct RecordLayer {
column_names: Vec<String>,
// These are the raw records in the current layer. The sole reason we keep this around is so we can return the original value
// if it's being peeked. Otherwise we could accept an iterator over it.
// or if it could be Clonable we could do that anyway;
// cause it would keep memory footprint lower while keep everything working
// (yee would make return O(n); we would need to traverse iterator once again; but maybe worth it)
record_values: Vec<Vec<Value>>,
// This is the text representation of the record values (the actual text that will be displayed to users).
// It's an Option because we need configuration to set it and we (currently) don't have access to configuration when things are created.
record_text: Option<Vec<Vec<NuText>>>,
orientation: Orientation,
name: Option<String>,
was_transposed: bool,
cursor: WindowCursor2D,
}
impl<'a> RecordLayer<'a> {
fn new(
columns: impl Into<Cow<'a, [String]>>,
records: impl Into<Cow<'a, [Vec<Value>]>>,
) -> Self {
let columns = columns.into();
let records = records.into();
impl RecordLayer {
fn new(columns: Vec<String>, records: Vec<Vec<Value>>) -> Self {
// TODO: refactor so this is fallible and returns a Result instead of panicking
let cursor =
WindowCursor2D::new(records.len(), columns.len()).expect("Failed to create cursor");
let column_names = columns.iter().map(|s| strip_string(s)).collect();
Self {
columns,
records,
column_names,
record_values: records,
record_text: None,
cursor,
orientation: Orientation::Top,
name: None,
@ -369,21 +378,21 @@ impl<'a> RecordLayer<'a> {
fn count_rows(&self) -> usize {
match self.orientation {
Orientation::Top => self.records.len(),
Orientation::Left => self.columns.len(),
Orientation::Top => self.record_values.len(),
Orientation::Left => self.column_names.len(),
}
}
fn count_columns(&self) -> usize {
match self.orientation {
Orientation::Top => self.columns.len(),
Orientation::Left => self.records.len(),
Orientation::Top => self.column_names.len(),
Orientation::Left => self.record_values.len(),
}
}
fn get_column_header(&self) -> Option<String> {
let col = self.cursor.column();
self.columns.get(col).map(|header| header.to_string())
self.column_names.get(col).map(|header| header.to_string())
}
fn reset_cursor(&mut self) {
@ -570,13 +579,13 @@ fn handle_key_event_cursor_mode(
}
}
fn create_layer(value: Value) -> Result<RecordLayer<'static>> {
fn create_layer(value: Value) -> Result<RecordLayer> {
let (columns, values) = collect_input(value)?;
Ok(RecordLayer::new(columns, values))
}
fn push_layer(view: &mut RecordView<'_>, mut next_layer: RecordLayer<'static>) {
fn push_layer(view: &mut RecordView, mut next_layer: RecordLayer) {
let layer = view.get_layer_last();
let header = layer.get_column_header();
@ -599,9 +608,9 @@ fn estimate_page_size(area: Rect, show_head: bool) -> u16 {
}
/// scroll to the end of the data
fn tail_data(state: &mut RecordView<'_>, page_size: usize) {
fn tail_data(state: &mut RecordView, page_size: usize) {
let layer = state.get_layer_last_mut();
let count_rows = layer.records.len();
let count_rows = layer.count_rows();
if count_rows > page_size {
layer
.cursor
@ -620,6 +629,7 @@ fn convert_records_to_string(
row.iter()
.map(|value| {
let text = value.clone().to_abbreviated_string(cfg);
let text = strip_string(&text);
let float_precision = cfg.float_precision as usize;
make_styled_string(style_computer, text, Some(value), float_precision)
@ -649,12 +659,16 @@ fn build_last_value(v: &RecordView) -> Value {
fn build_table_as_list(v: &RecordView) -> Value {
let layer = v.get_layer_last();
let cols = &layer.columns;
let vals = layer
.records
.record_values
.iter()
.map(|vals| {
let record = cols.iter().cloned().zip(vals.iter().cloned()).collect();
let record = layer
.column_names
.iter()
.cloned()
.zip(vals.iter().cloned())
.collect();
Value::record(record, NuSpan::unknown())
})
.collect();
@ -665,16 +679,15 @@ fn build_table_as_list(v: &RecordView) -> Value {
fn build_table_as_record(v: &RecordView) -> Value {
let layer = v.get_layer_last();
let record = if let Some(row) = layer.records.first() {
layer
.columns
let mut record = Record::new();
if let Some(row) = layer.record_values.first() {
record = layer
.column_names
.iter()
.cloned()
.zip(row.iter().cloned())
.collect()
} else {
Record::new()
};
.collect();
}
Value::record(record, NuSpan::unknown())
}
@ -708,17 +721,12 @@ fn get_percentage(value: usize, max: usize) -> usize {
((value as f32 / max as f32) * 100.0).floor() as usize
}
fn transpose_table(layer: &mut RecordLayer<'_>) {
let count_rows = layer.records.len();
let count_columns = layer.columns.len();
fn transpose_table(layer: &mut RecordLayer) {
let count_rows = layer.record_values.len();
let count_columns = layer.column_names.len();
if layer.was_transposed {
let data = match &mut layer.records {
Cow::Owned(data) => data,
Cow::Borrowed(_) => unreachable!("must never happen"),
};
let headers = pop_first_column(data);
let headers = pop_first_column(&mut layer.record_values);
let headers = headers
.into_iter()
.map(|value| match value {
@ -727,22 +735,24 @@ fn transpose_table(layer: &mut RecordLayer<'_>) {
})
.collect();
let data = _transpose_table(data, count_rows, count_columns - 1);
let data = _transpose_table(&layer.record_values, count_rows, count_columns - 1);
layer.records = Cow::Owned(data);
layer.columns = Cow::Owned(headers);
} else {
let mut data = _transpose_table(&layer.records, count_rows, count_columns);
layer.record_values = data;
layer.column_names = headers;
for (column, column_name) in layer.columns.iter().enumerate() {
return;
}
let mut data = _transpose_table(&layer.record_values, count_rows, count_columns);
for (column, column_name) in layer.column_names.iter().enumerate() {
let value = Value::string(column_name, NuSpan::unknown());
data[column].insert(0, value);
}
layer.records = Cow::Owned(data);
layer.columns = (1..count_rows + 1 + 1).map(|i| i.to_string()).collect();
}
layer.record_values = data;
layer.column_names = (1..count_rows + 1 + 1).map(|i| i.to_string()).collect();
layer.was_transposed = !layer.was_transposed;
}
@ -770,3 +780,9 @@ fn _transpose_table(
data
}
fn strip_string(text: &str) -> String {
String::from_utf8(strip_ansi_escapes::strip(text))
.map_err(|_| ())
.unwrap_or_else(|_| text.to_owned())
}

88
crates/nu-explore/src/views/record/table_widget.rs
View File

@ -13,15 +13,12 @@ use ratatui::{
text::Span,
widgets::{Block, Borders, Paragraph, StatefulWidget, Widget},
};
use std::{
borrow::Cow,
cmp::{max, Ordering},
};
use std::cmp::{max, Ordering};
#[derive(Debug, Clone)]
pub struct TableWidget<'a> {
columns: Cow<'a, [String]>,
data: Cow<'a, [Vec<NuText>]>,
columns: &'a [String],
data: &'a [Vec<NuText>],
index_row: usize,
index_column: usize,
config: TableConfig,
@ -39,8 +36,8 @@ pub enum Orientation {
impl<'a> TableWidget<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
columns: impl Into<Cow<'a, [String]>>,
data: impl Into<Cow<'a, [Vec<NuText>]>>,
columns: &'a [String],
data: &'a [Vec<NuText>],
style_computer: &'a StyleComputer<'a>,
index_row: usize,
index_column: usize,
@ -48,8 +45,8 @@ impl<'a> TableWidget<'a> {
header_position: Orientation,
) -> Self {
Self {
columns: columns.into(),
data: data.into(),
columns,
data,
style_computer,
index_row,
index_column,
@ -197,22 +194,25 @@ impl<'a> TableWidget<'a> {
}
if show_head {
let mut header = [head_row_text(&head, self.style_computer)];
let head_style = head_style(&head, self.style_computer);
if head_width > use_space as usize {
truncate_str(&mut header[0].0, use_space as usize)
truncate_str(&mut head, use_space as usize)
}
let head_iter = [(&head, head_style)].into_iter();
let mut w = width;
w += render_space(buf, w, head_y, 1, padding_l);
w += render_column(buf, w, head_y, use_space, &header);
w += render_column(buf, w, head_y, use_space, head_iter);
w += render_space(buf, w, head_y, 1, padding_r);
let x = w - padding_r - use_space;
state.layout.push(&header[0].0, x, head_y, use_space, 1);
state.layout.push(&head, x, head_y, use_space, 1);
}
let head_rows = column.iter().map(|(t, s)| (t, *s));
width += render_space(buf, width, data_y, data_height, padding_l);
width += render_column(buf, width, data_y, use_space, &column);
width += render_column(buf, width, data_y, use_space, head_rows);
width += render_space(buf, width, data_y, data_height, padding_r);
for (row, (text, _)) in column.iter().enumerate() {
@ -305,10 +305,9 @@ impl<'a> TableWidget<'a> {
return;
}
let columns = columns
let columns_iter = columns
.iter()
.map(|s| head_row_text(s, self.style_computer))
.collect::<Vec<_>>();
.map(|s| (s.clone(), head_style(s, self.style_computer)));
if !show_index {
let x = area.x + left_w;
@ -326,12 +325,12 @@ impl<'a> TableWidget<'a> {
let x = area.x + left_w;
left_w += render_space(buf, x, area.y, 1, padding_l);
let x = area.x + left_w;
left_w += render_column(buf, x, area.y, columns_width as u16, &columns);
left_w += render_column(buf, x, area.y, columns_width as u16, columns_iter);
let x = area.x + left_w;
left_w += render_space(buf, x, area.y, 1, padding_r);
let layout_x = left_w - padding_r - columns_width as u16;
for (i, (text, _)) in columns.iter().enumerate() {
for (i, text) in columns.iter().enumerate() {
state
.layout
.push(text, layout_x, area.y + i as u16, columns_width as u16, 1);
@ -377,10 +376,12 @@ impl<'a> TableWidget<'a> {
break;
}
let head_rows = column.iter().map(|(t, s)| (t, *s));
let x = area.x + left_w;
left_w += render_space(buf, x, area.y, area.height, padding_l);
let x = area.x + left_w;
left_w += render_column(buf, x, area.y, column_width, &column);
left_w += render_column(buf, x, area.y, column_width, head_rows);
let x = area.x + left_w;
left_w += render_space(buf, x, area.y, area.height, padding_r);
@ -619,14 +620,14 @@ fn repeat_vertical(
c: char,
style: TextStyle,
) {
let text = std::iter::repeat(c)
.take(width as usize)
.collect::<String>();
let text = String::from(c);
let style = text_style_to_tui_style(style);
let span = Span::styled(text, style);
let span = Span::styled(&text, style);
for row in 0..height {
buf.set_span(x, y + row, &span, width);
for col in 0..width {
buf.set_span(x + col, y + row, &span, 1);
}
}
}
@ -676,35 +677,28 @@ fn calculate_column_width(column: &[NuText]) -> usize {
.unwrap_or(0)
}
fn render_column(
fn render_column<T, S>(
buf: &mut ratatui::buffer::Buffer,
x: u16,
y: u16,
available_width: u16,
rows: &[NuText],
) -> u16 {
for (row, (text, style)) in rows.iter().enumerate() {
let style = text_style_to_tui_style(*style);
let text = strip_string(text);
let span = Span::styled(text, style);
rows: impl Iterator<Item = (T, S)>,
) -> u16
where
T: AsRef<str>,
S: Into<TextStyle>,
{
for (row, (text, style)) in rows.enumerate() {
let style = text_style_to_tui_style(style.into());
let span = Span::styled(text.as_ref(), style);
buf.set_span(x, y + row as u16, &span, available_width);
}
available_width
}
fn strip_string(text: &str) -> String {
String::from_utf8(strip_ansi_escapes::strip(text))
.map_err(|_| ())
.unwrap_or_else(|_| text.to_owned())
}
fn head_row_text(head: &str, style_computer: &StyleComputer) -> NuText {
(
String::from(head),
TextStyle::with_style(
Alignment::Center,
style_computer.compute("header", &Value::string(head, nu_protocol::Span::unknown())),
),
)
fn head_style(head: &str, style_computer: &StyleComputer) -> TextStyle {
let style =
style_computer.compute("header", &Value::string(head, nu_protocol::Span::unknown()));
TextStyle::with_style(Alignment::Center, style)
}

20
crates/nu-explore/src/views/try.rs
View File

@ -16,21 +16,21 @@ use ratatui::{
};
use std::cmp::min;
pub struct TryView<'a> {
pub struct TryView {
input: Value,
command: String,
reactive: bool,
table: Option<RecordView<'a>>,
immediate: bool,
table: Option<RecordView>,
view_mode: bool,
border_color: Style,
}
impl<'a> TryView<'a> {
impl TryView {
pub fn new(input: Value) -> Self {
Self {
input,
table: None,
reactive: false,
immediate: false,
border_color: Style::default(),
view_mode: false,
command: String::new(),
@ -48,7 +48,7 @@ impl<'a> TryView<'a> {
}
}
impl View for TryView<'_> {
impl View for TryView {
fn draw(&mut self, f: &mut Frame, area: Rect, cfg: ViewConfig<'_>, layout: &mut Layout) {
let border_color = self.border_color;
@ -178,7 +178,7 @@ impl View for TryView<'_> {
if !self.command.is_empty() {
self.command.pop();
if self.reactive {
if self.immediate {
match self.try_run(engine_state, stack) {
Ok(_) => info.report = Some(Report::default()),
Err(err) => info.report = Some(Report::error(format!("Error: {err}"))),
@ -191,7 +191,7 @@ impl View for TryView<'_> {
KeyCode::Char(c) => {
self.command.push(*c);
if self.reactive {
if self.immediate {
match self.try_run(engine_state, stack) {
Ok(_) => info.report = Some(Report::default()),
Err(err) => info.report = Some(Report::error(format!("Error: {err}"))),
@ -235,7 +235,7 @@ impl View for TryView<'_> {
fn setup(&mut self, config: ViewConfig<'_>) {
self.border_color = nu_style_to_tui(config.explore_config.table.separator_style);
self.reactive = config.explore_config.try_reactive;
self.immediate = config.explore_config.try_reactive;
let mut r = RecordView::new(vec![], vec![]);
r.setup(config);
@ -253,7 +253,7 @@ fn run_command(
input: &Value,
engine_state: &EngineState,
stack: &mut Stack,
) -> Result<RecordView<'static>> {
) -> Result<RecordView> {
let pipeline = run_command_with_value(command, input, engine_state, stack)?;
let is_record = matches!(pipeline, PipelineData::Value(Value::Record { .. }, ..));

9
crates/nu-table/src/table.rs
View File

@ -259,9 +259,7 @@ fn draw_table(
table.with(width_ctrl);
}
let tablewidth = table.total_width();
table_to_string(table, tablewidth, termwidth)
table_to_string(table, termwidth)
}
fn set_indent(table: &mut Table, left: usize, right: usize) {
@ -289,7 +287,9 @@ fn set_border_head(table: &mut Table, with_footer: bool, wctrl: TableWidthCtrl)
}
}
fn table_to_string(table: Table, total_width: usize, termwidth: usize) -> Option<String> {
fn table_to_string(table: Table, termwidth: usize) -> Option<String> {
let total_width = table.total_width();
if total_width > termwidth {
None
} else {
@ -318,6 +318,7 @@ impl TableOption<NuRecords, CompleteDimensionVecRecords<'_>, ColoredConfig> for
dim: &mut CompleteDimensionVecRecords<'_>,
) {
let total_width = get_total_width2(&self.width, cfg);
if total_width > self.max {
TableTrim {
max: self.max,