Merge pull request #8 from elferherrera/lite-parser

Comments with a newline dont get together

crates/nu-parser/src/lex.rs

@@ -273,7 +273,15 @@ pub fn lex(
                 if *input == b'\n' || *input == b'\r' {
                     output.push(Token::new(
                         TokenContents::Comment,
-                        Span::new(start, curr_offset),
+                        Span::new(start, curr_offset - 1),
+                    ));
+
+                    // Adding an end of line token after a comment
+                    // This helps during lite_parser to avoid losing a command
+                    // in a statement
+                    output.push(Token::new(
+                        TokenContents::Eol,
+                        Span::new(curr_offset - 1, curr_offset),
                     ));
                     start = curr_offset;
 
@@ -307,4 +315,4 @@ pub fn lex(
         }
     }
     (output, error)
-}
+}

crates/nu-parser/src/lite_parse.rs

@@ -96,13 +96,15 @@ pub fn lite_parse(tokens: &[Token]) -> (LiteBlock, Option<ParseError>) {
             TokenContents::Eol | TokenContents::Semicolon => {
                 if !curr_command.is_empty() {
                     curr_pipeline.push(curr_command);
+
+                    curr_command = LiteCommand::new();
                 }
-                curr_command = LiteCommand::new();
 
                 if !curr_pipeline.is_empty() {
                     block.push(curr_pipeline);
+
+                    curr_pipeline = LiteStatement::new();
                 }
-                curr_pipeline = LiteStatement::new();
             }
             TokenContents::Comment => {
                 curr_command.comments.push(token.span);

crates/nu-parser/tests/test_lex.rs

@@ -57,7 +57,7 @@ fn lex_comment() {
         output.0.get(4).unwrap(),
         &Token {
             contents: TokenContents::Comment,
-            span: Span { start: 12, end: 25 }
+            span: Span { start: 12, end: 24 }
         }
     );
 }
@@ -91,3 +91,47 @@ fn lex_incomplete_quote() {
     let err = output.1.unwrap();
     assert!(matches!(err, ParseError::UnexpectedEof(v, _) if v == "'"));
 }
+
+#[test]
+fn lex_comments() {
+    // Comments should keep the end of line token
+    // Code:
+    // let z = 4
+    // let x = 4 #comment
+    // let y = 1 # comment
+    let file = b"let z = 4 #comment \n let x = 4 # comment\n let y = 1 # comment";
+
+    let output = lex(file, 0, &[], &[]);
+
+    assert_eq!(
+        output.0.get(4).unwrap(),
+        &Token {
+            contents: TokenContents::Comment,
+            span: Span { start: 10, end: 19 }
+        }
+    );
+    assert_eq!(
+        output.0.get(5).unwrap(),
+        &Token {
+            contents: TokenContents::Eol,
+            span: Span { start: 19, end: 20 }
+        }
+    );
+
+    // When there is no space between the comment and the new line the span
+    // for the command and the EOL overlaps
+    assert_eq!(
+        output.0.get(10).unwrap(),
+        &Token {
+            contents: TokenContents::Comment,
+            span: Span { start: 31, end: 40 }
+        }
+    );
+    assert_eq!(
+        output.0.get(11).unwrap(),
+        &Token {
+            contents: TokenContents::Eol,
+            span: Span { start: 40, end: 41 }
+        }
+    );
+}

crates/nu-parser/tests/test_lite_parser.rs

@@ -16,6 +16,9 @@ fn lite_parse_helper(input: &[u8]) -> Result<LiteBlock, ParseError> {
 
 #[test]
 fn comment_before() -> Result<(), ParseError> {
+    // Code:
+    // # this is a comment
+    // def foo bar
     let input = b"# this is a comment\ndef foo bar";
 
     let lite_block = lite_parse_helper(input)?;
@@ -25,11 +28,18 @@ fn comment_before() -> Result<(), ParseError> {
     assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
     assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
 
+    assert_eq!(
+        lite_block.block[0].commands[0].comments[0],
+        Span { start: 0, end: 19 }
+    );
+
     Ok(())
 }
 
 #[test]
 fn comment_beside() -> Result<(), ParseError> {
+    // Code:
+    // def foo bar # this is a comment
     let input = b"def foo bar # this is a comment";
 
     let lite_block = lite_parse_helper(input)?;
@@ -39,44 +49,72 @@ fn comment_beside() -> Result<(), ParseError> {
     assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
     assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
 
+    assert_eq!(
+        lite_block.block[0].commands[0].comments[0],
+        Span { start: 12, end: 31 }
+    );
+
     Ok(())
 }
 
 #[test]
 fn comments_stack() -> Result<(), ParseError> {
+    // Code:
+    // # this is a comment
+    // # another comment
+    // # def foo bar
     let input = b"# this is a comment\n# another comment\ndef foo bar ";
 
     let lite_block = lite_parse_helper(input)?;
 
     assert_eq!(lite_block.block.len(), 1);
-    assert_eq!(lite_block.block[0].commands.len(), 1);
     assert_eq!(lite_block.block[0].commands[0].comments.len(), 2);
     assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
 
+    assert_eq!(
+        lite_block.block[0].commands[0].comments[0],
+        Span { start: 0, end: 19 }
+    );
+
+    assert_eq!(
+        lite_block.block[0].commands[0].comments[1],
+        Span { start: 20, end: 37 }
+    );
+
     Ok(())
 }
 
 #[test]
 fn separated_comments_dont_stack() -> Result<(), ParseError> {
+    // Code:
+    // # this is a comment
+    //
+    // # another comment
+    // # def foo bar
     let input = b"# this is a comment\n\n# another comment\ndef foo bar ";
 
     let lite_block = lite_parse_helper(input)?;
 
     assert_eq!(lite_block.block.len(), 1);
-    assert_eq!(lite_block.block[0].commands.len(), 1);
-    assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[0].commands[0].comments.len(), 2);
+    assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
+
     assert_eq!(
         lite_block.block[0].commands[0].comments[0],
-        Span { start: 21, end: 39 }
+        Span { start: 0, end: 19 }
+    );
+
+    assert_eq!(
+        lite_block.block[0].commands[0].comments[1],
+        Span { start: 21, end: 38 }
     );
-    assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
 
     Ok(())
 }
 
 #[test]
 fn multiple_statements() -> Result<(), ParseError> {
-    // Code :
+    // Code:
     // # A comment
     // let a = ( 3 + (
     // 4 +
@@ -87,13 +125,15 @@ fn multiple_statements() -> Result<(), ParseError> {
     let lite_block = lite_parse_helper(input)?;
 
     assert_eq!(lite_block.block.len(), 2);
-    assert_eq!(lite_block.block[0].commands.len(), 1);
+    assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[0].commands[0].parts.len(), 4);
     assert_eq!(
         lite_block.block[0].commands[0].comments[0],
-        Span { start: 0, end: 11 }
+        Span { start: 0, end: 10 }
     );
 
-    assert_eq!(lite_block.block[1].commands.len(), 1);
+    assert_eq!(lite_block.block[1].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[1].commands[0].parts.len(), 4);
     assert_eq!(
         lite_block.block[1].commands[0].comments[0],
         Span { start: 52, end: 61 }
@@ -105,7 +145,7 @@ fn multiple_statements() -> Result<(), ParseError> {
 #[test]
 fn multiple_commands() -> Result<(), ParseError> {
     // Pipes add commands to the lite parser
-    // Code :
+    // Code:
     // let a = ls | where name == 1
     // let b = 1 # comment
     let input = b"let a = ls | where name == 1 \n let b = 1 # comment";
@@ -123,3 +163,84 @@ fn multiple_commands() -> Result<(), ParseError> {
 
     Ok(())
 }
+
+#[test]
+fn multiple_commands_with_comment() -> Result<(), ParseError> {
+    // Pipes add commands to the lite parser
+    // The comments are attached to the commands next to them
+    // Code:
+    // let a = ls | where name == 1 # comment
+    // let b = 1 # comment
+    //let a = ls | where name == 1 # comment \n let b = 1 # comment
+    let input = b"let a = ls | where name == 1 # comment\n let b = 1 # comment";
+
+    let lite_block = lite_parse_helper(input)?;
+
+    assert_eq!(lite_block.block.len(), 2);
+    assert_eq!(lite_block.block[0].commands.len(), 2);
+    assert_eq!(lite_block.block[1].commands.len(), 1);
+
+    assert_eq!(
+        lite_block.block[0].commands[1].comments[0],
+        Span { start: 29, end: 38 }
+    );
+
+    Ok(())
+}
+
+#[test]
+fn multiple_commands_with_pipes() -> Result<(), ParseError> {
+    // The comments inside () get encapsulated in the whole item
+    // Code:
+    // # comment 1
+    // # comment 2
+    // let a = ( ls
+    // | where name =~ some    # another comment
+    // | each { |file| rm file.name } # final comment
+    // )
+    // # comment A
+    // let b = 0;
+    let input = b"# comment 1
+# comment 2
+let a = ( ls
+| where name =~ some # another comment
+| each { |file| rm file.name }) # final comment
+# comment A
+let b = 0
+";
+
+    let lite_block = lite_parse_helper(input)?;
+
+    assert_eq!(lite_block.block.len(), 2);
+    assert_eq!(lite_block.block[0].commands[0].comments.len(), 3);
+    assert_eq!(lite_block.block[0].commands[0].parts.len(), 4);
+
+    assert_eq!(
+        lite_block.block[0].commands[0].parts[3],
+        Span {
+            start: 32,
+            end: 107
+        }
+    );
+
+    assert_eq!(
+        lite_block.block[0].commands[0].comments[2],
+        Span {
+            start: 108,
+            end: 123
+        }
+    );
+
+    assert_eq!(lite_block.block[1].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[1].commands[0].parts.len(), 4);
+
+    assert_eq!(
+        lite_block.block[1].commands[0].comments[0],
+        Span {
+            start: 124,
+            end: 135
+        }
+    );
+
+    Ok(())
+}