comments with a newline dont get together

crates/nu-parser/src/lex.rs

@@ -273,7 +273,15 @@ pub fn lex(
                 if *input == b'\n' || *input == b'\r' {
                     output.push(Token::new(
                         TokenContents::Comment,
-                        Span::new(start, curr_offset),
+                        Span::new(start, curr_offset - 1),
+                    ));
+
+                    // Adding an end of line token after a comment
+                    // This helps during lite_parser to avoid losing a command
+                    // in a statement
+                    output.push(Token::new(
+                        TokenContents::Eol,
+                        Span::new(curr_offset - 1, curr_offset),
                     ));
                     start = curr_offset;
 
@@ -307,4 +315,4 @@ pub fn lex(
         }
     }
     (output, error)
-}
+}

crates/nu-parser/src/lite_parse.rs

@@ -27,6 +27,10 @@ impl LiteCommand {
     pub fn is_empty(&self) -> bool {
         self.parts.is_empty()
     }
+
+    pub fn is_empty_comments(&self) -> bool {
+        self.comments.is_empty()
+    }
 }
 
 #[derive(Debug)]
@@ -94,7 +98,7 @@ pub fn lite_parse(tokens: &[Token]) -> (LiteBlock, Option<ParseError>) {
                 }
             }
             TokenContents::Eol | TokenContents::Semicolon => {
-                if !curr_command.is_empty() {
+                if !curr_command.is_empty() || !curr_command.is_empty_comments() {
                     curr_pipeline.push(curr_command);
                 }
                 curr_command = LiteCommand::new();

crates/nu-parser/tests/test_lex.rs

@@ -57,7 +57,7 @@ fn lex_comment() {
         output.0.get(4).unwrap(),
         &Token {
             contents: TokenContents::Comment,
-            span: Span { start: 12, end: 25 }
+            span: Span { start: 12, end: 24 }
         }
     );
 }
@@ -91,3 +91,47 @@ fn lex_incomplete_quote() {
     let err = output.1.unwrap();
     assert!(matches!(err, ParseError::UnexpectedEof(v, _) if v == "'"));
 }
+
+#[test]
+fn lex_comments() {
+    // Comments should keep the end of line token
+    // Code:
+    // let z = 4
+    // let x = 4 #comment
+    // let y = 1 # comment
+    let file = b"let z = 4 #comment \n let x = 4 # comment\n let y = 1 # comment";
+
+    let output = lex(file, 0, &[], &[]);
+
+    assert_eq!(
+        output.0.get(4).unwrap(),
+        &Token {
+            contents: TokenContents::Comment,
+            span: Span { start: 10, end: 19 }
+        }
+    );
+    assert_eq!(
+        output.0.get(5).unwrap(),
+        &Token {
+            contents: TokenContents::Eol,
+            span: Span { start: 19, end: 20 }
+        }
+    );
+
+    // When there is no space between the comment and the new line the span
+    // for the command and the EOL overlaps
+    assert_eq!(
+        output.0.get(10).unwrap(),
+        &Token {
+            contents: TokenContents::Comment,
+            span: Span { start: 31, end: 40 }
+        }
+    );
+    assert_eq!(
+        output.0.get(11).unwrap(),
+        &Token {
+            contents: TokenContents::Eol,
+            span: Span { start: 40, end: 41 }
+        }
+    );
+}

crates/nu-parser/tests/test_lite_parser.rs

@@ -16,20 +16,25 @@ fn lite_parse_helper(input: &[u8]) -> Result<LiteBlock, ParseError> {
 
 #[test]
 fn comment_before() -> Result<(), ParseError> {
+    // Code:
+    // # this is a comment
+    // def foo bar
     let input = b"# this is a comment\ndef foo bar";
 
     let lite_block = lite_parse_helper(input)?;
 
-    assert_eq!(lite_block.block.len(), 1);
+    assert_eq!(lite_block.block.len(), 2);
     assert_eq!(lite_block.block[0].commands.len(), 1);
     assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
-    assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
+    assert_eq!(lite_block.block[1].commands[0].parts.len(), 3);
 
     Ok(())
 }
 
 #[test]
 fn comment_beside() -> Result<(), ParseError> {
+    // Code:
+    // def foo bar # this is a comment
     let input = b"def foo bar # this is a comment";
 
     let lite_block = lite_parse_helper(input)?;
@@ -44,39 +49,63 @@ fn comment_beside() -> Result<(), ParseError> {
 
 #[test]
 fn comments_stack() -> Result<(), ParseError> {
+    // Code:
+    // # this is a comment
+    // # another comment
+    // # def foo bar
     let input = b"# this is a comment\n# another comment\ndef foo bar ";
 
     let lite_block = lite_parse_helper(input)?;
 
-    assert_eq!(lite_block.block.len(), 1);
-    assert_eq!(lite_block.block[0].commands.len(), 1);
-    assert_eq!(lite_block.block[0].commands[0].comments.len(), 2);
-    assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
+    assert_eq!(lite_block.block.len(), 3);
+    assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[0].commands[0].parts.len(), 0);
+
+    assert_eq!(lite_block.block[1].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[1].commands[0].parts.len(), 0);
+
+    assert_eq!(lite_block.block[2].commands[0].comments.len(), 0);
+    assert_eq!(lite_block.block[2].commands[0].parts.len(), 3);
 
     Ok(())
 }
 
 #[test]
 fn separated_comments_dont_stack() -> Result<(), ParseError> {
+    // Code:
+    // # this is a comment
+    //
+    // # another comment
+    // # def foo bar
     let input = b"# this is a comment\n\n# another comment\ndef foo bar ";
 
     let lite_block = lite_parse_helper(input)?;
 
-    assert_eq!(lite_block.block.len(), 1);
-    assert_eq!(lite_block.block[0].commands.len(), 1);
+    assert_eq!(lite_block.block.len(), 3);
     assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[0].commands[0].parts.len(), 0);
+
+    assert_eq!(lite_block.block[1].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[1].commands[0].parts.len(), 0);
+
+    assert_eq!(lite_block.block[2].commands[0].comments.len(), 0);
+    assert_eq!(lite_block.block[2].commands[0].parts.len(), 3);
+
     assert_eq!(
         lite_block.block[0].commands[0].comments[0],
-        Span { start: 21, end: 39 }
+        Span { start: 0, end: 19 }
+    );
+    assert_eq!(
+        lite_block.block[1].commands[0].comments[0],
+        Span { start: 21, end: 38 }
     );
-    assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
 
     Ok(())
 }
 
 #[test]
 fn multiple_statements() -> Result<(), ParseError> {
-    // Code :
+    // Code:
     // # A comment
     // let a = ( 3 + (
     // 4 +
@@ -86,16 +115,20 @@ fn multiple_statements() -> Result<(), ParseError> {
 
     let lite_block = lite_parse_helper(input)?;
 
-    assert_eq!(lite_block.block.len(), 2);
-    assert_eq!(lite_block.block[0].commands.len(), 1);
+    assert_eq!(lite_block.block.len(), 3);
+    assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
     assert_eq!(
         lite_block.block[0].commands[0].comments[0],
-        Span { start: 0, end: 11 }
+        Span { start: 0, end: 10 }
     );
 
-    assert_eq!(lite_block.block[1].commands.len(), 1);
+    assert_eq!(lite_block.block[1].commands[0].comments.len(), 0);
+    assert_eq!(lite_block.block[1].commands[0].parts.len(), 4);
+
+    assert_eq!(lite_block.block[2].commands[0].comments.len(), 1);
+    assert_eq!(lite_block.block[2].commands[0].parts.len(), 4);
     assert_eq!(
-        lite_block.block[1].commands[0].comments[0],
+        lite_block.block[2].commands[0].comments[0],
         Span { start: 52, end: 61 }
     );
 
@@ -105,7 +138,7 @@ fn multiple_statements() -> Result<(), ParseError> {
 #[test]
 fn multiple_commands() -> Result<(), ParseError> {
     // Pipes add commands to the lite parser
-    // Code :
+    // Code:
     // let a = ls | where name == 1
     // let b = 1 # comment
     let input = b"let a = ls | where name == 1 \n let b = 1 # comment";
@@ -123,3 +156,27 @@ fn multiple_commands() -> Result<(), ParseError> {
 
     Ok(())
 }
+
+#[test]
+fn multiple_commands_with_comment() -> Result<(), ParseError> {
+    // Pipes add commands to the lite parser
+    // The comments are attached to the commands next to them
+    // Code:
+    // let a = ls | where name == 1 # comment
+    // let b = 1 # comment
+    //let a = ls | where name == 1 # comment \n let b = 1 # comment
+    let input = b"let a = ls | where name == 1 # comment\n let b = 1 # comment";
+
+    let lite_block = lite_parse_helper(input)?;
+
+    assert_eq!(lite_block.block.len(), 2);
+    assert_eq!(lite_block.block[0].commands.len(), 2);
+    assert_eq!(lite_block.block[1].commands.len(), 1);
+
+    assert_eq!(
+        lite_block.block[0].commands[1].comments[0],
+        Span { start: 29, end: 38 }
+    );
+
+    Ok(())
+}