Limit recursion to avoid stack overflow (#7657)

Add recursion limit to `def` and `block`.
Summary of this PR , it will detect if `def` call itself or not .
Then execute by using `stack` which I think best choice to use with this
design and core as it is available in all crates and mutable and
calculate the recursion limit on calling `def`.
Set 50 as recursion limit on `Config`.
Add some tests too .

Fixes #5899

Co-authored-by: Reilly Wood <reilly.wood@icloud.com>

crates/nu-protocol/src/ast/block.rs

@@ -11,6 +11,7 @@ pub struct Block {
     pub captures: Vec<VarId>,
     pub redirect_env: bool,
     pub span: Option<Span>, // None option encodes no span to avoid using test_span()
+    pub recursive: Option<bool>, // does the block call itself?
 }
 
 impl Block {
@@ -51,6 +52,7 @@ impl Block {
             captures: vec![],
             redirect_env: false,
             span: None,
+            recursive: None,
         }
     }
 }
@@ -66,6 +68,7 @@ where
             captures: vec![],
             redirect_env: false,
             span: None,
+            recursive: None,
         }
     }
 }

crates/nu-protocol/src/engine/stack.rs

@@ -34,6 +34,7 @@ pub struct Stack {
     pub env_hidden: HashMap<String, HashSet<String>>,
     /// List of active overlays
     pub active_overlays: Vec<String>,
+    pub recursion_count: Box<u64>,
 }
 
 impl Stack {
@@ -43,6 +44,7 @@ impl Stack {
             env_vars: vec![],
             env_hidden: HashMap::new(),
             active_overlays: vec![DEFAULT_OVERLAY_NAME.to_string()],
+            recursion_count: Box::new(0),
         }
     }
 
@@ -123,6 +125,7 @@ impl Stack {
             env_vars,
             env_hidden: HashMap::new(),
             active_overlays: self.active_overlays.clone(),
+            recursion_count: self.recursion_count.to_owned(),
         }
     }
 
@@ -147,6 +150,7 @@ impl Stack {
             env_vars,
             env_hidden: HashMap::new(),
             active_overlays: self.active_overlays.clone(),
+            recursion_count: self.recursion_count.to_owned(),
         }
     }
 

crates/nu-protocol/src/shell_error.rs

@@ -903,6 +903,19 @@ Either make sure {0} is a string, or add a 'to_string' entry for it in ENV_CONVE
     /// Return event, which may become an error if used outside of a function
     #[error("Return used outside of function")]
     Return(#[label = "used outside of function"] Span, Box<Value>),
+
+    /// The code being executed called itself too many times.
+    ///
+    /// ## Resolution
+    ///
+    /// Adjust your Nu code to
+    #[error("Recursion limit ({recursion_limit}) reached")]
+    #[diagnostic(code(nu::shell::recursion_limit_reached), url(docsrs))]
+    RecursionLimitReached {
+        recursion_limit: u64,
+        #[label("This called itself too many times")]
+        span: Option<Span>,
+    },
 }
 
 impl From<std::io::Error> for ShellError {