eval.c: Add new var `lisp-eval-depth-reserve`

Rather than blindly increase `max-lisp-eval-depth` when entering the
debugger or running `signal-hook-function`, use this new "reserve"
to keep track of how much we have grown the stack for "debugger"
purposes so that for example recursive calls to `signal-hook-function`
can't eat up the whole C stack.

* src/eval.c (max_ensure_room): Rewrite.
(restore_stack_limits): Move before `max_ensure_room`.  Rewrite.
(call_debugger, signal_or_quit): Adjust calls accordingly.
Also grow `max-lisp-eval-depth` for `hander-bind` handlers.
(init_eval_once): Don't initialize `max_lisp_eval_depth` here.
(syms_of_eval): Initialize it here instead.
Add new var `lisp-eval-depth-reserve`.

* doc/lispref/eval.texi (Eval): Add `lisp-eval-depth-reserve`.

3 files changed, 53 insertions, 26 deletions

diff --git a/doc/lispref/eval.texi b/doc/lispref/eval.texi
index f4c99640143..b42020f43af 100644
--- a/doc/lispref/eval.texi
+++ b/doc/lispref/eval.texi
@@ -844,11 +844,24 @@ function body forms, as well as explicit calls in Lisp code.
 
 The default value of this variable is 1600.  If you set it to a value
 less than 100, Lisp will reset it to 100 if the given value is
-reached.  Entry to the Lisp debugger increases the value, if there is
-little room left, to make sure the debugger itself has room to
-execute.
+reached.
 @end defopt
 
+@defopt lisp-eval-depth-reserve
+In order to be able to debug infinite recursion errors, when invoking the
+Lisp debugger, Emacs increases temporarily the value of
+@code{max-lisp-eval-depth}, if there is little room left, to make sure
+the debugger itself has room to execute.  The same happens when
+running the handler of a @code{handler-bind}.  @xref{Handling Errors}.
+
+The variable @code{lisp-eval-depth-reserve} bounds the extra depth
+that Emacs can add to @code{max-lisp-eval-depth} for those
+exceptional circumstances.
+
+The default value of this variable is 200.
+@end defopt
+
+
 @defvar values
 The value of this variable is a list of the values returned by all the
 expressions that were read, evaluated, and printed from buffers
diff --git a/etc/NEWS b/etc/NEWS
index db3b838c380..7bbfbf9512d 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -1396,6 +1396,11 @@ is already loaded, in which case it either signals an error or
 forcibly reloads the file that defines the feature.
 
 +++
+** New variable 'lisp-eval-depth-reserve'.
+It puts a limit to the amount by which Emacs can temporarily increase
+'max-lisp-eval-depth' when handling signals.
+
++++
 ** New special form 'handler-bind'.
 Provides a functionality similar to `condition-case` except it runs the
 handler code without unwinding the stack, such that we can record the
diff --git a/src/eval.c b/src/eval.c
index 7e578a1aa05..b982c124184 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -212,7 +212,6 @@ void
 init_eval_once (void)
 {
   /* Don't forget to update docs (lispref node "Eval").  */
-  max_lisp_eval_depth = 1600;
   Vrun_hooks = Qnil;
   pdumper_do_now_and_after_load (init_eval_once_for_pdumper);
 }
@@ -248,22 +247,29 @@ init_eval (void)
   redisplay_deep_handler = NULL;
 }
 
-/* Ensure that *M is at least A + B if possible, or is its maximum
-   value otherwise.  */
-
 static void
-max_ensure_room (intmax_t *m, intmax_t a, intmax_t b)
+restore_stack_limits (Lisp_Object data)
 {
-  intmax_t sum = ckd_add (&sum, a, b) ? INTMAX_MAX : sum;
-  *m = max (*m, sum);
+  intmax_t old_depth;
+  integer_to_intmax (data, &old_depth);
+  lisp_eval_depth_reserve += max_lisp_eval_depth - old_depth;
+  max_lisp_eval_depth = old_depth;
 }
 
-/* Unwind-protect function used by call_debugger.  */
+/* Try and ensure that we have at least B dpeth available.  */
 
 static void
-restore_stack_limits (Lisp_Object data)
+max_ensure_room (intmax_t b)
 {
-  integer_to_intmax (data, &max_lisp_eval_depth);
+  intmax_t sum = ckd_add (&sum, lisp_eval_depth, b) ? INTMAX_MAX : sum;
+  intmax_t diff = min (sum - max_lisp_eval_depth, lisp_eval_depth_reserve);
+  if (diff <= 0)
+    return;
+  intmax_t old_depth = max_lisp_eval_depth;
+  max_lisp_eval_depth += diff;
+  lisp_eval_depth_reserve -= diff;
+  /* Restore limits after leaving the debugger.  */
+  record_unwind_protect (restore_stack_limits, make_int (old_depth));
 }
 
 /* Call the Lisp debugger, giving it argument ARG.  */
@@ -274,16 +280,12 @@ call_debugger (Lisp_Object arg)
   bool debug_while_redisplaying;
   specpdl_ref count = SPECPDL_INDEX ();
   Lisp_Object val;
-  intmax_t old_depth = max_lisp_eval_depth;
 
   /* The previous value of 40 is too small now that the debugger
      prints using cl-prin1 instead of prin1.  Printing lists nested 8
      deep (which is the value of print-level used in the debugger)
      currently requires 77 additional frames.  See bug#31919.  */
-  max_ensure_room (&max_lisp_eval_depth, lisp_eval_depth, 100);
-
-  /* Restore limits after leaving the debugger.  */
-  record_unwind_protect (restore_stack_limits, make_int (old_depth));
+  max_ensure_room (100);
 
 #ifdef HAVE_WINDOW_SYSTEM
   if (display_hourglass_p)
@@ -1802,16 +1804,13 @@ signal_or_quit (Lisp_Object error_symbol, Lisp_Object data, bool keyboard_quit)
 
   /* This hook is used by edebug.  */
   if (! NILP (Vsignal_hook_function)
-      && ! NILP (error_symbol)
-      /* Don't try to call a lisp function if we've already overflowed
-         the specpdl stack.  */
-      && specpdl_ptr < specpdl_end)
+      && ! NILP (error_symbol))
     {
-      /* Edebug takes care of restoring these variables when it exits.  */
-      max_ensure_room (&max_lisp_eval_depth, lisp_eval_depth, 20);
-
+      specpdl_ref count = SPECPDL_INDEX ();
+      max_ensure_room (20);
       /* FIXME: 'handler-bind' makes `signal-hook-function' obsolete?  */
       call2 (Vsignal_hook_function, error_symbol, data);
+      unbind_to (count, Qnil);
     }
 
   conditions = Fget (real_error_symbol, Qerror_conditions);
@@ -1849,9 +1848,12 @@ signal_or_quit (Lisp_Object error_symbol, Lisp_Object data, bool keyboard_quit)
                 Lisp_Object error_data
                   = (NILP (error_symbol)
                      ? data : Fcons (error_symbol, data));
+                specpdl_ref count = SPECPDL_INDEX ();
+                max_ensure_room (20);
                 push_handler (make_fixnum (skip + h->bytecode_dest),
                               SKIP_CONDITIONS);
                 call1 (h->val, error_data);
+                unbind_to (count, Qnil);
                 pop_handler ();
               }
             continue;
@@ -1901,8 +1903,8 @@ signal_or_quit (Lisp_Object error_symbol, Lisp_Object data, bool keyboard_quit)
       && NILP (Vinhibit_debugger)
       && !NILP (Ffboundp (Qdebug_early)))
     {
-      max_ensure_room (&max_lisp_eval_depth, lisp_eval_depth, 100);
       specpdl_ref count = SPECPDL_INDEX ();
+      max_ensure_room (100);
       AUTO_STRING (redisplay_trace, "*Redisplay-trace*");
       Lisp_Object redisplay_trace_buffer;
       AUTO_STRING (gap, "\n\n\n\n"); /* Separates things in *Redisplay-trace* */
@@ -4345,6 +4347,13 @@ actual stack overflow in C, which would be fatal for Emacs.
 You can safely make it considerably larger than its default value,
 if that proves inconveniently small.  However, if you increase it too far,
 Emacs could overflow the real C stack, and crash.  */);
+  max_lisp_eval_depth = 1600;
+
+  DEFVAR_INT ("lisp-eval-depth-reserve", lisp_eval_depth_reserve,
+              doc: /* Extra depth that can be allocated to handle errors.
+This is the max depth that the system will add to `max-lisp-eval-depth'
+when calling debuggers or `handler-bind' handlers.  */);
+  lisp_eval_depth_reserve = 200;
 
   DEFVAR_LISP ("quit-flag", Vquit_flag,
                doc: /* Non-nil causes `eval' to abort, unless `inhibit-quit' is non-nil.