Support Eshell iterative evaluation in the background

This really just generalizes Eshell's previous support for iterative
evaluation of a single current command to a list of multiple commands,
of which at most one can be in the foreground (bug#66066).

* lisp/eshell/esh-cmd.el (eshell-last-async-procs)
(eshell-current-command): Make obsolete in favor of...
(eshell-foreground-command): ... this
(eshell-background-commands): New variable.
(eshell-interactive-process-p): Make obsolete.
(eshell-head-process, eshell-tail-process): Use
'eshell-foreground-command'.
(eshell-cmd-initialize): Initialize new variables.
(eshell-add-command, eshell-remove-command)
(eshell-commands-for-process): New functions.
(eshell-parse-command): Make 'eshell-do-subjob' the outermost call.
(eshell-do-subjob): Call 'eshell-resume-eval' to split this command
off from its parent forms.
(eshell-eval-command): Use 'eshell-add-command'.
(eshell-resume-command): Use 'eshell-commands-for-process'.
(eshell-resume-eval): Take a COMMAND argument.  Return
':eshell-background' form for deferred background commands.
(eshell-do-eval): Remove check for 'eshell-current-subjob-p'.  This is
handled differently now.

* lisp/eshell/eshell.el (eshell-command): Wait for all processes to
exit when running synchronously.

* lisp/eshell/esh-mode.el (eshell-intercept-commands)
(eshell-watch-for-password-prompt):
* lisp/eshell/em-cmpl.el (eshell-complete-parse-arguments):
* lisp/eshell/em-smart.el (eshell-smart-display-move): Use
'eshell-foreground-command'.

* test/lisp/eshell/esh-cmd-tests.el
(esh-cmd-test/background/simple-command)
(esh-cmd-test/background/subcommand): New tests.
(esh-cmd-test/throw): Use 'eshell-foreground-command'.

* test/lisp/eshell/eshell-tests.el (eshell-test/queue-input): Use
'eshell-foreground-command'.

* test/lisp/eshell/em-script-tests.el
(em-script-test/source-script/background): Make the test script more
complex.

* test/lisp/eshell/eshell-tests.el
(eshell-test/eshell-command/pipeline-wait): New test.

* doc/misc/eshell.texi (Bugs and ideas): Remove implemented feature.

9 files changed, 165 insertions, 73 deletions

diff --git a/doc/misc/eshell.texi b/doc/misc/eshell.texi
index 8b3eb72aa66..cc94f610615 100644
--- a/doc/misc/eshell.texi
+++ b/doc/misc/eshell.texi
@@ -2568,8 +2568,6 @@ A special associate array, which can take references of the form
 @samp{$=[REGEXP]}.  It indexes into the directory ring.
 @end table
 
-@item Eshell scripts can't execute in the background
-
 @item Support zsh's ``Parameter Expansion'' syntax, i.e., @samp{$@{@var{name}:-@var{val}@}}
 
 @item Create a mode @code{eshell-browse}
diff --git a/lisp/eshell/em-cmpl.el b/lisp/eshell/em-cmpl.el
index 25dccbd695c..61f1237b907 100644
--- a/lisp/eshell/em-cmpl.el
+++ b/lisp/eshell/em-cmpl.el
@@ -343,7 +343,7 @@ to writing a completion function."
 (defun eshell-complete-parse-arguments ()
   "Parse the command line arguments for `pcomplete-argument'."
   (when (and eshell-no-completion-during-jobs
-             (eshell-interactive-process-p))
+             eshell-foreground-command)
     (eshell--pcomplete-insert-tab))
   (let ((end (point-marker))
         (begin (save-excursion (beginning-of-line) (point)))
diff --git a/lisp/eshell/em-smart.el b/lisp/eshell/em-smart.el
index d5002a59d14..4c39a991ec6 100644
--- a/lisp/eshell/em-smart.el
+++ b/lisp/eshell/em-smart.el
@@ -294,7 +294,7 @@ and the end of the buffer are still visible."
        ((eq this-command 'self-insert-command)
         (if (eq last-command-event ? )
             (if (and eshell-smart-space-goes-to-end
-                     eshell-current-command)
+                     eshell-foreground-command)
                 (if (not (pos-visible-in-window-p (point-max)))
                     (setq this-command 'scroll-up)
                   (setq this-command 'eshell-smart-goto-end))
diff --git a/lisp/eshell/esh-cmd.el b/lisp/eshell/esh-cmd.el
index fc7d54a758d..990d2ca1122 100644
--- a/lisp/eshell/esh-cmd.el
+++ b/lisp/eshell/esh-cmd.el
@@ -263,7 +263,24 @@ command line.")
 
 ;;; Internal Variables:
 
-(defvar eshell-current-command nil)
+;; These variables have been merged into `eshell-foreground-command'.
+;; Outside of this file, the most-common use for them is to check
+;; whether they're nil.
+(define-obsolete-variable-alias 'eshell-last-async-procs
+  'eshell-foreground-command "30.1")
+(define-obsolete-variable-alias 'eshell-current-command
+  'eshell-foreground-command "30.1")
+
+(defvar eshell-foreground-command nil
+  "The currently-running foreground command, if any.
+This is a list of the form (FORM PROCESSES).  FORM is the Eshell
+command form.  PROCESSES is a list of processes that deferred the
+command.")
+(defvar eshell-background-commands nil
+  "A list of currently-running deferred commands.
+Each element is of the form (FORM PROCESSES), as with
+`eshell-foreground-command' (which see).")
+
 (defvar eshell-command-name nil)
 (defvar eshell-command-arguments nil)
 (defvar eshell-in-pipeline-p nil
@@ -273,11 +290,6 @@ otherwise t.")
 (defvar eshell-in-subcommand-p nil)
 (defvar eshell-last-arguments nil)
 (defvar eshell-last-command-name nil)
-(defvar eshell-last-async-procs nil
-  "The currently-running foreground process(es).
-When executing a pipeline, this is a list of all the pipeline's
-processes, with the first usually reading from stdin and last
-usually writing to stdout.")
 
 (defvar eshell-allow-commands t
   "If non-nil, allow evaluating command forms (including Lisp forms).
@@ -294,29 +306,30 @@ also `eshell-complete-parse-arguments'.")
 
 (defsubst eshell-interactive-process-p ()
   "Return non-nil if there is a currently running command process."
-  eshell-last-async-procs)
+  (declare (obsolete 'eshell-foreground-command "30.1"))
+  eshell-foreground-command)
 
 (defsubst eshell-head-process ()
   "Return the currently running process at the head of any pipeline.
 This only returns external (non-Lisp) processes."
-  (car eshell-last-async-procs))
+  (caadr eshell-foreground-command))
 
 (defsubst eshell-tail-process ()
   "Return the currently running process at the tail of any pipeline.
 This only returns external (non-Lisp) processes."
-  (car (last eshell-last-async-procs)))
+  (car (last (cadr eshell-foreground-command))))
 
 (define-obsolete-function-alias 'eshell-interactive-process
   'eshell-tail-process "29.1")
 
 (defun eshell-cmd-initialize ()     ;Called from `eshell-mode' via intern-soft!
   "Initialize the Eshell command processing module."
-  (setq-local eshell-current-command nil)
+  (setq-local eshell-foreground-command nil)
+  (setq-local eshell-background-commands nil)
   (setq-local eshell-command-name nil)
   (setq-local eshell-command-arguments nil)
   (setq-local eshell-last-arguments nil)
   (setq-local eshell-last-command-name nil)
-  (setq-local eshell-last-async-procs nil)
 
   (add-hook 'eshell-kill-hook #'eshell-resume-command nil t)
   (add-hook 'eshell-parse-argument-hook
@@ -337,6 +350,47 @@ This only returns external (non-Lisp) processes."
       (throw 'pcomplete-completions
              (all-completions pcomplete-stub obarray 'boundp)))))
 
+;; Current command management
+
+(defun eshell-add-command (form &optional background)
+  "Add a command FORM to our list of known commands and return the new entry.
+If non-nil, BACKGROUND indicates that this is a command running
+in the background.  The result is a command entry in the
+form (BACKGROUND FORM PROCESSES), where PROCESSES is initially
+nil."
+  (cons (when background 'background)
+        (if background
+            (car (push (list form nil) eshell-background-commands))
+          (cl-assert (null eshell-foreground-command))
+          (setq eshell-foreground-command (list form nil)))))
+
+(defun eshell-remove-command (command)
+  "Remove COMMAND from our list of known commands.
+COMMAND should be a list of the form (BACKGROUND FORM PROCESSES),
+as returned by `eshell-add-command' (which see)."
+  (let ((background (car command))
+        (entry (cdr command)))
+    (if background
+        (setq eshell-background-commands
+              (delq entry eshell-background-commands))
+      (cl-assert (eq eshell-foreground-command entry))
+      (setq eshell-foreground-command nil))))
+
+(defun eshell-commands-for-process (process)
+  "Return all commands associated with a PROCESS.
+Each element will have the form (BACKGROUND FORM PROCESSES), as
+returned by `eshell-add-command' (which see).
+
+Usually, there should only be one element in this list, but it's
+theoretically possible to have more than one associated command
+for a given process."
+  (nconc (when (memq process (cadr eshell-foreground-command))
+           (list (cons nil eshell-foreground-command)))
+         (seq-keep (lambda (cmd)
+                     (when (memq process (cadr cmd))
+                       (cons 'background cmd)))
+                   eshell-background-commands)))
+
 ;; Command parsing
 
 (defsubst eshell--region-p (object)
@@ -407,8 +461,6 @@ command hooks should be run before and after the command."
        (lambda (cmd)
          (let ((sep (pop sep-terms)))
            (setq cmd (eshell-parse-pipeline cmd))
-           (when (equal sep "&")
-             (setq cmd `(eshell-do-subjob (cons :eshell-background ,cmd))))
            (unless eshell-in-pipeline-p
              (setq cmd `(eshell-trap-errors ,cmd)))
            ;; Copy I/O handles so each full statement can manipulate
@@ -416,6 +468,8 @@ command hooks should be run before and after the command."
            ;; command in the list; we won't use the originals again
            ;; anyway.
            (setq cmd `(eshell-with-copied-handles ,cmd ,(not sep)))
+           (when (equal sep "&")
+             (setq cmd `(eshell-do-subjob ,cmd)))
            cmd))
        sub-chains)))
     (if toplevel
@@ -740,13 +794,13 @@ if none)."
 
 (defmacro eshell-do-subjob (object)
   "Evaluate a command OBJECT as a subjob.
-We indicate that the process was run in the background by returning it
-ensconced in a list."
+We indicate that the process was run in the background by
+returning it as (:eshell-background . PROCESSES)."
   `(let ((eshell-current-subjob-p t)
          ;; Print subjob messages.  This could have been cleared
          ;; (e.g. by `eshell-source-file', which see).
          (eshell-subjob-messages t))
-     ,object))
+     (eshell-resume-eval (eshell-add-command ',object 'background))))
 
 (defmacro eshell-commands (object &optional silent)
   "Place a valid set of handles, and context, around command OBJECT."
@@ -980,12 +1034,12 @@ Return the process (or head and tail processes) created by
 COMMAND, if any.  If COMMAND is a background command, return the
 process(es) in a cons cell like:
 
-  (:eshell-background . PROCESS)"
-  (if eshell-current-command
+  (:eshell-background . PROCESSES)"
+  (if eshell-foreground-command
       (progn
         ;; We can just stick the new command at the end of the current
         ;; one, and everything will happen as it should.
-        (setcdr (last (cdr eshell-current-command))
+        (setcdr (last (cdar eshell-foreground-command))
                 (list `(let ((here (and (eobp) (point))))
                          ,(and input
                                `(insert-and-inherit ,(concat input "\n")))
@@ -994,56 +1048,61 @@ process(es) in a cons cell like:
                          (eshell-do-eval ',command))))
         (eshell-debug-command 'form
           "enqueued command form for %S\n\n%s"
-          (or input "<no string>") (eshell-stringify eshell-current-command)))
+          (or input "<no string>")
+          (eshell-stringify (car eshell-foreground-command))))
     (eshell-debug-command-start input)
-    (setq eshell-current-command command)
     (let* (result
            (delim (catch 'eshell-incomplete
-                    (ignore (setq result (eshell-resume-eval))))))
+                    (ignore (setq result (eshell-resume-eval
+                                          (eshell-add-command command)))))))
       (when delim
         (error "Unmatched delimiter: %S" delim))
       result)))
 
 (defun eshell-resume-command (proc status)
-  "Resume the current command when a pipeline ends."
-  (when (and proc
-             ;; Make sure PROC is one of our foreground processes and
-             ;; that all of those processes are now dead.
-             (member proc eshell-last-async-procs)
-             (not (seq-some #'eshell-process-active-p eshell-last-async-procs)))
-    (if (and ;; Check STATUS to determine whether we want to resume or
-             ;; abort the command.
-             (stringp status)
-             (not (string= "stopped" status))
-             (not (string-match eshell-reset-signals status)))
-        (eshell-resume-eval)
-      (setq eshell-last-async-procs nil)
-      (setq eshell-current-command nil)
-      (declare-function eshell-reset "esh-mode" (&optional no-hooks))
-      (eshell-reset))))
-
-(defun eshell-resume-eval ()
-  "Destructively evaluate a form which may need to be deferred."
-  (setq eshell-last-async-procs nil)
-  (when eshell-current-command
-    (eshell-condition-case err
-        (let (retval procs)
-          (unwind-protect
-              (progn
-                (setq procs (catch 'eshell-defer
-                              (ignore (setq retval
-                                            (eshell-do-eval
-                                             eshell-current-command)))))
-                (when retval
-                  (cadr retval)))
-            (setq eshell-last-async-procs procs)
+  "Resume the current command when a pipeline ends.
+PROC is the process that invoked this from its sentinel, and
+STATUS is its status."
+  (when proc
+    (dolist (command (eshell-commands-for-process proc))
+      (unless (seq-some #'eshell-process-active-p (nth 2 command))
+        (setf (nth 2 command) nil) ; Clear processes from command.
+        (if (and ;; Check STATUS to determine whether we want to resume or
+                 ;; abort the command.
+                 (stringp status)
+                 (not (string= "stopped" status))
+                 (not (string-match eshell-reset-signals status)))
+            (eshell-resume-eval command)
+          (eshell-remove-command command)
+          (declare-function eshell-reset "esh-mode" (&optional no-hooks))
+          (eshell-reset))))))
+
+(defun eshell-resume-eval (command)
+  "Destructively evaluate a COMMAND which may need to be deferred.
+COMMAND is a command entry of the form (BACKGROUND FORM
+PROCESSES) (see `eshell-add-command').
+
+Return the result of COMMAND's FORM if it wasn't deferred.  If
+BACKGROUND is non-nil and Eshell defers COMMAND, return a list of
+the form (:eshell-background . PROCESSES)."
+  (eshell-condition-case err
+      (let (retval procs)
+        (unwind-protect
+            (progn
+              (setq procs
+                    (catch 'eshell-defer
+                      (ignore (setq retval (eshell-do-eval (cadr command))))))
+              (cond
+               (retval (cadr retval))
+               ((car command) (cons :eshell-background procs))))
+          (if procs
+              (setf (nth 2 command) procs)
             ;; If we didn't defer this command, clear it out.  This
             ;; applies both when the command has finished normally,
             ;; and when a signal or thrown value causes us to unwind.
-            (unless procs
-              (setq eshell-current-command nil))))
-      (error
-       (error (error-message-string err))))))
+            (eshell-remove-command command))))
+    (error
+     (error (error-message-string err)))))
 
 (defmacro eshell-manipulate (form tag &rest body)
   "Manipulate a command FORM with BODY, using TAG as a debug identifier."
@@ -1272,7 +1331,6 @@ have been replaced by constants."
                     (setcdr form (cdr new-form)))
                   (eshell-do-eval form synchronous-p))
               (if-let (((memq (car form) eshell-deferrable-commands))
-                       ((not eshell-current-subjob-p))
                        (procs (eshell-make-process-list result)))
                   (if synchronous-p
                       (apply #'eshell/wait procs)
diff --git a/lisp/eshell/esh-mode.el b/lisp/eshell/esh-mode.el
index 0c381dbb86a..2b560afb92c 100644
--- a/lisp/eshell/esh-mode.el
+++ b/lisp/eshell/esh-mode.el
@@ -453,7 +453,7 @@ and the hook `eshell-exit-hook'."
                      last-command-event))))
 
 (defun eshell-intercept-commands ()
-  (when (and (eshell-interactive-process-p)
+  (when (and eshell-foreground-command
              (not (and (integerp last-input-event)
                        (memq last-input-event '(?\C-x ?\C-c)))))
     (let ((possible-events (where-is-internal this-command))
@@ -967,7 +967,7 @@ buffer's process if STRING contains a password prompt defined by
 `eshell-password-prompt-regexp'.
 
 This function could be in the list `eshell-output-filter-functions'."
-  (when (eshell-interactive-process-p)
+  (when eshell-foreground-command
     (save-excursion
       (let ((case-fold-search t))
         (goto-char eshell-last-output-block-begin)
diff --git a/lisp/eshell/eshell.el b/lisp/eshell/eshell.el
index a3f80f453eb..8765ba499a1 100644
--- a/lisp/eshell/eshell.el
+++ b/lisp/eshell/eshell.el
@@ -315,9 +315,8 @@ argument), then insert output into the current buffer at point."
         ;; make the output as attractive as possible, with no
         ;; extraneous newlines
         (when intr
-          (if (eshell-interactive-process-p)
-              (eshell-wait-for-process (eshell-tail-process)))
-          (cl-assert (not (eshell-interactive-process-p)))
+          (apply #'eshell-wait-for-process (cadr eshell-foreground-command))
+          (cl-assert (not eshell-foreground-command))
           (goto-char (point-max))
           (while (and (bolp) (not (bobp)))
             (delete-char -1)))
diff --git a/test/lisp/eshell/em-script-tests.el b/test/lisp/eshell/em-script-tests.el
index 191755dcc3e..02e4125d827 100644
--- a/test/lisp/eshell/em-script-tests.el
+++ b/test/lisp/eshell/em-script-tests.el
@@ -67,14 +67,14 @@
   "Test sourcing a script in the background."
   (skip-unless (executable-find "echo"))
   (ert-with-temp-file temp-file
-    :text "*echo hi"
+    :text "*echo hi\nif {[ foo = foo ]} {*echo bye}"
     (eshell-with-temp-buffer bufname "old"
       (with-temp-eshell
        (eshell-match-command-output
         (format "source %s > #<%s> &" temp-file bufname)
         "\\`\\'")
        (eshell-wait-for-subprocess t))
-      (should (equal (buffer-string) "hi\n")))))
+      (should (equal (buffer-string) "hi\nbye\n")))))
 
 (ert-deftest em-script-test/source-script/arg-vars ()
   "Test sourcing script with $0, $1, ... variables."
diff --git a/test/lisp/eshell/esh-cmd-tests.el b/test/lisp/eshell/esh-cmd-tests.el
index 643038f89ff..e0783b26ad6 100644
--- a/test/lisp/eshell/esh-cmd-tests.el
+++ b/test/lisp/eshell/esh-cmd-tests.el
@@ -104,6 +104,32 @@ bug#59469."
     "value\nexternal\nvalue\n")))
 
 
+;; Background command invocation
+
+(ert-deftest esh-cmd-test/background/simple-command ()
+  "Test invocation with a simple background command."
+  (skip-unless (executable-find "echo"))
+  (eshell-with-temp-buffer bufname ""
+    (with-temp-eshell
+     (eshell-match-command-output
+      (format "*echo hi > #<%s> &" bufname)
+      (rx "[echo" (? ".exe") "] " (+ digit) "\n"))
+     (eshell-wait-for-subprocess t))
+    (should (equal (buffer-string) "hi\n"))))
+
+(ert-deftest esh-cmd-test/background/subcommand ()
+  "Test invocation with a background command containing subcommands."
+  (skip-unless (and (executable-find "echo")
+                    (executable-find "rev")))
+  (eshell-with-temp-buffer bufname ""
+    (with-temp-eshell
+     (eshell-match-command-output
+      (format "*echo ${*echo hello | rev} > #<%s> &" bufname)
+      (rx "[echo" (? ".exe") "] " (+ digit) "\n"))
+     (eshell-wait-for-subprocess t))
+    (should (equal (buffer-string) "olleh\n"))))
+
+
 ;; Lisp forms
 
 (ert-deftest esh-cmd-test/quoted-lisp-form ()
@@ -453,8 +479,7 @@ This tests when `eshell-lisp-form-nil-is-failure' is nil."
                  "echo hi; (throw 'tag 42); echo bye"))
               42))
    (should (eshell-match-output "\\`hi\n\\'"))
-   (should-not eshell-current-command)
-   (should-not eshell-last-async-procs)
+   (should-not eshell-foreground-command)
    ;; Make sure we can call another command after throwing.
    (eshell-match-command-output "echo again" "\\`again\n")))
 
diff --git a/test/lisp/eshell/eshell-tests.el b/test/lisp/eshell/eshell-tests.el
index 25c8cfd389c..b02e5fca592 100644
--- a/test/lisp/eshell/eshell-tests.el
+++ b/test/lisp/eshell/eshell-tests.el
@@ -58,6 +58,18 @@ This test uses a pipeline for the command."
         (eshell-command "*echo hi | *cat" t)
         (should (equal (buffer-string) "hi\n"))))))
 
+(ert-deftest eshell-test/eshell-command/pipeline-wait ()
+  "Check that `eshell-command' waits for all its processes before returning."
+  (skip-unless (and (executable-find "echo")
+                    (executable-find "sh")
+                    (executable-find "rev")))
+  (ert-with-temp-directory eshell-directory-name
+    (let ((eshell-history-file-name nil))
+      (with-temp-buffer
+        (eshell-command
+         "*echo hello | sh -c 'sleep 1; rev' 1>&2 | *echo goodbye" t)
+        (should (equal (buffer-string) "goodbye\nolleh\n"))))))
+
 (ert-deftest eshell-test/eshell-command/background ()
   "Test that `eshell-command' works for background commands."
   (skip-unless (executable-find "echo"))
@@ -132,7 +144,7 @@ insert the queued one at the next prompt, and finally run it."
    (eshell-insert-command "sleep 1; echo slept")
    (eshell-insert-command "echo alpha" #'eshell-queue-input)
    (let ((start (marker-position (eshell-beginning-of-output))))
-     (eshell-wait-for (lambda () (not eshell-current-command)))
+     (eshell-wait-for (lambda () (not eshell-foreground-command)))
      (should (string-match "^slept\n.*echo alpha\nalpha\n$"
                            (buffer-substring-no-properties
                             start (eshell-end-of-output)))))))