Allow creating processes where only one of stdin or stdout is a PTY

* src/lisp.h (emacs_spawn):
* src/callproc.c (emacs_spawn): Add PTY_IN and PTY_OUT arguments to
specify which streams should be set up as a PTY.
(call_process): Adjust call to 'emacs_spawn'.

* src/process.h (Lisp_Process): Replace 'pty_flag' with 'pty_in' and
'pty_out'.

* src/process.c (is_pty_from_symbol): New function.
(make-process): Allow :connection-type to be a cons cell, and allow
using a stderr process with a PTY for stdin/stdout.
(create_process): Handle creating a process where only one of stdin or
stdout is a PTY.

* lisp/eshell/esh-proc.el (eshell-needs-pipe, eshell-needs-pipe-p):
Remove.
(eshell-gather-process-output): Use 'make-process' and set
':connection-type' as needed by the value of 'eshell-in-pipeline-p'.

* lisp/net/tramp.el (tramp-handle-make-process):
* lisp/net/tramp-adb.el (tramp-adb-handle-make-process):
* lisp/net/tramp-sh.el (tramp-sh-handle-make-process): Don't signal an
error when ':connection-type' is a cons cell.

* test/src/process-tests.el
(process-test-sentinel-wait-function-working-p): Allow passing PROC
in, and rework into...
(process-test-wait-for-sentinel): ... this.
(process-test-sentinel-accept-process-output)
(process-test-sentinel-sit-for, process-test-quoted-batfile)
(process-test-stderr-filter): Use 'process-test-wait-for-sentinel'.
(make/process/test-connection-type): New function.
(make-process/connection-type/pty, make-process/connection-type/pty-2)
(make-process/connection-type/pipe)
(make-process/connection-type/pipe-2)
(make-process/connection-type/in-pty)
(make-process/connection-type/out-pty)
(make-process/connection-type/pty-with-stderr-buffer)
(make-process/connection-type/out-pty-with-stderr-buffer): New tests.

* test/lisp/eshell/esh-proc-tests.el (esh-proc-test--detect-pty-cmd):
New variable.
(esh-proc-test/pipeline-connection-type/no-pipeline)
(esh-proc-test/pipeline-connection-type/first)
(esh-proc-test/pipeline-connection-type/middle)
(esh-proc-test/pipeline-connection-type/last): New tests.

* doc/lispref/processes.texi (Asynchronous Processes): Document new
':connection-type' behavior.
(Output from Processes): Remove caveat about ':stderr' forcing
'make-process' to use pipes.

* etc/NEWS: Announce this change (bug#56025).

4 files changed, 107 insertions, 67 deletions

diff --git a/src/callproc.c b/src/callproc.c
index dd162f36a6c..aec0a2f5a58 100644
--- a/src/callproc.c
+++ b/src/callproc.c
@@ -650,7 +650,7 @@ call_process (ptrdiff_t nargs, Lisp_Object *args, int filefd,
 
   child_errno
     = emacs_spawn (&pid, filefd, fd_output, fd_error, new_argv, env,
-                   SSDATA (current_dir), NULL, &oldset);
+                   SSDATA (current_dir), NULL, false, false, &oldset);
   eassert ((child_errno == 0) == (0 < pid));
 
   if (pid > 0)
@@ -1412,14 +1412,15 @@ emacs_posix_spawn_init_attributes (posix_spawnattr_t *attributes,
 int
 emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
              char **argv, char **envp, const char *cwd,
-             const char *pty, const sigset_t *oldset)
+             const char *pty_name, bool pty_in, bool pty_out,
+             const sigset_t *oldset)
 {
 #if USABLE_POSIX_SPAWN
   /* Prefer the simpler `posix_spawn' if available.  `posix_spawn'
      doesn't yet support setting up pseudoterminals, so we fall back
      to `vfork' if we're supposed to use a pseudoterminal.  */
 
-  bool use_posix_spawn = pty == NULL;
+  bool use_posix_spawn = pty_name == NULL;
 
   posix_spawn_file_actions_t actions;
   posix_spawnattr_t attributes;
@@ -1473,7 +1474,9 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
   /* vfork, and prevent local vars from being clobbered by the vfork.  */
   pid_t *volatile newpid_volatile = newpid;
   const char *volatile cwd_volatile = cwd;
-  const char *volatile pty_volatile = pty;
+  const char *volatile ptyname_volatile = pty_name;
+  bool volatile ptyin_volatile = pty_in;
+  bool volatile ptyout_volatile = pty_out;
   char **volatile argv_volatile = argv;
   int volatile stdin_volatile = std_in;
   int volatile stdout_volatile = std_out;
@@ -1495,7 +1498,9 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
 
   newpid = newpid_volatile;
   cwd = cwd_volatile;
-  pty = pty_volatile;
+  pty_name = ptyname_volatile;
+  pty_in = ptyin_volatile;
+  pty_out = ptyout_volatile;
   argv = argv_volatile;
   std_in = stdin_volatile;
   std_out = stdout_volatile;
@@ -1506,13 +1511,12 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
   if (pid == 0)
 #endif /* not WINDOWSNT */
     {
-      bool pty_flag = pty != NULL;
       /* Make the pty be the controlling terminal of the process.  */
 #ifdef HAVE_PTYS
       dissociate_controlling_tty ();
 
       /* Make the pty's terminal the controlling terminal.  */
-      if (pty_flag && std_in >= 0)
+      if (pty_in && std_in >= 0)
         {
 #ifdef TIOCSCTTY
           /* We ignore the return value
@@ -1521,7 +1525,7 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
 #endif
         }
 #if defined (LDISC1)
-      if (pty_flag && std_in >= 0)
+      if (pty_in && std_in >= 0)
         {
           struct termios t;
           tcgetattr (std_in, &t);
@@ -1531,7 +1535,7 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
         }
 #else
 #if defined (NTTYDISC) && defined (TIOCSETD)
-      if (pty_flag && std_in >= 0)
+      if (pty_in && std_in >= 0)
         {
           /* Use new line discipline.  */
           int ldisc = NTTYDISC;
@@ -1548,18 +1552,21 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
      both TIOCSCTTY is defined.  */
         /* Now close the pty (if we had it open) and reopen it.
            This makes the pty the controlling terminal of the subprocess.  */
-      if (pty_flag)
+      if (pty_name)
         {
 
           /* I wonder if emacs_close (emacs_open (pty, ...))
              would work?  */
-          if (std_in >= 0)
+          if (pty_in && std_in >= 0)
             emacs_close (std_in);
-          std_out = std_in = emacs_open_noquit (pty, O_RDWR, 0);
-
+          int ptyfd = emacs_open_noquit (pty_name, O_RDWR, 0);
+          if (pty_in)
+            std_in = ptyfd;
+          if (pty_out)
+            std_out = ptyfd;
           if (std_in < 0)
             {
-              emacs_perror (pty);
+              emacs_perror (pty_name);
               _exit (EXIT_CANCELED);
             }
 
@@ -1599,7 +1606,7 @@ emacs_spawn (pid_t *newpid, int std_in, int std_out, int std_err,
       /* Stop blocking SIGCHLD in the child.  */
       unblock_child_signal (oldset);
 
-      if (pty_flag)
+      if (pty_out)
         child_setup_tty (std_out);
 #endif
 
diff --git a/src/lisp.h b/src/lisp.h
index 8e36620fe53..fe6e98843d1 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -4943,7 +4943,8 @@ extern void setup_process_coding_systems (Lisp_Object);
 #endif
 
 extern int emacs_spawn (pid_t *, int, int, int, char **, char **,
-                        const char *, const char *, const sigset_t *);
+                        const char *, const char *, bool, bool,
+                        const sigset_t *);
 extern char **make_environment_block (Lisp_Object) ATTRIBUTE_RETURNS_NONNULL;
 extern void init_callproc_1 (void);
 extern void init_callproc (void);
diff --git a/src/process.c b/src/process.c
index 1ac5a509e56..68dbd8b68bd 100644
--- a/src/process.c
+++ b/src/process.c
@@ -1316,6 +1316,19 @@ set_process_filter_masks (struct Lisp_Process *p)
     add_process_read_fd (p->infd);
 }
 
+static bool
+is_pty_from_symbol (Lisp_Object symbol)
+{
+  if (EQ (symbol, Qpty))
+    return true;
+  else if (EQ (symbol, Qpipe))
+    return false;
+  else if (NILP (symbol))
+    return !NILP (Vprocess_connection_type);
+  else
+    report_file_error ("Unknown connection type", symbol);
+}
+
 DEFUN ("set-process-filter", Fset_process_filter, Sset_process_filter,
        2, 2, 0,
        doc: /* Give PROCESS the filter function FILTER; nil means default.
@@ -1741,15 +1754,18 @@ signals to stop and continue a process.
 :connection-type TYPE -- TYPE is control type of device used to
 communicate with subprocesses.  Values are `pipe' to use a pipe, `pty'
 to use a pty, or nil to use the default specified through
-`process-connection-type'.
+`process-connection-type'.  If TYPE is a cons (INPUT . OUTPUT), then
+INPUT will be used for standard input and OUTPUT for standard output
+(and standard error if `:stderr' is nil).
 
 :filter FILTER -- Install FILTER as the process filter.
 
 :sentinel SENTINEL -- Install SENTINEL as the process sentinel.
 
 :stderr STDERR -- STDERR is either a buffer or a pipe process attached
-to the standard error of subprocess.  Specifying this implies
-`:connection-type' is set to `pipe'.  If STDERR is nil, standard error
+to the standard error of subprocess.  When specifying this, the
+subprocess's standard error will always communicate via a pipe, no
+matter the value of `:connection-type'.  If STDERR is nil, standard error
 is mixed with standard output and sent to BUFFER or FILTER.  (Note
 that specifying :stderr will create a new, separate (but associated)
 process, with its own filter and sentinel.  See
@@ -1845,22 +1861,20 @@ usage: (make-process &rest ARGS)  */)
   CHECK_TYPE (NILP (tem), Qnull, tem);
 
   tem = plist_get (contact, QCconnection_type);
-  if (EQ (tem, Qpty))
-    XPROCESS (proc)->pty_flag = true;
-  else if (EQ (tem, Qpipe))
-    XPROCESS (proc)->pty_flag = false;
-  else if (NILP (tem))
-    XPROCESS (proc)->pty_flag = !NILP (Vprocess_connection_type);
+  if (CONSP (tem))
+    {
+      XPROCESS (proc)->pty_in = is_pty_from_symbol (XCAR (tem));
+      XPROCESS (proc)->pty_out = is_pty_from_symbol (XCDR (tem));
+    }
   else
-    report_file_error ("Unknown connection type", tem);
-
-  if (!NILP (stderrproc))
     {
-      pset_stderrproc (XPROCESS (proc), stderrproc);
-
-      XPROCESS (proc)->pty_flag = false;
+      XPROCESS (proc)->pty_in = XPROCESS (proc)->pty_out =
+        is_pty_from_symbol (tem);
     }
 
+  if (!NILP (stderrproc))
+    pset_stderrproc (XPROCESS (proc), stderrproc);
+
 #ifdef HAVE_GNUTLS
   /* AKA GNUTLS_INITSTAGE(proc).  */
   verify (GNUTLS_STAGE_EMPTY == 0);
@@ -2099,66 +2113,80 @@ static void
 create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
 {
   struct Lisp_Process *p = XPROCESS (process);
-  int inchannel, outchannel;
+  int inchannel = -1, outchannel = -1;
   pid_t pid = -1;
   int vfork_errno;
   int forkin, forkout, forkerr = -1;
-  bool pty_flag = 0;
+  bool pty_in = false, pty_out = false;
   char pty_name[PTY_NAME_SIZE];
   Lisp_Object lisp_pty_name = Qnil;
+  int ptychannel = -1, pty_tty = -1;
   sigset_t oldset;
 
   /* Ensure that the SIGCHLD handler can notify
      `wait_reading_process_output'.  */
   child_signal_init ();
 
-  inchannel = outchannel = -1;
-
-  if (p->pty_flag)
-    outchannel = inchannel = allocate_pty (pty_name);
+  if (p->pty_in || p->pty_out)
+    ptychannel = allocate_pty (pty_name);
 
-  if (inchannel >= 0)
+  if (ptychannel >= 0)
     {
-      p->open_fd[READ_FROM_SUBPROCESS] = inchannel;
 #if ! defined (USG) || defined (USG_SUBTTY_WORKS)
       /* On most USG systems it does not work to open the pty's tty here,
          then close it and reopen it in the child.  */
       /* Don't let this terminal become our controlling terminal
          (in case we don't have one).  */
-      forkout = forkin = emacs_open (pty_name, O_RDWR | O_NOCTTY, 0);
-      if (forkin < 0)
+      pty_tty = emacs_open (pty_name, O_RDWR | O_NOCTTY, 0);
+      if (pty_tty < 0)
         report_file_error ("Opening pty", Qnil);
-      p->open_fd[SUBPROCESS_STDIN] = forkin;
-#else
-      forkin = forkout = -1;
 #endif /* not USG, or USG_SUBTTY_WORKS */
-      pty_flag = 1;
+      pty_in = p->pty_in;
+      pty_out = p->pty_out;
       lisp_pty_name = build_string (pty_name);
     }
+
+  /* Set up stdin for the child process.  */
+  if (ptychannel >= 0 && p->pty_in)
+    {
+      p->open_fd[SUBPROCESS_STDIN] = forkin = pty_tty;
+      outchannel = ptychannel;
+    }
   else
     {
-      if (emacs_pipe (p->open_fd + SUBPROCESS_STDIN) != 0
-          || emacs_pipe (p->open_fd + READ_FROM_SUBPROCESS) != 0)
+      if (emacs_pipe (p->open_fd + SUBPROCESS_STDIN) != 0)
         report_file_error ("Creating pipe", Qnil);
       forkin = p->open_fd[SUBPROCESS_STDIN];
       outchannel = p->open_fd[WRITE_TO_SUBPROCESS];
+    }
+
+  /* Set up stdout for the child process.  */
+  if (ptychannel >= 0 && p->pty_out)
+    {
+      forkout = pty_tty;
+      p->open_fd[READ_FROM_SUBPROCESS] = inchannel = ptychannel;
+    }
+  else
+    {
+      if (emacs_pipe (p->open_fd + READ_FROM_SUBPROCESS) != 0)
+        report_file_error ("Creating pipe", Qnil);
       inchannel = p->open_fd[READ_FROM_SUBPROCESS];
       forkout = p->open_fd[SUBPROCESS_STDOUT];
 
 #if defined(GNU_LINUX) && defined(F_SETPIPE_SZ)
       fcntl (inchannel, F_SETPIPE_SZ, read_process_output_max);
 #endif
+    }
 
-      if (!NILP (p->stderrproc))
-        {
-          struct Lisp_Process *pp = XPROCESS (p->stderrproc);
+  if (!NILP (p->stderrproc))
+    {
+      struct Lisp_Process *pp = XPROCESS (p->stderrproc);
 
-          forkerr = pp->open_fd[SUBPROCESS_STDOUT];
+      forkerr = pp->open_fd[SUBPROCESS_STDOUT];
 
-          /* Close unnecessary file descriptors.  */
-          close_process_fd (&pp->open_fd[WRITE_TO_SUBPROCESS]);
-          close_process_fd (&pp->open_fd[SUBPROCESS_STDIN]);
-        }
+      /* Close unnecessary file descriptors.  */
+      close_process_fd (&pp->open_fd[WRITE_TO_SUBPROCESS]);
+      close_process_fd (&pp->open_fd[SUBPROCESS_STDIN]);
     }
 
   if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel)
@@ -2183,7 +2211,8 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
      we just reopen the device (see emacs_get_tty_pgrp) as this is
      more portable (see USG_SUBTTY_WORKS above).  */
 
-  p->pty_flag = pty_flag;
+  p->pty_in = pty_in;
+  p->pty_out = pty_out;
   pset_status (p, Qrun);
 
   if (!EQ (p->command, Qt)
@@ -2199,13 +2228,15 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
   block_input ();
   block_child_signal (&oldset);
 
-  pty_flag = p->pty_flag;
-  eassert (pty_flag == ! NILP (lisp_pty_name));
+  pty_in = p->pty_in;
+  pty_out = p->pty_out;
+  eassert ((pty_in || pty_out) == ! NILP (lisp_pty_name));
 
   vfork_errno
     = emacs_spawn (&pid, forkin, forkout, forkerr, new_argv, env,
                    SSDATA (current_dir),
-                   pty_flag ? SSDATA (lisp_pty_name) : NULL, &oldset);
+                   pty_in || pty_out ? SSDATA (lisp_pty_name) : NULL,
+                   pty_in, pty_out, &oldset);
 
   eassert ((vfork_errno == 0) == (0 < pid));
 
@@ -2263,7 +2294,7 @@ create_pty (Lisp_Object process)
 {
   struct Lisp_Process *p = XPROCESS (process);
   char pty_name[PTY_NAME_SIZE];
-  int pty_fd = !p->pty_flag ? -1 : allocate_pty (pty_name);
+  int pty_fd = !(p->pty_in || p->pty_out) ? -1 : allocate_pty (pty_name);
 
   if (pty_fd >= 0)
     {
@@ -2301,7 +2332,7 @@ create_pty (Lisp_Object process)
          we just reopen the device (see emacs_get_tty_pgrp) as this is
          more portable (see USG_SUBTTY_WORKS above).  */
 
-      p->pty_flag = 1;
+      p->pty_in = p->pty_out = true;
       pset_status (p, Qrun);
       setup_process_coding_systems (process);
 
@@ -2412,7 +2443,7 @@ usage:  (make-pipe-process &rest ARGS)  */)
     p->kill_without_query = 1;
   if (tem = plist_get (contact, QCstop), !NILP (tem))
     pset_command (p, Qt);
-  eassert (! p->pty_flag);
+  eassert (! p->pty_in && ! p->pty_out);
 
   if (!EQ (p->command, Qt)
       && !EQ (p->filter, Qt))
@@ -3147,7 +3178,7 @@ usage:  (make-serial-process &rest ARGS)  */)
     p->kill_without_query = 1;
   if (tem = plist_get (contact, QCstop), !NILP (tem))
     pset_command (p, Qt);
-  eassert (! p->pty_flag);
+  eassert (! p->pty_in && ! p->pty_out);
 
   if (!EQ (p->command, Qt)
       && !EQ (p->filter, Qt))
@@ -6808,7 +6839,7 @@ process_send_signal (Lisp_Object process, int signo, Lisp_Object current_group,
     error ("Process %s is not active",
            SDATA (p->name));
 
-  if (!p->pty_flag)
+  if (! p->pty_in)
     current_group = Qnil;
 
   /* If we are using pgrps, get a pgrp number and make it negative.  */
@@ -7177,7 +7208,7 @@ process has been transmitted to the serial port.  */)
       send_process (proc, "", 0, Qnil);
     }
 
-  if (XPROCESS (proc)->pty_flag)
+  if (XPROCESS (proc)->pty_in)
     send_process (proc, "\004", 1, Qnil);
   else if (EQ (XPROCESS (proc)->type, Qserial))
     {
diff --git a/src/process.h b/src/process.h
index 392b661ce69..92baf0c4cb9 100644
--- a/src/process.h
+++ b/src/process.h
@@ -156,8 +156,9 @@ struct Lisp_Process
     /* True means kill silently if Emacs is exited.
        This is the inverse of the `query-on-exit' flag.  */
     bool_bf kill_without_query : 1;
-    /* True if communicating through a pty.  */
-    bool_bf pty_flag : 1;
+    /* True if communicating through a pty for input or output.  */
+    bool_bf pty_in : 1;
+    bool_bf pty_out : 1;
     /* Flag to set coding-system of the process buffer from the
        coding_system used to decode process output.  */
     bool_bf inherit_coding_system_flag : 1;