Fix "not a tty" bug on Solaris 10

* configure.ac (PTY_OPEN): Define to plain 'open'
on SVR4-derived hosts, so that the O_CLOEXEC flag isn't set.
* src/process.c (allocate_pty): Set the O_CLOEXEC flag after
calling PTY_TTY_NAME_SPRINTF, for the benefit of SVR4-derived
hosts that call grantpt which does its work via a setuid subcommand
(Bug#19191, Bug#19927, Bug#20555, Bug#20686).
Also, set O_CLOEXEC even if PTY_OPEN is not defined, since it
seems relevant in that case too.

2 files changed, 15 insertions, 10 deletions

diff --git a/configure.ac b/configure.ac
index 9c6a74aface..070b0612471 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4397,14 +4397,17 @@ case $opsys in
     ;;
 
   sol2* )
-    dnl On SysVr4, grantpt(3) forks a subprocess, so keep sigchld_handler()
+    dnl On SysVr4, grantpt(3) forks a subprocess, so do not use
+    dnl O_CLOEXEC when opening the pty, and keep the SIGCHLD handler
     dnl from intercepting that death.  If any child but grantpt's should die
     dnl within, it should be caught after sigrelse(2).
+    AC_DEFINE(PTY_OPEN, [fd = open (pty_name, O_RDWR | O_NONBLOCK)])
     AC_DEFINE(PTY_TTY_NAME_SPRINTF, [{ char *ptsname (int), *ptyname; int grantpt_result; sigset_t blocked; sigemptyset (&blocked); sigaddset (&blocked, SIGCHLD); pthread_sigmask (SIG_BLOCK, &blocked, 0); grantpt_result = grantpt (fd); pthread_sigmask (SIG_UNBLOCK, &blocked, 0); if (grantpt_result == -1 || unlockpt (fd) == -1 || !(ptyname = ptsname (fd))) { emacs_close (fd); return -1; } snprintf (pty_name, PTY_NAME_SIZE, "%s", ptyname); }])
     ;;
 
   unixware )
     dnl Comments are as per sol2*.
+    AC_DEFINE(PTY_OPEN, [fd = open (pty_name, O_RDWR | O_NONBLOCK)])
     AC_DEFINE(PTY_TTY_NAME_SPRINTF, [{ char *ptsname (int), *ptyname; int grantpt_result; sigset_t blocked; sigemptyset (&blocked); sigaddset (&blocked, SIGCHLD); pthread_sigmask (SIG_BLOCK, &blocked, 0); grantpt_result = grantpt (fd); pthread_sigmask (SIG_UNBLOCK, &blocked, 0); if (grantpt_result == -1) fatal("could not grant slave pty"); if (unlockpt(fd) == -1) fatal("could not unlock slave pty"); if (!(ptyname = ptsname(fd))) fatal ("could not enable slave pty"); snprintf (pty_name, PTY_NAME_SIZE, "%s", ptyname); }])
     ;;
 esac
diff --git a/src/process.c b/src/process.c
index 17fe708a954..b4f979fd484 100644
--- a/src/process.c
+++ b/src/process.c
@@ -658,22 +658,24 @@ allocate_pty (char pty_name[PTY_NAME_SIZE])
 
         if (fd >= 0)
           {
-#ifdef PTY_OPEN
+#ifdef PTY_TTY_NAME_SPRINTF
+            PTY_TTY_NAME_SPRINTF
+#else
+            sprintf (pty_name, "/dev/tty%c%x", c, i);
+#endif /* no PTY_TTY_NAME_SPRINTF */
+
             /* Set FD's close-on-exec flag.  This is needed even if
                PT_OPEN calls posix_openpt with O_CLOEXEC, since POSIX
                doesn't require support for that combination.
+               Do this after PTY_TTY_NAME_SPRINTF, which on some platforms
+               doesn't work if the close-on-exec flag is set (Bug#20555).
                Multithreaded platforms where posix_openpt ignores
                O_CLOEXEC (or where PTY_OPEN doesn't call posix_openpt)
                have a race condition between the PTY_OPEN and here.  */
             fcntl (fd, F_SETFD, FD_CLOEXEC);
-#endif
-            /* Check to make certain that both sides are available
-               this avoids a nasty yet stupid bug in rlogins.  */
-#ifdef PTY_TTY_NAME_SPRINTF
-            PTY_TTY_NAME_SPRINTF
-#else
-            sprintf (pty_name, "/dev/tty%c%x", c, i);
-#endif /* no PTY_TTY_NAME_SPRINTF */
+
+            /* Check to make certain that both sides are available.
+               This avoids a nasty yet stupid bug in rlogins.  */
             if (faccessat (AT_FDCWD, pty_name, R_OK | W_OK, AT_EACCESS) != 0)
               {
                 emacs_close (fd);