Fix "not a tty" bug on Solaris 10

* configure.ac (PTY_OPEN): Define to plain 'open'
on SVR4-derived hosts, so that the O_CLOEXEC flag isn't set.
* src/process.c (allocate_pty): Set the O_CLOEXEC flag after
calling PTY_TTY_NAME_SPRINTF, for the benefit of SVR4-derived
hosts that call grantpt which does its work via a setuid subcommand
(Bug#19191, Bug#19927, Bug#20555, Bug#20686).
Also, set O_CLOEXEC even if PTY_OPEN is not defined, since it
seems relevant in that case too.

1 files changed, 11 insertions, 9 deletions

diff --git a/src/process.c b/src/process.c
index 17fe708a954..b4f979fd484 100644
--- a/src/process.c
+++ b/src/process.c
@@ -658,22 +658,24 @@ allocate_pty (char pty_name[PTY_NAME_SIZE])
 
         if (fd >= 0)
           {
-#ifdef PTY_OPEN
+#ifdef PTY_TTY_NAME_SPRINTF
+            PTY_TTY_NAME_SPRINTF
+#else
+            sprintf (pty_name, "/dev/tty%c%x", c, i);
+#endif /* no PTY_TTY_NAME_SPRINTF */
+
             /* Set FD's close-on-exec flag.  This is needed even if
                PT_OPEN calls posix_openpt with O_CLOEXEC, since POSIX
                doesn't require support for that combination.
+               Do this after PTY_TTY_NAME_SPRINTF, which on some platforms
+               doesn't work if the close-on-exec flag is set (Bug#20555).
                Multithreaded platforms where posix_openpt ignores
                O_CLOEXEC (or where PTY_OPEN doesn't call posix_openpt)
                have a race condition between the PTY_OPEN and here.  */
             fcntl (fd, F_SETFD, FD_CLOEXEC);
-#endif
-            /* Check to make certain that both sides are available
-               this avoids a nasty yet stupid bug in rlogins.  */
-#ifdef PTY_TTY_NAME_SPRINTF
-            PTY_TTY_NAME_SPRINTF
-#else
-            sprintf (pty_name, "/dev/tty%c%x", c, i);
-#endif /* no PTY_TTY_NAME_SPRINTF */
+
+            /* Check to make certain that both sides are available.
+               This avoids a nasty yet stupid bug in rlogins.  */
             if (faccessat (AT_FDCWD, pty_name, R_OK | W_OK, AT_EACCESS) != 0)
               {
                 emacs_close (fd);