Manually limit file descriptors that we select on to FD_SETSIZE.

This works even if another thread or process resets the resource limit
for open file descriptors, e.g., using 'prlimit' on GNU/Linux.

* src/process.c (create_process, create_pty, Fmake_pipe_process)
(Fmake_serial_process, connect_network_socket)
(server_accept_connection): Limit file descriptors to FD_SETSIZE.
* test/src/process-tests.el (process-tests--with-raised-rlimit): New
helper macro.
(process-tests--fd-setsize-test): Rename from
'process-tests--with-many-pipes'.  Increase resource limit during test
if possible.
(process-tests/fd-setsize-no-crash/make-process)
(process-tests/fd-setsize-no-crash/make-pipe-process)
(process-tests/fd-setsize-no-crash/make-network-process)
(process-tests--new-pty): Rename callers.

1 files changed, 25 insertions, 0 deletions

diff --git a/src/process.c b/src/process.c
index 7c56366d8e1..ba2bb3c9e46 100644
--- a/src/process.c
+++ b/src/process.c
@@ -2114,6 +2114,9 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
         }
     }
 
+  if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel)
+    report_file_errno ("Creating pipe", Qnil, EMFILE);
+
 #ifndef WINDOWSNT
   if (emacs_pipe (p->open_fd + READ_FROM_EXEC_MONITOR) != 0)
     report_file_error ("Creating pipe", Qnil);
@@ -2210,6 +2213,8 @@ create_pty (Lisp_Object process)
   if (pty_fd >= 0)
     {
       p->open_fd[SUBPROCESS_STDIN] = pty_fd;
+      if (FD_SETSIZE <= pty_fd)
+        report_file_errno ("Opening pty", Qnil, EMFILE);
 #if ! defined (USG) || defined (USG_SUBTTY_WORKS)
       /* On most USG systems it does not work to open the pty's tty here,
          then close it and reopen it in the child.  */
@@ -2317,6 +2322,9 @@ usage:  (make-pipe-process &rest ARGS)  */)
   outchannel = p->open_fd[WRITE_TO_SUBPROCESS];
   inchannel = p->open_fd[READ_FROM_SUBPROCESS];
 
+  if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel)
+    report_file_errno ("Creating pipe", Qnil, EMFILE);
+
   fcntl (inchannel, F_SETFL, O_NONBLOCK);
   fcntl (outchannel, F_SETFL, O_NONBLOCK);
 
@@ -3059,6 +3067,8 @@ usage:  (make-serial-process &rest ARGS)  */)
 
   fd = serial_open (port);
   p->open_fd[SUBPROCESS_STDIN] = fd;
+  if (FD_SETSIZE <= fd)
+    report_file_errno ("Opening serial port", Qnil, EMFILE);
   p->infd = fd;
   p->outfd = fd;
   if (fd > max_desc)
@@ -3280,6 +3290,7 @@ connect_network_socket (Lisp_Object proc, Lisp_Object addrinfos,
   if (!NILP (use_external_socket_p))
     {
       socket_to_use = external_sock_fd;
+      eassert (socket_to_use < FD_SETSIZE);
 
       /* Ensure we don't consume the external socket twice.  */
       external_sock_fd = -1;
@@ -3321,6 +3332,13 @@ connect_network_socket (Lisp_Object proc, Lisp_Object addrinfos,
               xerrno = errno;
               continue;
             }
+          if (FD_SETSIZE <= s)
+            {
+              emacs_close (s);
+              s = -1;
+              xerrno = EMFILE;
+              continue;
+            }
         }
 
       if (p->is_non_blocking_client && ! (SOCK_NONBLOCK && socket_to_use < 0))
@@ -4782,6 +4800,13 @@ server_accept_connection (Lisp_Object server, int channel)
 
   s = accept4 (channel, &saddr.sa, &len, SOCK_CLOEXEC);
 
+  if (FD_SETSIZE <= s)
+    {
+      emacs_close (s);
+      s = -1;
+      errno = EMFILE;
+    }
+
   if (s < 0)
     {
       int code = errno;