Restore file descriptor limit in subprocesses

Problem reported by Philipp Stephani (Bug#24869).
* src/callproc.c (child_setup) [!DOS_NT]:
Call restore_nofile_limit in the child.
* src/process.c (nofile_limit) [HAVE_SETRLIMIT]: New static var.
(restore_nofile_limit): New function.
(init_process_emacs) [HAVE_SETRLIMIT]: Set the new var.

3 files changed, 27 insertions, 4 deletions

diff --git a/src/callproc.c b/src/callproc.c
index 8ed28556e0d..dc3ca4ac102 100644
--- a/src/callproc.c
+++ b/src/callproc.c
@@ -1315,6 +1315,9 @@ child_setup (int in, int out, int err, char **new_argv, bool set_pgrp,
 #else  /* not WINDOWSNT */
 
 #ifndef MSDOS
+
+  restore_nofile_limit ();
+
   /* Redirect file descriptors and clear the close-on-exec flag on the
      redirected ones.  IN, OUT, and ERR are close-on-exec so they
      need not be closed explicitly.  */
diff --git a/src/process.c b/src/process.c
index d27b57d560f..d68c930dd6f 100644
--- a/src/process.c
+++ b/src/process.c
@@ -42,6 +42,11 @@ along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.  */
 
 #ifdef HAVE_SETRLIMIT
 # include <sys/resource.h>
+
+/* If NOFILE_LIMIT.rlim_cur is greater than FD_SETSIZE, then
+   NOFILE_LIMIT is the initial limit on the number of open files,
+   which should be restored in child processes.  */
+static struct rlimit nofile_limit;
 #endif
 
 /* Are local (unix) sockets supported?  */
@@ -7770,6 +7775,17 @@ catch_child_signal (void)
 }
 #endif  /* subprocesses */
 
+/* Limit the number of open files to the value it had at startup.  */
+
+void
+restore_nofile_limit (void)
+{
+#ifdef HAVE_SETRLIMIT
+  if (FD_SETSIZE < nofile_limit.rlim_cur)
+    setrlimit (RLIMIT_NOFILE, &nofile_limit);
+#endif
+}
+
 
 /* This is not called "init_process" because that is the name of a
    Mach system call, so it would cause problems on Darwin systems.  */
@@ -7796,12 +7812,15 @@ init_process_emacs (int sockfd)
     }
 
 #ifdef HAVE_SETRLIMIT
-  /* Don't allocate more than FD_SETSIZE file descriptors.  */
-  struct rlimit rlim;
-  if (getrlimit (RLIMIT_NOFILE, &rlim) == 0 && FD_SETSIZE < rlim.rlim_cur)
+  /* Don't allocate more than FD_SETSIZE file descriptors for Emacs itself.  */
+  if (getrlimit (RLIMIT_NOFILE, &nofile_limit) != 0)
+    nofile_limit.rlim_cur = 0;
+  else if (FD_SETSIZE < nofile_limit.rlim_cur)
     {
+      struct rlimit rlim = nofile_limit;
       rlim.rlim_cur = FD_SETSIZE;
-      setrlimit (RLIMIT_NOFILE, &rlim);
+      if (setrlimit (RLIMIT_NOFILE, &rlim) != 0)
+        nofile_limit.rlim_cur = 0;
     }
 #endif
 
diff --git a/src/process.h b/src/process.h
index 9926050b9c3..24c628231a0 100644
--- a/src/process.h
+++ b/src/process.h
@@ -265,6 +265,7 @@ extern void delete_read_fd (int fd);
 extern void add_write_fd (int fd, fd_callback func, void *data);
 extern void delete_write_fd (int fd);
 extern void catch_child_signal (void);
+extern void restore_nofile_limit (void);
 
 #ifdef WINDOWSNT
 extern Lisp_Object network_interface_list (void);