Limit RLIMIT_NOFILE to FD_SETSIZE on macOS

* src/nsterm.m ([EmacsApp applicationDidFinishLaunching:]): Call
CoreFoundation functions that increase RLIMIT_NOFILE behind our back
during startup, and then set RLIMIT_NOFILE back to FD_SETSIZE to avoid
crashes in setup_process_coding_system  (Bug#39164).

1 files changed, 15 insertions, 0 deletions

diff --git a/src/nsterm.m b/src/nsterm.m
index e92e3d5a6f5..ac467840a25 100644
--- a/src/nsterm.m
+++ b/src/nsterm.m
@@ -5812,6 +5812,21 @@ ns_term_shutdown (int sig)
 #endif
 
 #ifdef NS_IMPL_COCOA
+  /* Some functions/methods in CoreFoundation/Foundation increase the
+     maximum number of open files for the process in their first call.
+     We make dummy calls to them and then reduce the resource limit
+     here, since pselect cannot handle file descriptors that are
+     greater than or equal to FD_SETSIZE.  */
+  CFSocketGetTypeID ();
+  CFFileDescriptorGetTypeID ();
+  [[NSFileHandle alloc] init];
+  struct rlimit rlim;
+  if (getrlimit (RLIMIT_NOFILE, &rlim) == 0
+      && rlim.rlim_cur > FD_SETSIZE)
+    {
+      rlim.rlim_cur = FD_SETSIZE;
+      setrlimit (RLIMIT_NOFILE, &rlim);
+    }
   if ([NSApp activationPolicy] == NSApplicationActivationPolicyProhibited) {
     /* Set the app's activation policy to regular when we run outside
        of a bundle.  This is already done for us by Info.plist when we