Backport fcntl.h AIX fix from the trunk.

This fixes a bug with the shell freezing.  See:
http://debbugs.gnu.org/cgi/bugreport.cgi?bug=17598#185
Merge from gnulib, incorporating:
2014-05-31 dup2, fcntl, fcntl-h: port to AIX 7.1
* lib/fcntl.in.h, m4/dup2.m4, m4/fcntl.m4:
Update from gnulib.

2 files changed, 21 insertions, 3 deletions

diff --git a/m4/dup2.m4 b/m4/dup2.m4
index 89638a0bfe3..6498fc20245 100644
--- a/m4/dup2.m4
+++ b/m4/dup2.m4
@@ -8,6 +8,7 @@ AC_DEFUN([gl_FUNC_DUP2],
 [
   AC_REQUIRE([gl_UNISTD_H_DEFAULTS])
   AC_REQUIRE([AC_CANONICAL_HOST])
+  AC_CHECK_FUNCS_ONCE([getdtablesize])
   m4_ifdef([gl_FUNC_DUP2_OBSOLETE], [
     AC_CHECK_FUNCS_ONCE([dup2])
     if test $ac_cv_func_dup2 = no; then
@@ -23,6 +24,11 @@ AC_DEFUN([gl_FUNC_DUP2],
 #include <fcntl.h>
 #include <errno.h>]],
            [int result = 0;
+#ifdef HAVE_GETDTABLESIZE
+            int bad_fd = getdtablesize ();
+#else
+            int bad_fd = 1000000;
+#endif
 #ifdef FD_CLOEXEC
             if (fcntl (1, F_SETFD, FD_CLOEXEC) == -1)
               result |= 1;
@@ -37,7 +43,7 @@ AC_DEFUN([gl_FUNC_DUP2],
             if (dup2 (0, 0) != -1)
               result |= 8;
             /* Many gnulib modules require POSIX conformance of EBADF.  */
-            if (dup2 (2, 1000000) == -1 && errno != EBADF)
+            if (dup2 (2, bad_fd) == -1 && errno != EBADF)
               result |= 16;
             /* Flush out some cygwin core dumps.  */
             if (dup2 (2, -1) != -1 || errno != EBADF)
@@ -56,7 +62,9 @@ AC_DEFUN([gl_FUNC_DUP2],
            linux*) # On linux between 2008-07-27 and 2009-05-11, dup2 of a
                    # closed fd may yield -EBADF instead of -1 / errno=EBADF.
              gl_cv_func_dup2_works="guessing no" ;;
-           freebsd*) # on FreeBSD 6.1, dup2(1,1000000) gives EMFILE, not EBADF.
+           aix* | freebsd*)
+                   # on AIX 7.1 and FreeBSD 6.1, dup2 (1,toobig) gives EMFILE,
+                   # not EBADF.
              gl_cv_func_dup2_works="guessing no" ;;
            haiku*) # on Haiku alpha 2, dup2(1, 1) resets FD_CLOEXEC.
              gl_cv_func_dup2_works="guessing no" ;;
diff --git a/m4/fcntl.m4 b/m4/fcntl.m4
index 4a2771fae35..f824beb6597 100644
--- a/m4/fcntl.m4
+++ b/m4/fcntl.m4
@@ -19,7 +19,7 @@ AC_DEFUN([gl_FUNC_FCNTL],
   AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
   AC_REQUIRE([gl_FCNTL_H_DEFAULTS])
   AC_REQUIRE([AC_CANONICAL_HOST])
-  AC_CHECK_FUNCS_ONCE([fcntl])
+  AC_CHECK_FUNCS_ONCE([fcntl getdtablesize])
   if test $ac_cv_func_fcntl = no; then
     gl_REPLACE_FCNTL
   else
@@ -28,11 +28,21 @@ AC_DEFUN([gl_FUNC_FCNTL],
     AC_CACHE_CHECK([whether fcntl handles F_DUPFD correctly],
       [gl_cv_func_fcntl_f_dupfd_works],
       [AC_RUN_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_GETDTABLESIZE
+# include <unistd.h>
+#endif
 #include <fcntl.h>
 #include <errno.h>
 ]], [[int result = 0;
+#ifdef HAVE_GETDTABLESIZE
+      int bad_fd = getdtablesize ();
+#else
+      int bad_fd = 1000000;
+#endif
       if (fcntl (0, F_DUPFD, -1) != -1) result |= 1;
       if (errno != EINVAL) result |= 2;
+      if (fcntl (0, F_DUPFD, bad_fd) != -1) result |= 4;
+      if (errno != EINVAL) result |= 8;
       return result;
          ]])],
          [gl_cv_func_fcntl_f_dupfd_works=yes],