Backport fcntl.h AIX fix from the trunk.

This fixes a bug with the shell freezing.  See:
http://debbugs.gnu.org/cgi/bugreport.cgi?bug=17598#185
Merge from gnulib, incorporating:
2014-05-31 dup2, fcntl, fcntl-h: port to AIX 7.1
* lib/fcntl.in.h, m4/dup2.m4, m4/fcntl.m4:
Update from gnulib.

4 files changed, 47 insertions, 3 deletions

diff --git a/ChangeLog b/ChangeLog
index 77e54096116..d13c8aeabe9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2014-06-11  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Backport fcntl.h AIX fix from the trunk (Bug#17598).
+        This fixes a bug with the shell freezing.  See:
+        http://debbugs.gnu.org/cgi/bugreport.cgi?bug=17598#185
+        Merge from gnulib, incorporating:
+        2014-05-31 dup2, fcntl, fcntl-h: port to AIX 7.1
+        * lib/fcntl.in.h, m4/dup2.m4, m4/fcntl.m4:
+        Update from gnulib.
+
 2014-06-07  Paul Eggert  <eggert@cs.ucla.edu>
 
         Port better to AIX (Bug#17598).
diff --git a/lib/fcntl.in.h b/lib/fcntl.in.h
index 99f75e60fd4..e23b4b2bcdd 100644
--- a/lib/fcntl.in.h
+++ b/lib/fcntl.in.h
@@ -186,6 +186,22 @@ _GL_WARN_ON_USE (openat, "openat is not portable - "
 
 /* Fix up the O_* macros.  */
 
+/* AIX 7.1 with XL C 12.1 defines O_CLOEXEC, O_NOFOLLOW, and O_TTY_INIT
+   to values outside 'int' range, so omit these misdefinitions.
+   But avoid namespace pollution on non-AIX systems.  */
+#ifdef _AIX
+# include <limits.h>
+# if defined O_CLOEXEC && ! (INT_MIN <= O_CLOEXEC && O_CLOEXEC <= INT_MAX)
+#  undef O_CLOEXEC
+# endif
+# if defined O_NOFOLLOW && ! (INT_MIN <= O_NOFOLLOW && O_NOFOLLOW <= INT_MAX)
+#  undef O_NOFOLLOW
+# endif
+# if defined O_TTY_INIT && ! (INT_MIN <= O_TTY_INIT && O_TTY_INIT <= INT_MAX)
+#  undef O_TTY_INIT
+# endif
+#endif
+
 #if !defined O_DIRECT && defined O_DIRECTIO
 /* Tru64 spells it 'O_DIRECTIO'.  */
 # define O_DIRECT O_DIRECTIO
diff --git a/m4/dup2.m4 b/m4/dup2.m4
index 89638a0bfe3..6498fc20245 100644
--- a/m4/dup2.m4
+++ b/m4/dup2.m4
@@ -8,6 +8,7 @@ AC_DEFUN([gl_FUNC_DUP2],
 [
   AC_REQUIRE([gl_UNISTD_H_DEFAULTS])
   AC_REQUIRE([AC_CANONICAL_HOST])
+  AC_CHECK_FUNCS_ONCE([getdtablesize])
   m4_ifdef([gl_FUNC_DUP2_OBSOLETE], [
     AC_CHECK_FUNCS_ONCE([dup2])
     if test $ac_cv_func_dup2 = no; then
@@ -23,6 +24,11 @@ AC_DEFUN([gl_FUNC_DUP2],
 #include <fcntl.h>
 #include <errno.h>]],
            [int result = 0;
+#ifdef HAVE_GETDTABLESIZE
+            int bad_fd = getdtablesize ();
+#else
+            int bad_fd = 1000000;
+#endif
 #ifdef FD_CLOEXEC
             if (fcntl (1, F_SETFD, FD_CLOEXEC) == -1)
               result |= 1;
@@ -37,7 +43,7 @@ AC_DEFUN([gl_FUNC_DUP2],
             if (dup2 (0, 0) != -1)
               result |= 8;
             /* Many gnulib modules require POSIX conformance of EBADF.  */
-            if (dup2 (2, 1000000) == -1 && errno != EBADF)
+            if (dup2 (2, bad_fd) == -1 && errno != EBADF)
               result |= 16;
             /* Flush out some cygwin core dumps.  */
             if (dup2 (2, -1) != -1 || errno != EBADF)
@@ -56,7 +62,9 @@ AC_DEFUN([gl_FUNC_DUP2],
            linux*) # On linux between 2008-07-27 and 2009-05-11, dup2 of a
                    # closed fd may yield -EBADF instead of -1 / errno=EBADF.
              gl_cv_func_dup2_works="guessing no" ;;
-           freebsd*) # on FreeBSD 6.1, dup2(1,1000000) gives EMFILE, not EBADF.
+           aix* | freebsd*)
+                   # on AIX 7.1 and FreeBSD 6.1, dup2 (1,toobig) gives EMFILE,
+                   # not EBADF.
              gl_cv_func_dup2_works="guessing no" ;;
            haiku*) # on Haiku alpha 2, dup2(1, 1) resets FD_CLOEXEC.
              gl_cv_func_dup2_works="guessing no" ;;
diff --git a/m4/fcntl.m4 b/m4/fcntl.m4
index 4a2771fae35..f824beb6597 100644
--- a/m4/fcntl.m4
+++ b/m4/fcntl.m4
@@ -19,7 +19,7 @@ AC_DEFUN([gl_FUNC_FCNTL],
   AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
   AC_REQUIRE([gl_FCNTL_H_DEFAULTS])
   AC_REQUIRE([AC_CANONICAL_HOST])
-  AC_CHECK_FUNCS_ONCE([fcntl])
+  AC_CHECK_FUNCS_ONCE([fcntl getdtablesize])
   if test $ac_cv_func_fcntl = no; then
     gl_REPLACE_FCNTL
   else
@@ -28,11 +28,21 @@ AC_DEFUN([gl_FUNC_FCNTL],
     AC_CACHE_CHECK([whether fcntl handles F_DUPFD correctly],
       [gl_cv_func_fcntl_f_dupfd_works],
       [AC_RUN_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_GETDTABLESIZE
+# include <unistd.h>
+#endif
 #include <fcntl.h>
 #include <errno.h>
 ]], [[int result = 0;
+#ifdef HAVE_GETDTABLESIZE
+      int bad_fd = getdtablesize ();
+#else
+      int bad_fd = 1000000;
+#endif
       if (fcntl (0, F_DUPFD, -1) != -1) result |= 1;
       if (errno != EINVAL) result |= 2;
+      if (fcntl (0, F_DUPFD, bad_fd) != -1) result |= 4;
+      if (errno != EINVAL) result |= 8;
       return result;
          ]])],
          [gl_cv_func_fcntl_f_dupfd_works=yes],