Fix bignum comparisons with NaN

* src/data.c (isnan): Move earlier.
(bignumcompare): Explicitly handle NaN.
* test/src/data-tests.el (data-tests-min): Add NaN tests
for bignum.
(data-check-sign): Fix for previous patch.
* test/src/fns-tests.el (test-bignum-eql): Add NaN test.

1 files changed, 18 insertions, 6 deletions

diff --git a/src/data.c b/src/data.c
index 3d55d9d17d5..4388a2b0ffc 100644
--- a/src/data.c
+++ b/src/data.c
@@ -2397,6 +2397,10 @@ bool-vector.  IDX starts at 0.  */)
 
 /* Arithmetic functions */
 
+#ifndef isnan
+# define isnan(x) ((x) != (x))
+#endif
+
 static Lisp_Object
 bignumcompare (Lisp_Object num1, Lisp_Object num2,
                enum Arith_Comparison comparison)
@@ -2407,7 +2411,13 @@ bignumcompare (Lisp_Object num1, Lisp_Object num2,
   if (BIGNUMP (num1))
     {
       if (FLOATP (num2))
-        cmp = mpz_cmp_d (XBIGNUM (num1)->value, XFLOAT_DATA (num2));
+        {
+          /* Note that GMP doesn't define comparisons against NaN, so
+             we need to handle them specially.  */
+          if (isnan (XFLOAT_DATA (num2)))
+            return Qnil;
+          cmp = mpz_cmp_d (XBIGNUM (num1)->value, XFLOAT_DATA (num2));
+        }
       else if (FIXNUMP (num2))
         {
           if (sizeof (EMACS_INT) > sizeof (long) && XINT (num2) > LONG_MAX)
@@ -2431,7 +2441,13 @@ bignumcompare (Lisp_Object num1, Lisp_Object num2,
     {
       eassume (BIGNUMP (num2));
       if (FLOATP (num1))
-        cmp = - mpz_cmp_d (XBIGNUM (num2)->value, XFLOAT_DATA (num1));
+        {
+          /* Note that GMP doesn't define comparisons against NaN, so
+             we need to handle them specially.  */
+          if (isnan (XFLOAT_DATA (num1)))
+            return Qnil;
+          cmp = - mpz_cmp_d (XBIGNUM (num2)->value, XFLOAT_DATA (num1));
+        }
       else
         {
           eassume (FIXNUMP (num1));
@@ -3021,10 +3037,6 @@ arith_driver (enum arithop code, ptrdiff_t nargs, Lisp_Object *args)
   return unbind_to (count, make_number (accum));
 }
 
-#ifndef isnan
-# define isnan(x) ((x) != (x))
-#endif
-
 static Lisp_Object
 float_arith_driver (double accum, ptrdiff_t argnum, enum arithop code,
                     ptrdiff_t nargs, Lisp_Object *args)