Fix bignum comparisons with NaN

* src/data.c (isnan): Move earlier.
(bignumcompare): Explicitly handle NaN.
* test/src/data-tests.el (data-tests-min): Add NaN tests
for bignum.
(data-check-sign): Fix for previous patch.
* test/src/fns-tests.el (test-bignum-eql): Add NaN test.

3 files changed, 24 insertions, 9 deletions

diff --git a/src/data.c b/src/data.c
index 3d55d9d17d5..4388a2b0ffc 100644
--- a/src/data.c
+++ b/src/data.c
@@ -2397,6 +2397,10 @@ bool-vector.  IDX starts at 0.  */)
 
 /* Arithmetic functions */
 
+#ifndef isnan
+# define isnan(x) ((x) != (x))
+#endif
+
 static Lisp_Object
 bignumcompare (Lisp_Object num1, Lisp_Object num2,
                enum Arith_Comparison comparison)
@@ -2407,7 +2411,13 @@ bignumcompare (Lisp_Object num1, Lisp_Object num2,
   if (BIGNUMP (num1))
     {
       if (FLOATP (num2))
-        cmp = mpz_cmp_d (XBIGNUM (num1)->value, XFLOAT_DATA (num2));
+        {
+          /* Note that GMP doesn't define comparisons against NaN, so
+             we need to handle them specially.  */
+          if (isnan (XFLOAT_DATA (num2)))
+            return Qnil;
+          cmp = mpz_cmp_d (XBIGNUM (num1)->value, XFLOAT_DATA (num2));
+        }
       else if (FIXNUMP (num2))
         {
           if (sizeof (EMACS_INT) > sizeof (long) && XINT (num2) > LONG_MAX)
@@ -2431,7 +2441,13 @@ bignumcompare (Lisp_Object num1, Lisp_Object num2,
     {
       eassume (BIGNUMP (num2));
       if (FLOATP (num1))
-        cmp = - mpz_cmp_d (XBIGNUM (num2)->value, XFLOAT_DATA (num1));
+        {
+          /* Note that GMP doesn't define comparisons against NaN, so
+             we need to handle them specially.  */
+          if (isnan (XFLOAT_DATA (num1)))
+            return Qnil;
+          cmp = - mpz_cmp_d (XBIGNUM (num2)->value, XFLOAT_DATA (num1));
+        }
       else
         {
           eassume (FIXNUMP (num1));
@@ -3021,10 +3037,6 @@ arith_driver (enum arithop code, ptrdiff_t nargs, Lisp_Object *args)
   return unbind_to (count, make_number (accum));
 }
 
-#ifndef isnan
-# define isnan(x) ((x) != (x))
-#endif
-
 static Lisp_Object
 float_arith_driver (double accum, ptrdiff_t argnum, enum arithop code,
                     ptrdiff_t nargs, Lisp_Object *args)
diff --git a/test/src/data-tests.el b/test/src/data-tests.el
index 07159df48cf..ee6a3eb9222 100644
--- a/test/src/data-tests.el
+++ b/test/src/data-tests.el
@@ -105,7 +105,9 @@
   (should (isnan (min 0.0e+NaN)))
   (should (isnan (min 0.0e+NaN 1 2)))
   (should (isnan (min 1.0 0.0e+NaN)))
-  (should (isnan (min 1.0 0.0e+NaN 1.1))))
+  (should (isnan (min 1.0 0.0e+NaN 1.1)))
+  (should (isnan (min 1.0 0.0e+NaN 1.1 (1+ most-positive-fixnum))))
+  (should (isnan (max 1.0 0.0e+NaN 1.1 (1+ most-positive-fixnum)))))
 
 (defun data-tests-popcnt (byte)
   "Calculate the Hamming weight of BYTE."
@@ -618,6 +620,6 @@ comparing the subr with a much slower lisp implementation."
   (should (= (ash most-negative-fixnum 1)
              (* most-negative-fixnum 2)))
   (should (= (lsh most-negative-fixnum 1)
-             (* (abs most-negative-fixnum) 2))))
+             (* most-negative-fixnum 2))))
 
 ;;; data-tests.el ends here
diff --git a/test/src/fns-tests.el b/test/src/fns-tests.el
index f5f3b892441..d440cfabda4 100644
--- a/test/src/fns-tests.el
+++ b/test/src/fns-tests.el
@@ -599,6 +599,7 @@
         (y (+ most-positive-fixnum 1)))
     (should (eq x x))
     (should (eql x y))
-    (should (equal x y))))
+    (should (equal x y))
+    (should-not (eql x 0.0e+NaN))))
 
 (provide 'fns-tests)