Avoid overexposing fixnums for hash codes

Following a suggestion by Stefan Monnier in:
https://lists.gnu.org/r/emacs-devel/2019-07/msg00530.html
* doc/lispref/hash.texi (Creating Hash, Defining Hash):
* src/fns.c (Fsxhash_eq, Fsxhash_eql, Fsxhash_equal, Fmake_hash_table):
Don’t insist that hash codes be fixnums, reverting
the recent doc changes to the contrary.
* src/bytecode.c (exec_byte_code): Special-case only the eq case,
as the others aren’t worth tuning now that we treat bignum hashes
like fixnums.
* src/fns.c (hashfn_user_defined): If the hash code is a bignum,
reduce its hash down to a fixnum.

2 files changed, 11 insertions, 15 deletions

diff --git a/src/bytecode.c b/src/bytecode.c
index d668a9a6a15..9aad1eb642b 100644
--- a/src/bytecode.c
+++ b/src/bytecode.c
@@ -1406,18 +1406,12 @@ exec_byte_code (Lisp_Object bytestr, Lisp_Object vector, Lisp_Object maxdepth,
 
             /* h->count is a faster approximation for HASH_TABLE_SIZE (h)
                here. */
-            if (h->count <= 5)
+            if (h->count <= 5 && !h->test.cmpfn)
               { /* Do a linear search if there are not many cases
                    FIXME: 5 is arbitrarily chosen.  */
-                Lisp_Object hash_code
-                  = h->test.cmpfn ? h->test.hashfn (v1, h) : Qnil;
-
-                for (i = h->count; 0 <= --i; )
-                  if (EQ (v1, HASH_KEY (h, i))
-                      || (h->test.cmpfn
-                          && EQ (hash_code, HASH_HASH (h, i))
-                          && !NILP (h->test.cmpfn (v1, HASH_KEY (h, i), h))))
-                    break;
+                for (i = h->count; 0 <= --i; )
+                  if (EQ (v1, HASH_KEY (h, i)))
+                    break;
               }
             else
               i = hash_lookup (h, v1, NULL);
diff --git a/src/fns.c b/src/fns.c
index 734a2e253c7..d28d437df9c 100644
--- a/src/fns.c
+++ b/src/fns.c
@@ -47,6 +47,7 @@ static void sort_vector_copy (Lisp_Object, ptrdiff_t,
 enum equal_kind { EQUAL_NO_QUIT, EQUAL_PLAIN, EQUAL_INCLUDING_PROPERTIES };
 static bool internal_equal (Lisp_Object, Lisp_Object,
                             enum equal_kind, int, Lisp_Object);
+static EMACS_UINT sxhash_bignum (struct Lisp_Bignum *);
 
 DEFUN ("identity", Fidentity, Sidentity, 1, 1, 0,
        doc: /* Return the argument unchanged.  */
@@ -4021,7 +4022,8 @@ Lisp_Object
 hashfn_user_defined (Lisp_Object key, struct Lisp_Hash_Table *h)
 {
   Lisp_Object args[] = { h->test.user_hash_function, key };
-  return hash_table_user_defined_call (ARRAYELTS (args), args, h);
+  Lisp_Object hash = hash_table_user_defined_call (ARRAYELTS (args), args, h);
+  return BIGNUMP (hash) ? make_fixnum (sxhash_bignum (XBIGNUM (hash))) : hash;
 }
 
 struct hash_table_test const
@@ -4707,7 +4709,7 @@ sxhash (Lisp_Object obj, int depth)
  ***********************************************************************/
 
 DEFUN ("sxhash-eq", Fsxhash_eq, Ssxhash_eq, 1, 1, 0,
-       doc: /* Return a fixnum hash code for OBJ suitable for `eq'.
+       doc: /* Return an integer hash code for OBJ suitable for `eq'.
 If (eq A B), then (= (sxhash-eq A) (sxhash-eq B)).
 
 Hash codes are not guaranteed to be preserved across Emacs sessions.  */)
@@ -4717,7 +4719,7 @@ Hash codes are not guaranteed to be preserved across Emacs sessions.  */)
 }
 
 DEFUN ("sxhash-eql", Fsxhash_eql, Ssxhash_eql, 1, 1, 0,
-       doc: /* Return a fixnum hash code for OBJ suitable for `eql'.
+       doc: /* Return an integer hash code for OBJ suitable for `eql'.
 If (eql A B), then (= (sxhash-eql A) (sxhash-eql B)).
 
 Hash codes are not guaranteed to be preserved across Emacs sessions.  */)
@@ -4727,7 +4729,7 @@ Hash codes are not guaranteed to be preserved across Emacs sessions.  */)
 }
 
 DEFUN ("sxhash-equal", Fsxhash_equal, Ssxhash_equal, 1, 1, 0,
-       doc: /* Return a fixnum hash code for OBJ suitable for `equal'.
+       doc: /* Return an integer hash code for OBJ suitable for `equal'.
 If (equal A B), then (= (sxhash-equal A) (sxhash-equal B)).
 
 Hash codes are not guaranteed to be preserved across Emacs sessions.  */)
@@ -4751,7 +4753,7 @@ keys.  Default is `eql'.  Predefined are the tests `eq', `eql', and
 Default is 65.
 
 :rehash-size REHASH-SIZE - Indicates how to expand the table when it
-fills up.  If REHASH-SIZE is a fixnum, increase the size by that
+fills up.  If REHASH-SIZE is an integer, increase the size by that
 amount.  If it is a float, it must be > 1.0, and the new size is the
 old size multiplied by that factor.  Default is 1.5.