Avoid undefined behavior in color table hashing.

* image.c (CT_HASH_RGB) [COLOR_TABLE_SUPPORT]: Remove, replacing with ...
(ct_hash_rgb) [COLOR_TABLE_SUPPORT]: New function.  All uses changed.
This function avoids undefined behavior with signed shift overflow.

2 files changed, 13 insertions, 2 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index d44de652dde..2be24fa851b 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,10 @@
+2014-11-13  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Avoid undefined behavior in color table hashing.
+        * image.c (CT_HASH_RGB) [COLOR_TABLE_SUPPORT]: Remove, replacing with ...
+        (ct_hash_rgb) [COLOR_TABLE_SUPPORT]: New function.  All uses changed.
+        This function avoids undefined behavior with signed shift overflow.
+
 2014-11-10  Eli Zaretskii  <eliz@gnu.org>
 
         * fileio.c (Finsert_file_contents): Invalidate buffer caches also
diff --git a/src/image.c b/src/image.c
index 4b73a5fe80c..1a2c0e29dde 100644
--- a/src/image.c
+++ b/src/image.c
@@ -4294,7 +4294,11 @@ struct ct_color
 
 /* Value is a hash of the RGB color given by R, G, and B.  */
 
-#define CT_HASH_RGB(R, G, B) (((R) << 16) ^ ((G) << 8) ^ (B))
+static unsigned
+ct_hash_rgb (unsigned r, unsigned g, unsigned b)
+{
+  return (r << 16) ^ (g << 8) ^ b;
+}
 
 /* The color hash table.  */
 
@@ -4349,7 +4353,7 @@ free_color_table (void)
 static unsigned long
 lookup_rgb_color (struct frame *f, int r, int g, int b)
 {
-  unsigned hash = CT_HASH_RGB (r, g, b);
+  unsigned hash = ct_hash_rgb (r, g, b);
   int i = hash % CT_SIZE;
   struct ct_color *p;
   Display_Info *dpyinfo;