Avoid undefined behavior with signed left shift.

Caught by 'gcc -fsanitize=undefined'.
* dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
* dispnew.c (line_hash_code, scrolling):
* scroll.c (calculate_scrolling, calculate_direct_scrolling):
Use 'unsigned', not 'int', for line hashes.
(scrolling_max_lines_saved): Avoid mystery constants for hash sizes.

4 files changed, 28 insertions, 16 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 28367ed34f5..dce13035766 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,13 @@
+2014-08-03  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Avoid undefined behavior with signed left shift.
+        Caught by 'gcc -fsanitize=undefined'.
+        * dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
+        * dispnew.c (line_hash_code, scrolling):
+        * scroll.c (calculate_scrolling, calculate_direct_scrolling):
+        Use 'unsigned', not 'int', for line hashes.
+        (scrolling_max_lines_saved): Avoid mystery constants for hash sizes.
+
 2014-08-02  Paul Eggert  <eggert@cs.ucla.edu>
 
         Make compare-strings more compatible with old behavior (Bug#17903).
diff --git a/src/dispextern.h b/src/dispextern.h
index bb6f1eb2b5e..ebd4260d408 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -3507,13 +3507,13 @@ extern void tty_append_glyph (struct it *);
 
 /* Defined in scroll.c */
 
-extern int scrolling_max_lines_saved (int, int, int *, int *, int *);
+extern int scrolling_max_lines_saved (int, int, unsigned *, unsigned *, int *);
 extern void do_line_insertion_deletion_costs (struct frame *, const char *,
                                               const char *, const char *,
                                               const char *, const char *,
                                               const char *, int);
-void scrolling_1 (struct frame *, int, int, int, int *, int *, int *,
-                  int *, int);
+void scrolling_1 (struct frame *, int, int, int, int *, int *, unsigned *,
+                  unsigned *, int);
 
 /* Defined in frame.c */
 
diff --git a/src/dispnew.c b/src/dispnew.c
index 70862985e66..78df22ddbe2 100644
--- a/src/dispnew.c
+++ b/src/dispnew.c
@@ -1108,10 +1108,10 @@ prepare_desired_row (struct window *w, struct glyph_row *row, bool mode_line_p)
 /* Return a hash code for glyph row ROW, which may
    be from current or desired matrix of frame F.  */
 
-static int
+static unsigned
 line_hash_code (struct frame *f, struct glyph_row *row)
 {
-  int hash = 0;
+  unsigned hash = 0;
 
   if (row->enabled_p)
     {
@@ -4647,8 +4647,8 @@ scrolling (struct frame *frame)
   int unchanged_at_top, unchanged_at_bottom;
   int window_size;
   int changed_lines;
-  int *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
-  int *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *old_draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   register int i;
diff --git a/src/scroll.c b/src/scroll.c
index 3da236ca8a6..6c559663f80 100644
--- a/src/scroll.c
+++ b/src/scroll.c
@@ -90,7 +90,7 @@ calculate_scrolling (struct frame *frame,
                      /* matrix is of size window_size + 1 on each side.  */
                      struct matrix_elt *matrix,
                      int window_size, int lines_below,
-                     int *draw_cost, int *old_hash, int *new_hash,
+                     int *draw_cost, unsigned *old_hash, unsigned *new_hash,
                      int free_at_end)
 {
   register int i, j;
@@ -427,7 +427,7 @@ calculate_direct_scrolling (struct frame *frame,
                             struct matrix_elt *matrix,
                             int window_size, int lines_below,
                             int *draw_cost, int *old_draw_cost,
-                            int *old_hash, int *new_hash,
+                            unsigned *old_hash, unsigned *new_hash,
                             int free_at_end)
 {
   register int i, j;
@@ -794,7 +794,7 @@ do_direct_scrolling (struct frame *frame, struct glyph_matrix *current_matrix,
 void
 scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
              int unchanged_at_bottom, int *draw_cost, int *old_draw_cost,
-             int *old_hash, int *new_hash, int free_at_end)
+             unsigned *old_hash, unsigned *new_hash, int free_at_end)
 {
   struct matrix_elt *matrix
     = alloca ((window_size + 1) * (window_size + 1) * sizeof *matrix);
@@ -829,12 +829,14 @@ scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
 
 int
 scrolling_max_lines_saved (int start, int end,
-                           int *oldhash, int *newhash,
+                           unsigned *oldhash, unsigned *newhash,
                            int *cost)
 {
-  struct { int hash; int count; } lines[01000];
-  register int i, h;
-  register int matchcount = 0;
+  enum { LOG2_NLINES = 9 };
+  enum { NLINES = 1 << LOG2_NLINES };
+  struct { unsigned hash; int count; } lines[NLINES];
+  int i, h;
+  int matchcount = 0;
   int avg_length = 0;
   int threshold;
 
@@ -855,7 +857,7 @@ scrolling_max_lines_saved (int start, int end,
     {
       if (cost[i] > threshold)
         {
-          h = newhash[i] & 0777;
+          h = newhash[i] & (NLINES - 1);
           lines[h].hash = newhash[i];
           lines[h].count++;
         }
@@ -865,7 +867,7 @@ scrolling_max_lines_saved (int start, int end,
      matches between old lines and new.  */
   for (i = start; i < end; i++)
     {
-      h = oldhash[i] & 0777;
+      h = oldhash[i] & (NLINES - 1);
       if (oldhash[i] == lines[h].hash)
         {
           matchcount++;