Integer overflow fixes for scrolling, etc.

6 files changed, 61 insertions, 31 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 6113c2362ee..741bd6e3d53 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,39 @@
+2011-09-04  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Integer overflow fixes for scrolling, etc.
+        Without this fix, Emacs silently mishandles large integers sometimes.
+        For example, "C-u 4294967297 M-x recenter" was be treated as if
+        it were "C-u 1 M-x recenter" on a typical 64-bit host.
+
+        * xdisp.c: Integer overflow fix.
+        (try_window_id): Check Emacs fixnum range before converting to 'int'.
+
+        * window.c: Integer overflow fixes.
+        (window_scroll_line_based, Frecenter):
+        Check that an Emacs fixnum is in range before assigning it to 'int'.
+        (Frecenter, Fmove_to_window_line): Use EMACS_INT, not int, for
+        values converted from Emacs fixnums.
+        (Frecenter): Don't wrap around a line count if it is out of 'int'
+        range; instead, treat it as an extreme value.
+        (Fset_window_configuration, compare_window_configurations):
+        Use ptrdiff_t, not int, for index that might exceed 2 GiB.
+
+        * search.c: Integer overflow fixes
+        (Freplace_match): Use ptrdiff_t, not int, for indexes that can
+        exceed INT_MAX.  Check that EMACS_INT value is in range before
+        assigning it to the (possibly-narrower) index.
+        (match_limit): Don't assume that a fixnum can fit in 'int'.
+
+        * print.c: Integer overflow fix.
+        (print_object): Use ptrdiff_t, not int, for index that can
+        exceed INT_MAX.
+
+        * indent.c: Integer overflow fixes.
+        (position_indentation): Now takes ptrdiff_t, not int.
+        (Fvertical_motion): Don't wrap around LINES values that don't fit
+        in 'int'.  Instead, treat them as extreme values.  This is good
+        enough for windows, which can't have more than INT_MAX lines anyway.
+
 2011-09-03  Lars Magne Ingebrigtsen  <larsi@gnus.org>
 
         * Require libxml/parser.h to avoid compilation warning.
diff --git a/src/indent.c b/src/indent.c
index 313315e9081..6e602d28f60 100644
--- a/src/indent.c
+++ b/src/indent.c
@@ -56,7 +56,7 @@ EMACS_INT last_known_column_point;
 static int last_known_column_modified;
 
 static EMACS_INT current_column_1 (void);
-static EMACS_INT position_indentation (int);
+static EMACS_INT position_indentation (ptrdiff_t);
 
 /* Cache of beginning of line found by the last call of
    current_column. */
@@ -855,7 +855,7 @@ following any initial whitespace.  */)
 }
 
 static EMACS_INT
-position_indentation (register int pos_byte)
+position_indentation (ptrdiff_t pos_byte)
 {
   register EMACS_INT column = 0;
   int tab_width = SANE_TAB_WIDTH (current_buffer);
@@ -2063,7 +2063,7 @@ whether or not it is currently displayed in some window.  */)
           /* Do this even if LINES is 0, so that we move back to the
              beginning of the current line as we ought.  */
           if (XINT (lines) == 0 || IT_CHARPOS (it) > 0)
-            move_it_by_lines (&it, XINT (lines));
+            move_it_by_lines (&it, max (INT_MIN, XINT (lines)));
         }
       else
         {
@@ -2083,7 +2083,7 @@ whether or not it is currently displayed in some window.  */)
                       && it.c == '\n'))
                 move_it_by_lines (&it, -1);
               it.vpos = 0;
-              move_it_by_lines (&it, XINT (lines));
+              move_it_by_lines (&it, min (INT_MAX, XINT (lines)));
             }
           else
             {
@@ -2099,12 +2099,12 @@ whether or not it is currently displayed in some window.  */)
                       move_it_by_lines (&it, 1);
                     }
                   if (XINT (lines) > 1)
-                    move_it_by_lines (&it, XINT (lines) - 1);
+                    move_it_by_lines (&it, min (INT_MAX, XINT (lines) - 1));
                 }
               else
                 {
                   it.vpos = 0;
-                  move_it_by_lines (&it, XINT (lines));
+                  move_it_by_lines (&it, min (INT_MAX, XINT (lines)));
                 }
             }
         }
diff --git a/src/print.c b/src/print.c
index 35f89860843..913a14b3e42 100644
--- a/src/print.c
+++ b/src/print.c
@@ -1697,7 +1697,7 @@ print_object (Lisp_Object obj, register Lisp_Object printcharfun, int escapeflag
         }
       else if (BOOL_VECTOR_P (obj))
         {
-          register int i;
+          ptrdiff_t i;
           register unsigned char c;
           struct gcpro gcpro1;
           EMACS_INT size_in_chars
diff --git a/src/search.c b/src/search.c
index d892792cbaa..b3d67e6c431 100644
--- a/src/search.c
+++ b/src/search.c
@@ -2404,7 +2404,7 @@ since only regular expressions have distinguished subexpressions.  */)
   int some_uppercase;
   int some_nonuppercase_initial;
   register int c, prevc;
-  int sub;
+  ptrdiff_t sub;
   EMACS_INT opoint, newpoint;
 
   CHECK_STRING (newtext);
@@ -2423,9 +2423,9 @@ since only regular expressions have distinguished subexpressions.  */)
   else
     {
       CHECK_NUMBER (subexp);
-      sub = XINT (subexp);
-      if (sub < 0 || sub >= search_regs.num_regs)
+      if (! (0 <= XINT (subexp) && XINT (subexp) < search_regs.num_regs))
         args_out_of_range (subexp, make_number (search_regs.num_regs));
+      sub = XINT (subexp);
     }
 
   if (NILP (string))
@@ -2662,7 +2662,7 @@ since only regular expressions have distinguished subexpressions.  */)
           unsigned char str[MAX_MULTIBYTE_LENGTH];
           const unsigned char *add_stuff = NULL;
           ptrdiff_t add_len = 0;
-          int idx = -1;
+          ptrdiff_t idx = -1;
 
           if (str_multibyte)
             {
@@ -2813,7 +2813,7 @@ since only regular expressions have distinguished subexpressions.  */)
 static Lisp_Object
 match_limit (Lisp_Object num, int beginningp)
 {
-  register int n;
+  EMACS_INT n;
 
   CHECK_NUMBER (num);
   n = XINT (num);
diff --git a/src/window.c b/src/window.c
index 96b1144acf2..e3850387a64 100644
--- a/src/window.c
+++ b/src/window.c
@@ -4662,14 +4662,9 @@ window_scroll_line_based (Lisp_Object window, int n, int whole, int noerror)
 
   if (pos < ZV)
     {
-      int this_scroll_margin = scroll_margin;
-
       /* Don't use a scroll margin that is negative or too large.  */
-      if (this_scroll_margin < 0)
-        this_scroll_margin = 0;
-
-      if (XINT (w->total_lines) < 4 * scroll_margin)
-        this_scroll_margin = XINT (w->total_lines) / 4;
+      int this_scroll_margin =
+        max (0, min (scroll_margin, XINT (w->total_lines) / 4));
 
       set_marker_restricted_both (w->start, w->buffer, pos, pos_byte);
       w->start_at_line_beg = bolp;
@@ -5057,7 +5052,7 @@ and redisplay normally--don't erase and redraw the frame.  */)
   struct buffer *obuf = current_buffer;
   int center_p = 0;
   EMACS_INT charpos, bytepos;
-  int iarg IF_LINT (= 0);
+  EMACS_INT iarg IF_LINT (= 0);
   int this_scroll_margin;
 
   /* If redisplay is suppressed due to an error, try again.  */
@@ -5096,9 +5091,8 @@ and redisplay normally--don't erase and redraw the frame.  */)
 
   /* Do this after making BUF current
      in case scroll_margin is buffer-local.  */
-  this_scroll_margin = max (0, scroll_margin);
-  this_scroll_margin = min (this_scroll_margin,
-                            XFASTINT (w->total_lines) / 4);
+  this_scroll_margin =
+    max (0, min (scroll_margin, XFASTINT (w->total_lines) / 4));
 
   /* Handle centering on a graphical frame specially.  Such frames can
      have variable-height lines and centering point on the basis of
@@ -5122,7 +5116,7 @@ and redisplay normally--don't erase and redraw the frame.  */)
         {
           struct it it;
           struct text_pos pt;
-          int nlines = -iarg;
+          int nlines = min (INT_MAX, -iarg);
           int extra_line_spacing;
           int h = window_box_height (w);
           void *itdata = bidi_shelve_cache ();
@@ -5288,15 +5282,14 @@ zero means top of window, negative means relative to bottom of window.  */)
   lines = displayed_window_lines (w);
 
 #if 0
-  this_scroll_margin = max (0, scroll_margin);
-  this_scroll_margin = min (this_scroll_margin, lines / 4);
+  this_scroll_margin = max (0, min (scroll_margin, lines / 4));
 #endif
 
   if (NILP (arg))
     XSETFASTINT (arg, lines / 2);
   else
     {
-      int iarg = XINT (Fprefix_numeric_value (arg));
+      EMACS_INT iarg = XINT (Fprefix_numeric_value (arg));
 
       if (iarg < 0)
         iarg = iarg + lines;
@@ -5468,7 +5461,8 @@ the return value is nil.  Otherwise the value is t.  */)
       struct window *root_window;
       struct window **leaf_windows;
       int n_leaf_windows;
-      int k, i, n;
+      ptrdiff_t k;
+      int i, n;
 
       /* If the frame has been resized since this window configuration was
          made, we change the frame to the size specified in the
@@ -6344,7 +6338,7 @@ compare_window_configurations (Lisp_Object configuration1, Lisp_Object configura
 {
   register struct save_window_data *d1, *d2;
   struct Lisp_Vector *sws1, *sws2;
-  int i;
+  ptrdiff_t i;
 
   CHECK_WINDOW_CONFIGURATION (configuration1);
   CHECK_WINDOW_CONFIGURATION (configuration2);
diff --git a/src/xdisp.c b/src/xdisp.c
index 1716cc82188..f11362c1ae6 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -16919,8 +16919,8 @@ try_window_id (struct window *w)
   {
     int this_scroll_margin, cursor_height;
 
-    this_scroll_margin = max (0, scroll_margin);
-    this_scroll_margin = min (this_scroll_margin, WINDOW_TOTAL_LINES (w) / 4);
+    this_scroll_margin =
+      max (0, min (scroll_margin, WINDOW_TOTAL_LINES (w) / 4));
     this_scroll_margin *= FRAME_LINE_HEIGHT (it.f);
     cursor_height = MATRIX_ROW (w->desired_matrix, w->cursor.vpos)->height;