Improve buffer-overflow checking.

* fileio.c (Finsert_file_contents):
* insdel.c (insert_from_buffer_1, replace_range, replace_range_2):
Remove the old (too-loose) buffer overflow checks.
They weren't needed, since make_gap checks for buffer overflow.
* insdel.c (make_gap_larger): Catch buffer overflows that were missed.
The old code merely checked for Emacs fixnum overflow, and relied
on undefined (wraparound) behavior.  The new code avoids undefined
behavior, and also checks for ptrdiff_t and/or size_t overflow.

3 files changed, 19 insertions, 36 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 011f5beefe0..5f18c8d0062 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,15 @@
 2011-06-16  Paul Eggert  <eggert@cs.ucla.edu>
 
+        Improve buffer-overflow checking.
+        * fileio.c (Finsert_file_contents):
+        * insdel.c (insert_from_buffer_1, replace_range, replace_range_2):
+        Remove the old (too-loose) buffer overflow checks.
+        They weren't needed, since make_gap checks for buffer overflow.
+        * insdel.c (make_gap_larger): Catch buffer overflows that were missed.
+        The old code merely checked for Emacs fixnum overflow, and relied
+        on undefined (wraparound) behavior.  The new code avoids undefined
+        behavior, and also checks for ptrdiff_t and/or size_t overflow.
+
         * editfns.c (Finsert_char): Don't dump core with very negative counts.
         Tune.  Don't use wider integers than needed.  Don't use alloca.
         Use a bigger 'string' buffer.  Rewrite to avoid 'n > 0' test.
diff --git a/src/fileio.c b/src/fileio.c
index 4458a3a4807..dd34872c263 100644
--- a/src/fileio.c
+++ b/src/fileio.c
@@ -3800,16 +3800,7 @@ variable `last-coding-system-used' to the coding system actually used.  */)
     }
 
   if (! not_regular)
-    {
-      register Lisp_Object temp;
-
-      total = XINT (end) - XINT (beg);
-
-      /* Make sure point-max won't overflow after this insertion.  */
-      XSETINT (temp, total);
-      if (total != XINT (temp))
-        buffer_overflow ();
-    }
+    total = XINT (end) - XINT (beg);
   else
     /* For a special file, all we can do is guess.  */
     total = READ_BUF_SIZE;
diff --git a/src/insdel.c b/src/insdel.c
index ca53177a3e1..bc95e3ad9e8 100644
--- a/src/insdel.c
+++ b/src/insdel.c
@@ -406,16 +406,16 @@ make_gap_larger (EMACS_INT nbytes_added)
   EMACS_INT real_gap_loc;
   EMACS_INT real_gap_loc_byte;
   EMACS_INT old_gap_size;
+  EMACS_INT current_size = Z_BYTE - BEG_BYTE + GAP_SIZE;
+  enum { enough_for_a_while = 2000 };
 
-  /* If we have to get more space, get enough to last a while.  */
-  nbytes_added += 2000;
+  if (BUF_BYTES_MAX - current_size < nbytes_added)
+    buffer_overflow ();
 
-  { EMACS_INT total_size = Z_BYTE - BEG_BYTE + GAP_SIZE + nbytes_added;
-    if (total_size < 0
-        /* Don't allow a buffer size that won't fit in a Lisp integer.  */
-        || total_size != XINT (make_number (total_size)))
-      buffer_overflow ();
-  }
+  /* If we have to get more space, get enough to last a while;
+     but do not exceed the maximum buffer size.  */
+  nbytes_added = min (nbytes_added + enough_for_a_while,
+                      BUF_BYTES_MAX - current_size);
 
   enlarge_buffer_text (current_buffer, nbytes_added);
 
@@ -1069,7 +1069,6 @@ static void
 insert_from_buffer_1 (struct buffer *buf,
                       EMACS_INT from, EMACS_INT nchars, int inherit)
 {
-  register Lisp_Object temp;
   EMACS_INT chunk, chunk_expanded;
   EMACS_INT from_byte = buf_charpos_to_bytepos (buf, from);
   EMACS_INT to_byte = buf_charpos_to_bytepos (buf, from + nchars);
@@ -1108,11 +1107,6 @@ insert_from_buffer_1 (struct buffer *buf,
       outgoing_nbytes = outgoing_before_gap + outgoing_after_gap;
     }
 
-  /* Make sure point-max won't overflow after this insertion.  */
-  XSETINT (temp, outgoing_nbytes + Z);
-  if (outgoing_nbytes + Z != XINT (temp))
-    buffer_overflow ();
-
   /* Do this before moving and increasing the gap,
      because the before-change hooks might move the gap
      or make it smaller.  */
@@ -1309,7 +1303,6 @@ replace_range (EMACS_INT from, EMACS_INT to, Lisp_Object new,
   EMACS_INT insbytes = SBYTES (new);
   EMACS_INT from_byte, to_byte;
   EMACS_INT nbytes_del, nchars_del;
-  register Lisp_Object temp;
   struct gcpro gcpro1;
   INTERVAL intervals;
   EMACS_INT outgoing_insbytes = insbytes;
@@ -1353,11 +1346,6 @@ replace_range (EMACS_INT from, EMACS_INT to, Lisp_Object new,
     outgoing_insbytes
       = count_size_as_multibyte (SDATA (new), insbytes);
 
-  /* Make sure point-max won't overflow after this insertion.  */
-  XSETINT (temp, Z_BYTE - nbytes_del + outgoing_insbytes);
-  if (Z_BYTE - nbytes_del + outgoing_insbytes != XINT (temp))
-    buffer_overflow ();
-
   GCPRO1 (new);
 
   /* Make sure the gap is somewhere in or next to what we are deleting.  */
@@ -1488,7 +1476,6 @@ replace_range_2 (EMACS_INT from, EMACS_INT from_byte,
                  int markers)
 {
   EMACS_INT nbytes_del, nchars_del;
-  Lisp_Object temp;
 
   CHECK_MARKERS ();
 
@@ -1498,11 +1485,6 @@ replace_range_2 (EMACS_INT from, EMACS_INT from_byte,
   if (nbytes_del <= 0 && insbytes == 0)
     return;
 
-  /* Make sure point-max won't overflow after this insertion.  */
-  XSETINT (temp, Z_BYTE - nbytes_del + insbytes);
-  if (Z_BYTE - nbytes_del + insbytes != XINT (temp))
-    buffer_overflow ();
-
   /* Make sure the gap is somewhere in or next to what we are deleting.  */
   if (from > GPT)
     gap_right (from, from_byte);