Check for buffer and string overflow more precisely.

* buffer.h (BUF_BYTES_MAX): New macro.
* lisp.h (STRING_BYTES_MAX): New macro.
* alloc.c (Fmake_string):
* character.c (string_escape_byte8):
* coding.c (coding_alloc_by_realloc):
* doprnt.c (doprnt):
* editfns.c (Fformat):
* eval.c (verror):
Use STRING_BYTES_MAX, not MOST_POSITIVE_FIXNUM,
since they may not be the same number.
* editfns.c (Finsert_char):
* fileio.c (Finsert_file_contents):
Likewise for BUF_BYTES_MAX.

10 files changed, 37 insertions, 11 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 8333bf2dedd..374790b51f6 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,20 @@
 2011-06-04  Paul Eggert  <eggert@cs.ucla.edu>
 
+        Check for buffer and string overflow more precisely.
+        * buffer.h (BUF_BYTES_MAX): New macro.
+        * lisp.h (STRING_BYTES_MAX): New macro.
+        * alloc.c (Fmake_string):
+        * character.c (string_escape_byte8):
+        * coding.c (coding_alloc_by_realloc):
+        * doprnt.c (doprnt):
+        * editfns.c (Fformat):
+        * eval.c (verror):
+        Use STRING_BYTES_MAX, not MOST_POSITIVE_FIXNUM,
+        since they may not be the same number.
+        * editfns.c (Finsert_char):
+        * fileio.c (Finsert_file_contents):
+        Likewise for BUF_BYTES_MAX.
+
         Use ptrdiff_t, not int, for sizes.
         * image.c (slurp_file): Switch from int to ptrdiff_t.
         All uses changed.
diff --git a/src/alloc.c b/src/alloc.c
index 0c18fca1755..d20ee1e59e3 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -2205,7 +2205,7 @@ INIT must be an integer that represents a character.  */)
       int len = CHAR_STRING (c, str);
       EMACS_INT string_len = XINT (length);
 
-      if (string_len > MOST_POSITIVE_FIXNUM / len)
+      if (string_len > STRING_BYTES_MAX / len)
         string_overflow ();
       nbytes = len * string_len;
       val = make_uninit_multibyte_string (string_len, nbytes);
diff --git a/src/buffer.h b/src/buffer.h
index 8c64a24e804..3c91bdfe570 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -306,6 +306,11 @@ do								\
   }                                                             \
 while (0)
 
+/* Maximum number of bytes in a buffer.
+   A buffer cannot contain more bytes than a 1-origin fixnum can represent,
+   nor can it be so large that C pointer arithmetic stops working.  */
+#define BUF_BYTES_MAX min (MOST_POSITIVE_FIXNUM - 1, min (SIZE_MAX, PTRDIFF_MAX))
+
 /* Return the address of byte position N in current buffer.  */
 
 #define BYTE_POS_ADDR(n) \
diff --git a/src/character.c b/src/character.c
index 4aa1b75cd8c..cff00f44660 100644
--- a/src/character.c
+++ b/src/character.c
@@ -838,7 +838,7 @@ string_escape_byte8 (Lisp_Object string)
   if (multibyte)
     {
       if ((MOST_POSITIVE_FIXNUM - nchars) / 3 < byte8_count
-          || (MOST_POSITIVE_FIXNUM - nbytes) / 2 < byte8_count)
+          || (STRING_BYTES_MAX - nbytes) / 2 < byte8_count)
         string_overflow ();
 
       /* Convert 2-byte sequence of byte8 chars to 4-byte octal.  */
@@ -847,7 +847,7 @@ string_escape_byte8 (Lisp_Object string)
     }
   else
     {
-      if ((MOST_POSITIVE_FIXNUM - nchars) / 3 < byte8_count)
+      if ((STRING_BYTES_MAX - nchars) / 3 < byte8_count)
         string_overflow ();
 
       /* Convert 1-byte sequence of byte8 chars to 4-byte octal.  */
diff --git a/src/coding.c b/src/coding.c
index 6ccaf354c74..64e8e41a5a1 100644
--- a/src/coding.c
+++ b/src/coding.c
@@ -1071,8 +1071,8 @@ coding_set_destination (struct coding_system *coding)
 static void
 coding_alloc_by_realloc (struct coding_system *coding, EMACS_INT bytes)
 {
-  if (coding->dst_bytes >= MOST_POSITIVE_FIXNUM - bytes)
-    error ("Maximum size of buffer or string exceeded");
+  if (STRING_BYTES_MAX - coding->dst_bytes < bytes)
+    string_overflow ();
   coding->destination = (unsigned char *) xrealloc (coding->destination,
                                                     coding->dst_bytes + bytes);
   coding->dst_bytes += bytes;
diff --git a/src/doprnt.c b/src/doprnt.c
index d2abc119912..5ca3ea89be6 100644
--- a/src/doprnt.c
+++ b/src/doprnt.c
@@ -329,7 +329,7 @@ doprnt (char *buffer, register size_t bufsize, const char *format,
                 minlen = atoi (&fmtcpy[1]);
               string = va_arg (ap, char *);
               tem = strlen (string);
-              if (tem > MOST_POSITIVE_FIXNUM)
+              if (tem > STRING_BYTES_MAX)
                 error ("String for %%s or %%S format is too long");
               width = strwidth (string, tem);
               goto doit1;
@@ -338,7 +338,7 @@ doprnt (char *buffer, register size_t bufsize, const char *format,
             doit:
               /* Coming here means STRING contains ASCII only.  */
               tem = strlen (string);
-              if (tem > MOST_POSITIVE_FIXNUM)
+              if (tem > STRING_BYTES_MAX)
                 error ("Format width or precision too large");
               width = tem;
             doit1:
diff --git a/src/editfns.c b/src/editfns.c
index b18a35fe295..b7951c45fac 100644
--- a/src/editfns.c
+++ b/src/editfns.c
@@ -2341,7 +2341,7 @@ from adjoining text, if those properties are sticky.  */)
     len = CHAR_STRING (XFASTINT (character), str);
   else
     str[0] = XFASTINT (character), len = 1;
-  if (MOST_POSITIVE_FIXNUM / len < XINT (count))
+  if (BUF_BYTES_MAX / len < XINT (count))
     error ("Maximum buffer size would be exceeded");
   n = XINT (count) * len;
   if (n <= 0)
@@ -3588,7 +3588,7 @@ usage: (format STRING &rest OBJECTS)  */)
   char initial_buffer[4000];
   char *buf = initial_buffer;
   EMACS_INT bufsize = sizeof initial_buffer;
-  EMACS_INT max_bufsize = min (MOST_POSITIVE_FIXNUM + 1, SIZE_MAX);
+  EMACS_INT max_bufsize = STRING_BYTES_MAX + 1;
   char *p;
   Lisp_Object buf_save_value IF_LINT (= {0});
   register char *format, *end, *format_start;
diff --git a/src/eval.c b/src/eval.c
index f8bc0a9f6aa..ef5abac17ae 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -1994,7 +1994,7 @@ verror (const char *m, va_list ap)
 {
   char buf[4000];
   size_t size = sizeof buf;
-  size_t size_max = min (MOST_POSITIVE_FIXNUM + 1, SIZE_MAX);
+  size_t size_max = STRING_BYTES_MAX + 1;
   size_t mlen = strlen (m);
   char *buffer = buf;
   size_t used;
diff --git a/src/fileio.c b/src/fileio.c
index 2f7716d5b54..d9bc28d8c37 100644
--- a/src/fileio.c
+++ b/src/fileio.c
@@ -3248,7 +3248,7 @@ variable `last-coding-system-used' to the coding system actually used.  */)
   /* Check whether the size is too large or negative, which can happen on a
      platform that allows file sizes greater than the maximum off_t value.  */
   if (! not_regular
-      && ! (0 <= st.st_size && st.st_size <= MOST_POSITIVE_FIXNUM))
+      && ! (0 <= st.st_size && st.st_size <= BUF_BYTES_MAX))
     error ("Maximum buffer size exceeded");
 
   /* Prevent redisplay optimizations.  */
diff --git a/src/lisp.h b/src/lisp.h
index 1defda151ae..ad4614c7b16 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -766,6 +766,12 @@ extern EMACS_INT string_bytes (struct Lisp_String *);
 
 #endif /* not GC_CHECK_STRING_BYTES */
 
+/* A string cannot contain more bytes than a fixnum can represent,
+   nor can it be so long that C pointer arithmetic stops working on
+   the string plus a terminating null.  */
+#define STRING_BYTES_MAX  \
+  min (MOST_POSITIVE_FIXNUM, min (SIZE_MAX, PTRDIFF_MAX) - 1)
+
 /* Mark STR as a unibyte string.  */
 #define STRING_SET_UNIBYTE(STR)  \
   do { if (EQ (STR, empty_multibyte_string))  \