[ChangeLog]

Malloc failure behavior now depends on size of allocation.
* lib/allocator.h (struct allocator.die): New size arg.
* lib/careadlinkat.c (careadlinkat): Pass size to 'die' function.
If the actual problem is an ssize_t limitation, not a size_t or
malloc failure, fail with errno == ENAMETOOLONG instead of calling 'die'.
[src/ChangeLog]
Malloc failure behavior now depends on size of allocation.
* alloc.c (buffer_memory_full, memory_full): New arg NBYTES.
* lisp.h: Change signatures accordingly.
* alloc.c, buffer.c, editfns.c, menu.c, minibuf.c, xterm.c:
All callers changed.

11 files changed, 86 insertions, 41 deletions

diff --git a/ChangeLog b/ChangeLog
index 879ce2834e7..4ca48fa91c4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2011-05-30  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Malloc failure behavior now depends on size of allocation.
+        * lib/allocator.h (struct allocator.die): New size arg.
+        * lib/careadlinkat.c (careadlinkat): Pass size to 'die' function.
+        If the actual problem is an ssize_t limitation, not a size_t or
+        malloc failure, fail with errno == ENAMETOOLONG instead of calling 'die'.
+
 2011-05-29  Paul Eggert  <eggert@cs.ucla.edu>
 
         Adjust to recent gnulib change for @GUARD_PREFIX@.
diff --git a/lib/allocator.h b/lib/allocator.h
index 953117da83f..b8de95c0f50 100644
--- a/lib/allocator.h
+++ b/lib/allocator.h
@@ -45,10 +45,11 @@ struct allocator
   /* Call FREE to free memory, like 'free'.  */
   void (*free) (void *);
 
-  /* If nonnull, call DIE if MALLOC or REALLOC fails.  DIE should not
-     return.  DIE can be used by code that detects memory overflow
-     while calculating sizes to be passed to MALLOC or REALLOC.  */
-  void (*die) (void);
+  /* If nonnull, call DIE (SIZE) if MALLOC (SIZE) or REALLOC (...,
+     SIZE) fails.  DIE should not return.  SIZE should equal SIZE_MAX
+     if size_t overflow was detected while calculating sizes to be
+     passed to MALLOC or REALLOC.  */
+  void (*die) (size_t);
 };
 
 /* An allocator using the stdlib functions and a null DIE function.  */
diff --git a/lib/careadlinkat.c b/lib/careadlinkat.c
index e2909c766d5..6e4aa1395ff 100644
--- a/lib/careadlinkat.c
+++ b/lib/careadlinkat.c
@@ -135,6 +135,7 @@ careadlinkat (int fd, char const *filename,
           if (buf == stack_buf)
             {
               char *b = (char *) alloc->allocate (link_size);
+              buf_size = link_size;
               if (! b)
                 break;
               memcpy (b, buf, link_size);
@@ -158,6 +159,11 @@ careadlinkat (int fd, char const *filename,
         buf_size *= 2;
       else if (buf_size < buf_size_max)
         buf_size = buf_size_max;
+      else if (buf_size_max < SIZE_MAX)
+        {
+          errno = ENAMETOOLONG;
+          return NULL;
+        }
       else
         break;
       buf = (char *) alloc->allocate (buf_size);
@@ -165,7 +171,7 @@ careadlinkat (int fd, char const *filename,
   while (buf);
 
   if (alloc->die)
-    alloc->die ();
+    alloc->die (buf_size);
   errno = ENOMEM;
   return NULL;
 }
diff --git a/src/ChangeLog b/src/ChangeLog
index 7b0c73adbc9..aa034b3493f 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,11 @@
 2011-05-30  Paul Eggert  <eggert@cs.ucla.edu>
 
+        Malloc failure behavior now depends on size of allocation.
+        * alloc.c (buffer_memory_full, memory_full): New arg NBYTES.
+        * lisp.h: Change signatures accordingly.
+        * alloc.c, buffer.c, editfns.c, menu.c, minibuf.c, xterm.c:
+        All callers changed.
+
         * gnutls.c: Use Emacs's memory allocators.
         Without this change, the gnutls library would invoke malloc etc.
         directly, which causes problems on non-SYNC_INPUT hosts, and which
diff --git a/src/alloc.c b/src/alloc.c
index 8215cc53cd3..be045be2ab4 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -471,7 +471,7 @@ display_malloc_warning (void)
 /* Called if we can't allocate relocatable space for a buffer.  */
 
 void
-buffer_memory_full (void)
+buffer_memory_full (EMACS_INT nbytes)
 {
   /* If buffers use the relocating allocator, no need to free
      spare_memory, because we may have plenty of malloc space left
@@ -481,7 +481,7 @@ buffer_memory_full (void)
      malloc.  */
 
 #ifndef REL_ALLOC
-  memory_full ();
+  memory_full (nbytes);
 #endif
 
   /* This used to call error, but if we've run out of memory, we could
@@ -677,7 +677,7 @@ xmalloc (size_t size)
   MALLOC_UNBLOCK_INPUT;
 
   if (!val && size)
-    memory_full ();
+    memory_full (size);
   return val;
 }
 
@@ -698,7 +698,8 @@ xrealloc (POINTER_TYPE *block, size_t size)
     val = (POINTER_TYPE *) realloc (block, size);
   MALLOC_UNBLOCK_INPUT;
 
-  if (!val && size) memory_full ();
+  if (!val && size)
+    memory_full (size);
   return val;
 }
 
@@ -791,7 +792,7 @@ lisp_malloc (size_t nbytes, enum mem_type type)
 
   MALLOC_UNBLOCK_INPUT;
   if (!val && nbytes)
-    memory_full ();
+    memory_full (nbytes);
   return val;
 }
 
@@ -938,7 +939,7 @@ lisp_align_malloc (size_t nbytes, enum mem_type type)
       if (base == 0)
         {
           MALLOC_UNBLOCK_INPUT;
-          memory_full ();
+          memory_full (ABLOCKS_BYTES);
         }
 
       aligned = (base == abase);
@@ -964,7 +965,7 @@ lisp_align_malloc (size_t nbytes, enum mem_type type)
               lisp_malloc_loser = base;
               free (base);
               MALLOC_UNBLOCK_INPUT;
-              memory_full ();
+              memory_full (SIZE_MAX);
             }
         }
 #endif
@@ -3270,35 +3271,58 @@ make_event_array (register int nargs, Lisp_Object *args)
  ************************************************************************/
 
 
-/* Called if malloc returns zero.  */
+/* Called if malloc (NBYTES) returns zero.  If NBYTES == SIZE_MAX,
+   there may have been size_t overflow so that malloc was never
+   called, or perhaps malloc was invoked successfully but the
+   resulting pointer had problems fitting into a tagged EMACS_INT.  In
+   either case this counts as memory being full even though malloc did
+   not fail.  */
 
 void
-memory_full (void)
+memory_full (size_t nbytes)
 {
-  int i;
+  /* Do not go into hysterics merely because a large request failed.  */
+  int enough_free_memory = 0;
+  if (SPARE_MEMORY < nbytes)
+    {
+      void *p = malloc (SPARE_MEMORY);
+      if (p)
+        {
+          if (spare_memory[0])
+            free (p);
+          else
+            spare_memory[0] = p;
+          enough_free_memory = 1;
+        }
+    }
 
-  Vmemory_full = Qt;
+  if (! enough_free_memory)
+    {
+      int i;
 
-  memory_full_cons_threshold = sizeof (struct cons_block);
+      Vmemory_full = Qt;
 
-  /* The first time we get here, free the spare memory.  */
-  for (i = 0; i < sizeof (spare_memory) / sizeof (char *); i++)
-    if (spare_memory[i])
-      {
-        if (i == 0)
-          free (spare_memory[i]);
-        else if (i >= 1 && i <= 4)
-          lisp_align_free (spare_memory[i]);
-        else
-          lisp_free (spare_memory[i]);
-        spare_memory[i] = 0;
-      }
+      memory_full_cons_threshold = sizeof (struct cons_block);
+
+      /* The first time we get here, free the spare memory.  */
+      for (i = 0; i < sizeof (spare_memory) / sizeof (char *); i++)
+        if (spare_memory[i])
+          {
+            if (i == 0)
+              free (spare_memory[i]);
+            else if (i >= 1 && i <= 4)
+              lisp_align_free (spare_memory[i]);
+            else
+              lisp_free (spare_memory[i]);
+            spare_memory[i] = 0;
+          }
 
-  /* Record the space now used.  When it decreases substantially,
-     we can refill the memory reserve.  */
+      /* Record the space now used.  When it decreases substantially,
+         we can refill the memory reserve.  */
 #if !defined SYSTEM_MALLOC && !defined SYNC_INPUT
-  bytes_used_when_full = BYTES_USED;
+      bytes_used_when_full = BYTES_USED;
 #endif
+    }
 
   /* This used to call error, but if we've run out of memory, we could
      get infinite recursion trying to build the string.  */
diff --git a/src/buffer.c b/src/buffer.c
index 05bd129976f..e9ff8f492ba 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -328,7 +328,7 @@ even if it is dead.  The return value is never nil.  */)
   alloc_buffer_text (b, BUF_GAP_SIZE (b) + 1);
   UNBLOCK_INPUT;
   if (! BUF_BEG_ADDR (b))
-    buffer_memory_full ();
+    buffer_memory_full (BUF_GAP_SIZE (b) + 1);
 
   b->pt = BEG;
   b->begv = BEG;
@@ -4892,7 +4892,7 @@ alloc_buffer_text (struct buffer *b, size_t nbytes)
   if (p == NULL)
     {
       UNBLOCK_INPUT;
-      memory_full ();
+      memory_full (nbytes);
     }
 
   b->text->beg = (unsigned char *) p;
@@ -4920,7 +4920,7 @@ enlarge_buffer_text (struct buffer *b, EMACS_INT delta)
   if (p == NULL)
     {
       UNBLOCK_INPUT;
-      memory_full ();
+      memory_full (nbytes);
     }
 
   BUF_BEG_ADDR (b) = (unsigned char *) p;
diff --git a/src/editfns.c b/src/editfns.c
index 8b48355fbfa..0e40fde9ca4 100644
--- a/src/editfns.c
+++ b/src/editfns.c
@@ -3636,7 +3636,7 @@ usage: (format STRING &rest OBJECTS)  */)
   {
     EMACS_INT i;
     if ((SIZE_MAX - formatlen) / sizeof (struct info) <= nargs)
-      memory_full ();
+      memory_full (SIZE_MAX);
     SAFE_ALLOCA (info, struct info *, (nargs + 1) * sizeof *info + formatlen);
     discarded = (char *) &info[nargs + 1];
     for (i = 0; i < nargs + 1; i++)
diff --git a/src/lisp.h b/src/lisp.h
index 26d09c6d555..ceaf7f49eb2 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -2685,8 +2685,8 @@ extern void allocate_string_data (struct Lisp_String *, EMACS_INT, EMACS_INT);
 extern void reset_malloc_hooks (void);
 extern void uninterrupt_malloc (void);
 extern void malloc_warning (const char *);
-extern void memory_full (void) NO_RETURN;
-extern void buffer_memory_full (void) NO_RETURN;
+extern void memory_full (size_t) NO_RETURN;
+extern void buffer_memory_full (EMACS_INT) NO_RETURN;
 extern int survives_gc_p (Lisp_Object);
 extern void mark_object (Lisp_Object);
 #if defined REL_ALLOC && !defined SYSTEM_MALLOC
diff --git a/src/menu.c b/src/menu.c
index e4338f349f6..7eda4c6ebb5 100644
--- a/src/menu.c
+++ b/src/menu.c
@@ -178,7 +178,7 @@ static void
 grow_menu_items (void)
 {
   if ((INT_MAX - MENU_ITEMS_PANE_LENGTH) / 2 < menu_items_allocated)
-    memory_full ();
+    memory_full (SIZE_MAX);
   menu_items_allocated *= 2;
   menu_items = larger_vector (menu_items, menu_items_allocated, Qnil);
 }
diff --git a/src/minibuf.c b/src/minibuf.c
index 3f8bd835211..fd51b0a2358 100644
--- a/src/minibuf.c
+++ b/src/minibuf.c
@@ -245,7 +245,7 @@ read_minibuf_noninteractive (Lisp_Object map, Lisp_Object initial,
              len == size - 1 && line[len - 1] != '\n'))
     {
       if ((size_t) -1 / 2 < size)
-        memory_full ();
+        memory_full (SIZE_MAX);
       size *= 2;
       line = (char *) xrealloc (line, size);
     }
diff --git a/src/xterm.c b/src/xterm.c
index 3b8112d972b..56d2ce0e9e5 100644
--- a/src/xterm.c
+++ b/src/xterm.c
@@ -4225,7 +4225,7 @@ x_send_scroll_bar_event (Lisp_Object window, int part, int portion, int whole)
       size_t old_nbytes = scroll_bar_windows_size * sizeof *scroll_bar_windows;
 
       if ((size_t) -1 / sizeof *scroll_bar_windows < new_size)
-        memory_full ();
+        memory_full (SIZE_MAX);
       scroll_bar_windows = (struct window **) xrealloc (scroll_bar_windows,
                                                         nbytes);
       memset (&scroll_bar_windows[i], 0, nbytes - old_nbytes);