Merge from emacs-24; up to 2014-04-01T20:18:12Z!eggert@cs.ucla.edu

8 files changed, 298 insertions, 32 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index b4722839fe3..d0d133c4989 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,37 @@
+2014-04-07  Stefan Monnier  <monnier@iro.umontreal.ca>
+
+        * lisp.h (struct Lisp_Symbol): New bitfield `pinned'.
+
+        * alloc.c: Keep track of symbols referenced from pure space (bug#17168).
+        (symbol_block_pinned): New var.
+        (Fmake_symbol): Initialize `pinned'.
+        (purecopy): New function, extracted from Fpurecopy.  Mark symbols as
+        pinned and signal an error for un-purifiable objects.
+        (pure_cons): Use it.
+        (Fpurecopy): Use it, except for objects that can't be purified.
+        (mark_pinned_symbols): New function.
+        (Fgarbage_collect): Use it.
+        (gc_sweep): Remove hack made unnecessary.
+
+2014-04-05  Glenn Morris  <rgm@gnu.org>
+
+        * keyboard.c (Fopen_dribble_file): Doc tweak.
+
+2014-04-04  Jan Djärv  <jan.h.d@swipnet.se>
+
+        Backport from trunk.
+        * nsterm.m (updateFrameSize:): If waiting for the tool bar and tool
+        bar is zero height, just return (Bug#16976).
+        (initFrameFromEmacs:): Initialize wait_for_tool_bar.
+        * nsterm.h (EmacsView): Add wait_for_tool_bar.
+        * nsmenu.m (update_frame_tool_bar): Return early if view or toolbar
+        is nil.  If waiting for toolbar to complete, force a redraw.
+        (free_frame_tool_bar): Set wait_for_tool_bar = NO (Bug#16976).
+
+2014-04-03  Ken Brown  <kbrown@cornell.edu>
+
+        * Makefile.in (EMACS_MANIFEST): Update comment.  (Bug#17176)
+
 2014-04-07  Paul Eggert  <eggert@cs.ucla.edu>
 
         * alloc.c: Simplify by removing use of HAVE_EXECINFO_H.
@@ -4490,7 +4524,7 @@
         * frame.c (delete_frame): Block/unblock input to overcome race
         condition (Bug#15475).
 
-2013-09-29  Andreas Politz  <politza@hochschule-trier.de>  (tiny change)
+2013-09-29  Andreas Politz  <politza@hochschule-trier.de>
 
         * frame.c (delete_frame): Record selected frame only after
         calling Qdelete_frame_functions (Bug#15477).
diff --git a/src/Makefile.in b/src/Makefile.in
index 388923596c9..7e71d030279 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -259,7 +259,7 @@ W32_LIBS=@W32_LIBS@
 
 ## emacs.res if HAVE_W32
 EMACSRES = @EMACSRES@
-## emacs-*.manifest if HAVE_W32
+## emacs-*.manifest if WINDOWSNT
 EMACS_MANIFEST = @EMACS_MANIFEST@
 ## If HAVE_W32, compiler arguments for including
 ## the resource file in the binary.
diff --git a/src/alloc.c b/src/alloc.c
index dbd1ece5d49..ea8d81648d7 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -3357,6 +3357,13 @@ struct symbol_block
 
 static struct symbol_block *symbol_block;
 static int symbol_block_index = SYMBOL_BLOCK_SIZE;
+/* Pointer to the first symbol_block that contains pinned symbols.
+   Tests for 24.4 showed that at dump-time, Emacs contains about 15K symbols,
+   10K of which are pinned (and all but 250 of them are interned in obarray),
+   whereas a "typical session" has in the order of 30K symbols.
+   `symbol_block_pinned' lets mark_pinned_symbols scan only 15K symbols rather
+   than 30K to find the 10K symbols we need to mark.  */
+static struct symbol_block *symbol_block_pinned;
 
 /* List of free symbols.  */
 
@@ -3409,10 +3416,11 @@ Its value is void, and its function definition and property list are nil.  */)
   SET_SYMBOL_VAL (p, Qunbound);
   set_symbol_function (val, Qnil);
   set_symbol_next (val, NULL);
-  p->gcmarkbit = 0;
+  p->gcmarkbit = false;
   p->interned = SYMBOL_UNINTERNED;
   p->constant = 0;
-  p->declared_special = 0;
+  p->declared_special = false;
+  p->pinned = false;
   consing_since_gc += sizeof (struct Lisp_Symbol);
   symbols_consed++;
   total_free_symbols--;
@@ -5214,6 +5222,8 @@ make_pure_c_string (const char *data, ptrdiff_t nchars)
   return string;
 }
 
+static Lisp_Object purecopy (Lisp_Object obj);
+
 /* Return a cons allocated from pure space.  Give it pure copies
    of CAR as car and CDR as cdr.  */
 
@@ -5223,8 +5233,8 @@ pure_cons (Lisp_Object car, Lisp_Object cdr)
   Lisp_Object new;
   struct Lisp_Cons *p = pure_alloc (sizeof *p, Lisp_Cons);
   XSETCONS (new, p);
-  XSETCAR (new, Fpurecopy (car));
-  XSETCDR (new, Fpurecopy (cdr));
+  XSETCAR (new, purecopy (car));
+  XSETCDR (new, purecopy (cdr));
   return new;
 }
 
@@ -5265,9 +5275,19 @@ Does not copy symbols.  Copies strings without text properties.  */)
 {
   if (NILP (Vpurify_flag))
     return obj;
-
-  if (PURE_POINTER_P (XPNTR (obj)))
+  else if (MARKERP (obj) || OVERLAYP (obj)
+           || HASH_TABLE_P (obj) || SYMBOLP (obj))
+    /* Can't purify those.  */
     return obj;
+  else
+    return purecopy (obj);
+}
+
+static Lisp_Object
+purecopy (Lisp_Object obj)
+{
+  if (PURE_POINTER_P (XPNTR (obj)) || INTEGERP (obj) || SUBRP (obj))
+    return obj;    /* Already pure.  */
 
   if (HASH_TABLE_P (Vpurify_flag)) /* Hash consing.  */
     {
@@ -5295,7 +5315,7 @@ Does not copy symbols.  Copies strings without text properties.  */)
         size &= PSEUDOVECTOR_SIZE_MASK;
       vec = XVECTOR (make_pure_vector (size));
       for (i = 0; i < size; i++)
-        vec->contents[i] = Fpurecopy (AREF (obj, i));
+        vec->contents[i] = purecopy (AREF (obj, i));
       if (COMPILEDP (obj))
         {
           XSETPVECTYPE (vec, PVEC_COMPILED);
@@ -5304,11 +5324,23 @@ Does not copy symbols.  Copies strings without text properties.  */)
       else
         XSETVECTOR (obj, vec);
     }
-  else if (MARKERP (obj))
-    error ("Attempt to copy a marker to pure storage");
+  else if (SYMBOLP (obj))
+    {
+      if (!XSYMBOL (obj)->pinned)
+        { /* We can't purify them, but they appear in many pure objects.
+             Mark them as `pinned' so we know to mark them at every GC cycle.  */
+          XSYMBOL (obj)->pinned = true;
+          symbol_block_pinned = symbol_block;
+        }
+      return obj;
+    }
   else
-    /* Not purified, don't hash-cons.  */
-    return obj;
+    {
+      Lisp_Object args[2];
+      args[0] = build_pure_c_string ("Don't know how to purify: %S");
+      args[1] = obj;
+      Fsignal (Qerror, (Fcons (Fformat (2, args), Qnil)));
+    }
 
   if (HASH_TABLE_P (Vpurify_flag)) /* Hash consing.  */
     Fputhash (obj, obj, Vpurify_flag);
@@ -5471,6 +5503,24 @@ compact_undo_list (Lisp_Object list)
   return list;
 }
 
+static void
+mark_pinned_symbols (void)
+{
+  struct symbol_block *sblk;
+  int lim = (symbol_block_pinned == symbol_block
+             ? symbol_block_index : SYMBOL_BLOCK_SIZE);
+
+  for (sblk = symbol_block_pinned; sblk; sblk = sblk->next)
+    {
+      union aligned_Lisp_Symbol *sym = sblk->symbols, *end = sym + lim;
+      for (; sym < end; ++sym)
+        if (sym->s.pinned)
+          mark_object (make_lisp_ptr (&sym->s, Lisp_Symbol));
+
+      lim = SYMBOL_BLOCK_SIZE;
+    }
+}
+
 DEFUN ("garbage-collect", Fgarbage_collect, Sgarbage_collect, 0, 0, "",
        doc: /* Reclaim storage for Lisp objects no longer needed.
 Garbage collection happens automatically if you cons more than
@@ -5573,6 +5623,7 @@ See Info node `(elisp)Garbage Collection'.  */)
   for (i = 0; i < staticidx; i++)
     mark_object (*staticvec[i]);
 
+  mark_pinned_symbols ();
   mark_specpdl ();
   mark_terminals ();
   mark_kboards ();
@@ -6574,12 +6625,7 @@ sweep_symbols (void)
 
       for (; sym < end; ++sym)
         {
-          /* Check if the symbol was created during loadup.  In such a case
-             it might be pointed to by pure bytecode which we don't trace,
-             so we conservatively assume that it is live.  */
-          bool pure_p = PURE_POINTER_P (XSTRING (sym->s.name));
-
-          if (!sym->s.gcmarkbit && !pure_p)
+          if (!sym->s.gcmarkbit)
             {
               if (sym->s.redirect == SYMBOL_LOCALIZED)
                 xfree (SYMBOL_BLV (&sym->s));
@@ -6593,8 +6639,6 @@ sweep_symbols (void)
           else
             {
               ++num_used;
-              if (!pure_p)
-                eassert (!STRING_MARKED_P (XSTRING (sym->s.name)));
               sym->s.gcmarkbit = 0;
             }
         }
diff --git a/src/bidi.c b/src/bidi.c
index b96cc24bbd1..53c2dad1b6b 100644
--- a/src/bidi.c
+++ b/src/bidi.c
@@ -22,9 +22,16 @@ along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.  */
    A sequential implementation of the Unicode Bidirectional algorithm,
    (UBA) as per UAX#9, a part of the Unicode Standard.
 
-   Unlike the reference and most other implementations, this one is
-   designed to be called once for every character in the buffer or
-   string.
+   Unlike the Reference Implementation and most other implementations,
+   this one is designed to be called once for every character in the
+   buffer or string.  That way, we can leave intact the design of the
+   Emacs display engine, whereby an iterator object is used to
+   traverse buffer or string text character by character, and generate
+   the necessary data for displaying each character in 'struct glyph'
+   objects.  (See xdisp.c for the details of that iteration.)  The
+   functions on this file replace the original linear iteration in the
+   logical order of the text with a non-linear iteration in the visual
+   order, i.e. in the order characters should be shown on display.
 
    The main entry point is bidi_move_to_visually_next.  Each time it
    is called, it finds the next character in the visual order, and
@@ -52,7 +59,182 @@ along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.  */
    A note about references to UAX#9 rules: if the reference says
    something like "X9/Retaining", it means that you need to refer to
    rule X9 and to its modifications described in the "Implementation
-   Notes" section of UAX#9, under "Retaining Format Codes".  */
+   Notes" section of UAX#9, under "Retaining Format Codes".
+
+   Here's the overview of the design of the reordering engine
+   implemented by this file.
+
+   Basic implementation structure
+   ------------------------------
+
+   The sequential processing steps described by UAX#9 are implemented
+   as recursive levels of processing, all of which examine the next
+   character in the logical order.  This hierarchy of processing looks
+   as follows, from the innermost (deepest) to the outermost level,
+   omitting some subroutines used by each level:
+
+     bidi_fetch_char         -- fetch next character
+     bidi_resolve_explicit   -- resolve explicit levels and directions
+     bidi_resolve_weak       -- resolve weak types
+     bidi_resolve_neutral    -- resolve neutral types
+     bidi_level_of_next_char -- resolve implicit levels
+
+   Each level calls the level below it, and works on the result
+   returned by the lower level, including all of its sub-levels.
+
+   Unlike all the levels below it, bidi_level_of_next_char can return
+   the information about either the next or previous character in the
+   logical order, depending on the current direction of scanning the
+   buffer or string.  For the next character, it calls all the levels
+   below it; for the previous character, it uses the cache, described
+   below.
+
+   Thus, the result of calling bidi_level_of_next_char is the resolved
+   level of the next or the previous character in the logical order.
+   Based on this information, the function bidi_move_to_visually_next
+   finds the next character in the visual order and updates the
+   direction in which the buffer is scanned, either forward or
+   backward, to find the next character to be displayed.  (Text is
+   scanned backwards when it needs to be reversed for display, i.e. if
+   the visual order is the inverse of the logical order.)  This
+   implements the last, reordering steps of the UBA, by successively
+   calling bidi_level_of_next_char until the character of the required
+   embedding level is found; the scan direction is dynamically updated
+   as a side effect.  See the commentary before the 'while' loop in
+   bidi_move_to_visually_next, for the details.
+
+   Fetching characters
+   -------------------
+
+   In a nutshell, fetching the next character boils down to calling
+   STRING_CHAR_AND_LENGTH, passing it the address of a buffer or
+   string position.  See bidi_fetch_char.  However, if the next
+   character is "covered" by a display property of some kind,
+   bidi_fetch_char returns the u+FFFC "object replacement character"
+   that represents the entire run of text covered by the display
+   property.  (The ch_len and nchars members of 'struct bidi_it'
+   reflect the length in bytes and characters of that text.)  This is
+   so we reorder text on both sides of the display property as
+   appropriate for an image or embedded string.  Similarly, text
+   covered by a display spec of the form '(space ...)', is replaced
+   with the u+2029 paragraph separator character, so such display
+   specs produce the same effect as a TAB under UBA.  Both these
+   special characters are not actually displayed -- the display
+   property is displayed instead -- but just used to compute the
+   embedding level of the surrounding text so as to produce the
+   required effect.
+
+   Bidi iterator states
+   --------------------
+
+   The UBA is highly context dependent in some of its parts,
+   i.e. results of processing a character can generally depend on
+   characters very far away.  The UAX#9 description of the UBA
+   prescribes a stateful processing of each character, whereby the
+   results of this processing depend on various state variables, such
+   as the current embedding level, level stack, and directional
+   override status.  In addition, the UAX#9 description includes many
+   passages like this (from rule W2 in this case):
+
+     Search backward from each instance of a European number until the
+     first strong type (R, L, AL, or sos) is found. If an AL is found,
+     change the type of the European number to Arabic number.
+
+   To support this, we use a bidi iterator object, 'struct bidi_it',
+   which is a sub-structure of 'struct it' used by xdisp.c (see
+   dispextern.h for the definition of both of these structures).  The
+   bidi iterator holds the entire state of the iteration required by
+   the UBA, and is updated as the text is traversed.  In particular,
+   the embedding level of the current character being resolved is
+   recorded in the iterator state.  To avoid costly searches backward
+   in support of rules like W2 above, the necessary character types
+   are also recorded in the iterator state as they are found during
+   the forward scan, and then used when such rules need to be applied.
+   (Forward scans cannot be avoided in this way; they need to be
+   performed at least once, and the results recorded in the iterator
+   state, to be reused until the forward scan oversteps the recorded
+   position.)
+
+   In this manner, the iterator state acts as a mini-cache of
+   contextual information required for resolving the level of the
+   current character by various UBA rules.
+
+   Caching of bidi iterator states
+   -------------------------------
+
+   As described above, the reordering engine uses the information
+   recorded in the bidi iterator state in order to resolve the
+   embedding level of the current character.  When the reordering
+   engine needs to process the next character in the logical order, it
+   fetches it and applies to it all the UBA levels, updating the
+   iterator state as it goes.  But when the buffer or string is
+   scanned backwards, i.e. in the reverse order of buffer/string
+   positions, the scanned characters were already processed during the
+   preceding forward scan (see bidi_find_other_level_edge).  To avoid
+   costly re-processing of characters that were already processed
+   during the forward scan, the iterator states computed while
+   scanning forward are cached.
+
+   The cache is just a linear array of 'struct bidi_it' objects, which
+   is dynamically allocated and reallocated as needed, since the size
+   of the cache depends on the text being processed.  We only need the
+   cache while processing embedded levels higher than the base
+   paragraph embedding level, because these higher levels require
+   changes in scan direction.  Therefore, as soon as we are back to
+   the base embedding level, we can free the cache; see the calls to
+   bidi_cache_reset and bidi_cache_shrink, for the conditions to do
+   this.
+
+   The cache maintains the index of the next unused cache slot -- this
+   is where the next iterator state will be cached.  The function
+   bidi_cache_iterator_state saves an instance of the state in the
+   cache and increments the unused slot index.  The companion function
+   bidi_cache_find looks up a cached state that corresponds to a given
+   buffer/string position.  All of the cached states must correspond
+   1:1 to the buffer or string region whose processing they reflect;
+   bidi.c will abort if it finds cache slots that violate this 1:1
+   correspondence.
+
+   When the parent iterator 'struct it' is pushed (see push_it in
+   xdisp.c) to pause the current iteration and start iterating over a
+   different object (e.g., a 'display' string that covers some buffer
+   text), the bidi iterator cache needs to be "pushed" as well, so
+   that a new empty cache could be used while iterating over the new
+   object.  Later, when the new object is exhausted, and xdisp.c calls
+   pop_it, we need to "pop" the bidi cache as well and return to the
+   original cache.  See bidi_push_it and bidi_pop_it for how this is
+   done.
+
+   Some functions of the display engine save copies of 'struct it' in
+   local variables, and restore them later.  For examples, see
+   pos_visible_p and move_it_in_display_line_to in xdisp.c, and
+   window_scroll_pixel_based in window.c.  When this happens, we need
+   to save and restore the bidi cache as well, because conceptually
+   the cache is part of the 'struct it' state, and needs to be in
+   perfect sync with the portion of the buffer/string that is being
+   processed.  This saving and restoring of the cache state is handled
+   by bidi_shelve_cache and bidi_unshelve_cache, and the helper macros
+   SAVE_IT and RESTORE_IT defined on xdisp.c.
+
+   Note that, because reordering is implemented below the level in
+   xdisp.c that breaks glyphs into screen lines, we are violating
+   paragraph 3.4 of UAX#9. which mandates that line breaking shall be
+   done before reordering each screen line separately.  However,
+   following UAX#9 to the letter in this matter goes against the basic
+   design of the Emacs display engine, and so we choose here this
+   minor deviation from the UBA letter in preference to redesign of
+   the display engine.  The effect of this is only seen in continued
+   lines that are broken into screen lines in the middle of a run
+   whose direction is opposite to the paragraph's base direction.
+
+   Important design and implementation note: when the code needs to
+   scan far ahead, be sure to avoid such scans as much as possible
+   when the buffer/string doesn't contain any RTL characters.  Users
+   of left-to-right scripts will never forgive you if you introduce
+   some slow-down due to bidi in situations that don't involve any
+   bidirectional text.  See the large comment near the beginning of
+   bidi_resolve_neutral, for one situation where such shortcut was
+   necessary.  */
 
 #include <config.h>
 #include <stdio.h>
diff --git a/src/buffer.h b/src/buffer.h
index d461415e036..de117eb9c61 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -847,8 +847,8 @@ struct buffer
   struct region_cache *width_run_cache;
   struct region_cache *bidi_paragraph_cache;
 
-  /* Non-zero means don't use redisplay optimizations for
-     displaying this buffer.  */
+  /* Non-zero means disable redisplay optimizations when rebuilding the glyph
+     matrices (but not when redrawing).  */
   bool_bf prevent_redisplay_optimizations_p : 1;
 
   /* Non-zero whenever the narrowing is changed in this buffer.  */
diff --git a/src/frame.h b/src/frame.h
index 62dc33511b5..ff696df9eff 100644
--- a/src/frame.h
+++ b/src/frame.h
@@ -375,7 +375,8 @@ struct frame
      set this directly, use SET_FRAME_ICONIFIED instead.  */
   bool_bf iconified : 1;
 
-  /* True if this frame should be redrawn.  */
+  /* True if this frame should be fully redisplayed.  Disables all
+     optimizations while rebuilding matrices and redrawing.  */
   bool_bf garbaged : 1;
 
   /* False means, if this frame has just one window,
diff --git a/src/keyboard.c b/src/keyboard.c
index 277d4f1047c..87a2f91a17c 100644
--- a/src/keyboard.c
+++ b/src/keyboard.c
@@ -1,7 +1,6 @@
 /* Keyboard and mouse input; editor command loop.
 
-Copyright (C) 1985-1989, 1993-1997, 1999-2014 Free Software Foundation,
-Inc.
+Copyright (C) 1985-1989, 1993-1997, 1999-2014 Free Software Foundation, Inc.
 
 This file is part of GNU Emacs.
 
@@ -2379,7 +2378,7 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
    -2 means do neither.
    1 means do both.  */
 
-/* The arguments MAP is for menu prompting.  MAP is a keymap.
+/* The argument MAP is a keymap for menu prompting.
 
    PREV_EVENT is the previous input event, or nil if we are reading
    the first event of a key sequence (or not reading a key sequence).
@@ -10068,7 +10067,10 @@ DEFUN ("open-dribble-file", Fopen_dribble_file, Sopen_dribble_file, 1, 1,
        "FOpen dribble file: ",
        doc: /* Start writing all keyboard characters to a dribble file called FILE.
 If FILE is nil, close any open dribble file.
-The file will be closed when Emacs exits.  */)
+The file will be closed when Emacs exits.
+
+Be aware that this records ALL characters you type!
+This may include sensitive information such as passwords.  */)
   (Lisp_Object file)
 {
   if (dribble)
diff --git a/src/lisp.h b/src/lisp.h
index 03c0d99fb27..264b8832092 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -1572,6 +1572,9 @@ struct Lisp_Symbol
      special (with `defvar' etc), and shouldn't be lexically bound.  */
   bool_bf declared_special : 1;
 
+  /* True if pointed to from purespace and hence can't be GC'd.  */
+  bool_bf pinned : 1;
+
   /* The symbol's name, as a Lisp string.  */
   Lisp_Object name;