Merge from trunk.

6 files changed, 121 insertions, 40 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 4f1edc95668..4f0a0b9ab74 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,4 +1,4 @@
-2011-09-27  Paul Eggert  <eggert@cs.ucla.edu>
+2011-09-28  Paul Eggert  <eggert@cs.ucla.edu>
 
         * alloc.c (pure_bytes_used_lisp, pure_bytes_used_non_lisp):
         (allocate_vectorlike, buffer_memory_full, struct sdata, SDATA_SIZE)
@@ -775,6 +775,27 @@
         rather than rolling our own approximation.
         (SCROLL_BAR_VEC_SIZE): Remove; not used.
 
+2011-09-28  Eli Zaretskii  <eliz@gnu.org>
+
+        * xdisp.c (compute_display_string_end): If there's no display
+        string at CHARPOS, return -1.
+
+        * bidi.c (bidi_fetch_char): When compute_display_string_end
+        returns a negative value, treat the character as a normal
+        character not covered by a display string.  (Bug#9624)
+
+2011-09-28  Juanma Barranquero  <lekktu@gmail.com>
+
+        * lread.c (Fread_from_string): Fix typo in docstring.
+
+2011-09-27  Eli Zaretskii  <eliz@gnu.org>
+
+        * xdisp.c (handle_invisible_prop): If invisible text ends on a
+        newline, reseat the iterator instead of bidi-iterating there one
+        character at a time.  (Bug#9610)
+        (BUFFER_POS_REACHED_P, move_it_in_display_line_to): Bail when past
+        TO_CHARPOS if the bidi iterator is at base embedding level.
+
 2011-09-27  Andreas Schwab  <schwab@linux-m68k.org>
 
         * lread.c (readevalloop): Use correct code for NBSP.
diff --git a/src/ChangeLog.11 b/src/ChangeLog.11
index b64736459c8..0a9df7d1aee 100644
--- a/src/ChangeLog.11
+++ b/src/ChangeLog.11
@@ -4547,7 +4547,7 @@
 
         * composite.c (composition_compute_stop_pos): In forward search,
         pay attention to the possibility that some character after ENDPOS
-        will be composed with charactrs before ENDPOS.
+        will be composed with characters before ENDPOS.
 
 2010-08-24  Chong Yidong  <cyd@stupidchicken.com>
 
diff --git a/src/ChangeLog.9 b/src/ChangeLog.9
index f25434087c1..ceec5da3296 100644
--- a/src/ChangeLog.9
+++ b/src/ChangeLog.9
@@ -10460,7 +10460,7 @@
         (Fkey_description): Likewise.
 
         * lread.c (read1): On reading multibyte string, be sure to make
-        all 8-bit chararacters in valid multibyte form.
+        all 8-bit characters in valid multibyte form.
         (readchar): Use FETCH_STRING_CHAR_ADVANCE unconditionally.
 
         * print.c (print_object): Use FETCH_STRING_CHAR_ADVANCE
diff --git a/src/bidi.c b/src/bidi.c
index c7b3a770597..a43490f4110 100644
--- a/src/bidi.c
+++ b/src/bidi.c
@@ -974,6 +974,15 @@ bidi_fetch_char (ptrdiff_t bytepos, ptrdiff_t charpos, ptrdiff_t *disp_pos,
           ch = 0xFFFC;
         }
       disp_end_pos = compute_display_string_end (*disp_pos, string);
+      if (disp_end_pos < 0)
+        {
+          /* Somebody removed the display string from the buffer
+             behind our back.  Recover by processing this buffer
+             position as if no display property were present there to
+             begin with.  */
+          *disp_prop = 0;
+          goto normal_char;
+        }
       *nchars = disp_end_pos - *disp_pos;
       if (*nchars <= 0)
         abort ();
@@ -988,6 +997,7 @@ bidi_fetch_char (ptrdiff_t bytepos, ptrdiff_t charpos, ptrdiff_t *disp_pos,
     }
   else
     {
+    normal_char:
       if (string->s)
         {
           int len;
diff --git a/src/lread.c b/src/lread.c
index 1179d63436f..91195a88e63 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -1971,7 +1971,7 @@ DEFUN ("read-from-string", Fread_from_string, Sread_from_string, 1, 3, 0,
        doc: /* Read one Lisp expression which is represented as text by STRING.
 Returns a cons: (OBJECT-READ . FINAL-STRING-INDEX).
 FINAL-STRING-INDEX is an integer giving the position of the next
- remaining chararacter in STRING.
+ remaining character in STRING.
 START and END optionally delimit a substring of STRING from which to read;
  they default to 0 and (length STRING) respectively.  */)
   (Lisp_Object string, Lisp_Object start, Lisp_Object end)
diff --git a/src/xdisp.c b/src/xdisp.c
index e6a38f76dc5..ce240922010 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -3389,9 +3389,10 @@ compute_display_string_pos (struct text_pos *position,
 }
 
 /* Return the character position of the end of the display string that
-   started at CHARPOS.  A display string is either an overlay with
-   `display' property whose value is a string or a `display' text
-   property whose value is a string.  */
+   started at CHARPOS.  If there's no display string at CHARPOS,
+   return -1.  A display string is either an overlay with `display'
+   property whose value is a string or a `display' text property whose
+   value is a string.  */
 ptrdiff_t
 compute_display_string_end (ptrdiff_t charpos, struct bidi_string_data *string)
 {
@@ -3405,8 +3406,22 @@ compute_display_string_end (ptrdiff_t charpos, struct bidi_string_data *string)
   if (charpos >= eob || (string->s && !STRINGP (object)))
     return eob;
 
+  /* It could happen that the display property or overlay was removed
+     since we found it in compute_display_string_pos above.  One way
+     this can happen is if JIT font-lock was called (through
+     handle_fontified_prop), and jit-lock-functions remove text
+     properties or overlays from the portion of buffer that includes
+     CHARPOS.  Muse mode is known to do that, for example.  In this
+     case, we return -1 to the caller, to signal that no display
+     string is actually present at CHARPOS.  See bidi_fetch_char for
+     how this is handled.
+
+     An alternative would be to never look for display properties past
+     it->stop_charpos.  But neither compute_display_string_pos nor
+     bidi_fetch_char that calls it know or care where the next
+     stop_charpos is.  */
   if (NILP (Fget_char_property (pos, Qdisplay, object)))
-    abort ();
+    return -1;
 
   /* Look forward for the first character where the `display' property
      changes.  */
@@ -4059,40 +4074,67 @@ handle_invisible_prop (struct it *it)
           /* The position newpos is now either ZV or on visible text.  */
           if (it->bidi_p && newpos < ZV)
             {
-              /* With bidi iteration, the region of invisible text
-                 could start and/or end in the middle of a non-base
-                 embedding level.  Therefore, we need to skip
-                 invisible text using the bidi iterator, starting at
-                 IT's current position, until we find ourselves
-                 outside the invisible text.  Skipping invisible text
-                 _after_ bidi iteration avoids affecting the visual
-                 order of the displayed text when invisible properties
-                 are added or removed.  */
-              if (it->bidi_it.first_elt && it->bidi_it.charpos < ZV)
+              ptrdiff_t bpos = CHAR_TO_BYTE (newpos);
+
+              if (FETCH_BYTE (bpos) == '\n'
+                  || (newpos > BEGV && FETCH_BYTE (bpos - 1) == '\n'))
                 {
-                  /* If we were `reseat'ed to a new paragraph,
-                     determine the paragraph base direction.  We need
-                     to do it now because next_element_from_buffer may
-                     not have a chance to do it, if we are going to
-                     skip any text at the beginning, which resets the
-                     FIRST_ELT flag.  */
-                  bidi_paragraph_init (it->paragraph_embedding,
-                                       &it->bidi_it, 1);
+                  /* If the invisible text ends on a newline or the
+                     character after a newline, we can avoid the
+                     costly, character by character, bidi iteration to
+                     newpos, and instead simply reseat the iterator
+                     there.  That's because all bidi reordering
+                     information is tossed at the newline.  This is a
+                     big win for modes that hide complete lines, like
+                     Outline, Org, etc.  (Implementation note: the
+                     call to reseat_1 is necessary, because it signals
+                     to the bidi iterator that it needs to reinit its
+                     internal information when the next element for
+                     display is requested.  */
+                  struct text_pos tpos;
+
+                  SET_TEXT_POS (tpos, newpos, bpos);
+                  reseat_1 (it, tpos, 0);
                 }
-              do
+              else      /* Must use the slow method.  */
                 {
-                  bidi_move_to_visually_next (&it->bidi_it);
+                  /* With bidi iteration, the region of invisible text
+                     could start and/or end in the middle of a
+                     non-base embedding level.  Therefore, we need to
+                     skip invisible text using the bidi iterator,
+                     starting at IT's current position, until we find
+                     ourselves outside the invisible text.  Skipping
+                     invisible text _after_ bidi iteration avoids
+                     affecting the visual order of the displayed text
+                     when invisible properties are added or
+                     removed.  */
+                  if (it->bidi_it.first_elt && it->bidi_it.charpos < ZV)
+                    {
+                      /* If we were `reseat'ed to a new paragraph,
+                         determine the paragraph base direction.  We
+                         need to do it now because
+                         next_element_from_buffer may not have a
+                         chance to do it, if we are going to skip any
+                         text at the beginning, which resets the
+                         FIRST_ELT flag.  */
+                      bidi_paragraph_init (it->paragraph_embedding,
+                                           &it->bidi_it, 1);
+                    }
+                  do
+                    {
+                      bidi_move_to_visually_next (&it->bidi_it);
+                    }
+                  while (it->stop_charpos <= it->bidi_it.charpos
+                         && it->bidi_it.charpos < newpos);
+                  IT_CHARPOS (*it) = it->bidi_it.charpos;
+                  IT_BYTEPOS (*it) = it->bidi_it.bytepos;
+                  /* If we overstepped NEWPOS, record its position in
+                     the iterator, so that we skip invisible text if
+                     later the bidi iteration lands us in the
+                     invisible region again. */
+                  if (IT_CHARPOS (*it) >= newpos)
+                    it->prev_stop = newpos;
                 }
-              while (it->stop_charpos <= it->bidi_it.charpos
-                     && it->bidi_it.charpos < newpos);
-              IT_CHARPOS (*it) = it->bidi_it.charpos;
-              IT_BYTEPOS (*it) = it->bidi_it.bytepos;
-              /* If we overstepped NEWPOS, record its position in the
-                 iterator, so that we skip invisible text if later the
-                 bidi iteration lands us in the invisible region
-                 again. */
-              if (IT_CHARPOS (*it) >= newpos)
-                it->prev_stop = newpos;
             }
           else
             {
@@ -7886,7 +7928,9 @@ move_it_in_display_line_to (struct it *it,
   ((op & MOVE_TO_POS) != 0                                      \
    && BUFFERP (it->object)                                      \
    && (IT_CHARPOS (*it) == to_charpos                           \
-       || (!it->bidi_p && IT_CHARPOS (*it) > to_charpos)        \
+       || ((!it->bidi_p                                         \
+            || BIDI_AT_BASE_LEVEL (it->bidi_it))                \
+           && IT_CHARPOS (*it) > to_charpos)                    \
        || (it->what == IT_COMPOSITION                           \
            && ((IT_CHARPOS (*it) > to_charpos                   \
                 && to_charpos >= it->cmp_it.charpos)            \
@@ -7918,7 +7962,13 @@ move_it_in_display_line_to (struct it *it,
       if ((op & MOVE_TO_POS) != 0
           && BUFFERP (it->object)
           && it->method == GET_FROM_BUFFER
-          && ((!it->bidi_p && IT_CHARPOS (*it) > to_charpos)
+          && (((!it->bidi_p
+                /* When the iterator is at base embedding level, we
+                   are guaranteed that characters are delivered for
+                   display in strictly increasing order of their
+                   buffer positions.  */
+                || BIDI_AT_BASE_LEVEL (it->bidi_it))
+               && IT_CHARPOS (*it) > to_charpos)
               || (it->bidi_p
                   && (prev_method == GET_FROM_IMAGE
                       || prev_method == GET_FROM_STRETCH