Merge from trunk.

6 files changed, 257 insertions, 124 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 7b21a47d59a..34914c96c6c 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,4 +1,4 @@
-2011-10-17  Paul Eggert  <eggert@cs.ucla.edu>
+2011-10-23  Paul Eggert  <eggert@cs.ucla.edu>
 
         Fix integer width and related bugs.
         * alloc.c (pure_bytes_used_lisp, pure_bytes_used_non_lisp):
@@ -789,6 +789,83 @@
         rather than rolling our own approximation.
         (SCROLL_BAR_VEC_SIZE): Remove; not used.
 
+2011-10-20  Eli Zaretskii  <eliz@gnu.org>
+
+        * dispextern.h (struct bidi_it): New member next_en_type.
+
+        * bidi.c (bidi_line_init): Initialize the next_en_type member.
+        (bidi_resolve_explicit_1): When next_en_pos is valid for the
+        current character, check also for next_en_type being WEAK_EN.
+        (bidi_resolve_weak): Don't enter the expensive loop if the current
+        position is before next_en_pos.  Record the bidi type of the first
+        non-ET, non-BN character we find, in addition to its position.
+        (bidi_level_of_next_char): Invalidate next_en_type when
+        next_en_pos is over-stepped.
+
+2011-10-20  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Time zone name fixes for non-ASCII locales (Bug#641, Bug#9794)
+        * editfns.c: Rewrite current-time-zone so that it invokes
+        the equivalent of (format-time-string "%Z") to get the time zone name.
+        This fixes a bug when the time zone name contains characters that
+        need converting from the system time locale to Emacs internal format.
+        This fixes a shortcoming that I introduced in my 1999-10-19 patch:
+        that patch fixed format-time-string to do the conversion, but
+        I forgot to fix current-time-zone.
+        (format_time_string): New function, containing most of
+        what Fformat_time_string used to contain.
+        (Fformat_time_string): Rewrite in terms of format_time_string.
+        This doesn't change this function's behavior.
+        (current-time-zone): Rewrite to use format_time_string.
+        This fixes the bug reported by Michael Schierl in
+        <http://lists.gnu.org/archive/html/emacs-devel/2007-06/msg00334.html>.
+        Jason Rumney's 2007-06-07 change worked around this bug, but
+        didn't fix it.
+        * systime.h (tzname, timezone): Remove no-longer-used declarations.
+
+2011-10-19  Eli Zaretskii  <eliz@gnu.org>
+
+        * xdisp.c (start_display): If the character at POS is displayed
+        via a display vector, reset IT->current.dpvec_index to zero.
+        (try_window_reusing_current_matrix): If a line ends in a display
+        vector or the next line starts in a display vector, continue
+        redrawing the window even though the character position of
+        start_row was reached.
+        (Bug#9771, part 2)
+
+2011-10-18  Chong Yidong  <cyd@gnu.org>
+
+        * xdisp.c (get_next_display_element): Handle U+2010 and U+2011
+        with nobreak-char-display too.
+
+2011-10-18  Eli Zaretskii  <eliz@gnu.org>
+
+        Fix part 3 of bug#9771.
+        * bidi.c (bidi_line_init): Initialize next_en_pos to zero, not -1.
+        (bidi_resolve_neutral): Don't enter the expensive loop looking for
+        non-neutral characters if the current character is a paragraph
+        separator (a.k.a. Newline).  This avoids running the same
+        expensive loop twice, once when we consume the preceding newline
+        and the other time when the line actually needs to be displayed.
+        Avoid the loop when we see neutrals on the base embedding level
+        following a character whose directionality is the same as the
+        paragraph's.  This avoids running the expensive loop when a line
+        ends in a long sequence of neutrals, like control characters.
+        Add assertion against STRONG_AL type.  Slightly rearrange code
+        that determines the type of a neutral given the first non-neutral
+        that follows it.
+        (bidi_level_of_next_char): Set next_en_pos to zero when
+        invalidating its info.
+
+2011-10-17  Eli Zaretskii  <eliz@gnu.org>
+
+        * xdisp.c (push_display_prop): Determine whether to record string
+        or buffer position by IT->string, not by IT->method.  Allow
+        GET_FROM_DISPLAY_VECTOR as IT->method on entry.  (Bug#9771, part 4)
+        (move_it_vertically_backward): Don't look for character position
+        immediately after the newline when in a continuation line.
+        (Bug#9771, part 1)
+
 2011-10-15  Martin Rudalics  <rudalics@gmx.at>
 
         * window.c (coordinates_in_window): Rewrite and delabelize
diff --git a/src/bidi.c b/src/bidi.c
index c6d7db96576..e8f2df89a9e 100644
--- a/src/bidi.c
+++ b/src/bidi.c
@@ -846,7 +846,10 @@ bidi_line_init (struct bidi_it *bidi_it)
   bidi_it->level_stack[0].override = NEUTRAL_DIR; /* X1 */
   bidi_it->invalid_levels = 0;
   bidi_it->invalid_rl_levels = -1;
-  bidi_it->next_en_pos = -1;
+  /* Setting this to zero will force its recomputation the first time
+     we need it for W5.  */
+  bidi_it->next_en_pos = 0;
+  bidi_it->next_en_type = UNKNOWN_BT;
   bidi_it->next_for_ws.type = UNKNOWN_BT;
   bidi_set_sor_type (bidi_it,
                      (bidi_it->paragraph_dir == R2L ? 1 : 0),
@@ -1435,7 +1438,8 @@ bidi_resolve_explicit_1 (struct bidi_it *bidi_it)
               }
           }
         else if (bidi_it->prev.type_after_w1 == WEAK_EN /* W5/Retaining */
-                 || bidi_it->next_en_pos > bidi_it->charpos)
+                 || (bidi_it->next_en_pos > bidi_it->charpos
+                     && bidi_it->next_en_type == WEAK_EN))
           type = WEAK_EN;
         break;
       case LRE: /* X3 */
@@ -1471,7 +1475,8 @@ bidi_resolve_explicit_1 (struct bidi_it *bidi_it)
               }
           }
         else if (bidi_it->prev.type_after_w1 == WEAK_EN /* W5/Retaining */
-                 || bidi_it->next_en_pos > bidi_it->charpos)
+                 || (bidi_it->next_en_pos > bidi_it->charpos
+                     && bidi_it->next_en_type == WEAK_EN))
           type = WEAK_EN;
         break;
       case PDF: /* X7 */
@@ -1497,7 +1502,8 @@ bidi_resolve_explicit_1 (struct bidi_it *bidi_it)
               }
           }
         else if (bidi_it->prev.type_after_w1 == WEAK_EN /* W5/Retaining */
-                 || bidi_it->next_en_pos > bidi_it->charpos)
+                 || (bidi_it->next_en_pos > bidi_it->charpos
+                     && bidi_it->next_en_type == WEAK_EN))
           type = WEAK_EN;
         break;
       default:
@@ -1729,10 +1735,15 @@ bidi_resolve_weak (struct bidi_it *bidi_it)
       else if (type == WEAK_ET  /* W5: ET with EN before or after it */
                || type == WEAK_BN)      /* W5/Retaining */
         {
-          if (bidi_it->prev.type_after_w1 == WEAK_EN /* ET/BN w/EN before it */
-              || bidi_it->next_en_pos > bidi_it->charpos)
+          if (bidi_it->prev.type_after_w1 == WEAK_EN) /* ET/BN w/EN before it */
             type = WEAK_EN;
-          else                  /* W5: ET/BN with EN after it.  */
+          else if (bidi_it->next_en_pos > bidi_it->charpos
+                   && bidi_it->next_en_type != WEAK_BN)
+            {
+              if (bidi_it->next_en_type == WEAK_EN) /* ET/BN with EN after it */
+                type = WEAK_EN;
+            }
+          else if (bidi_it->next_en_pos >=0)
             {
               ptrdiff_t en_pos = bidi_it->charpos + bidi_it->nchars;
               const unsigned char *s = (STRINGP (bidi_it->string.lstring)
@@ -1761,20 +1772,27 @@ bidi_resolve_weak (struct bidi_it *bidi_it)
                   en_pos = bidi_it->charpos;
                   bidi_copy_it (bidi_it, &saved_it);
                 }
+              /* Remember this position, to speed up processing of the
+                 next ETs.  */
+              bidi_it->next_en_pos = en_pos;
               if (type_of_next == WEAK_EN)
                 {
                   /* If the last strong character is AL, the EN we've
                      found will become AN when we get to it (W2). */
-                  if (bidi_it->last_strong.type_after_w1 != STRONG_AL)
-                    {
-                      type = WEAK_EN;
-                      /* Remember this EN position, to speed up processing
-                         of the next ETs.  */
-                      bidi_it->next_en_pos = en_pos;
-                    }
+                  if (bidi_it->last_strong.type_after_w1 == STRONG_AL)
+                    type_of_next = WEAK_AN;
                   else if (type == WEAK_BN)
                     type = NEUTRAL_ON; /* W6/Retaining */
+                  else
+                    type = WEAK_EN;
                 }
+              else if (type_of_next == NEUTRAL_B)
+                /* Record the fact that there are no more ENs from
+                   here to the end of paragraph, to avoid entering the
+                   loop above ever again in this paragraph.  */
+                bidi_it->next_en_pos = -1;
+              /* Record the type of the character where we ended our search.  */
+              bidi_it->next_en_type = type_of_next;
             }
         }
     }
@@ -1843,13 +1861,45 @@ bidi_resolve_neutral (struct bidi_it *bidi_it)
         || type == NEUTRAL_ON))
     abort ();
 
-  if (bidi_get_category (type) == NEUTRAL
+  if ((type != NEUTRAL_B /* Don't risk entering the long loop below if
+                            we are already at paragraph end.  */
+       && bidi_get_category (type) == NEUTRAL)
       || (type == WEAK_BN && prev_level == current_level))
     {
       if (bidi_it->next_for_neutral.type != UNKNOWN_BT)
         type = bidi_resolve_neutral_1 (bidi_it->prev_for_neutral.type,
                                        bidi_it->next_for_neutral.type,
                                        current_level);
+      /* The next two "else if" clauses are shortcuts for the
+         important special case when we have a long sequence of
+         neutral or WEAK_BN characters, such as whitespace or nulls or
+         other control characters, on the base embedding level of the
+         paragraph, and that sequence goes all the way to the end of
+         the paragraph and follows a character whose resolved
+         directionality is identical to the base embedding level.
+         (This is what happens in a buffer with plain L2R text that
+         happens to include long sequences of control characters.)  By
+         virtue of N1, the result of examining this long sequence will
+         always be either STRONG_L or STRONG_R, depending on the base
+         embedding level.  So we use this fact directly instead of
+         entering the expensive loop in the "else" clause.  */
+      else if (current_level == 0
+               && bidi_it->prev_for_neutral.type == STRONG_L
+               && !bidi_explicit_dir_char (bidi_it->ch))
+        type = bidi_resolve_neutral_1 (bidi_it->prev_for_neutral.type,
+                                       STRONG_L, current_level);
+      else if (/* current level is 1 */
+               current_level == 1
+               /* base embedding level is also 1 */
+               && bidi_it->level_stack[0].level == 1
+               /* previous character is one of those considered R for
+                  the purposes of W5 */
+               && (bidi_it->prev_for_neutral.type == STRONG_R
+                   || bidi_it->prev_for_neutral.type == WEAK_EN
+                   || bidi_it->prev_for_neutral.type == WEAK_AN)
+               && !bidi_explicit_dir_char (bidi_it->ch))
+        type = bidi_resolve_neutral_1 (bidi_it->prev_for_neutral.type,
+                                       STRONG_R, current_level);
       else
         {
           /* Arrrgh!!  The UAX#9 algorithm is too deeply entrenched in
@@ -1900,6 +1950,9 @@ bidi_resolve_neutral (struct bidi_it *bidi_it)
               case STRONG_L:
               case STRONG_R:
               case STRONG_AL:
+                /* Actually, STRONG_AL cannot happen here, because
+                   bidi_resolve_weak converts it to STRONG_R, per W3.  */
+                xassert (type != STRONG_AL);
                 next_type = type;
                 break;
               case WEAK_EN:
@@ -1907,7 +1960,6 @@ bidi_resolve_neutral (struct bidi_it *bidi_it)
                 /* N1: ``European and Arabic numbers are treated as
                    though they were R.''  */
                 next_type = STRONG_R;
-                saved_it.next_for_neutral.type = STRONG_R;
                 break;
               case WEAK_BN:
                 if (!bidi_explicit_dir_char (bidi_it->ch))
@@ -1920,11 +1972,7 @@ bidi_resolve_neutral (struct bidi_it *bidi_it)
                    member.  */
                 if (saved_it.type != WEAK_BN
                     || bidi_get_category (bidi_it->prev.type_after_w1) == NEUTRAL)
-                  {
-                    next_type = bidi_it->prev_for_neutral.type;
-                    saved_it.next_for_neutral.type = next_type;
-                    bidi_check_type (next_type);
-                  }
+                  next_type = bidi_it->prev_for_neutral.type;
                 else
                   {
                     /* This is a BN which does not adjoin neutrals.
@@ -1938,7 +1986,9 @@ bidi_resolve_neutral (struct bidi_it *bidi_it)
             }
           type = bidi_resolve_neutral_1 (saved_it.prev_for_neutral.type,
                                          next_type, current_level);
+          saved_it.next_for_neutral.type = next_type;
           saved_it.type = type;
+          bidi_check_type (next_type);
           bidi_check_type (type);
           bidi_copy_it (bidi_it, &saved_it);
         }
@@ -2014,7 +2064,10 @@ bidi_level_of_next_char (struct bidi_it *bidi_it)
         bidi_it->next_for_neutral.type = UNKNOWN_BT;
       if (bidi_it->next_en_pos >= 0
           && bidi_it->charpos >= bidi_it->next_en_pos)
-        bidi_it->next_en_pos = -1;
+        {
+          bidi_it->next_en_pos = 0;
+          bidi_it->next_en_type = UNKNOWN_BT;
+        }
       if (bidi_it->next_for_ws.type != UNKNOWN_BT
           && bidi_it->charpos >= bidi_it->next_for_ws.charpos)
         bidi_it->next_for_ws.type = UNKNOWN_BT;
@@ -2140,7 +2193,7 @@ bidi_level_of_next_char (struct bidi_it *bidi_it)
     }
 
   /* Resolve implicit levels, with a twist: PDFs get the embedding
-     level of the enbedding they terminate.  See below for the
+     level of the embedding they terminate.  See below for the
      reason.  */
   if (bidi_it->orig_type == PDF
       /* Don't do this if this formatting code didn't change the
diff --git a/src/dispextern.h b/src/dispextern.h
index dfef2884297..5f2c844b58d 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -1856,7 +1856,8 @@ struct bidi_it {
   struct bidi_saved_info next_for_neutral; /* surrounding characters for... */
   struct bidi_saved_info prev_for_neutral; /* ...resolving neutrals */
   struct bidi_saved_info next_for_ws; /* character after sequence of ws */
-  ptrdiff_t next_en_pos;        /* position of next EN char for ET */
+  ptrdiff_t next_en_pos;        /* pos. of next char for determining ET type */
+  bidi_type_t next_en_type;     /* type of char at next_en_pos */
   ptrdiff_t ignore_bn_limit;    /* position until which to ignore BNs */
   bidi_dir_t sor;               /* direction of start-of-run in effect */
   int scan_dir;                 /* direction of text scan, 1: forw, -1: back */
diff --git a/src/editfns.c b/src/editfns.c
index b376c3a7c51..16e552afe1d 100644
--- a/src/editfns.c
+++ b/src/editfns.c
@@ -85,6 +85,8 @@ extern Lisp_Object w32_get_internal_run_time (void);
 #endif
 
 static void time_overflow (void) NO_RETURN;
+static Lisp_Object format_time_string (char const *, ptrdiff_t, Lisp_Object,
+                                       int, time_t *, struct tm **);
 static int tm_diff (struct tm *, struct tm *);
 static void update_buffer_properties (ptrdiff_t, ptrdiff_t);
 
@@ -1687,32 +1689,40 @@ For example, to produce full ISO 8601 format, use "%Y-%m-%dT%T%z".
 usage: (format-time-string FORMAT-STRING &optional TIME UNIVERSAL)  */)
   (Lisp_Object format_string, Lisp_Object timeval, Lisp_Object universal)
 {
-  time_t value;
+  time_t t;
+  struct tm *tm;
+
+  CHECK_STRING (format_string);
+  format_string = code_convert_string_norecord (format_string,
+                                                Vlocale_coding_system, 1);
+  return format_time_string (SSDATA (format_string), SBYTES (format_string),
+                             timeval, ! NILP (universal), &t, &tm);
+}
+
+static Lisp_Object
+format_time_string (char const *format, ptrdiff_t formatlen,
+                    Lisp_Object timeval, int ut, time_t *tval, struct tm **tmp)
+{
   ptrdiff_t size;
   int usec;
   int ns;
   struct tm *tm;
-  int ut = ! NILP (universal);
-
-  CHECK_STRING (format_string);
 
-  if (! lisp_time_argument (timeval, &value, &usec))
+  if (! lisp_time_argument (timeval, tval, &usec))
     error ("Invalid time specification");
   ns = usec * 1000;
 
-  format_string = code_convert_string_norecord (format_string,
-                                                Vlocale_coding_system, 1);
-
   /* This is probably enough.  */
-  size = SBYTES (format_string);
+  size = formatlen;
   if (size <= (STRING_BYTES_BOUND - 50) / 6)
     size = size * 6 + 50;
 
   BLOCK_INPUT;
-  tm = ut ? gmtime (&value) : localtime (&value);
+  tm = ut ? gmtime (tval) : localtime (tval);
   UNBLOCK_INPUT;
   if (! tm)
     time_overflow ();
+  *tmp = tm;
 
   synchronize_system_time_locale ();
 
@@ -1723,9 +1733,7 @@ usage: (format-time-string FORMAT-STRING &optional TIME UNIVERSAL)  */)
 
       buf[0] = '\1';
       BLOCK_INPUT;
-      result = emacs_nmemftime (buf, size, SSDATA (format_string),
-                                SBYTES (format_string),
-                                tm, ut, ns);
+      result = emacs_nmemftime (buf, size, format, formatlen, tm, ut, ns);
       UNBLOCK_INPUT;
       if ((result > 0 && result < size) || (result == 0 && buf[0] == '\0'))
         return code_convert_string_norecord (make_unibyte_string (buf, result),
@@ -1733,9 +1741,7 @@ usage: (format-time-string FORMAT-STRING &optional TIME UNIVERSAL)  */)
 
       /* If buffer was too small, make it bigger and try again.  */
       BLOCK_INPUT;
-      result = emacs_nmemftime (NULL, (size_t) -1,
-                                SSDATA (format_string),
-                                SBYTES (format_string),
+      result = emacs_nmemftime (NULL, (size_t) -1, format, formatlen,
                                 tm, ut, ns);
       UNBLOCK_INPUT;
       if (STRING_BYTES_BOUND <= result)
@@ -1983,51 +1989,34 @@ the data it can't find.  */)
 {
   time_t value;
   struct tm *t;
-  struct tm gmt;
-
-  if (!lisp_time_argument (specified_time, &value, NULL))
-    t = NULL;
-  else
-    {
-      BLOCK_INPUT;
-      t = gmtime (&value);
-      if (t)
-        {
-          gmt = *t;
-          t = localtime (&value);
-        }
-      UNBLOCK_INPUT;
-    }
+  struct tm localtm;
+  struct tm *localt;
+  Lisp_Object zone_offset, zone_name;
+
+  zone_offset = Qnil;
+  zone_name = format_time_string ("%Z", sizeof "%Z" - 1, specified_time,
+                                  0, &value, &localt);
+  localtm = *localt;
+  BLOCK_INPUT;
+  t = gmtime (&value);
+  UNBLOCK_INPUT;
 
   if (t)
     {
-      int offset = tm_diff (t, &gmt);
-      char *s = 0;
-      char buf[sizeof "+00" + INT_STRLEN_BOUND (int)];
-
-#ifdef HAVE_TM_ZONE
-      if (t->tm_zone)
-        s = (char *)t->tm_zone;
-#else /* not HAVE_TM_ZONE */
-#ifdef HAVE_TZNAME
-      if (t->tm_isdst == 0 || t->tm_isdst == 1)
-        s = tzname[t->tm_isdst];
-#endif
-#endif /* not HAVE_TM_ZONE */
-
-      if (!s)
+      int offset = tm_diff (&localtm, t);
+      zone_offset = make_number (offset);
+      if (SCHARS (zone_name) == 0)
         {
           /* No local time zone name is available; use "+-NNNN" instead.  */
           int m = offset / 60;
           int am = offset < 0 ? - m : m;
+          char buf[sizeof "+00" + INT_STRLEN_BOUND (int)];
           sprintf (buf, "%c%02d%02d", (offset < 0 ? '-' : '+'), am/60, am%60);
-          s = buf;
+          zone_name = build_string (buf);
         }
-
-      return Fcons (make_number (offset), Fcons (build_string (s), Qnil));
     }
-  else
-    return Fmake_list (make_number (2), Qnil);
+
+  return list2 (zone_offset, zone_name);
 }
 
 /* This holds the value of `environ' produced by the previous
diff --git a/src/systime.h b/src/systime.h
index bed9ed4aa71..b90372dbe20 100644
--- a/src/systime.h
+++ b/src/systime.h
@@ -38,17 +38,6 @@ typedef unsigned long Time;
 # endif
 #endif
 
-#ifdef HAVE_TZNAME
-#ifndef tzname          /* For SGI.  */
-extern char *tzname[];  /* RS6000 and others want it this way.  */
-#endif
-#endif
-
-/* SVr4 doesn't actually declare this in its #include files.  */
-#ifdef USG5_4
-extern time_t timezone;
-#endif
-
 /* On some configurations (hpux8.0, X11R4), sys/time.h and X11/Xos.h
    disagree about the name of the guard symbol.  */
 #ifdef HPUX
diff --git a/src/xdisp.c b/src/xdisp.c
index a264da3892a..cb68969ae2d 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -2854,6 +2854,13 @@ start_display (struct it *it, struct window *w, struct text_pos pos)
 
               it->continuation_lines_width += it->current_x;
             }
+          /* If the character at POS is displayed via a display
+             vector, move_it_to above stops at the final glyph of
+             IT->dpvec.  To make the caller redisplay that character
+             again (a.k.a. start at POS), we need to reset the
+             dpvec_index to the beginning of IT->dpvec.  */
+          else if (it->current.dpvec_index >= 0)
+            it->current.dpvec_index = 0;
 
           /* We're starting a new display line, not affected by the
              height of the continued line, so clear the appropriate
@@ -6379,8 +6386,8 @@ get_next_display_element (struct it *it)
         {
           Lisp_Object dv;
           struct charset *unibyte = CHARSET_FROM_ID (charset_unibyte);
-          enum { char_is_other = 0, char_is_nbsp, char_is_soft_hyphen }
-          nbsp_or_shy = char_is_other;
+          int nonascii_space_p = 0;
+          int nonascii_hyphen_p = 0;
           int c = it->c;        /* This is the character to display.  */
 
           if (! it->multibyte_p && ! ASCII_CHAR_P (c))
@@ -6432,10 +6439,15 @@ get_next_display_element (struct it *it)
               goto get_next;
             }
 
+          /* If `nobreak-char-display' is non-nil, we display
+             non-ASCII spaces and hyphens specially.  */
           if (! ASCII_CHAR_P (c) && ! NILP (Vnobreak_char_display))
-            nbsp_or_shy = (c == 0xA0   ? char_is_nbsp
-                           : c == 0xAD ? char_is_soft_hyphen
-                           :             char_is_other);
+            {
+              if (c == 0xA0)
+                nonascii_space_p = 1;
+              else if (c == 0xAD || c == 0x2010 || c == 0x2011)
+                nonascii_hyphen_p = 1;
+            }
 
           /* Translate control characters into `\003' or `^C' form.
              Control characters coming from a display table entry are
@@ -6443,7 +6455,8 @@ get_next_display_element (struct it *it)
              the translation.  This could easily be changed but I
              don't believe that it is worth doing.
 
-             NBSP and SOFT-HYPEN are property translated too.
+             The characters handled by `nobreak-char-display' must be
+             translated too.
 
              Non-printable characters and raw-byte characters are also
              translated to octal form.  */
@@ -6454,14 +6467,15 @@ get_next_display_element (struct it *it)
                       && it->glyph_row
                       && (it->glyph_row->mode_line_p || it->avoid_cursor_p))
                   || (c != '\n' && c != '\t'))
-               : (nbsp_or_shy
+               : (nonascii_space_p
+                  || nonascii_hyphen_p
                   || CHAR_BYTE8_P (c)
                   || ! CHAR_PRINTABLE_P (c))))
             {
-              /* C is a control character, NBSP, SOFT-HYPEN, raw-byte,
-                 or a non-printable character which must be displayed
-                 either as '\003' or as `^C' where the '\\' and '^'
-                 can be defined in the display table.  Fill
+              /* C is a control character, non-ASCII space/hyphen,
+                 raw-byte, or a non-printable character which must be
+                 displayed either as '\003' or as `^C' where the '\\'
+                 and '^' can be defined in the display table.  Fill
                  IT->ctl_chars with glyphs for what we have to
                  display.  Then, set IT->dpvec to these glyphs.  */
               Lisp_Object gc;
@@ -6509,17 +6523,14 @@ get_next_display_element (struct it *it)
                   goto display_control;
                 }
 
-              /* Handle non-break space in the mode where it only gets
+              /* Handle non-ascii space in the mode where it only gets
                  highlighting.  */
 
-              if (EQ (Vnobreak_char_display, Qt)
-                  && nbsp_or_shy == char_is_nbsp)
+              if (nonascii_space_p && EQ (Vnobreak_char_display, Qt))
                 {
-                  /* Merge the no-break-space face into the current face.  */
+                  /* Merge `nobreak-space' into the current face.  */
                   face_id = merge_faces (it->f, Qnobreak_space, 0,
                                          it->face_id);
-
-                  c = ' ';
                   XSETINT (it->ctl_chars[0], ' ');
                   ctl_len = 1;
                   goto display_control;
@@ -6558,25 +6569,21 @@ get_next_display_element (struct it *it)
                   last_escape_glyph_merged_face_id = face_id;
                 }
 
-              /* Handle soft hyphens in the mode where they only get
-                 highlighting.  */
+              /* Draw non-ASCII hyphen with just highlighting: */
 
-              if (EQ (Vnobreak_char_display, Qt)
-                  && nbsp_or_shy == char_is_soft_hyphen)
+              if (nonascii_hyphen_p && EQ (Vnobreak_char_display, Qt))
                 {
                   XSETINT (it->ctl_chars[0], '-');
                   ctl_len = 1;
                   goto display_control;
                 }
 
-              /* Handle non-break space and soft hyphen
-                 with the escape glyph.  */
+              /* Draw non-ASCII space/hyphen with escape glyph: */
 
-              if (nbsp_or_shy)
+              if (nonascii_space_p || nonascii_hyphen_p)
                 {
                   XSETINT (it->ctl_chars[0], escape_glyph);
-                  c = (nbsp_or_shy == char_is_nbsp ? ' ' : '-');
-                  XSETINT (it->ctl_chars[1], c);
+                  XSETINT (it->ctl_chars[1], nonascii_space_p ? ' ' : '-');
                   ctl_len = 2;
                   goto display_control;
                 }
@@ -8774,7 +8781,10 @@ move_it_vertically_backward (struct it *it, int dy)
          reordering.  We want to get to the character position
          that is immediately after the newline of the previous
          line.  */
-      if (it->bidi_p && IT_CHARPOS (*it) > BEGV
+      if (it->bidi_p
+          && !it->continuation_lines_width
+          && !STRINGP (it->string)
+          && IT_CHARPOS (*it) > BEGV
           && FETCH_BYTE (IT_BYTEPOS (*it) - 1) != '\n')
         {
           ptrdiff_t nl_pos =
@@ -16075,13 +16085,20 @@ try_window_reusing_current_matrix (struct window *w)
 
               start_vpos = MATRIX_ROW_VPOS (start_row, w->current_matrix);
             }
-          /* If we have reached alignment,
-             we can copy the rest of the rows.  */
-          if (IT_CHARPOS (it) == CHARPOS (start))
+          /* If we have reached alignment, we can copy the rest of the
+             rows.  */
+          if (IT_CHARPOS (it) == CHARPOS (start)
+              /* Don't accept "alignment" inside a display vector,
+                 since start_row could have started in the middle of
+                 that same display vector (thus their character
+                 positions match), and we have no way of telling if
+                 that is the case.  */
+              && it.current.dpvec_index < 0)
             break;
 
           if (display_line (&it))
             last_text_row = it.glyph_row - 1;
+
         }
 
       /* A value of current_y < last_visible_y means that we stopped
@@ -18434,9 +18451,10 @@ static int
 push_display_prop (struct it *it, Lisp_Object prop)
 {
   struct text_pos pos =
-    (it->method == GET_FROM_STRING) ? it->current.string_pos : it->current.pos;
+    STRINGP (it->string) ? it->current.string_pos : it->current.pos;
 
   xassert (it->method == GET_FROM_BUFFER
+           || it->method == GET_FROM_DISPLAY_VECTOR
            || it->method == GET_FROM_STRING);
 
   /* We need to save the current buffer/string position, so it will be
@@ -28004,12 +28022,18 @@ The face used for trailing whitespace is `trailing-whitespace'.  */);
   Vshow_trailing_whitespace = Qnil;
 
   DEFVAR_LISP ("nobreak-char-display", Vnobreak_char_display,
-    doc: /* *Control highlighting of nobreak space and soft hyphen.
-A value of t means highlight the character itself (for nobreak space,
-use face `nobreak-space').
-A value of nil means no highlighting.
-Other values mean display the escape glyph followed by an ordinary
-space or ordinary hyphen.  */);
+    doc: /* Control highlighting of non-ASCII space and hyphen chars.
+If the value is t, Emacs highlights non-ASCII chars which have the
+same appearance as an ASCII space or hyphen, using the `nobreak-space'
+or `escape-glyph' face respectively.
+
+U+00A0 (no-break space), U+00AD (soft hyphen), U+2010 (hyphen), and
+U+2011 (non-breaking hyphen) are affected.
+
+Any other non-nil value means to display these characters as a escape
+glyph followed by an ordinary space or hyphen.
+
+A value of nil means no special handling of these characters.  */);
   Vnobreak_char_display = Qt;
 
   DEFVAR_LISP ("void-text-area-pointer", Vvoid_text_area_pointer,