6 files changed, 175 insertions, 113 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index d21e6383764..3f9b5beeab6 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,52 @@
+2011-10-20  Eli Zaretskii  <eliz@gnu.org>
+
+        * dispextern.h (struct bidi_it): New member next_en_type.
+
+        * bidi.c (bidi_line_init): Initialize the next_en_type member.
+        (bidi_resolve_explicit_1): When next_en_pos is valid for the
+        current character, check also for next_en_type being WEAK_EN.
+        (bidi_resolve_weak): Don't enter the expensive loop if the current
+        position is before next_en_pos.  Record the bidi type of the first
+        non-ET, non-BN character we find, in addition to its position.
+        (bidi_level_of_next_char): Invalidate next_en_type when
+        next_en_pos is over-stepped.
+
+2011-10-20  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Time zone name fixes for non-ASCII locales (Bug#641, Bug#9794)
+        * editfns.c: Rewrite current-time-zone so that it invokes
+        the equivalent of (format-time-string "%Z") to get the time zone name.
+        This fixes a bug when the time zone name contains characters that
+        need converting from the system time locale to Emacs internal format.
+        This fixes a shortcoming that I introduced in my 1999-10-19 patch:
+        that patch fixed format-time-string to do the conversion, but
+        I forgot to fix current-time-zone.
+        (format_time_string): New function, containing most of
+        what Fformat_time_string used to contain.
+        (Fformat_time_string): Rewrite in terms of format_time_string.
+        This doesn't change this function's behavior.
+        (current-time-zone): Rewrite to use format_time_string.
+        This fixes the bug reported by Michael Schierl in
+        <http://lists.gnu.org/archive/html/emacs-devel/2007-06/msg00334.html>.
+        Jason Rumney's 2007-06-07 change worked around this bug, but
+        didn't fix it.
+        * systime.h (tzname, timezone): Remove no-longer-used declarations.
+
+2011-10-19  Eli Zaretskii  <eliz@gnu.org>
+
+        * xdisp.c (start_display): If the character at POS is displayed
+        via a display vector, reset IT->current.dpvec_index to zero.
+        (try_window_reusing_current_matrix): If a line ends in a display
+        vector or the next line starts in a display vector, continue
+        redrawing the window even though the character position of
+        start_row was reached.
+        (Bug#9771, part 2)
+
+2011-10-18  Chong Yidong  <cyd@gnu.org>
+
+        * xdisp.c (get_next_display_element): Handle U+2010 and U+2011
+        with nobreak-char-display too.
+
 2011-10-18  Eli Zaretskii  <eliz@gnu.org>
 
         Fix part 3 of bug#9771.
diff --git a/src/bidi.c b/src/bidi.c
index 29e3c817318..e688637a408 100644
--- a/src/bidi.c
+++ b/src/bidi.c
@@ -849,6 +849,7 @@ bidi_line_init (struct bidi_it *bidi_it)
   /* Setting this to zero will force its recomputation the first time
      we need it for W5.  */
   bidi_it->next_en_pos = 0;
+  bidi_it->next_en_type = UNKNOWN_BT;
   bidi_it->next_for_ws.type = UNKNOWN_BT;
   bidi_set_sor_type (bidi_it,
                      (bidi_it->paragraph_dir == R2L ? 1 : 0),
@@ -1437,7 +1438,8 @@ bidi_resolve_explicit_1 (struct bidi_it *bidi_it)
               }
           }
         else if (bidi_it->prev.type_after_w1 == WEAK_EN /* W5/Retaining */
-                 || bidi_it->next_en_pos > bidi_it->charpos)
+                 || (bidi_it->next_en_pos > bidi_it->charpos
+                     && bidi_it->next_en_type == WEAK_EN))
           type = WEAK_EN;
         break;
       case LRE: /* X3 */
@@ -1473,7 +1475,8 @@ bidi_resolve_explicit_1 (struct bidi_it *bidi_it)
               }
           }
         else if (bidi_it->prev.type_after_w1 == WEAK_EN /* W5/Retaining */
-                 || bidi_it->next_en_pos > bidi_it->charpos)
+                 || (bidi_it->next_en_pos > bidi_it->charpos
+                     && bidi_it->next_en_type == WEAK_EN))
           type = WEAK_EN;
         break;
       case PDF: /* X7 */
@@ -1499,7 +1502,8 @@ bidi_resolve_explicit_1 (struct bidi_it *bidi_it)
               }
           }
         else if (bidi_it->prev.type_after_w1 == WEAK_EN /* W5/Retaining */
-                 || bidi_it->next_en_pos > bidi_it->charpos)
+                 || (bidi_it->next_en_pos > bidi_it->charpos
+                     && bidi_it->next_en_type == WEAK_EN))
           type = WEAK_EN;
         break;
       default:
@@ -1731,10 +1735,15 @@ bidi_resolve_weak (struct bidi_it *bidi_it)
       else if (type == WEAK_ET  /* W5: ET with EN before or after it */
                || type == WEAK_BN)      /* W5/Retaining */
         {
-          if (bidi_it->prev.type_after_w1 == WEAK_EN /* ET/BN w/EN before it */
-              || bidi_it->next_en_pos > bidi_it->charpos)
+          if (bidi_it->prev.type_after_w1 == WEAK_EN) /* ET/BN w/EN before it */
             type = WEAK_EN;
-          else if (bidi_it->next_en_pos >=0) /* W5: ET/BN with EN after it.  */
+          else if (bidi_it->next_en_pos > bidi_it->charpos
+                   && bidi_it->next_en_type != WEAK_BN)
+            {
+              if (bidi_it->next_en_type == WEAK_EN) /* ET/BN with EN after it */
+                type = WEAK_EN;
+            }
+          else if (bidi_it->next_en_pos >=0)
             {
               EMACS_INT en_pos = bidi_it->charpos + bidi_it->nchars;
               const unsigned char *s = (STRINGP (bidi_it->string.lstring)
@@ -1763,25 +1772,27 @@ bidi_resolve_weak (struct bidi_it *bidi_it)
                   en_pos = bidi_it->charpos;
                   bidi_copy_it (bidi_it, &saved_it);
                 }
+              /* Remember this position, to speed up processing of the
+                 next ETs.  */
+              bidi_it->next_en_pos = en_pos;
               if (type_of_next == WEAK_EN)
                 {
                   /* If the last strong character is AL, the EN we've
                      found will become AN when we get to it (W2). */
-                  if (bidi_it->last_strong.type_after_w1 != STRONG_AL)
-                    {
-                      type = WEAK_EN;
-                      /* Remember this EN position, to speed up processing
-                         of the next ETs.  */
-                      bidi_it->next_en_pos = en_pos;
-                    }
+                  if (bidi_it->last_strong.type_after_w1 == STRONG_AL)
+                    type_of_next = WEAK_AN;
                   else if (type == WEAK_BN)
                     type = NEUTRAL_ON; /* W6/Retaining */
+                  else
+                    type = WEAK_EN;
                 }
               else if (type_of_next == NEUTRAL_B)
                 /* Record the fact that there are no more ENs from
                    here to the end of paragraph, to avoid entering the
                    loop above ever again in this paragraph.  */
                 bidi_it->next_en_pos = -1;
+              /* Record the type of the character where we ended our search.  */
+              bidi_it->next_en_type = type_of_next;
             }
         }
     }
@@ -2053,7 +2064,10 @@ bidi_level_of_next_char (struct bidi_it *bidi_it)
         bidi_it->next_for_neutral.type = UNKNOWN_BT;
       if (bidi_it->next_en_pos >= 0
           && bidi_it->charpos >= bidi_it->next_en_pos)
-        bidi_it->next_en_pos = 0;
+        {
+          bidi_it->next_en_pos = 0;
+          bidi_it->next_en_type = UNKNOWN_BT;
+        }
       if (bidi_it->next_for_ws.type != UNKNOWN_BT
           && bidi_it->charpos >= bidi_it->next_for_ws.charpos)
         bidi_it->next_for_ws.type = UNKNOWN_BT;
@@ -2179,7 +2193,7 @@ bidi_level_of_next_char (struct bidi_it *bidi_it)
     }
 
   /* Resolve implicit levels, with a twist: PDFs get the embedding
-     level of the enbedding they terminate.  See below for the
+     level of the embedding they terminate.  See below for the
      reason.  */
   if (bidi_it->orig_type == PDF
       /* Don't do this if this formatting code didn't change the
diff --git a/src/dispextern.h b/src/dispextern.h
index bdc5008c94c..40e41923554 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -1866,7 +1866,8 @@ struct bidi_it {
   struct bidi_saved_info next_for_neutral; /* surrounding characters for... */
   struct bidi_saved_info prev_for_neutral; /* ...resolving neutrals */
   struct bidi_saved_info next_for_ws; /* character after sequence of ws */
-  EMACS_INT next_en_pos;        /* position of next EN char for ET */
+  EMACS_INT next_en_pos;        /* pos. of next char for determining ET type */
+  bidi_type_t next_en_type;     /* type of char at next_en_pos */
   EMACS_INT ignore_bn_limit;    /* position until which to ignore BNs */
   bidi_dir_t sor;               /* direction of start-of-run in effect */
   int scan_dir;                 /* direction of text scan, 1: forw, -1: back */
diff --git a/src/editfns.c b/src/editfns.c
index 5f89391ce22..83cd4bd5535 100644
--- a/src/editfns.c
+++ b/src/editfns.c
@@ -85,6 +85,8 @@ extern Lisp_Object w32_get_internal_run_time (void);
 #endif
 
 static void time_overflow (void) NO_RETURN;
+static Lisp_Object format_time_string (char const *, ptrdiff_t, Lisp_Object,
+                                       int, time_t *, struct tm **);
 static int tm_diff (struct tm *, struct tm *);
 static void update_buffer_properties (EMACS_INT, EMACS_INT);
 
@@ -1700,33 +1702,41 @@ For example, to produce full ISO 8601 format, use "%Y-%m-%dT%T%z".
 usage: (format-time-string FORMAT-STRING &optional TIME UNIVERSAL)  */)
   (Lisp_Object format_string, Lisp_Object timeval, Lisp_Object universal)
 {
-  time_t value;
+  time_t t;
+  struct tm *tm;
+
+  CHECK_STRING (format_string);
+  format_string = code_convert_string_norecord (format_string,
+                                                Vlocale_coding_system, 1);
+  return format_time_string (SSDATA (format_string), SBYTES (format_string),
+                             timeval, ! NILP (universal), &t, &tm);
+}
+
+static Lisp_Object
+format_time_string (char const *format, ptrdiff_t formatlen,
+                    Lisp_Object timeval, int ut, time_t *tval, struct tm **tmp)
+{
   ptrdiff_t size;
   int usec;
   int ns;
   struct tm *tm;
-  int ut = ! NILP (universal);
-
-  CHECK_STRING (format_string);
 
-  if (! (lisp_time_argument (timeval, &value, &usec)
+  if (! (lisp_time_argument (timeval, tval, &usec)
          && 0 <= usec && usec < 1000000))
     error ("Invalid time specification");
   ns = usec * 1000;
 
-  format_string = code_convert_string_norecord (format_string,
-                                                Vlocale_coding_system, 1);
-
   /* This is probably enough.  */
-  size = SBYTES (format_string);
+  size = formatlen;
   if (size <= (STRING_BYTES_BOUND - 50) / 6)
     size = size * 6 + 50;
 
   BLOCK_INPUT;
-  tm = ut ? gmtime (&value) : localtime (&value);
+  tm = ut ? gmtime (tval) : localtime (tval);
   UNBLOCK_INPUT;
   if (! tm)
     time_overflow ();
+  *tmp = tm;
 
   synchronize_system_time_locale ();
 
@@ -1737,9 +1747,7 @@ usage: (format-time-string FORMAT-STRING &optional TIME UNIVERSAL)  */)
 
       buf[0] = '\1';
       BLOCK_INPUT;
-      result = emacs_nmemftime (buf, size, SSDATA (format_string),
-                                SBYTES (format_string),
-                                tm, ut, ns);
+      result = emacs_nmemftime (buf, size, format, formatlen, tm, ut, ns);
       UNBLOCK_INPUT;
       if ((result > 0 && result < size) || (result == 0 && buf[0] == '\0'))
         return code_convert_string_norecord (make_unibyte_string (buf, result),
@@ -1747,9 +1755,7 @@ usage: (format-time-string FORMAT-STRING &optional TIME UNIVERSAL)  */)
 
       /* If buffer was too small, make it bigger and try again.  */
       BLOCK_INPUT;
-      result = emacs_nmemftime (NULL, (size_t) -1,
-                                SSDATA (format_string),
-                                SBYTES (format_string),
+      result = emacs_nmemftime (NULL, (size_t) -1, format, formatlen,
                                 tm, ut, ns);
       UNBLOCK_INPUT;
       if (STRING_BYTES_BOUND <= result)
@@ -1994,51 +2000,34 @@ the data it can't find.  */)
 {
   time_t value;
   struct tm *t;
-  struct tm gmt;
-
-  if (!lisp_time_argument (specified_time, &value, NULL))
-    t = NULL;
-  else
-    {
-      BLOCK_INPUT;
-      t = gmtime (&value);
-      if (t)
-        {
-          gmt = *t;
-          t = localtime (&value);
-        }
-      UNBLOCK_INPUT;
-    }
+  struct tm localtm;
+  struct tm *localt;
+  Lisp_Object zone_offset, zone_name;
+
+  zone_offset = Qnil;
+  zone_name = format_time_string ("%Z", sizeof "%Z" - 1, specified_time,
+                                  0, &value, &localt);
+  localtm = *localt;
+  BLOCK_INPUT;
+  t = gmtime (&value);
+  UNBLOCK_INPUT;
 
   if (t)
     {
-      int offset = tm_diff (t, &gmt);
-      char *s = 0;
-      char buf[sizeof "+00" + INT_STRLEN_BOUND (int)];
-
-#ifdef HAVE_TM_ZONE
-      if (t->tm_zone)
-        s = (char *)t->tm_zone;
-#else /* not HAVE_TM_ZONE */
-#ifdef HAVE_TZNAME
-      if (t->tm_isdst == 0 || t->tm_isdst == 1)
-        s = tzname[t->tm_isdst];
-#endif
-#endif /* not HAVE_TM_ZONE */
-
-      if (!s)
+      int offset = tm_diff (&localtm, t);
+      zone_offset = make_number (offset);
+      if (SCHARS (zone_name) == 0)
         {
           /* No local time zone name is available; use "+-NNNN" instead.  */
           int m = offset / 60;
           int am = offset < 0 ? - m : m;
+          char buf[sizeof "+00" + INT_STRLEN_BOUND (int)];
           sprintf (buf, "%c%02d%02d", (offset < 0 ? '-' : '+'), am/60, am%60);
-          s = buf;
+          zone_name = build_string (buf);
         }
-
-      return Fcons (make_number (offset), Fcons (build_string (s), Qnil));
     }
-  else
-    return Fmake_list (make_number (2), Qnil);
+
+  return list2 (zone_offset, zone_name);
 }
 
 /* This holds the value of `environ' produced by the previous
diff --git a/src/systime.h b/src/systime.h
index bed9ed4aa71..b90372dbe20 100644
--- a/src/systime.h
+++ b/src/systime.h
@@ -38,17 +38,6 @@ typedef unsigned long Time;
 # endif
 #endif
 
-#ifdef HAVE_TZNAME
-#ifndef tzname          /* For SGI.  */
-extern char *tzname[];  /* RS6000 and others want it this way.  */
-#endif
-#endif
-
-/* SVr4 doesn't actually declare this in its #include files.  */
-#ifdef USG5_4
-extern time_t timezone;
-#endif
-
 /* On some configurations (hpux8.0, X11R4), sys/time.h and X11/Xos.h
    disagree about the name of the guard symbol.  */
 #ifdef HPUX
diff --git a/src/xdisp.c b/src/xdisp.c
index 1f4c98f8388..e0b36dbe100 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -2856,6 +2856,13 @@ start_display (struct it *it, struct window *w, struct text_pos pos)
 
               it->continuation_lines_width += it->current_x;
             }
+          /* If the character at POS is displayed via a display
+             vector, move_it_to above stops at the final glyph of
+             IT->dpvec.  To make the caller redisplay that character
+             again (a.k.a. start at POS), we need to reset the
+             dpvec_index to the beginning of IT->dpvec.  */
+          else if (it->current.dpvec_index >= 0)
+            it->current.dpvec_index = 0;
 
           /* We're starting a new display line, not affected by the
              height of the continued line, so clear the appropriate
@@ -6409,8 +6416,8 @@ get_next_display_element (struct it *it)
         {
           Lisp_Object dv;
           struct charset *unibyte = CHARSET_FROM_ID (charset_unibyte);
-          enum { char_is_other = 0, char_is_nbsp, char_is_soft_hyphen }
-          nbsp_or_shy = char_is_other;
+          int nonascii_space_p = 0;
+          int nonascii_hyphen_p = 0;
           int c = it->c;        /* This is the character to display.  */
 
           if (! it->multibyte_p && ! ASCII_CHAR_P (c))
@@ -6462,10 +6469,15 @@ get_next_display_element (struct it *it)
               goto get_next;
             }
 
+          /* If `nobreak-char-display' is non-nil, we display
+             non-ASCII spaces and hyphens specially.  */
           if (! ASCII_CHAR_P (c) && ! NILP (Vnobreak_char_display))
-            nbsp_or_shy = (c == 0xA0   ? char_is_nbsp
-                           : c == 0xAD ? char_is_soft_hyphen
-                           :             char_is_other);
+            {
+              if (c == 0xA0)
+                nonascii_space_p = 1;
+              else if (c == 0xAD || c == 0x2010 || c == 0x2011)
+                nonascii_hyphen_p = 1;
+            }
 
           /* Translate control characters into `\003' or `^C' form.
              Control characters coming from a display table entry are
@@ -6473,7 +6485,8 @@ get_next_display_element (struct it *it)
              the translation.  This could easily be changed but I
              don't believe that it is worth doing.
 
-             NBSP and SOFT-HYPEN are property translated too.
+             The characters handled by `nobreak-char-display' must be
+             translated too.
 
              Non-printable characters and raw-byte characters are also
              translated to octal form.  */
@@ -6484,14 +6497,15 @@ get_next_display_element (struct it *it)
                       && it->glyph_row
                       && (it->glyph_row->mode_line_p || it->avoid_cursor_p))
                   || (c != '\n' && c != '\t'))
-               : (nbsp_or_shy
+               : (nonascii_space_p
+                  || nonascii_hyphen_p
                   || CHAR_BYTE8_P (c)
                   || ! CHAR_PRINTABLE_P (c))))
             {
-              /* C is a control character, NBSP, SOFT-HYPEN, raw-byte,
-                 or a non-printable character which must be displayed
-                 either as '\003' or as `^C' where the '\\' and '^'
-                 can be defined in the display table.  Fill
+              /* C is a control character, non-ASCII space/hyphen,
+                 raw-byte, or a non-printable character which must be
+                 displayed either as '\003' or as `^C' where the '\\'
+                 and '^' can be defined in the display table.  Fill
                  IT->ctl_chars with glyphs for what we have to
                  display.  Then, set IT->dpvec to these glyphs.  */
               Lisp_Object gc;
@@ -6540,17 +6554,14 @@ get_next_display_element (struct it *it)
                   goto display_control;
                 }
 
-              /* Handle non-break space in the mode where it only gets
+              /* Handle non-ascii space in the mode where it only gets
                  highlighting.  */
 
-              if (EQ (Vnobreak_char_display, Qt)
-                  && nbsp_or_shy == char_is_nbsp)
+              if (nonascii_space_p && EQ (Vnobreak_char_display, Qt))
                 {
-                  /* Merge the no-break-space face into the current face.  */
+                  /* Merge `nobreak-space' into the current face.  */
                   face_id = merge_faces (it->f, Qnobreak_space, 0,
                                          it->face_id);
-
-                  c = ' ';
                   XSETINT (it->ctl_chars[0], ' ');
                   ctl_len = 1;
                   goto display_control;
@@ -6590,25 +6601,21 @@ get_next_display_element (struct it *it)
                   last_escape_glyph_merged_face_id = face_id;
                 }
 
-              /* Handle soft hyphens in the mode where they only get
-                 highlighting.  */
+              /* Draw non-ASCII hyphen with just highlighting: */
 
-              if (EQ (Vnobreak_char_display, Qt)
-                  && nbsp_or_shy == char_is_soft_hyphen)
+              if (nonascii_hyphen_p && EQ (Vnobreak_char_display, Qt))
                 {
                   XSETINT (it->ctl_chars[0], '-');
                   ctl_len = 1;
                   goto display_control;
                 }
 
-              /* Handle non-break space and soft hyphen
-                 with the escape glyph.  */
+              /* Draw non-ASCII space/hyphen with escape glyph: */
 
-              if (nbsp_or_shy)
+              if (nonascii_space_p || nonascii_hyphen_p)
                 {
                   XSETINT (it->ctl_chars[0], escape_glyph);
-                  c = (nbsp_or_shy == char_is_nbsp ? ' ' : '-');
-                  XSETINT (it->ctl_chars[1], c);
+                  XSETINT (it->ctl_chars[1], nonascii_space_p ? ' ' : '-');
                   ctl_len = 2;
                   goto display_control;
                 }
@@ -16146,13 +16153,20 @@ try_window_reusing_current_matrix (struct window *w)
 
               start_vpos = MATRIX_ROW_VPOS (start_row, w->current_matrix);
             }
-          /* If we have reached alignment,
-             we can copy the rest of the rows.  */
-          if (IT_CHARPOS (it) == CHARPOS (start))
+          /* If we have reached alignment, we can copy the rest of the
+             rows.  */
+          if (IT_CHARPOS (it) == CHARPOS (start)
+              /* Don't accept "alignment" inside a display vector,
+                 since start_row could have started in the middle of
+                 that same display vector (thus their character
+                 positions match), and we have no way of telling if
+                 that is the case.  */
+              && it.current.dpvec_index < 0)
             break;
 
           if (display_line (&it))
             last_text_row = it.glyph_row - 1;
+
         }
 
       /* A value of current_y < last_visible_y means that we stopped
@@ -28239,12 +28253,18 @@ The face used for trailing whitespace is `trailing-whitespace'.  */);
   Vshow_trailing_whitespace = Qnil;
 
   DEFVAR_LISP ("nobreak-char-display", Vnobreak_char_display,
-    doc: /* *Control highlighting of nobreak space and soft hyphen.
-A value of t means highlight the character itself (for nobreak space,
-use face `nobreak-space').
-A value of nil means no highlighting.
-Other values mean display the escape glyph followed by an ordinary
-space or ordinary hyphen.  */);
+    doc: /* Control highlighting of non-ASCII space and hyphen chars.
+If the value is t, Emacs highlights non-ASCII chars which have the
+same appearance as an ASCII space or hyphen, using the `nobreak-space'
+or `escape-glyph' face respectively.
+
+U+00A0 (no-break space), U+00AD (soft hyphen), U+2010 (hyphen), and
+U+2011 (non-breaking hyphen) are affected.
+
+Any other non-nil value means to display these characters as a escape
+glyph followed by an ordinary space or hyphen.
+
+A value of nil means no special handling of these characters.  */);
   Vnobreak_char_display = Qt;
 
   DEFVAR_LISP ("void-text-area-pointer", Vvoid_text_area_pointer,