Retrospective commit from 2009-10-05.

Continue working on paragraph base direction.
Support per-buffer default paragraph direction.

 buffer.h (struct buffer): New member paragraph_direction.
 buffer.c (init_buffer_once): Initialize it.
 (syms_of_buffer): Declare Lisp variables
 default-paragraph-direction and paragraph-direction.
 dispextern.h (struct it): New member paragraph_embedding.
 xdisp.c (init_iterator): Initialize it from the buffer's value
 of paragraph-direction.
 <Qright_to_left, Qleft_to_right>: New variables.
 (syms_of_xdisp): Initialize and staticpro them.
 (set_iterator_to_next, next_element_from_buffer): Use the value of
 paragraph_embedding to determine the paragraph direction.
 bidi.c (bidi_line_init): Fix second argument to
 bidi_set_sor_type.
 (bidi_init_it): Initialize paragraph_dir to NEUTRAL_DIR.
 (bidi_get_next_char_visually): Record the last character of the
 separator in separator_limit, not the character after that.
 (bidi_find_paragraph_start): Accept character and byte positions
 instead of the whole iterator stricture.  All callers changed.

6 files changed, 104 insertions, 35 deletions

diff --git a/src/ChangeLog.bidi b/src/ChangeLog.bidi
index e581be7dc5b..db1d6c5bce0 100644
--- a/src/ChangeLog.bidi
+++ b/src/ChangeLog.bidi
@@ -1,3 +1,26 @@
+2009-10-05  Eli Zaretskii  <eliz@gnu.org>
+
+        * buffer.h (struct buffer): New member paragraph_direction.
+        * buffer.c (init_buffer_once): Initialize it.
+        (syms_of_buffer): Declare Lisp variables
+        default-paragraph-direction and paragraph-direction.
+
+        * dispextern.h (struct it): New member paragraph_embedding.
+        * xdisp.c (init_iterator): Initialize it from the buffer's value
+        of paragraph-direction.
+        <Qright_to_left, Qleft_to_right>: New variables.
+        (syms_of_xdisp): Initialize and staticpro them.
+        (set_iterator_to_next, next_element_from_buffer): Use the value of
+        paragraph_embedding to determine the paragraph direction.
+
+        * bidi.c (bidi_line_init): Fix second argument to
+        bidi_set_sor_type.
+        (bidi_init_it): Initialize paragraph_dir to NEUTRAL_DIR.
+        (bidi_get_next_char_visually): Record the last character of the
+        separator in separator_limit, not the character after that.
+        (bidi_find_paragraph_start): Accept character and byte positions
+        instead of the whole iterator stricture.  All callers changed.
+
 2009-10-04  Eli Zaretskii  <eliz@gnu.org>
 
         * bidi.c (bidi_at_paragraph_end): Check for paragraph-start if
diff --git a/src/bidi.c b/src/bidi.c
index 8d9e32d5c3b..fc7e326cfb5 100644
--- a/src/bidi.c
+++ b/src/bidi.c
@@ -807,7 +807,8 @@ bidi_line_init (struct bidi_it *bidi_it)
   bidi_it->invalid_rl_levels = -1;
   bidi_it->next_en_pos = -1;
   bidi_it->next_for_ws.type = UNKNOWN_BT;
-  bidi_set_sor_type (bidi_it, bidi_it->paragraph_dir,
+  bidi_set_sor_type (bidi_it,
+                     bidi_it->paragraph_dir == R2L ? 1 : 0,
                      bidi_it->level_stack[0].level); /* X10 */
 
   bidi_cache_reset ();
@@ -816,11 +817,9 @@ bidi_line_init (struct bidi_it *bidi_it)
 /* Find the beginning of this paragraph by looking back in the buffer.
    Value is the byte position of the paragraph's beginning.  */
 static EMACS_INT
-bidi_find_paragraph_start (struct bidi_it *bidi_it)
+bidi_find_paragraph_start (EMACS_INT pos, EMACS_INT pos_byte)
 {
   Lisp_Object re = Fbuffer_local_value (Qparagraph_start, Fcurrent_buffer ());
-  EMACS_INT pos = bidi_it->charpos;
-  EMACS_INT pos_byte = bidi_it->bytepos;
   EMACS_INT limit = ZV, limit_byte = ZV_BYTE;
 
   if (!STRINGP (re))
@@ -835,7 +834,13 @@ bidi_find_paragraph_start (struct bidi_it *bidi_it)
 }
 
 /* Determine the direction, a.k.a. base embedding level, of the
-   paragraph we are about to iterate through.  */
+   paragraph we are about to iterate through.  If DIR is either L2R or
+   R2L, just use that.  Otherwise, determine the paragraph direction
+   from the first strong character of the paragraph.
+
+   Note that this gives the paragraph separator the same direction as
+   the preceding paragraph, even though Emacs generally views the
+   separartor as not belonging to any paragraph.  */
 void
 bidi_paragraph_init (bidi_dir_t dir, struct bidi_it *bidi_it)
 {
@@ -868,24 +873,21 @@ bidi_paragraph_init (bidi_dir_t dir, struct bidi_it *bidi_it)
       if (bidi_it->charpos < bidi_it->separator_limit)
         return;
 
-      /* If we are before another paragraph separator, continue
-         through that with the previous paragraph direction.  */
-      sep_len = bidi_at_paragraph_end (bidi_it->charpos, bytepos);
-      if (sep_len >= 0)
+      /* If we are on a newline, get past it to where the next
+         paragraph might start.  */
+      if (FETCH_CHAR (bytepos) == '\n')
         {
-          bidi_it->separator_limit += sep_len + 1;
-          return;
+          bytepos++;
+          pos = bidi_it->charpos + 1;
         }
-      else if (sep_len == -2)
-        /* We are in the middle of a paragraph.  Search back to where
-           this paragraph starts.  */
-        bytepos = bidi_find_paragraph_start (bidi_it);
+
+      /* We are either at the beginning of a paragraph or in the
+         middle of it.  Find where this paragraph starts.  */
+      bytepos = bidi_find_paragraph_start (pos, bytepos);
 
       /* We should always be at the beginning of a new line at this
          point.  */
-      if (!(bytepos == BEGV_BYTE
-            || FETCH_CHAR (bytepos) == '\n'
-            || FETCH_CHAR (bytepos - 1) == '\n'))
+      if (!(bytepos == BEGV_BYTE || FETCH_CHAR (bytepos - 1) == '\n'))
         abort ();
 
       bidi_it->separator_limit = -1;
@@ -918,14 +920,15 @@ bidi_paragraph_init (bidi_dir_t dir, struct bidi_it *bidi_it)
   else
     abort ();
 
-  /* Contrary to UAX#9 clause P3, we only default to L2R if we have no
-     previous usable paragraph direction.  */
+  /* Contrary to UAX#9 clause P3, we only default the paragraph
+     direction to L2R if we have no previous usable paragraph
+     direction.  */
   if (bidi_it->paragraph_dir == NEUTRAL_DIR)
-    bidi_it->paragraph_dir = L2R; /* P3 */
+    bidi_it->paragraph_dir = L2R; /* P3 and ``higher protocols'' */
   if (bidi_it->paragraph_dir == R2L)
-    bidi_it->level_stack[0].level == 1;
+    bidi_it->level_stack[0].level = 1;
   else
-    bidi_it->level_stack[0].level == 0;
+    bidi_it->level_stack[0].level = 0;
 
   bidi_line_init (bidi_it);
 }
@@ -953,6 +956,7 @@ bidi_init_it (EMACS_INT charpos, EMACS_INT bytepos, struct bidi_it *bidi_it)
   bidi_set_paragraph_end (bidi_it);
   bidi_it->new_paragraph = 1;
   bidi_it->separator_limit = -1;
+  bidi_it->paragraph_dir = NEUTRAL_DIR;
   bidi_it->type = NEUTRAL_B;
   bidi_it->type_after_w1 = UNKNOWN_BT;
   bidi_it->orig_type = UNKNOWN_BT;
@@ -1943,9 +1947,13 @@ bidi_get_next_char_visually (struct bidi_it *bidi_it)
       next_level = bidi_level_of_next_char (bidi_it);
     }
 
-  /* Take note when we are at the end of the paragraph.  The next time
-     we are about to be called, set_iterator_to_next will
-     automatically reinit the paragraph direction, if needed.  */
+  /* Take note when we have just processed the newline that precedes
+     the end of the paragraph.  The next time we are about to be
+     called, set_iterator_to_next will automatically reinit the
+     paragraph direction, if needed.  We do this at the newline before
+     the paragraph separator, because the next character might not be
+     the first character of the next paragraph, due to the bidi
+     reordering.  */
   if (bidi_it->scan_dir == 1
       && bidi_it->orig_type == NEUTRAL_B
       && bidi_it->bytepos < ZV_BYTE)
@@ -1956,9 +1964,9 @@ bidi_get_next_char_visually (struct bidi_it *bidi_it)
       if (sep_len >= 0)
         {
           bidi_it->new_paragraph = 1;
-          /* Record the buffer position of the first character after
-             the paragraph separator.  */
-          bidi_it->separator_limit = bidi_it->charpos + 1 + sep_len + 1;
+          /* Record the buffer position of the last character of the
+             paragraph separator.  */
+          bidi_it->separator_limit = bidi_it->charpos + 1 + sep_len;
         }
     }
 
diff --git a/src/buffer.c b/src/buffer.c
index 2930465834d..8484abcdbb5 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -5188,6 +5188,7 @@ init_buffer_once ()
   buffer_defaults.ctl_arrow = Qt;
   buffer_defaults.bidi_display_reordering = Qnil;
   buffer_defaults.direction_reversed = Qnil;
+  buffer_defaults.paragraph_direction = Qnil;
   buffer_defaults.cursor_type = Qt;
   buffer_defaults.extra_line_spacing = Qnil;
   buffer_defaults.cursor_in_non_selected_windows = Qt;
@@ -5274,6 +5275,7 @@ init_buffer_once ()
   XSETFASTINT (buffer_local_flags.category_table, idx); ++idx;
   XSETFASTINT (buffer_local_flags.bidi_display_reordering, idx); ++idx;
   XSETFASTINT (buffer_local_flags.direction_reversed, idx); ++idx;
+  XSETFASTINT (buffer_local_flags.paragraph_direction, idx); ++idx;
   XSETFASTINT (buffer_local_flags.buffer_file_coding_system, idx);
   /* Make this one a permanent local.  */
   buffer_permanent_local_flags[idx++] = 1;
@@ -5545,6 +5547,11 @@ This is the same as (default-value 'direction-reversed).  */);
                      doc: /* *Default value of `enable-multibyte-characters' for buffers not overriding it.
 This is the same as (default-value 'enable-multibyte-characters).  */);
 
+  DEFVAR_LISP_NOPRO ("default-paragraph-direction",
+                     &buffer_defaults.paragraph_direction,
+                     doc: /* Default value of `paragraph-direction' for buffers that do not override it.
+This is the same as (default-value 'paragraph-direction).  */);
+
   DEFVAR_LISP_NOPRO ("default-buffer-file-coding-system",
                      &buffer_defaults.buffer_file_coding_system,
                      doc: /* Default value of `buffer-file-coding-system' for buffers not overriding it.
@@ -5806,6 +5813,18 @@ See also the variable `bidi-display-reordering'.  */);
                      doc: /*Non-nil means reorder bidirectional text for display in the visual order.
 See also the variable `direction-reversed'.  */);
 
+  DEFVAR_PER_BUFFER ("paragraph-direction",
+                     &current_buffer->paragraph_direction, Qnil,
+                     doc: /* *If non-nil, forces directionality of text paragraphs in the buffer.
+                             
+If this is nil (the default), the direction of each paragraph is
+determined by the first strong directional character of its text.
+The values of `right-to-left' and `left-to-right' override that.
+Any other value is treated as nil.
+                             
+This variable has no effect unless the buffer's value of
+\`bidi-display-reordering' is non-nil.  */);
+
  DEFVAR_PER_BUFFER ("truncate-lines", &current_buffer->truncate_lines, Qnil,
                      doc: /* *Non-nil means do not display continuation lines.
 Instead, give each line of text just one screen line.
diff --git a/src/buffer.h b/src/buffer.h
index c870f923e87..205bf865879 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -664,6 +664,10 @@ struct buffer
   /* Non-nil means set beginning of lines at the right edge of
      windows.  */
   Lisp_Object direction_reversed;
+  /* If non-nil, specifies which direction of text to force in each
+     paragraph.  Nil means determine paragraph direction dynamically
+     for each paragraph.  */
+  Lisp_Object paragraph_direction;
   /* Non-nil means do selective display;
      see doc string in syms_of_buffer (buffer.c) for details.  */
   Lisp_Object selective_display;
diff --git a/src/dispextern.h b/src/dispextern.h
index 6928d8ae1b8..d07b70df6cc 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -2327,6 +2327,7 @@ struct it
 
   /* For iterating over bidirectional text.  */
   struct bidi_it bidi_it;
+  bidi_dir_t paragraph_embedding;
 };
 
 
diff --git a/src/xdisp.c b/src/xdisp.c
index 7597b2c98ed..e8eb21c4e5a 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -248,6 +248,7 @@ Lisp_Object Qfontified;
 Lisp_Object Qgrow_only;
 Lisp_Object Qinhibit_eval_during_redisplay;
 Lisp_Object Qbuffer_position, Qposition, Qobject;
+Lisp_Object Qright_to_left, Qleft_to_right;
 
 /* Cursor shapes */
 Lisp_Object Qbar, Qhbar, Qbox, Qhollow;
@@ -2809,7 +2810,17 @@ init_iterator (it, w, charpos, bytepos, row, base_face_id)
   /* If we are to reorder bidirectional text, init the bidi
      iterator.  */
   if (it->bidi_p)
-    bidi_init_it (charpos, bytepos, &it->bidi_it);
+    {
+      /* Note the paragraph direction that this buffer wants to
+         use.  */
+      if (EQ (current_buffer->paragraph_direction, Qleft_to_right))
+        it->paragraph_embedding = L2R;
+      else if (EQ (current_buffer->paragraph_direction, Qright_to_left))
+        it->paragraph_embedding = R2L;
+      else
+        it->paragraph_embedding = NEUTRAL_DIR;
+      bidi_init_it (charpos, bytepos, &it->bidi_it);
+    }
 
   /* If a buffer position was specified, set the iterator there,
      getting overlays and face properties from that position.  */
@@ -6106,7 +6117,7 @@ set_iterator_to_next (it, reseat_p)
               /* If this is a new paragraph, determine its base
                  direction (a.k.a. its base embedding level).  */
               if (it->bidi_it.new_paragraph)
-                bidi_paragraph_init (NEUTRAL_DIR, &it->bidi_it);
+                bidi_paragraph_init (it->paragraph_embedding, &it->bidi_it);
               bidi_get_next_char_visually (&it->bidi_it);
               IT_BYTEPOS (*it) = it->bidi_it.bytepos;
               IT_CHARPOS (*it) = it->bidi_it.charpos;
@@ -6527,9 +6538,7 @@ next_element_from_buffer (it)
           || FETCH_CHAR (it->bidi_it.bytepos - 1) == '\n'
           || FETCH_CHAR (it->bidi_it.bytepos) == '\n')
         {
-          /* FIXME: NEUTRAL_DIR below should be user-definable and/or
-             come from some ``higher protocol''.  */
-          bidi_paragraph_init (NEUTRAL_DIR, &it->bidi_it);
+          bidi_paragraph_init (it->paragraph_embedding, &it->bidi_it);
           bidi_get_next_char_visually (&it->bidi_it);
         }
       else
@@ -6543,7 +6552,7 @@ next_element_from_buffer (it)
           IT_BYTEPOS (*it) = CHAR_TO_BYTE (IT_CHARPOS (*it));
           it->bidi_it.charpos = IT_CHARPOS (*it);
           it->bidi_it.bytepos = IT_BYTEPOS (*it);
-          bidi_paragraph_init (NEUTRAL_DIR, &it->bidi_it);
+          bidi_paragraph_init (it->paragraph_embedding, &it->bidi_it);
           do {
             /* Now return to buffer position where we were asked to
                get the next display element, and produce that.  */
@@ -24956,6 +24965,11 @@ syms_of_xdisp ()
   staticpro (&previous_help_echo_string);
   help_echo_pos = -1;
 
+  Qright_to_left = intern ("right-to-left");
+  staticpro (&Qright_to_left);
+  Qleft_to_right = intern ("left-to-right");
+  staticpro (&Qleft_to_right);
+
 #ifdef HAVE_WINDOW_SYSTEM
   DEFVAR_BOOL ("x-stretch-cursor", &x_stretch_cursor_p,
     doc: /* *Non-nil means draw block cursor as wide as the glyph under it.