merge from trunk

1 files changed, 186 insertions, 14 deletions

diff --git a/src/coding.c b/src/coding.c
index 32da72ab626..6cfcec905a1 100644
--- a/src/coding.c
+++ b/src/coding.c
@@ -286,6 +286,10 @@ encode_coding_XXX (struct coding_system *coding)
 #include <config.h>
 #include <stdio.h>
 
+#ifdef HAVE_WCHAR_H
+#include <wchar.h>
+#endif /* HAVE_WCHAR_H */
+
 #include "lisp.h"
 #include "character.h"
 #include "buffer.h"
@@ -6067,6 +6071,93 @@ complement_process_encoding_system (Lisp_Object coding_system)
 #define EOL_SEEN_CR     2
 #define EOL_SEEN_CRLF   4
 
+
+static Lisp_Object adjust_coding_eol_type (struct coding_system *coding, int eol_seen);
+
+
+/* Return 1 if all the source bytes are ASCII, and return 0 otherwize.
+   By side effects, set coding->head_ascii and coding->eol_seen.  The
+   value of coding->eol_seen is "logical or" of EOL_SEEN_LF,
+   EOL_SEEN_CR, and EOL_SEEN_CRLF, but the value is reliable only when
+   all the source bytes are ASCII.  */
+
+static bool
+detect_ascii (struct coding_system *coding)
+{
+  const unsigned char *src, *end;
+  Lisp_Object eol_type = CODING_ID_EOL_TYPE (coding->id);
+  int eol_seen;
+
+  eol_seen = (VECTORP (eol_type) ? EOL_SEEN_NONE
+              : EQ (eol_type, Qunix) ? EOL_SEEN_LF
+              : EQ (eol_type, Qdos) ? EOL_SEEN_CRLF
+              : EOL_SEEN_CR);
+  coding_set_source (coding);
+  src = coding->source;
+  end = src + coding->src_bytes;
+
+  if (inhibit_eol_conversion)
+    {
+      /* We don't have to check EOL format.  */
+      while (src < end && !( *src & 0x80)) src++;
+      eol_seen = EOL_SEEN_LF;
+      adjust_coding_eol_type (coding, eol_seen);
+    }
+  else if (eol_seen != EOL_SEEN_NONE)
+    {
+      /* We don't have to check EOL format either.  */
+      while (src < end && !(*src & 0x80)) src++;
+    }
+  else
+    {
+      end--;                    /* We look ahead one byte.  */
+      while (src < end)
+        {
+          int c = *src;
+
+          if (c & 0x80)
+            break;
+          src++;
+          if (c < 0x20)
+            {
+              if (c == '\r')
+                {
+                  if (*src == '\n')
+                    {
+                      eol_seen |= EOL_SEEN_CRLF;
+                      src++;
+                    }
+                  else
+                    eol_seen |= EOL_SEEN_CR;
+                }
+              else if (c == '\n')
+                eol_seen |= EOL_SEEN_LF;
+            }
+        }
+      if (src > end)
+        /* The last two bytes are CR LF, which means that we have
+           scanned all bytes. */
+        end++;
+      else if (src == end)
+        {
+          end++;
+          if (! (*src & 0x80))
+            {
+              if (*src == '\r')
+                eol_seen |= EOL_SEEN_CR;
+              else if (*src  == '\n')
+                eol_seen |= EOL_SEEN_LF;
+              src++;
+            }
+        }
+      adjust_coding_eol_type (coding, eol_seen);
+    }
+  coding->head_ascii = src - coding->source;
+  coding->eol_seen = eol_seen;
+  return (src == end);
+}
+
+
 /* Detect how end-of-line of a text of length SRC_BYTES pointed by
    SOURCE is encoded.  If CATEGORY is one of
    coding_category_utf_16_XXXX, assume that CR and LF are encoded by
@@ -6211,7 +6302,6 @@ detect_coding (struct coding_system *coding)
   coding_set_source (coding);
 
   src_end = coding->source + coding->src_bytes;
-  coding->head_ascii = 0;
 
   /* If we have not yet decided the text encoding type, detect it
      now.  */
@@ -6221,6 +6311,8 @@ detect_coding (struct coding_system *coding)
       struct coding_detection_info detect_info;
       bool null_byte_found = 0, eight_bit_found = 0;
 
+      coding->head_ascii = 0;
+      coding->eol_seen = EOL_SEEN_NONE;
       detect_info.checked = detect_info.found = detect_info.rejected = 0;
       for (src = coding->source; src < src_end; src++)
         {
@@ -6259,6 +6351,26 @@ detect_coding (struct coding_system *coding)
                   if (eight_bit_found)
                     break;
                 }
+              else if (! disable_ascii_optimization
+                       && ! inhibit_eol_conversion)
+                {
+                  if (c == '\r')
+                    {
+                      if (src < src_end && src[1] == '\n')
+                        {
+                          coding->eol_seen |= EOL_SEEN_CRLF;
+                          src++;
+                          coding->head_ascii++;
+                        }
+                      else
+                        coding->eol_seen |= EOL_SEEN_CR;
+                    }
+                  else if (c == '\n')
+                    {
+                      coding->eol_seen |= EOL_SEEN_LF;
+                    }
+                }
+
               if (! eight_bit_found)
                 coding->head_ascii++;
             }
@@ -6349,14 +6461,20 @@ detect_coding (struct coding_system *coding)
       coding_systems
         = AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_bom);
       detect_info.found = detect_info.rejected = 0;
-      coding->head_ascii = 0;
-      if (CONSP (coding_systems)
-          && detect_coding_utf_8 (coding, &detect_info))
+      if (detect_ascii (coding))
         {
-          if (detect_info.found & CATEGORY_MASK_UTF_8_SIG)
-            setup_coding_system (XCAR (coding_systems), coding);
-          else
-            setup_coding_system (XCDR (coding_systems), coding);
+          setup_coding_system (XCDR (coding_systems), coding);
+        }
+      else
+        {
+          if (CONSP (coding_systems)
+              && detect_coding_utf_8 (coding, &detect_info))
+            {
+              if (detect_info.found & CATEGORY_MASK_UTF_8_SIG)
+                setup_coding_system (XCAR (coding_systems), coding);
+              else
+                setup_coding_system (XCDR (coding_systems), coding);
+            }
         }
     }
   else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id)))
@@ -6369,6 +6487,7 @@ detect_coding (struct coding_system *coding)
         = AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_bom);
       detect_info.found = detect_info.rejected = 0;
       coding->head_ascii = 0;
+      coding->eol_seen = EOL_SEEN_NONE;
       if (CONSP (coding_systems)
           && detect_coding_utf_16 (coding, &detect_info))
         {
@@ -6806,7 +6925,7 @@ produce_chars (struct coding_system *coding, Lisp_Object translation_table,
 
   produced = dst - (coding->destination + coding->produced);
   if (BUFFERP (coding->dst_object) && produced_chars > 0)
-    insert_from_gap (produced_chars, produced);
+    insert_from_gap (produced_chars, produced, 0);
   coding->produced += produced;
   coding->produced_char += produced_chars;
   return carryover;
@@ -7391,7 +7510,7 @@ encode_coding (struct coding_system *coding)
   } while (coding->consumed_char < coding->src_chars);
 
   if (BUFFERP (coding->dst_object) && coding->produced_char > 0)
-    insert_from_gap (coding->produced_char, coding->produced);
+    insert_from_gap (coding->produced_char, coding->produced, 0);
 
   SAFE_FREE ();
 }
@@ -7487,8 +7606,6 @@ decode_coding_gap (struct coding_system *coding,
   ptrdiff_t count = SPECPDL_INDEX ();
   Lisp_Object attrs;
 
-  code_conversion_save (0, 0);
-
   coding->src_object = Fcurrent_buffer ();
   coding->src_chars = chars;
   coding->src_bytes = bytes;
@@ -7502,13 +7619,53 @@ decode_coding_gap (struct coding_system *coding,
 
   if (CODING_REQUIRE_DETECTION (coding))
     detect_coding (coding);
+  attrs = CODING_ID_ATTRS (coding->id);
+  if (! disable_ascii_optimization)
+    {
+      if (! NILP (CODING_ATTR_ASCII_COMPAT (attrs))
+          && NILP (CODING_ATTR_POST_READ (attrs))
+          && NILP (get_translation_table (attrs, 0, NULL))
+          && (coding->head_ascii >= 0 /* We've already called detect_coding */
+              ? coding->head_ascii == bytes
+              : detect_ascii (coding)))
+        {
+          if (coding->eol_seen == EOL_SEEN_CR)
+            {
+              unsigned char *src_end = GAP_END_ADDR;
+              unsigned char *src = src_end - coding->src_bytes;
+
+              while (src < src_end)
+                {
+                  if (*src++ == '\r')
+                    src[-1] = '\n';
+                }
+            }
+          else if (coding->eol_seen == EOL_SEEN_CRLF)
+            {
+              unsigned char *src = GAP_END_ADDR;
+              unsigned char *src_beg = src - coding->src_bytes;
+              unsigned char *dst = src;
+
+              while (src_beg < src)
+                {
+                  *--dst = *--src;
+                  if (*src == '\n')
+                    src--;
+                }
+              bytes -= dst - src;
+            }
+          coding->produced_char = coding->produced = bytes;
+          insert_from_gap (bytes, bytes, 1);
+          return;
+        }
+    }
+  code_conversion_save (0, 0);
 
   coding->mode |= CODING_MODE_LAST_BLOCK;
   current_buffer->text->inhibit_shrinking = 1;
   decode_coding (coding);
   current_buffer->text->inhibit_shrinking = 0;
 
-  attrs = CODING_ID_ATTRS (coding->id);
   if (! NILP (CODING_ATTR_POST_READ (attrs)))
     {
       ptrdiff_t prev_Z = Z, prev_Z_BYTE = Z_BYTE;
@@ -7966,11 +8123,21 @@ from_unicode (Lisp_Object str)
   return code_convert_string_norecord (str, Qutf_16le, 0);
 }
 
+Lisp_Object
+from_unicode_buffer (const wchar_t* wstr)
+{
+    return from_unicode (
+        make_unibyte_string (
+            (char*) wstr,
+            /* we get one of the two final 0 bytes for free. */
+            1 + sizeof (wchar_t) * wcslen (wstr)));
+}
+
 wchar_t *
 to_unicode (Lisp_Object str, Lisp_Object *buf)
 {
   *buf = code_convert_string_norecord (str, Qutf_16le, 1);
-  /* We need to make a another copy (in addition to the one made by
+  /* We need to make another copy (in addition to the one made by
      code_convert_string_norecord) to ensure that the final string is
      _doubly_ zero terminated --- that is, that the string is
      terminated by two zero bytes and one utf-16le null character.
@@ -10707,6 +10874,11 @@ from GNU Find and GNU Grep.  Emacs will then ignore the null bytes and
 decode text as usual.  */);
   inhibit_null_byte_detection = 0;
 
+  DEFVAR_BOOL ("disable-ascii-optimization", disable_ascii_optimization,
+               doc: /* If non-nil, Emacs does not optimize code decoder for ASCII files.
+Internal use only.  Removed after the experimental optimizer gets stable. */);
+  disable_ascii_optimization = 0;
+
   DEFVAR_LISP ("translation-table-for-input", Vtranslation_table_for_input,
                doc: /* Char table for translating self-inserting characters.
 This is applied to the result of input methods, not their input.