Merge from trunkpending

5 files changed, 80 insertions, 177 deletions

diff --git a/admin/unidata/Makefile.in b/admin/unidata/Makefile.in
index e1fe247631f..c890dad8903 100644
--- a/admin/unidata/Makefile.in
+++ b/admin/unidata/Makefile.in
@@ -23,7 +23,7 @@ EMACS = ../../src/emacs
 DSTDIR = ../../lisp/international
 RUNEMACS = ${EMACS} -Q -batch
 
-all: ${DSTDIR}/charprop.el ../../src/biditype.h ../../src/bidimirror.h
+all: ${DSTDIR}/charprop.el
 
 .el.elc:
         ${RUNEMACS} -batch -f batch-byte-compile $<
@@ -38,12 +38,6 @@ ${DSTDIR}/charprop.el: unidata-gen.elc unidata.txt
         cd ${DSTDIR}; \
         ${RUNEMACS} -batch --load $${ELC} -f unidata-gen-files $${DATADIR} $${DATA}
 
-../../src/biditype.h: UnicodeData.txt
-        gawk -F";" -f biditype.awk $< > $@
-
-../../src/bidimirror.h: BidiMirroring.txt
-        gawk -F"[; ]+" -f bidimirror.awk $< > $@
-
 install: charprop.el
         cp charprop.el ${DSTDIR}
         cp `sed -n 's/^;; FILE: //p' < charprop.el` ${DSTDIR}
diff --git a/admin/unidata/bidimirror.awk b/admin/unidata/bidimirror.awk
deleted file mode 100644
index fc3e8afaace..00000000000
--- a/admin/unidata/bidimirror.awk
+++ /dev/null
@@ -1,37 +0,0 @@
-# Generate data for bidi_mirroring_table, see src/bidi.c:bidi_initialize.
-
-# Copyright (C) 2010-2011  Free Software Foundation, Inc.
-
-# This file is part of GNU Emacs.
-
-# GNU Emacs is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# GNU Emacs is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
-
-# Written by Eli Zaretskii <eliz@gnu.org>
-
-BEGIN {
-     printf "  struct {\n    int from, to;\n  } bidi_mirror[] = {\n";
-     first = 1;
- }
-
-$1 !~ /^#/ && NF >= 2 {
-     if (!first)
-         printf ",\n";
-     else
-         first = 0;
-     printf "\t{ 0x%s, 0x%s }", $1, $2;
- }
-
-END {
-     printf " };\n";
- }
diff --git a/admin/unidata/biditype.awk b/admin/unidata/biditype.awk
deleted file mode 100644
index bb1aaad1973..00000000000
--- a/admin/unidata/biditype.awk
+++ /dev/null
@@ -1,93 +0,0 @@
-# Generate data for filling bidi_type_table, see src/bidi.c:bidi_initialize.
-
-# Copyright (C) 2010-2011  Free Software Foundation, Inc.
-
-# This file is part of GNU Emacs.
-
-# GNU Emacs is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# GNU Emacs is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
-
-# Written by Eli Zaretskii <eliz@gnu.org>
-
-function trtype(type)
-{
-    # Types are listed in the order of decresing use in UnicodeData.txt:
-    if (type == "ON")
-        return "NEUTRAL_ON";
-    else if (type == "NSM")
-        return "WEAK_NSM";
-    else if (type == "AL")
-        return "STRONG_AL";
-    else if (type == "R")
-        return "STRONG_R";
-    else if (type == "BN")
-        return "WEAK_BN";
-    else if (type == "EN")
-        return "WEAK_EN";
-    else if (type == "ET")
-        return "WEAK_ET";
-    else if (type == "AN")
-        return "WEAK_AN";
-    else if (type == "WS")
-        return "NEUTRAL_WS";
-    else if (type == "CS")
-        return "WEAK_CS";
-    else if (type == "ES")
-        return "WEAK_ES";
-    else if (type == "B")
-        return "NEUTRAL_B";
-    else if (type == "S")
-        return "NEUTRAL_S";
-    else if (type == "LRE" || type == "RLE" || type == "LRO" || type == "RLO" || type == "PDF")
-        return type;
-    else if (type == "L")
-        return "STRONG_L";
-    else
-    {
-        printf "Unknown type: %s\n", type > "/dev/stderr";
-        exit 1;
-    }
-}
-
-BEGIN {
-     otype = "";
-     startcode = "";
-     endcode = "";
-     printf "  struct {\n    int from, to;\n    bidi_type_t type;\n  } bidi_type[] = {\n";
-     first = 1;
- }
-
- {   code = $1;
-     ntype = $5;
-     if (ntype != otype)
-     {
-         # Don't output data for L, as that's the default value, see bidi.c.
-         if (otype != "L" && startcode != "")
-         {
-             if (!first)
-                 printf ",\n";
-             else
-                 first = 0;
-             printf "\t{ 0x%s, 0x%s, %s }", startcode, endcode, trtype(otype);
-         }
-         otype = ntype;
-         startcode = code;
-         endcode = code;
-     }
-     else
-         endcode = code;
- }
-
-END {
-     printf " };\n";
- }
diff --git a/admin/unidata/makefile.w32-in b/admin/unidata/makefile.w32-in
index 6a877e0c1d0..0e9b9f0d2bd 100644
--- a/admin/unidata/makefile.w32-in
+++ b/admin/unidata/makefile.w32-in
@@ -29,7 +29,7 @@ EMACSLOADPATH = $(lisp);$(lisp)/international;$(lisp)/emacs-lisp
 # Quote EMACS so it could be a file name with embedded whitespace
 RUNEMACS = "$(EMACS)" -Q -batch
 
-all: $(DSTDIR)/charprop.el ../../src/biditype.h ../../src/bidimirror.h
+all: $(DSTDIR)/charprop.el
 
 .el.elc:
         $(RUNEMACS) -f batch-byte-compile $<
@@ -51,16 +51,6 @@ charprop-CMD: unidata-gen.elc unidata.txt
 
 ${DSTDIR}/charprop.el: charprop-$(SHELLTYPE)
 
-../../src/biditype.h: UnicodeData.txt
-        gawk -F";" -f biditype.awk -v BINMODE=2 $< > biditype.h
-        $(CP) biditype.h $@
-        $(DEL) biditype.h
-
-../../src/bidimirror.h: BidiMirroring.txt
-        gawk -F"[; ]+" -f bidimirror.awk -v BINMODE=2 $< > bidimirror.h
-        $(CP) bidimirror.h $@
-        $(DEL) bidimirror.h
-
 clean:
-        - $(DEL) unidata-gen.elc unidata.txt biditype.h bidimirror.h
+        - $(DEL) unidata-gen.elc unidata.txt
 
diff --git a/admin/unidata/unidata-gen.el b/admin/unidata/unidata-gen.el
index ab1dcd134ac..d9277217f0e 100644
--- a/admin/unidata/unidata-gen.el
+++ b/admin/unidata/unidata-gen.el
@@ -146,7 +146,7 @@
     (setq unidata-list (cdr table))))
 
 ;; Alist of this form:
-;;   (PROP INDEX GENERATOR FILENAME DOCSTRING DESCRIBER VAL-LIST)
+;;   (PROP INDEX GENERATOR FILENAME DOCSTRING DESCRIBER DEFAULT VAL-LIST)
 ;; PROP: character property
 ;; INDEX: index to each element of unidata-list for PROP.
 ;;   It may be a function that generates an alist of character codes
@@ -155,14 +155,21 @@
 ;; FILENAME: filename to store the char-table
 ;; DOCSTRING: docstring for the property
 ;; DESCRIBER: function to call to get a description string of property value
-;; DEFAULT: the default value of the property
+;; DEFAULT: the default value of the property.  It may have the form
+;;   (VAL0 (FROM1 TO1 VAL1) ...) which indicates that the default
+;;   value is VAL0 except for characters in the ranges specified by
+;;   FROMn and TOn (inclusive).  The default value of characters
+;;   between FROMn and TOn is VALn.
 ;; VAL-LIST: list of specially ordered property values
 
 (defconst unidata-prop-alist
   '((name
      1 unidata-gen-table-name "uni-name.el"
      "Unicode character name.
-Property value is a string.")
+Property value is a string or nil.
+The value nil stands for the default value \"null string\")."
+     nil
+     nil)
     (general-category
      2 unidata-gen-table-symbol "uni-category.el"
      "Unicode general category.
@@ -170,7 +177,7 @@ Property value is one of the following symbols:
   Lu, Ll, Lt, Lm, Lo, Mn, Mc, Me, Nd, Nl, No, Pc, Pd, Ps, Pe, Pi, Pf, Po,
   Sm, Sc, Sk, So, Zs, Zl, Zp, Cc, Cf, Cs, Co, Cn"
      unidata-describe-general-category
-     nil
+     Cn
      ;; The order of elements must be in sync with unicode_category_t
      ;; in src/character.h.
      (Lu Ll Lt Lm Lo Mn Mc Me Nd Nl No Pc Pd Ps Pe Pi Pf Po
@@ -179,7 +186,8 @@ Property value is one of the following symbols:
      3 unidata-gen-table-integer "uni-combining.el"
      "Unicode canonical combining class.
 Property value is an integer."
-     unidata-describe-canonical-combining-class)
+     unidata-describe-canonical-combining-class
+     0)
     (bidi-class
      4 unidata-gen-table-symbol "uni-bidi.el"
      "Unicode bidi class.
@@ -187,7 +195,12 @@ Property value is one of the following symbols:
   L, LRE, LRO, R, AL, RLE, RLO, PDF, EN, ES, ET,
   AN, CS, NSM, BN, B, S, WS, ON"
      unidata-describe-bidi-class
-     L
+     ;; The assignment of default values to blocks of code points
+     ;; follows the file DerivedBidiClass.txt from the Unicode
+     ;; Character Database (UCD).
+     (L (#x0600 #x06FF AL) (#xFB50 #xFDFF AL) (#xFE70 #xFEFF AL)
+        (#x0590 #x05FF R) (#x07C0 #x08FF R)
+        (#xFB1D #xFB4F R) (#x10800 #x10FFF R) (#x1E800 #x1EFFF R))
      ;; The order of elements must be in sync with bidi_type_t in
      ;; src/dispextern.h.
      (L R EN AN BN B AL LRE LRO RLE RLO PDF ES ET CS NSM S WS ON))
@@ -202,23 +215,29 @@ one of these symbols representing compatibility formatting tag:
     (decimal-digit-value
      6 unidata-gen-table-integer "uni-decimal.el"
      "Unicode numeric value (decimal digit).
-Property value is an integer.")
+Property value is an integer 0..9, or nil.
+The value nil stands for NaN \"Numeric_Value\".")
     (digit-value
      7 unidata-gen-table-integer "uni-digit.el"
      "Unicode numeric value (digit).
-Property value is an integer.")
+Property value is an integer 0..9, or nil.
+The value nil stands for NaN \"Numeric_Value\".")
     (numeric-value
      8 unidata-gen-table-numeric "uni-numeric.el"
      "Unicode numeric value (numeric).
-Property value is an integer or a floating point.")
+Property value is an integer, a floating point, or nil.
+The value nil stands for NaN \"Numeric_Value\".")
     (mirrored
      9 unidata-gen-table-symbol "uni-mirrored.el"
      "Unicode bidi mirrored flag.
-Property value is a symbol `Y' or `N'.  See also the property `mirroring'.")
+Property value is a symbol `Y' or `N'.  See also the property `mirroring'."
+     nil
+     N)
     (old-name
      10 unidata-gen-table-name "uni-old-name.el"
      "Unicode old names as published in Unicode 1.0.
-Property value is a string.")
+Property value is a string or nil.
+The value nil stands for the default value \"null string\").")
     (iso-10646-comment
      11 unidata-gen-table-name "uni-comment.el"
      "Unicode ISO 10646 comment.
@@ -226,23 +245,30 @@ Property value is a string.")
     (uppercase
      12 unidata-gen-table-character "uni-uppercase.el"
      "Unicode simple uppercase mapping.
-Property value is a character."
+Property value is a character or nil.
+The value nil means that the actual property value of a character
+is the character itself."
      string)
     (lowercase
      13 unidata-gen-table-character "uni-lowercase.el"
      "Unicode simple lowercase mapping.
-Property value is a character."
+Property value is a character or nil.
+The value nil means that the actual property value of a character
+is the character itself."
      string)
     (titlecase
      14 unidata-gen-table-character "uni-titlecase.el"
      "Unicode simple titlecase mapping.
-Property value is a character."
+Property value is a character or nil.
+The value nil means that the actual property value of a character
+is the character itself."
      string)
     (mirroring
      unidata-gen-mirroring-list unidata-gen-table-character "uni-mirrored.el"
      "Unicode bidi-mirroring characters.
-Property value is a character that has the corresponding mirroring image,
-or nil for non-mirrored character.")))
+Property value is a character that has the corresponding mirroring image or nil.
+The value nil means that the actual property value of a character
+is the character itself.")))
 
 ;; Functions to access the above data.
 (defsubst unidata-prop-index (prop) (nth 1 (assq prop unidata-prop-alist)))
@@ -393,9 +419,18 @@ or nil for non-mirrored character.")))
     (while tail
       (setcar tail (cons (car tail) val-code))
       (setq tail (cdr tail) val-code (1+ val-code)))
-    (setq default-value (unidata-encode-val val-list default-value))
-    (set-char-table-range table t default-value)
-    (set-char-table-range table nil default-value)
+    (if (consp default-value)
+        (setq default-value (copy-sequence default-value))
+      (setq default-value (list default-value)))
+    (setcar default-value
+            (unidata-encode-val val-list (car default-value)))
+    (set-char-table-range table t (car default-value))
+    (set-char-table-range table nil (car default-value))
+    (dolist (elm (cdr default-value))
+      (setcar (nthcdr 2 elm)
+              (unidata-encode-val val-list (nth 2 elm)))
+      (set-char-table-range table (cons (car elm) (nth 1 elm)) (nth 2 elm)))
+
     (setq tail unidata-list)
     (while tail
       (setq elt (car tail) tail (cdr tail))
@@ -419,17 +454,27 @@ or nil for non-mirrored character.")))
                 (setq prev-range-data (cons (cons from to) val-code)))))
         (let* ((start (lsh (lsh range -7) 7))
                (limit (+ start 127))
-               str count new-val)
-          (fillarray vec 0)
-          ;; See the comment above.
-          (when (and prev-range-data
-                     (>= (cdr (car prev-range-data)) start))
-            (let ((from (car (car prev-range-data)))
-                  (to (cdr (car prev-range-data)))
-                  (vcode (cdr prev-range-data)))
+               str count new-val from to vcode)
+          (fillarray vec (car default-value))
+          (dolist (elm (cdr default-value))
+            (setq from (car elm) to (nth 1 elm))
+            (when (and (<= from limit)
+                       (or (>= from start) (>= to start)))
+              (setq from (max from start)
+                    to (min to limit)
+                    vcode (nth 2 elm))
               (while (<= from to)
                 (aset vec (- from start) vcode)
                 (setq from (1+ from)))))
+          ;; See the comment above.
+          (when (and prev-range-data
+                     (>= (cdr (car prev-range-data)) start))
+            (setq from (car (car prev-range-data))
+                  to (cdr (car prev-range-data))
+                  vcode (cdr prev-range-data))
+            (while (<= from to)
+              (aset vec (- from start) vcode)
+              (setq from (1+ from))))
           (setq prev-range-data nil)
           (if val-code
               (aset vec (- range start) val-code))
@@ -707,6 +752,9 @@ or nil for non-mirrored character.")))
 
 (defun unidata-get-decomposition (char val table)
   (cond
+   ((not val)
+    (list char))
+
    ((consp val)
     val)
 
@@ -747,7 +795,8 @@ or nil for non-mirrored character.")))
             (aset vec idx (nconc word-list tail-list)))
         (dotimes (i 128)
           (aset table (+ first-char i) (aref vec i)))
-        (aref vec (- char first-char)))))
+        (setq val (aref vec (- char first-char)))
+        (or val (list char)))))
 
    ;; Hangul syllable
    ((and (eq val 0) (>= char #xAC00) (<= char #xD7A3))