Support casing characters which map into multiple code points (bug#24603)

Implement unconditional special casing rules defined in Unicode standard.

Among other things, they deal with cases when a single code point is
replaced by multiple ones because single character does not exist (e.g.
‘fi’ ligature turning into ‘FL’) or is not commonly used (e.g. ß turning
into SS).

* admin/unidata/SpecialCasing.txt: New data file pulled from Unicode
standard distribution.
* admin/unidata/README: Mention SpecialCasing.txt.

* admin/unidata/unidata-get.el (unidata-gen-table-special-casing,
unidata-gen-table-special-casing--do-load): New functions generating
‘special-uppercase’, ‘special-lowercase’ and ‘special-titlecase’
character Unicode properties built from the SpecialCasing.txt Unicode
data file.

* src/casefiddle.c (struct casing_str_buf): New structure for
representing short strings used to handle one-to-many character
mappings.

(case_character_imlp): New function which can handle one-to-many
character mappings.
(case_character, case_single_character): Wrappers for the above
functions.  The former may map one character to multiple (or no)
code points while the latter does what the former used to do (i.e.
handles one-to-one mappings only).

(do_casify_natnum, do_casify_unibyte_string,
do_casify_unibyte_region): Use case_single_character.
(do_casify_multibyte_string, do_casify_multibyte_region): Support new
features of case_character.
* (do_casify_region): Updated to reflact do_casify_multibyte_string
changes.

(casify_word): Handle situation when one character-length of a word
can change affecting where end of the word is.

(upcase, capitalize, upcase-initials): Update documentation to mention
limitations when working on characters.

* test/src/casefiddle-tests.el (casefiddle-tests-char-properties):
Add test cases for the newly introduced character properties.
(casefiddle-tests-casing): Update test cases which are now passing.

* test/lisp/char-fold-tests.el (char-fold--ascii-upcase,
char-fold--ascii-downcase): New functions which behave like old ‘upcase’
and ‘downcase’.
(char-fold--test-match-exactly): Use the new functions.  This is needed
because otherwise fi and similar characters are turned into their multi-
-character representation.

* doc/lispref/strings.texi: Describe issue with casing characters versus
strings.
* doc/lispref/nonascii.texi: Describe the new character properties.

1 files changed, 37 insertions, 36 deletions

diff --git a/test/src/casefiddle-tests.el b/test/src/casefiddle-tests.el
index e83cb00059b..dd260633f4c 100644
--- a/test/src/casefiddle-tests.el
+++ b/test/src/casefiddle-tests.el
@@ -24,36 +24,40 @@
 
 (ert-deftest casefiddle-tests-char-properties ()
   "Sanity check of character Unicode properties."
-  (should-not
-   (let (errors)
-     ;;            character  uppercase  lowercase  titlecase
-     (dolist (test '((?A nil ?a nil)
-                     (?a ?A nil ?A)
-                     (?Ł nil ?ł nil)
-                     (?ł ?Ł nil ?Ł)
-
-                     (?DŽ nil ?dž ?Dž)
-                     (?Dž ?DŽ ?dž ?Dž)
-                     (?dž ?DŽ nil ?Dž)
-
-                     (?Σ nil ?σ nil)
-                     (?σ ?Σ nil ?Σ)
-                     (?ς ?Σ nil ?Σ)
-
-                     (?ⅷ ?Ⅷ nil ?Ⅷ)
-                     (?Ⅷ nil ?ⅷ nil)))
-       (let ((ch (car test))
-             (expected (cdr test))
-             (props '(uppercase lowercase titlecase)))
-         (while props
-           (let ((got (get-char-code-property ch (car props))))
-             (unless (equal (car expected) got)
-               (push (format "\n%c %s; expected: %s but got: %s"
-                             ch (car props) (car expected) got)
-                     errors)))
-           (setq props (cdr props) expected (cdr expected)))))
-     (when errors
-       (mapconcat (lambda (line) line) (nreverse errors) "")))))
+  (let ((props '(uppercase lowercase titlecase
+                 special-uppercase special-lowercase special-titlecase))
+        (tests '((?A nil ?a  nil  nil  nil  nil)
+                 (?a ?A  nil ?A   nil  nil  nil)
+                 (?Ł nil ?ł  nil  nil  nil  nil)
+                 (?ł ?Ł  nil ?Ł   nil  nil  nil)
+
+                 (?DŽ nil ?dž  ?Dž   nil  nil  nil)
+                 (?Dž ?DŽ  ?dž  ?Dž   nil  nil  nil)
+                 (?dž ?DŽ  nil ?Dž   nil  nil  nil)
+
+                 (?Σ nil ?σ  nil  nil  nil  nil)
+                 (?σ ?Σ  nil ?Σ   nil  nil  nil)
+                 (?ς ?Σ  nil ?Σ   nil  nil  nil)
+
+                 (?ⅷ ?Ⅷ  nil ?Ⅷ   nil  nil  nil)
+                 (?Ⅷ nil ?ⅷ  nil  nil  nil  nil)
+
+                 (?fi nil nil nil  "FI" nil "Fi")
+                 (?ß nil nil nil  "SS" nil "Ss")
+                 (?İ nil ?i  nil  nil "i\u0307" nil)))
+        errors)
+    (dolist (test tests)
+      (let ((ch (car test))
+            (expected (cdr test)))
+        (dolist (prop props)
+          (let ((got (get-char-code-property ch prop)))
+            (unless (equal (car expected) got)
+              (push (format "\n%c %s; expected: %s but got: %s"
+                            ch prop (car expected) got)
+                    errors)))
+          (setq expected (cdr expected)))))
+    (when errors
+      (ert-fail (mapconcat (lambda (line) line) (nreverse errors) "")))))
 
 
 (defconst casefiddle-tests--characters
@@ -188,16 +192,13 @@
         ("DŽUNGLA" "DŽUNGLA" "džungla" "Džungla" "DžUNGLA")
         ("Džungla" "DŽUNGLA" "džungla" "Džungla" "Džungla")
         ("džungla" "DŽUNGLA" "džungla" "Džungla" "Džungla")
+        ("define" "DEFINE" "define" "Define" "Define")
+        ("fish" "FISH" "fish" "Fish" "Fish")
+        ("Straße" "STRASSE" "straße" "Straße" "Straße")
         ;; FIXME(bug#24603): Everything below is broken at the moment.
         ;; Here’s what should happen:
-        ;;("define" "DEFINE" "define" "Define" "Define")
-        ;;("fish" "FIsh" "fish" "Fish" "Fish")
-        ;;("Straße" "STRASSE" "straße" "Straße" "Straße")
         ;;("ΌΣΟΣ" "ΌΣΟΣ" "όσος" "Όσος" "Όσος")
         ;; And here’s what is actually happening:
-        ("define" "DEfiNE" "define" "Define" "Define")
-        ("fish" "fiSH" "fish" "fish" "fish")
-        ("Straße" "STRAßE" "straße" "Straße" "Straße")
         ("ΌΣΟΣ" "ΌΣΟΣ" "όσοσ" "Όσοσ" "ΌΣΟΣ")
 
         ("όσος" "ΌΣΟΣ" "όσος" "Όσος" "Όσος"))))))