Improve error reporting when serializing non-Unicode strings to JSON

* src/coding.c (utf8_string_p): New helper function.
(syms_of_coding) <utf-8-unix>: Move from json.c.

* src/json.c (json_check_utf8): New helper function.
(lisp_to_json_toplevel_1, lisp_to_json): Use it.  To save a bit of
time, check for invalid UTF-8 strings only after encountering an
error, since Jansson already rejects them.

* test/src/json-tests.el (json-serialize/invalid-unicode): Adapt
expected error symbol.

1 files changed, 25 insertions, 8 deletions

diff --git a/src/json.c b/src/json.c
index 88db86ad2e3..93dcc730dae 100644
--- a/src/json.c
+++ b/src/json.c
@@ -316,6 +316,15 @@ json_check (json_t *object)
   return object;
 }
 
+/* If STRING is not a valid UTF-8 string, signal an error of type
+   `wrong-type-argument'.  STRING must be a unibyte string.  */
+
+static void
+json_check_utf8 (Lisp_Object string)
+{
+  CHECK_TYPE (utf8_string_p (string), Qutf_8_string_p, string);
+}
+
 static json_t *lisp_to_json (Lisp_Object);
 
 /* Convert a Lisp object to a toplevel JSON object (array or object).
@@ -363,9 +372,12 @@ lisp_to_json_toplevel_1 (Lisp_Object lisp, json_t **json)
             int status = json_object_set_new (*json, key_str,
                                               lisp_to_json (HASH_VALUE (h, i)));
             if (status == -1)
-              /* FIXME: A failure here might also indicate that the
-                 key is not a valid Unicode string.  */
-              json_out_of_memory ();
+              {
+                /* A failure can be caused either by an invalid key or
+                   by low memory.  */
+                json_check_utf8 (key);
+                json_out_of_memory ();
+              }
           }
       clear_unwind_protect (count);
       return unbind_to (count, Qnil);
@@ -447,9 +459,15 @@ lisp_to_json (Lisp_Object lisp)
   else if (STRINGP (lisp))
     {
       Lisp_Object encoded = json_encode (lisp);
-      /* FIXME: We might throw an out-of-memory error here if the
-         string is not valid Unicode.  */
-      return json_check (json_stringn (SSDATA (encoded), SBYTES (encoded)));
+      json_t *json = json_stringn (SSDATA (encoded), SBYTES (encoded));
+      if (json == NULL)
+        {
+          /* A failure can be caused either by an invalid string or by
+             low memory.  */
+          json_check_utf8 (encoded);
+          json_out_of_memory ();
+        }
+      return json;
     }
 
   /* LISP now must be a vector, hashtable, or alist.  */
@@ -863,8 +881,7 @@ syms_of_json (void)
 
   DEFSYM (Qstring_without_embedded_nulls_p, "string-without-embedded-nulls-p");
   DEFSYM (Qjson_value_p, "json-value-p");
-
-  DEFSYM (Qutf_8_unix, "utf-8-unix");
+  DEFSYM (Qutf_8_string_p, "utf-8-string-p");
 
   DEFSYM (Qjson_error, "json-error");
   DEFSYM (Qjson_out_of_memory, "json-out-of-memory");