Display pre-edit information from X input methods

This also repurposes the `pgtk-preedit-text' event to be
meaningful on X, renames it `preedit-text', and documents it.

* doc/lispref/commands.texi (Misc Events): Document
`preedit-text'.
* lisp/term/pgtk-win.el (pgtk-preedit-text): Bind to
`preedit-text' instead.
* lisp/term/x-win.el (x-preedit-overlay): New variable.
(x-preedit-text): New command, bound as a special event to
`preedit-text'.

* src/keyboard.c (kbd_buffer_get_event):
(make_lispy_event): Rename PGTK_PREEDIT_TEXT_EVENT
PREEDIT_TEXT_EVENT.
(syms_of_keyboard): New defsym `preedit-text'.
* src/pgtkterm.c (pgtk_enqueue_preedit): Use PREEDIT_TEXT_EVENT
instead.
* src/termhooks.h (enum event_kind): Rename
`PGTK_PREEDIT_TEXT_EVENT' `PREEDIT_TEXT_EVENT'.

* src/xfns.c (Xxic_preedit_draw_callback):
(Xxic_preedit_caret_callback):
(Xxic_preedit_done_callback):
(Xxic_preedit_start_callback): New callback variables.

(STYLE_OFFTHESPOT, STYLE_OVERTHESPOT):
(STYLE_ROOT, STYLE_CALLBACK, STYLE_NONE): New macros.
(supported_xim_styles): Use reasonable values.  This also serves
as a better fix for bug#10867.
(best_xim_style): Restore code deleted as part of the original
fix for bug#10867.
(create_frame_xic): Add preedit callbacks.
(xic_set_preeditarea): Add preedit callbacks.
(x_xic_to_frame):
(xic_preedit_start_callback):
(xic_preedit_caret_callback):
(xic_preedit_done_callback):
(x_xim_text_to_utf8_unix):
(xic_preedit_draw_callback): New functions.

* src/xterm.c (x_detect_focus_change): Fix type of XI event.
(x_free_frame_resources): Free preedit text buffer if still
present.
* src/xterm.h (struct x_output): New fields `preedit_size',
`preedit_chars' and `preedit_active'.

6 files changed, 345 insertions, 21 deletions

diff --git a/src/keyboard.c b/src/keyboard.c
index ec1b7cd85d3..a9f3257282b 100644
--- a/src/keyboard.c
+++ b/src/keyboard.c
@@ -3973,9 +3973,7 @@ kbd_buffer_get_event (KBOARD **kbp,
           *used_mouse_menu = true;
         FALLTHROUGH;
 #endif
-#ifdef HAVE_PGTK
-      case PGTK_PREEDIT_TEXT_EVENT:
-#endif
+      case PREEDIT_TEXT_EVENT:
 #ifdef HAVE_NTGUI
       case END_SESSION_EVENT:
       case LANGUAGE_CHANGE_EVENT:
@@ -6289,10 +6287,8 @@ make_lispy_event (struct input_event *event)
         return list3 (Qconfig_changed_event,
                       event->arg, event->frame_or_window);
 
-#ifdef HAVE_PGTK
-    case PGTK_PREEDIT_TEXT_EVENT:
-      return list2 (intern ("pgtk-preedit-text"), event->arg);
-#endif
+    case PREEDIT_TEXT_EVENT:
+      return list2 (Qpreedit_text, event->arg);
 
       /* The 'kind' field of the event is something we don't recognize.  */
     default:
@@ -12003,6 +11999,8 @@ syms_of_keyboard (void)
   DEFSYM (Qno_record, "no-record");
   DEFSYM (Qencoded, "encoded");
 
+  DEFSYM (Qpreedit_text, "preedit-text");
+
   button_down_location = make_nil_vector (5);
   staticpro (&button_down_location);
   staticpro (&frame_relative_event_pos);
@@ -12771,8 +12769,6 @@ keys_of_keyboard (void)
                             "ns-put-working-text");
   initial_define_lispy_key (Vspecial_event_map, "ns-unput-working-text",
                             "ns-unput-working-text");
-  initial_define_lispy_key (Vspecial_event_map, "pgtk-preedit-text",
-                            "pgtk-preedit-text");
   /* Here we used to use `ignore-event' which would simple set prefix-arg to
      current-prefix-arg, as is done in `handle-switch-frame'.
      But `handle-switch-frame is not run from the special-map.
diff --git a/src/pgtkterm.c b/src/pgtkterm.c
index 736fce09c4a..1d301d11f6f 100644
--- a/src/pgtkterm.c
+++ b/src/pgtkterm.c
@@ -5259,7 +5259,7 @@ pgtk_enqueue_preedit (struct frame *f, Lisp_Object preedit)
 {
   union buffered_input_event inev;
   EVENT_INIT (inev.ie);
-  inev.ie.kind = PGTK_PREEDIT_TEXT_EVENT;
+  inev.ie.kind = PREEDIT_TEXT_EVENT;
   inev.ie.arg = preedit;
   inev.ie.code = 0;
   XSETFRAME (inev.ie.frame_or_window, f);
diff --git a/src/termhooks.h b/src/termhooks.h
index 55f7aa5d1ae..518e855eae1 100644
--- a/src/termhooks.h
+++ b/src/termhooks.h
@@ -269,10 +269,8 @@ enum event_kind
   , FILE_NOTIFY_EVENT
 #endif
 
-#ifdef HAVE_PGTK
   /* Pre-edit text was changed. */
-  , PGTK_PREEDIT_TEXT_EVENT
-#endif
+  , PREEDIT_TEXT_EVENT
 
   /* Either the mouse wheel has been released without it being
      clicked, or the user has lifted his finger from a touchpad.
diff --git a/src/xfns.c b/src/xfns.c
index b94fe179224..d87e67f95b1 100644
--- a/src/xfns.c
+++ b/src/xfns.c
@@ -24,6 +24,7 @@ along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.  */
 #include <unistd.h>
 
 #include "lisp.h"
+#include "character.h"
 #include "xterm.h"
 #include "frame.h"
 #include "window.h"
@@ -2330,8 +2331,19 @@ hack_wm_protocols (struct frame *f, Widget widget)
 
 #ifdef HAVE_X_I18N
 
-static XFontSet xic_create_xfontset (struct frame *);
-static XIMStyle best_xim_style (XIMStyles *);
+static void xic_preedit_draw_callback (XIC, XPointer, XIMPreeditDrawCallbackStruct *);
+static void xic_preedit_caret_callback (XIC, XPointer, XIMPreeditCaretCallbackStruct *);
+static void xic_preedit_done_callback (XIC, XPointer, XPointer);
+static int xic_preedit_start_callback (XIC, XPointer, XPointer);
+
+static XIMCallback Xxic_preedit_draw_callback = { NULL,
+                                                  (XIMProc) xic_preedit_draw_callback };
+static XIMCallback Xxic_preedit_caret_callback = { NULL,
+                                                   (XIMProc) xic_preedit_caret_callback };
+static XIMCallback Xxic_preedit_done_callback = { NULL,
+                                                  (XIMProc) xic_preedit_done_callback };
+static XIMCallback Xxic_preedit_start_callback = { NULL,
+                                                   (void *) xic_preedit_start_callback };
 
 #if defined HAVE_X_WINDOWS && defined USE_X_TOOLKIT
 /* Create an X fontset on frame F with base font name BASE_FONTNAME.  */
@@ -2608,6 +2620,23 @@ xic_free_xfontset (struct frame *f)
   FRAME_XIC_FONTSET (f) = NULL;
 }
 
+/* Create XIC for frame F. */
+
+
+#define STYLE_OFFTHESPOT (XIMPreeditArea | XIMStatusArea)
+#define STYLE_OVERTHESPOT (XIMPreeditPosition | XIMStatusNothing)
+#define STYLE_ROOT (XIMPreeditNothing | XIMStatusNothing)
+#define STYLE_CALLBACK (XIMPreeditCallbacks | XIMStatusNothing)
+#define STYLE_NONE (XIMPreeditNothing | XIMStatusNothing)
+
+static const XIMStyle supported_xim_styles[] =
+  {
+    STYLE_CALLBACK,
+    STYLE_NONE,
+    STYLE_OVERTHESPOT,
+    STYLE_OFFTHESPOT,
+    STYLE_ROOT
+  };
 
 /* Value is the best input style, given user preferences USER (already
    checked to be supported by Emacs), and styles supported by the
@@ -2616,8 +2645,15 @@ xic_free_xfontset (struct frame *f)
 static XIMStyle
 best_xim_style (XIMStyles *xim)
 {
-  /* Return the default style. This is what GTK3 uses and
-     should work fine with all modern input methods.  */
+  int i, j;
+  int nr_supported = ARRAYELTS (supported_xim_styles);
+
+  for (i = 0; i < nr_supported; ++i)
+    for (j = 0; j < xim->count_styles; ++j)
+      if (supported_xim_styles[i] == xim->supported_styles[j])
+        return supported_xim_styles[i];
+
+  /* Return the default style.  */
   return XIMPreeditNothing | XIMStatusNothing;
 }
 
@@ -2692,6 +2728,22 @@ create_frame_xic (struct frame *f)
         goto out;
     }
 
+  if (xic_style & XIMPreeditCallbacks)
+    {
+      spot.x = 0;
+      spot.y = 0;
+      preedit_attr = XVaCreateNestedList (0,
+                                          XNSpotLocation, &spot,
+                                          XNPreeditStartCallback, &Xxic_preedit_start_callback,
+                                          XNPreeditDoneCallback, &Xxic_preedit_done_callback,
+                                          XNPreeditDrawCallback, &Xxic_preedit_draw_callback,
+                                          XNPreeditCaretCallback, &Xxic_preedit_caret_callback,
+                                          NULL);
+
+      if (!preedit_attr)
+        goto out;
+    }
+
   if (preedit_attr && status_attr)
     xic = XCreateIC (xim,
                      XNInputStyle, xic_style,
@@ -2768,7 +2820,12 @@ xic_set_preeditarea (struct window *w, int x, int y)
 
   spot.x = WINDOW_TO_FRAME_PIXEL_X (w, x) + WINDOW_LEFT_FRINGE_WIDTH (w) + WINDOW_LEFT_MARGIN_WIDTH(w);
   spot.y = WINDOW_TO_FRAME_PIXEL_Y (w, y) + FONT_BASE (FRAME_FONT (f));
-  attr = XVaCreateNestedList (0, XNSpotLocation, &spot, NULL);
+  attr = XVaCreateNestedList (0, XNSpotLocation, &spot,
+                              XNPreeditStartCallback, &Xxic_preedit_start_callback,
+                              XNPreeditDoneCallback, &Xxic_preedit_done_callback,
+                              XNPreeditDrawCallback, &Xxic_preedit_draw_callback,
+                              XNPreeditCaretCallback, &Xxic_preedit_caret_callback,
+                              NULL);
   XSetICValues (FRAME_XIC (f), XNPreeditAttributes, attr, NULL);
   XFree (attr);
 }
@@ -2816,9 +2873,273 @@ xic_set_statusarea (struct frame *f)
   XFree (attr);
 }
 
+static struct frame *
+x_xic_to_frame (XIC xic)
+{
+  Lisp_Object tail, tem;
+  struct frame *f;
+
+  FOR_EACH_FRAME (tail, tem)
+    {
+      f = XFRAME (tem);
+
+      if (FRAME_X_P (f) && FRAME_XIC (f) == xic)
+        return f;
+    }
+
+  return NULL;
+}
+
+static int
+xic_preedit_start_callback (XIC xic, XPointer client_data,
+                            XPointer call_data)
+{
+  struct frame *f = x_xic_to_frame (xic);
+  struct x_output *output;
+
+  if (f)
+    {
+      output = FRAME_X_OUTPUT (f);
+
+      output->preedit_size = 0;
+      output->preedit_active = true;
+
+      if (output->preedit_chars)
+        xfree (output->preedit_chars);
+
+      output->preedit_chars = NULL;
+    }
+
+  return -1;
+}
+
+static void
+xic_preedit_caret_callback (XIC xic, XPointer client_data,
+                            XIMPreeditCaretCallbackStruct *call_data)
+{
+
+}
+
+
+static void
+xic_preedit_done_callback (XIC xic, XPointer client_data,
+                           XPointer call_data)
+{
+  struct frame *f = x_xic_to_frame (xic);
+  struct x_output *output;
+  struct input_event ie;
+
+  if (f)
+    {
+      ie.kind = PREEDIT_TEXT_EVENT;
+      ie.arg = Qnil;
+      XSETFRAME (ie.frame_or_window, f);
+      XSETINT (ie.x, 0);
+      XSETINT (ie.y, 0);
+      kbd_buffer_store_event (&ie);
+
+      output = FRAME_X_OUTPUT (f);
+
+      if (output->preedit_chars)
+        xfree (output->preedit_chars);
+
+      output->preedit_size = 0;
+      output->preedit_active = false;
+      output->preedit_chars = NULL;
+    }
+}
+
+/* The string returned is not null-terminated.  */
+static char *
+x_xim_text_to_utf8_unix (XIMText *text, ptrdiff_t *length)
+{
+  unsigned char *wchar_buf;
+  ptrdiff_t wchar_actual_length, i;
+  ptrdiff_t nbytes;
+  struct coding_system coding;
+
+  if (text->encoding_is_wchar)
+    {
+      wchar_buf = xmalloc ((text->length + 1) * MAX_MULTIBYTE_LENGTH);
+      wchar_actual_length = 0;
+
+      for (i = 0; i < text->length; ++i)
+        wchar_actual_length += CHAR_STRING (text->string.wide_char[i],
+                                            wchar_buf + wchar_actual_length);
+      *length = wchar_actual_length;
+
+      return (char *) wchar_buf;
+    }
+
+  nbytes = strlen (text->string.multi_byte);
+  setup_coding_system (Qutf_8_unix, &coding);
+  coding.mode |= (CODING_MODE_LAST_BLOCK
+                  | CODING_MODE_SAFE_ENCODING);
+  coding.source = (const unsigned char *) text->string.multi_byte;
+  coding.dst_bytes = 2048;
+  coding.destination = xmalloc (2048);
+  decode_coding_object (&coding, Qnil, 0, 0, nbytes, nbytes, Qnil);
+
+  /* coding.destination has either been allocated by us, or
+     reallocated by decode_coding_object.  */
+
+  *length = coding.produced;
+  return (char *) coding.destination;
+}
+
+static void
+xic_preedit_draw_callback (XIC xic, XPointer client_data,
+                           XIMPreeditDrawCallbackStruct *call_data)
+{
+  struct frame *f = x_xic_to_frame (xic);
+  struct x_output *output;
+  ptrdiff_t text_length;
+  ptrdiff_t charpos;
+  ptrdiff_t original_size;
+  char *text;
+  char *chg_start, *chg_end;
+  struct input_event ie;
+
+  if (f)
+    {
+      output = FRAME_X_OUTPUT (f);
+
+      if (!output->preedit_active)
+        return;
+
+      if (call_data->text)
+        text = x_xim_text_to_utf8_unix (call_data->text, &text_length);
+      else
+        text = NULL;
+
+      original_size = output->preedit_size;
+
+      /* This is an ordinary insertion: reallocate the buffer to hold
+         enough for TEXT.  */
+      if (!call_data->chg_length)
+        {
+          if (!text)
+            goto im_abort;
+
+          if (output->preedit_chars)
+            output->preedit_chars = xrealloc (output->preedit_chars,
+                                              output->preedit_size += text_length);
+          else
+            output->preedit_chars = xmalloc (output->preedit_size += text_length);
+        }
+
+      chg_start = output->preedit_chars;
+
+      /* The IM sent bad data: the buffer is empty, but the change
+         position is more than 0.  */
+      if (!output->preedit_chars && call_data->chg_first)
+        goto im_abort;
+
+      /* Find the byte position for the character position where the
+         first change is to be made.  */
+      if (call_data->chg_first)
+        {
+          charpos = 0;
+
+          while (charpos < call_data->chg_first)
+            {
+              chg_start += BYTES_BY_CHAR_HEAD (*chg_start);
+
+              if ((chg_start - output->preedit_chars) > output->preedit_size)
+                /* The IM sent bad data: chg_start is larger than the
+                   current buffer.  */
+                goto im_abort;
+              ++charpos;
+            }
+        }
+
+      if (!call_data->chg_length)
+        {
+          if (!text)
+            goto im_abort;
+
+          memmove (chg_start + text_length, chg_start,
+                   original_size - (chg_start - output->preedit_chars));
+          memcpy (chg_start, text, text_length);
+        }
+      else
+        {
+          if (call_data->chg_length < 1)
+            goto im_abort;
+
+          charpos = 0;
+          chg_end = chg_start;
+
+          while (charpos < call_data->chg_length)
+            {
+              chg_end += BYTES_BY_CHAR_HEAD (*chg_end);
+
+              if ((chg_end - output->preedit_chars) > output->preedit_size)
+                /* The IM sent bad data: chg_end ends someplace outside
+                   the current buffer.  */
+                goto im_abort;
+              ++charpos;
+            }
+
+          memmove (chg_start, chg_end, ((output->preedit_chars
+                                         + output->preedit_size) - chg_end));
+          output->preedit_size -= (chg_end - chg_start);
+
+          if (text)
+            {
+              original_size = output->preedit_size;
+              output->preedit_chars = xrealloc (output->preedit_chars,
+                                                output->preedit_size += text_length);
+
+              /* Find chg_start again, since preedit_chars was reallocated.  */
+
+              chg_start = output->preedit_chars;
+              charpos = 0;
 
-/* Set X fontset for XIC of frame F, using base font name
-   BASE_FONTNAME.  Called when a new Emacs fontset is chosen.  */
+              while (charpos < call_data->chg_first)
+                {
+                  chg_start += BYTES_BY_CHAR_HEAD (*chg_start);
+
+                  if ((chg_start - output->preedit_chars) > output->preedit_size)
+                    /* The IM sent bad data: chg_start is larger than the
+                       current buffer.  */
+                    goto im_abort;
+                  ++charpos;
+                }
+
+              memmove (chg_start + text_length, chg_start,
+                       original_size - (chg_start - output->preedit_chars));
+              memcpy (chg_start, text, text_length);
+            }
+        }
+
+      if (text)
+        xfree (text);
+
+      /* This is okay because this callback is called from the big XIM
+         event filter, which runs inside XTread_socket.  */
+
+      ie.kind = PREEDIT_TEXT_EVENT;
+      XSETFRAME (ie.frame_or_window, f);
+      ie.arg = make_string_from_utf8 (output->preedit_chars,
+                                      output->preedit_size);
+      XSETINT (ie.x, 0);
+      XSETINT (ie.y, 0);
+
+      kbd_buffer_store_event (&ie);
+    }
+
+  return;
+
+ im_abort:
+  if (text)
+    xfree (text);
+  if (output->preedit_chars)
+    xfree (output->preedit_chars);
+  output->preedit_chars = NULL;
+  output->preedit_size = 0;
+  output->preedit_active = false;
+}
 
 void
 xic_set_xfontset (struct frame *f, const char *base_fontname)
diff --git a/src/xterm.c b/src/xterm.c
index 1d4c775753c..73c0bcf89ea 100644
--- a/src/xterm.c
+++ b/src/xterm.c
@@ -5198,7 +5198,7 @@ x_detect_focus_change (struct x_display_info *dpyinfo, struct frame *frame,
 #ifdef HAVE_XINPUT2
     case GenericEvent:
       {
-        XIEvent *xi_event = (XIEvent *) event;
+        XIEvent *xi_event = (XIEvent *) event->xcookie.data;
 
         struct frame *focus_frame = dpyinfo->x_focus_event_frame;
         int focus_state
@@ -14046,6 +14046,9 @@ x_free_frame_resources (struct frame *f)
 #ifdef HAVE_X_I18N
       if (FRAME_XIC (f))
         free_frame_xic (f);
+
+      if (f->output_data.x->preedit_chars)
+        xfree (f->output_data.x->preedit_chars);
 #endif
 
 #ifdef USE_CAIRO
diff --git a/src/xterm.h b/src/xterm.h
index d4600bdf800..dcac5732527 100644
--- a/src/xterm.h
+++ b/src/xterm.h
@@ -788,6 +788,12 @@ struct x_output
      They are used when creating the cairo surface next time.  */
   int cr_surface_desired_width, cr_surface_desired_height;
 #endif
+
+#ifdef HAVE_X_I18N
+  ptrdiff_t preedit_size;
+  char *preedit_chars;
+  bool preedit_active;
+#endif
 };
 
 enum