Stop relying on Xt hack to set window manager hints

* src/widget.c (get_wm_shell): Return WMShellWidget.
(update_wm_hints, widget_update_wm_size_hints): Return whether
or not size hints changed.
* src/widget.h: Delete `size_switch'.
* src/widgetprv.h (EmacsFramePart): Likewise.
* src/xterm.c (x_wm_set_size_hint): Set size hints the usual way
if Xt did not set them.  (bug#58412)

4 files changed, 40 insertions, 26 deletions

diff --git a/src/widget.c b/src/widget.c
index 5a75cdaca8e..aaab33b6d8e 100644
--- a/src/widget.c
+++ b/src/widget.c
@@ -195,7 +195,7 @@ round_size_to_char (EmacsFrame ew, Dimension in_width, Dimension in_height,
                       out_width, out_height);
 }
 
-static Widget
+static WMShellWidget
 get_wm_shell (Widget w)
 {
   Widget wmshell;
@@ -204,7 +204,7 @@ get_wm_shell (Widget w)
        wmshell && !XtIsWMShell (wmshell);
        wmshell = XtParent (wmshell));
 
-  return wmshell;
+  return (WMShellWidget) wmshell;
 }
 
 #if 0 /* Currently not used.  */
@@ -269,8 +269,8 @@ set_frame_size (EmacsFrame ew)
       (f, build_string ("set_frame_size"));
 }
 
-static void
-update_wm_hints (Widget wmshell, EmacsFrame ew)
+static bool
+update_wm_hints (WMShellWidget wmshell, EmacsFrame ew)
 {
   int cw;
   int ch;
@@ -280,6 +280,12 @@ update_wm_hints (Widget wmshell, EmacsFrame ew)
   int char_height;
   int base_width;
   int base_height;
+  char buffer[sizeof wmshell->wm.size_hints];
+  char *hints_ptr;
+
+  /* Copy the old size hints to the buffer.  */
+  memcpy (buffer, &wmshell->wm.size_hints,
+          sizeof wmshell->wm.size_hints);
 
   pixel_to_char_size (ew, ew->core.width, ew->core.height,
                       &char_width, &char_height);
@@ -292,27 +298,29 @@ update_wm_hints (Widget wmshell, EmacsFrame ew)
   base_height = (wmshell->core.height - ew->core.height
                  + (rounded_height - (char_height * ch)));
 
-  /* Ensure that Xt actually sets window manager hint flags specified
-     by the caller by making sure XtNminWidth (a relatively harmless
-     resource) always changes each time this function is invoked.  */
-  ew->emacs_frame.size_switch = !ew->emacs_frame.size_switch;
-
-  XtVaSetValues (wmshell,
+  XtVaSetValues ((Widget) wmshell,
                  XtNbaseWidth, (XtArgVal) base_width,
                  XtNbaseHeight, (XtArgVal) base_height,
                  XtNwidthInc, (XtArgVal) (frame_resize_pixelwise ? 1 : cw),
                  XtNheightInc, (XtArgVal) (frame_resize_pixelwise ? 1 : ch),
-                 XtNminWidth, (XtArgVal) (base_width
-                                          + ew->emacs_frame.size_switch),
-                 XtNminHeight, (XtArgVal) (base_height
-                                           + ew->emacs_frame.size_switch),
+                 XtNminWidth, (XtArgVal) base_width,
+                 XtNminHeight, (XtArgVal) base_height,
                  NULL);
+
+  /* Return if size hints really changed.  If they did not, then Xt
+     probably didn't set them either (or take the flags into
+     account.)  */
+  hints_ptr = (char *) &wmshell->wm.size_hints;
+
+  /* Skip flags, which is unsigned long.  */
+  return memcmp (hints_ptr + sizeof (long), buffer + sizeof (long),
+                 sizeof wmshell->wm.wm_hints - sizeof (long));
 }
 
-void
+bool
 widget_update_wm_size_hints (Widget widget, Widget frame)
 {
-  update_wm_hints (widget, (EmacsFrame) frame);
+  return update_wm_hints ((WMShellWidget) widget, (EmacsFrame) frame);
 }
 
 static void
@@ -357,8 +365,6 @@ EmacsFrameInitialize (Widget request, Widget new,
       exit (1);
     }
 
-  ew->emacs_frame.size_switch = 1;
-
   update_from_various_frame_slots (ew);
   set_frame_size (ew);
 }
diff --git a/src/widget.h b/src/widget.h
index 2906d5ff9ec..cf83cb10781 100644
--- a/src/widget.h
+++ b/src/widget.h
@@ -97,6 +97,6 @@ extern struct _DisplayContext *display_context;
 /* Special entry points */
 void EmacsFrameSetCharSize (Widget, int, int);
 void widget_store_internal_border (Widget widget);
-void widget_update_wm_size_hints (Widget widget, Widget frame);
+bool widget_update_wm_size_hints (Widget widget, Widget frame);
 
 #endif /* _EmacsFrame_h */
diff --git a/src/widgetprv.h b/src/widgetprv.h
index fe960326b03..3a4d9206ffe 100644
--- a/src/widgetprv.h
+++ b/src/widgetprv.h
@@ -49,9 +49,6 @@ typedef struct {
 
   Boolean       visual_bell;            /* flash instead of beep */
   int           bell_volume;            /* how loud is beep */
-  int           size_switch;            /* hack to make setting size
-                                           hints work correctly */
-
   /* private state */
 
 } EmacsFramePart;
diff --git a/src/xterm.c b/src/xterm.c
index 9c34fce7c5b..9059ad7136e 100644
--- a/src/xterm.c
+++ b/src/xterm.c
@@ -28006,6 +28006,7 @@ x_wm_set_size_hint (struct frame *f, long flags, bool user_position)
   Window window = FRAME_OUTER_WINDOW (f);
 #ifdef USE_X_TOOLKIT
   WMShellWidget shell;
+  bool hints_changed;
 #endif
 
   if (!window)
@@ -28032,8 +28033,9 @@ x_wm_set_size_hint (struct frame *f, long flags, bool user_position)
           shell->wm.size_hints.flags |= USPosition;
         }
 
-      widget_update_wm_size_hints (f->output_data.x->widget,
-                                   f->output_data.x->edit_widget);
+      hints_changed
+        = widget_update_wm_size_hints (f->output_data.x->widget,
+                                       f->output_data.x->edit_widget);
 
 #ifdef USE_MOTIF
       /* Do this all over again for the benefit of Motif, which always
@@ -28046,6 +28048,7 @@ x_wm_set_size_hint (struct frame *f, long flags, bool user_position)
           shell->wm.size_hints.flags &= ~PPosition;
           shell->wm.size_hints.flags |= USPosition;
         }
+#endif
 
       /* Drill hints into Motif, since it keeps setting its own.  */
       size_hints.flags = shell->wm.size_hints.flags;
@@ -28063,15 +28066,23 @@ x_wm_set_size_hint (struct frame *f, long flags, bool user_position)
       size_hints.min_aspect.y = shell->wm.size_hints.min_aspect.y;
       size_hints.max_aspect.x = shell->wm.size_hints.max_aspect.x;
       size_hints.max_aspect.y = shell->wm.size_hints.max_aspect.y;
-#ifdef HAVE_X11XTR6
       size_hints.base_width = shell->wm.base_width;
       size_hints.base_height = shell->wm.base_height;
       size_hints.win_gravity = shell->wm.win_gravity;
-#endif
 
+#ifdef USE_MOTIF
       XSetWMNormalHints (XtDisplay (f->output_data.x->widget),
                          XtWindow (f->output_data.x->widget),
                          &size_hints);
+#else
+      /* In many cases, widget_update_wm_size_hints will not have
+         updated the size hints if only flags changed.  When that
+         happens, set the WM hints manually.  */
+
+      if (!hints_changed)
+        XSetWMNormalHints (XtDisplay (f->output_data.x->widget),
+                           XtWindow (f->output_data.x->widget),
+                           &size_hints);
 #endif
 
       return;