* xselect.c: Relax test for outgoing X longs (Bug#9498).

(cons_to_x_long): New function.
(lisp_data_to_selection_data): Use it.  Correct the test for
short-versus-long data; it was negated.  Break out of vector
loop, for efficiency, when a long datum is discovered.

2 files changed, 36 insertions, 11 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 9ce2629d442..0ea45aed1a3 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,11 @@
+2011-09-16  Paul Eggert  <eggert@cs.ucla.edu>
+
+        * xselect.c: Relax test for outgoing X longs (Bug#9498).
+        (cons_to_x_long): New function.
+        (lisp_data_to_selection_data): Use it.  Correct the test for
+        short-versus-long data; it was negated.  Break out of vector
+        loop, for efficiency, when a long datum is discovered.
+
 2011-09-16  Stefan Monnier  <monnier@iro.umontreal.ca>
 
         * eval.c (Fquote): Document its non-consing behavior (bug#9482).
diff --git a/src/xselect.c b/src/xselect.c
index 29e8552bb9c..adee1872dba 100644
--- a/src/xselect.c
+++ b/src/xselect.c
@@ -116,6 +116,7 @@ static Lisp_Object Qx_lost_selection_functions, Qx_sent_selection_functions;
 #define X_SHRT_MIN (-1 - X_SHRT_MAX)
 #define X_LONG_MAX 0x7fffffff
 #define X_LONG_MIN (-1 - X_LONG_MAX)
+#define X_ULONG_MAX 0xffffffffUL
 
 /* If this is a smaller number than the max-request-size of the display,
    emacs will use INCR selection transfer when the selection is larger
@@ -378,7 +379,8 @@ x_own_selection (Lisp_Object selection_name, Lisp_Object selection_value,
 
 /* Given a selection-name and desired type, look up our local copy of
    the selection value and convert it to the type.
-   The value is nil or a string.
+   Return nil, a string, a vector, a symbol, an integer, or a cons
+   that CONS_TO_INTEGER could plausibly handle.
    This function is used both for remote requests (LOCAL_REQUEST is zero)
    and for local x-get-selection-internal (LOCAL_REQUEST is nonzero).
 
@@ -1718,6 +1720,21 @@ selection_data_to_lisp_data (Display *display, const unsigned char *data,
     }
 }
 
+/* Convert OBJ to an X long value, and return it as unsigned long.
+   OBJ should be an integer or a cons representing an integer.
+   Treat values in the range X_LONG_MAX + 1 .. X_ULONG_MAX as X
+   unsigned long values: in theory these values are supposed to be
+   signed but in practice unsigned 32-bit data are communicated via X
+   selections and we need to support that.  */
+static unsigned long
+cons_to_x_long (Lisp_Object obj)
+{
+  if (X_ULONG_MAX <= INTMAX_MAX
+      || XINT (INTEGERP (obj) ? obj : XCAR (obj)) < 0)
+    return cons_to_signed (obj, X_LONG_MIN, min (X_ULONG_MAX, INTMAX_MAX));
+  else
+    return cons_to_unsigned (obj, X_ULONG_MAX);
+}
 
 /* Use xfree, not XFree, to free the data obtained with this function.  */
 
@@ -1783,11 +1800,11 @@ lisp_data_to_selection_data (Display *display, Lisp_Object obj,
                    || (CONSP (XCDR (obj))
                        && INTEGERP (XCAR (XCDR (obj)))))))
     {
-      *data_ret = (unsigned char *) xmalloc (sizeof (long) + 1);
+      *data_ret = (unsigned char *) xmalloc (sizeof (unsigned long) + 1);
       *format_ret = 32;
       *size_ret = 1;
-      (*data_ret) [sizeof (long)] = 0;
-      (*(long **) data_ret) [0] = cons_to_signed (obj, X_LONG_MIN, X_LONG_MAX);
+      (*data_ret) [sizeof (unsigned long)] = 0;
+      (*(unsigned long **) data_ret) [0] = cons_to_x_long (obj);
       if (NILP (type)) type = QINTEGER;
     }
   else if (VECTORP (obj))
@@ -1822,15 +1839,15 @@ lisp_data_to_selection_data (Display *display, Lisp_Object obj,
           if (NILP (type)) type = QINTEGER;
           for (i = 0; i < size; i++)
             {
-              intmax_t v = cons_to_signed (XVECTOR (obj)->contents[i],
-                                           X_LONG_MIN, X_LONG_MAX);
-              if (X_SHRT_MIN <= v && v <= X_SHRT_MAX)
+              if (! RANGED_INTEGERP (X_SHRT_MIN, XVECTOR (obj)->contents[i],
+                                     X_SHRT_MAX))
                 {
                   /* Use sizeof (long) even if it is more than 32 bits.
                      See comment in x_get_window_property and
                      x_fill_property_data.  */
                   data_size = sizeof (long);
                   format = 32;
+                  break;
                 }
             }
           *data_ret = xnmalloc (size, data_size);
@@ -1838,12 +1855,12 @@ lisp_data_to_selection_data (Display *display, Lisp_Object obj,
           *size_ret = size;
           for (i = 0; i < size; i++)
             {
-              long v = cons_to_signed (XVECTOR (obj)->contents[i],
-                                       X_LONG_MIN, X_LONG_MAX);
               if (format == 32)
-                (*((long **) data_ret)) [i] = v;
+                (*((unsigned long **) data_ret)) [i] =
+                  cons_to_x_long (XVECTOR (obj)->contents[i]);
               else
-                (*((short **) data_ret)) [i] = v;
+                (*((short **) data_ret)) [i] =
+                  XINT (XVECTOR (obj)->contents[i]);
             }
         }
     }