Integer-related fixes for term.c etc.

* dispextern.h (face_tty_specified_color): New function.
* term.c (turn_on_face): Don't rely on undefined behavior when
assigning an out-of-range value to 'long'.  Simplify
test for toggling highlight.
(tty_capable_p): Omit last two (unused) args.  All callers changed.
* term.c (tty_capable_p, tty_menu_display, tty_menu_add_selection)
(read_menu_input, tty_menu_activate, tty_menu_show):
* xfaces.c (x_supports_face_attributes_p)
(tty_supports_face_attributes_p):
Use bool for boolean.  All callers changed.
(tty_supports_face_attributes_p): Omit defaults for color indices;
no longer needed.  Simplify tail call.

5 files changed, 69 insertions, 91 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 00ac6f6e28c..6f6b3a26f92 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,19 @@
+2013-11-06  Paul Eggert  <eggert@cs.ucla.edu>
+
+        Integer-related fixes for term.c etc.
+        * dispextern.h (face_tty_specified_color): New function.
+        * term.c (turn_on_face): Don't rely on undefined behavior when
+        assigning an out-of-range value to 'long'.  Simplify
+        test for toggling highlight.
+        (tty_capable_p): Omit last two (unused) args.  All callers changed.
+        * term.c (tty_capable_p, tty_menu_display, tty_menu_add_selection)
+        (read_menu_input, tty_menu_activate, tty_menu_show):
+        * xfaces.c (x_supports_face_attributes_p)
+        (tty_supports_face_attributes_p):
+        Use bool for boolean.  All callers changed.
+        (tty_supports_face_attributes_p): Omit defaults for color indices;
+        no longer needed.  Simplify tail call.
+
 2013-11-05  Stefan Monnier  <monnier@iro.umontreal.ca>
 
         * xdisp.c (prepare_menu_bars): Mark static.
diff --git a/src/dispextern.h b/src/dispextern.h
index 32c6a63e62f..d40febd207a 100644
--- a/src/dispextern.h
+++ b/src/dispextern.h
@@ -1720,6 +1720,15 @@ struct face
 
 #define FACE_TTY_DEFAULT_BG_COLOR ((unsigned long) -3)
 
+/* True if COLOR is a specified (i.e., nondefault) foreground or
+   background color for a tty face.  */
+
+INLINE bool
+face_tty_specified_color (unsigned long color)
+{
+  return color < FACE_TTY_DEFAULT_BG_COLOR;
+}
+
 /* Non-zero if FACE was realized for unibyte use.  */
 
 #define FACE_UNIBYTE_P(FACE) ((FACE)->charset < 0)
@@ -3489,8 +3498,7 @@ extern int string_cost (const char *);
 extern int per_line_cost (const char *);
 extern void calculate_costs (struct frame *);
 extern void produce_glyphs (struct it *);
-extern bool tty_capable_p (struct tty_display_info *, unsigned,
-                           unsigned long, unsigned long);
+extern bool tty_capable_p (struct tty_display_info *, unsigned);
 extern void set_tty_color_mode (struct tty_display_info *, struct frame *);
 extern struct terminal *get_named_tty (const char *);
 extern void create_tty_output (struct frame *);
diff --git a/src/menu.h b/src/menu.h
index 17d66290647..cd89f5eecea 100644
--- a/src/menu.h
+++ b/src/menu.h
@@ -51,7 +51,7 @@ extern Lisp_Object ns_menu_show (struct frame *, int, int, bool, bool,
                                  Lisp_Object, const char **);
 extern Lisp_Object xmenu_show (struct frame *, int, int, bool, bool,
                                Lisp_Object, const char **);
-extern Lisp_Object tty_menu_show (struct frame *, int, int, int, int,
-                                  Lisp_Object, int, const char **);
+extern Lisp_Object tty_menu_show (struct frame *, int, int, bool, bool,
+                                  Lisp_Object, bool, const char **);
 extern ptrdiff_t menu_item_width (const unsigned char *);
 #endif /* MENU_H */
diff --git a/src/term.c b/src/term.c
index 929280865a5..f86d71ac2f4 100644
--- a/src/term.c
+++ b/src/term.c
@@ -1892,55 +1892,18 @@ static void
 turn_on_face (struct frame *f, int face_id)
 {
   struct face *face = FACE_FROM_ID (f, face_id);
-  long fg = face->foreground;
-  long bg = face->background;
+  unsigned long fg = face->foreground;
+  unsigned long bg = face->background;
   struct tty_display_info *tty = FRAME_TTY (f);
 
-  /* Do this first because TS_end_standout_mode may be the same
+  /* Use reverse video if the face specifies that.
+     Do this first because TS_end_standout_mode may be the same
      as TS_exit_attribute_mode, which turns all appearances off. */
-  if (MAY_USE_WITH_COLORS_P (tty, NC_REVERSE))
-    {
-      if (tty->TN_max_colors > 0)
-        {
-          if (fg >= 0 && bg >= 0)
-            {
-              /* If the terminal supports colors, we can set them
-                 below without using reverse video.  The face's fg
-                 and bg colors are set as they should appear on
-                 the screen, i.e. they take the inverse-video'ness
-                 of the face already into account.  */
-            }
-          else if (inverse_video)
-            {
-              if (fg == FACE_TTY_DEFAULT_FG_COLOR
-                  || bg == FACE_TTY_DEFAULT_BG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-          else
-            {
-              if (fg == FACE_TTY_DEFAULT_BG_COLOR
-                  || bg == FACE_TTY_DEFAULT_FG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-        }
-      else
-        {
-          /* If we can't display colors, use reverse video
-             if the face specifies that.  */
-          if (inverse_video)
-            {
-              if (fg == FACE_TTY_DEFAULT_FG_COLOR
-                  || bg == FACE_TTY_DEFAULT_BG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-          else
-            {
-              if (fg == FACE_TTY_DEFAULT_BG_COLOR
-                  || bg == FACE_TTY_DEFAULT_FG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-        }
-    }
+  if (MAY_USE_WITH_COLORS_P (tty, NC_REVERSE)
+      && (inverse_video
+          ? fg == FACE_TTY_DEFAULT_FG_COLOR || bg == FACE_TTY_DEFAULT_BG_COLOR
+          : fg == FACE_TTY_DEFAULT_BG_COLOR || bg == FACE_TTY_DEFAULT_FG_COLOR))
+    tty_toggle_highlight (tty);
 
   if (face->tty_bold_p && MAY_USE_WITH_COLORS_P (tty, NC_BOLD))
     OUTPUT1_IF (tty, tty->TS_enter_bold_mode);
@@ -1965,7 +1928,7 @@ turn_on_face (struct frame *f, int face_id)
       char *p;
 
       ts = tty->standout_mode ? tty->TS_set_background : tty->TS_set_foreground;
-      if (fg >= 0 && ts)
+      if (face_tty_specified_color (fg) && ts)
         {
           p = tparam (ts, NULL, 0, fg, 0, 0, 0);
           OUTPUT (tty, p);
@@ -1973,7 +1936,7 @@ turn_on_face (struct frame *f, int face_id)
         }
 
       ts = tty->standout_mode ? tty->TS_set_foreground : tty->TS_set_background;
-      if (bg >= 0 && ts)
+      if (face_tty_specified_color (bg) && ts)
         {
           p = tparam (ts, NULL, 0, bg, 0, 0, 0);
           OUTPUT (tty, p);
@@ -2027,12 +1990,10 @@ turn_off_face (struct frame *f, int face_id)
 
 
 /* Return true if the terminal on frame F supports all of the
-   capabilities in CAPS simultaneously, with foreground and background
-   colors FG and BG.  */
+   capabilities in CAPS simultaneously.  */
 
 bool
-tty_capable_p (struct tty_display_info *tty, unsigned int caps,
-               unsigned long fg, unsigned long bg)
+tty_capable_p (struct tty_display_info *tty, unsigned int caps)
 {
 #define TTY_CAPABLE_P_TRY(tty, cap, TS, NC_bit)                         \
   if ((caps & (cap)) && (!(TS) || !MAY_USE_WITH_COLORS_P(tty, NC_bit))) \
@@ -2867,7 +2828,8 @@ tty_menu_search_pane (tty_menu *menu, int pane)
       {
         if (pane == menu->panenumber[i])
           return menu->submenu[i];
-        if ((try = tty_menu_search_pane (menu->submenu[i], pane)))
+        try = tty_menu_search_pane (menu->submenu[i], pane);
+        if (try)
           return try;
       }
   return (tty_menu *) 0;
@@ -2920,7 +2882,7 @@ mouse_get_xy (int *x, int *y)
 
 static void
 tty_menu_display (tty_menu *menu, int x, int y, int pn, int *faces,
-                  int mx, int my, int first_item, int disp_help)
+                  int mx, int my, int first_item, bool disp_help)
 {
   int i, face, width, enabled, mousehere, row, col;
   struct frame *sf = SELECTED_FRAME ();
@@ -2997,16 +2959,19 @@ tty_menu_add_pane (tty_menu *menu, const char *txt)
 
 /* Create a new item in a menu pane.  */
 
-static int
+static bool
 tty_menu_add_selection (tty_menu *menu, int pane,
-                        char *txt, int enable, char const *help_text)
+                        char *txt, bool enable, char const *help_text)
 {
   int len;
   unsigned char *p;
 
   if (pane)
-    if (!(menu = tty_menu_search_pane (menu, pane)))
-      return TTYM_FAILURE;
+    {
+      menu = tty_menu_search_pane (menu, pane);
+      if (! menu)
+        return 0;
+    }
   tty_menu_make_room (menu);
   menu->submenu[menu->count] = (tty_menu *) 0;
   menu->text[menu->count] = txt;
@@ -3027,7 +2992,7 @@ tty_menu_add_selection (tty_menu *menu, int pane,
   if (len > menu->width)
     menu->width = len;
 
-  return TTYM_SUCCESS;
+  return 1;
 }
 
 /* Decide where the menu would be placed if requested at (X,Y).  */
@@ -3155,7 +3120,7 @@ read_menu_input (struct frame *sf, int *x, int *y, int min_y, int max_y,
   else
     {
       Lisp_Object cmd;
-      int usable_input = 1;
+      bool usable_input = 1;
       mi_result st = MI_CONTINUE;
       struct tty_display_info *tty = FRAME_TTY (sf);
       Lisp_Object saved_mouse_tracking = do_mouse_tracking;
@@ -3215,10 +3180,11 @@ static int
 tty_menu_activate (tty_menu *menu, int *pane, int *selidx,
                    int x0, int y0, char **txt,
                    void (*help_callback)(char const *, int, int),
-                   int kbd_navigation)
+                   bool kbd_navigation)
 {
   struct tty_menu_state *state;
-  int statecount, x, y, i, leave, onepane;
+  int statecount, x, y, i;
+  bool leave, onepane;
   int result IF_LINT (= 0);
   int title_faces[4];           /* face to display the menu title */
   int faces[4], buffers_num_deleted = 0;
@@ -3285,7 +3251,8 @@ tty_menu_activate (tty_menu *menu, int *pane, int *selidx,
   tty_hide_cursor (tty);
   if (buffers_num_deleted)
     menu->text[0][7] = ' ';
-  if ((onepane = menu->count == 1 && menu->submenu[0]))
+  onepane = menu->count == 1 && menu->submenu[0];
+  if (onepane)
     {
       menu->width = menu->submenu[0]->width;
       state[0].menu = menu->submenu[0];
@@ -3585,8 +3552,8 @@ tty_menu_new_item_coords (struct frame *f, int which, int *x, int *y)
 }
 
 Lisp_Object
-tty_menu_show (struct frame *f, int x, int y, int for_click, int keymaps,
-               Lisp_Object title, int kbd_navigation, const char **error_name)
+tty_menu_show (struct frame *f, int x, int y, bool for_click, bool keymaps,
+               Lisp_Object title, bool kbd_navigation, const char **error_name)
 {
   tty_menu *menu;
   int pane, selidx, lpane, status;
@@ -3709,9 +3676,8 @@ tty_menu_show (struct frame *f, int x, int y, int for_click, int keymaps,
             item_data = SSDATA (item_name);
 
           if (lpane == TTYM_FAILURE
-              || (tty_menu_add_selection (menu, lpane, item_data,
-                                          !NILP (enable), help_string)
-                  == TTYM_FAILURE))
+              || (! tty_menu_add_selection (menu, lpane, item_data,
+                                            !NILP (enable), help_string)))
             {
               tty_menu_destroy (menu);
               *error_name = "Can't add selection to menu";
diff --git a/src/xfaces.c b/src/xfaces.c
index 6845e5c3c65..30658f9c4a8 100644
--- a/src/xfaces.c
+++ b/src/xfaces.c
@@ -4711,7 +4711,7 @@ DEFUN ("face-attributes-as-vector", Fface_attributes_as_vector,
 
 #ifdef HAVE_WINDOW_SYSTEM
 
-/* Return non-zero if all the face attributes in ATTRS are supported
+/* Return true if all the face attributes in ATTRS are supported
    on the window-system frame F.
 
    The definition of `supported' is somewhat heuristic, but basically means
@@ -4721,7 +4721,7 @@ DEFUN ("face-attributes-as-vector", Fface_attributes_as_vector,
     \(1) different in appearance than the default face, and
     \(2) `close in spirit' to what the attributes specify, if not exact.  */
 
-static int
+static bool
 x_supports_face_attributes_p (struct frame *f,
                               Lisp_Object attrs[LFACE_VECTOR_SIZE],
                               struct face *def_face)
@@ -4811,7 +4811,7 @@ x_supports_face_attributes_p (struct frame *f,
 
 #endif  /* HAVE_WINDOW_SYSTEM */
 
-/* Return non-zero if all the face attributes in ATTRS are supported
+/* Return true if all the face attributes in ATTRS are supported
    on the tty frame F.
 
    The definition of `supported' is somewhat heuristic, but basically means
@@ -4827,7 +4827,7 @@ x_supports_face_attributes_p (struct frame *f,
    will _not_ be satisfied by the tty display code's automatic
    substitution of a `dim' face for italic.  */
 
-static int
+static bool
 tty_supports_face_attributes_p (struct frame *f,
                                 Lisp_Object attrs[LFACE_VECTOR_SIZE],
                                 struct face *def_face)
@@ -4921,12 +4921,6 @@ tty_supports_face_attributes_p (struct frame *f,
 
   /* Color testing.  */
 
-  /* Default the color indices in FG_TTY_COLOR and BG_TTY_COLOR, since
-     we use them when calling `tty_capable_p' below, even if the face
-     specifies no colors.  */
-  fg_tty_color.pixel = FACE_TTY_DEFAULT_FG_COLOR;
-  bg_tty_color.pixel = FACE_TTY_DEFAULT_BG_COLOR;
-
   /* Check if foreground color is close enough.  */
   fg = attrs[LFACE_FOREGROUND_INDEX];
   if (STRINGP (fg))
@@ -4992,14 +4986,7 @@ tty_supports_face_attributes_p (struct frame *f,
 
   /* See if the capabilities we selected above are supported, with the
      given colors.  */
-  if (test_caps != 0 &&
-      ! tty_capable_p (FRAME_TTY (f), test_caps, fg_tty_color.pixel,
-                       bg_tty_color.pixel))
-    return 0;
-
-
-  /* Hmmm, everything checks out, this terminal must support this face.  */
-  return 1;
+  return tty_capable_p (FRAME_TTY (f), test_caps);
 }
 
 
@@ -5024,7 +5011,8 @@ satisfied by the tty display code's automatic substitution of a `dim'
 face for italic.  */)
   (Lisp_Object attributes, Lisp_Object display)
 {
-  int supports = 0, i;
+  bool supports = 0;
+  int i;
   Lisp_Object frame;
   struct frame *f;
   struct face *def_face;