Integer-related fixes for term.c etc.

* dispextern.h (face_tty_specified_color): New function.
* term.c (turn_on_face): Don't rely on undefined behavior when
assigning an out-of-range value to 'long'.  Simplify
test for toggling highlight.
(tty_capable_p): Omit last two (unused) args.  All callers changed.
* term.c (tty_capable_p, tty_menu_display, tty_menu_add_selection)
(read_menu_input, tty_menu_activate, tty_menu_show):
* xfaces.c (x_supports_face_attributes_p)
(tty_supports_face_attributes_p):
Use bool for boolean.  All callers changed.
(tty_supports_face_attributes_p): Omit defaults for color indices;
no longer needed.  Simplify tail call.

1 files changed, 34 insertions, 68 deletions

diff --git a/src/term.c b/src/term.c
index 929280865a5..f86d71ac2f4 100644
--- a/src/term.c
+++ b/src/term.c
@@ -1892,55 +1892,18 @@ static void
 turn_on_face (struct frame *f, int face_id)
 {
   struct face *face = FACE_FROM_ID (f, face_id);
-  long fg = face->foreground;
-  long bg = face->background;
+  unsigned long fg = face->foreground;
+  unsigned long bg = face->background;
   struct tty_display_info *tty = FRAME_TTY (f);
 
-  /* Do this first because TS_end_standout_mode may be the same
+  /* Use reverse video if the face specifies that.
+     Do this first because TS_end_standout_mode may be the same
      as TS_exit_attribute_mode, which turns all appearances off. */
-  if (MAY_USE_WITH_COLORS_P (tty, NC_REVERSE))
-    {
-      if (tty->TN_max_colors > 0)
-        {
-          if (fg >= 0 && bg >= 0)
-            {
-              /* If the terminal supports colors, we can set them
-                 below without using reverse video.  The face's fg
-                 and bg colors are set as they should appear on
-                 the screen, i.e. they take the inverse-video'ness
-                 of the face already into account.  */
-            }
-          else if (inverse_video)
-            {
-              if (fg == FACE_TTY_DEFAULT_FG_COLOR
-                  || bg == FACE_TTY_DEFAULT_BG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-          else
-            {
-              if (fg == FACE_TTY_DEFAULT_BG_COLOR
-                  || bg == FACE_TTY_DEFAULT_FG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-        }
-      else
-        {
-          /* If we can't display colors, use reverse video
-             if the face specifies that.  */
-          if (inverse_video)
-            {
-              if (fg == FACE_TTY_DEFAULT_FG_COLOR
-                  || bg == FACE_TTY_DEFAULT_BG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-          else
-            {
-              if (fg == FACE_TTY_DEFAULT_BG_COLOR
-                  || bg == FACE_TTY_DEFAULT_FG_COLOR)
-                tty_toggle_highlight (tty);
-            }
-        }
-    }
+  if (MAY_USE_WITH_COLORS_P (tty, NC_REVERSE)
+      && (inverse_video
+          ? fg == FACE_TTY_DEFAULT_FG_COLOR || bg == FACE_TTY_DEFAULT_BG_COLOR
+          : fg == FACE_TTY_DEFAULT_BG_COLOR || bg == FACE_TTY_DEFAULT_FG_COLOR))
+    tty_toggle_highlight (tty);
 
   if (face->tty_bold_p && MAY_USE_WITH_COLORS_P (tty, NC_BOLD))
     OUTPUT1_IF (tty, tty->TS_enter_bold_mode);
@@ -1965,7 +1928,7 @@ turn_on_face (struct frame *f, int face_id)
       char *p;
 
       ts = tty->standout_mode ? tty->TS_set_background : tty->TS_set_foreground;
-      if (fg >= 0 && ts)
+      if (face_tty_specified_color (fg) && ts)
         {
           p = tparam (ts, NULL, 0, fg, 0, 0, 0);
           OUTPUT (tty, p);
@@ -1973,7 +1936,7 @@ turn_on_face (struct frame *f, int face_id)
         }
 
       ts = tty->standout_mode ? tty->TS_set_foreground : tty->TS_set_background;
-      if (bg >= 0 && ts)
+      if (face_tty_specified_color (bg) && ts)
         {
           p = tparam (ts, NULL, 0, bg, 0, 0, 0);
           OUTPUT (tty, p);
@@ -2027,12 +1990,10 @@ turn_off_face (struct frame *f, int face_id)
 
 
 /* Return true if the terminal on frame F supports all of the
-   capabilities in CAPS simultaneously, with foreground and background
-   colors FG and BG.  */
+   capabilities in CAPS simultaneously.  */
 
 bool
-tty_capable_p (struct tty_display_info *tty, unsigned int caps,
-               unsigned long fg, unsigned long bg)
+tty_capable_p (struct tty_display_info *tty, unsigned int caps)
 {
 #define TTY_CAPABLE_P_TRY(tty, cap, TS, NC_bit)                         \
   if ((caps & (cap)) && (!(TS) || !MAY_USE_WITH_COLORS_P(tty, NC_bit))) \
@@ -2867,7 +2828,8 @@ tty_menu_search_pane (tty_menu *menu, int pane)
       {
         if (pane == menu->panenumber[i])
           return menu->submenu[i];
-        if ((try = tty_menu_search_pane (menu->submenu[i], pane)))
+        try = tty_menu_search_pane (menu->submenu[i], pane);
+        if (try)
           return try;
       }
   return (tty_menu *) 0;
@@ -2920,7 +2882,7 @@ mouse_get_xy (int *x, int *y)
 
 static void
 tty_menu_display (tty_menu *menu, int x, int y, int pn, int *faces,
-                  int mx, int my, int first_item, int disp_help)
+                  int mx, int my, int first_item, bool disp_help)
 {
   int i, face, width, enabled, mousehere, row, col;
   struct frame *sf = SELECTED_FRAME ();
@@ -2997,16 +2959,19 @@ tty_menu_add_pane (tty_menu *menu, const char *txt)
 
 /* Create a new item in a menu pane.  */
 
-static int
+static bool
 tty_menu_add_selection (tty_menu *menu, int pane,
-                        char *txt, int enable, char const *help_text)
+                        char *txt, bool enable, char const *help_text)
 {
   int len;
   unsigned char *p;
 
   if (pane)
-    if (!(menu = tty_menu_search_pane (menu, pane)))
-      return TTYM_FAILURE;
+    {
+      menu = tty_menu_search_pane (menu, pane);
+      if (! menu)
+        return 0;
+    }
   tty_menu_make_room (menu);
   menu->submenu[menu->count] = (tty_menu *) 0;
   menu->text[menu->count] = txt;
@@ -3027,7 +2992,7 @@ tty_menu_add_selection (tty_menu *menu, int pane,
   if (len > menu->width)
     menu->width = len;
 
-  return TTYM_SUCCESS;
+  return 1;
 }
 
 /* Decide where the menu would be placed if requested at (X,Y).  */
@@ -3155,7 +3120,7 @@ read_menu_input (struct frame *sf, int *x, int *y, int min_y, int max_y,
   else
     {
       Lisp_Object cmd;
-      int usable_input = 1;
+      bool usable_input = 1;
       mi_result st = MI_CONTINUE;
       struct tty_display_info *tty = FRAME_TTY (sf);
       Lisp_Object saved_mouse_tracking = do_mouse_tracking;
@@ -3215,10 +3180,11 @@ static int
 tty_menu_activate (tty_menu *menu, int *pane, int *selidx,
                    int x0, int y0, char **txt,
                    void (*help_callback)(char const *, int, int),
-                   int kbd_navigation)
+                   bool kbd_navigation)
 {
   struct tty_menu_state *state;
-  int statecount, x, y, i, leave, onepane;
+  int statecount, x, y, i;
+  bool leave, onepane;
   int result IF_LINT (= 0);
   int title_faces[4];           /* face to display the menu title */
   int faces[4], buffers_num_deleted = 0;
@@ -3285,7 +3251,8 @@ tty_menu_activate (tty_menu *menu, int *pane, int *selidx,
   tty_hide_cursor (tty);
   if (buffers_num_deleted)
     menu->text[0][7] = ' ';
-  if ((onepane = menu->count == 1 && menu->submenu[0]))
+  onepane = menu->count == 1 && menu->submenu[0];
+  if (onepane)
     {
       menu->width = menu->submenu[0]->width;
       state[0].menu = menu->submenu[0];
@@ -3585,8 +3552,8 @@ tty_menu_new_item_coords (struct frame *f, int which, int *x, int *y)
 }
 
 Lisp_Object
-tty_menu_show (struct frame *f, int x, int y, int for_click, int keymaps,
-               Lisp_Object title, int kbd_navigation, const char **error_name)
+tty_menu_show (struct frame *f, int x, int y, bool for_click, bool keymaps,
+               Lisp_Object title, bool kbd_navigation, const char **error_name)
 {
   tty_menu *menu;
   int pane, selidx, lpane, status;
@@ -3709,9 +3676,8 @@ tty_menu_show (struct frame *f, int x, int y, int for_click, int keymaps,
             item_data = SSDATA (item_name);
 
           if (lpane == TTYM_FAILURE
-              || (tty_menu_add_selection (menu, lpane, item_data,
-                                          !NILP (enable), help_string)
-                  == TTYM_FAILURE))
+              || (! tty_menu_add_selection (menu, lpane, item_data,
+                                            !NILP (enable), help_string)))
             {
               tty_menu_destroy (menu);
               *error_name = "Can't add selection to menu";