17 files changed, 842 insertions, 572 deletions

diff --git a/lisp/emacs-lisp/autoload.el b/lisp/emacs-lisp/autoload.el
index 11316f1d9d6..424b8e31936 100644
--- a/lisp/emacs-lisp/autoload.el
+++ b/lisp/emacs-lisp/autoload.el
@@ -500,41 +500,26 @@ Return non-nil in the case where no autoloads were added at point."
   (let ((generated-autoload-file buffer-file-name))
     (autoload-generate-file-autoloads file (current-buffer))))
 
-(defun autoload--split-prefixes-1 (strs)
-  (let ((prefixes ()))
-    (dolist (str strs)
-      (string-match "\\`[^-:/_]*[-:/_]*" str)
-      (let* ((prefix (match-string 0 str))
-             (tail (substring str (match-end 0)))
-             (cell (assoc prefix prefixes)))
-        (cond
-         ((null cell) (push (list prefix tail) prefixes))
-         ((equal (cadr cell) tail) nil)
-         (t (setcdr cell (cons tail (cdr cell)))))))
-    prefixes))
-
 (defvar autoload-compute-prefixes t
   "If non-nil, autoload will add code to register the prefixes used in a file.
 Standard prefixes won't be registered anyway.  I.e. if a file \"foo.el\" defines
 variables or functions that use \"foo-\" as prefix, that will not be registered.
 But all other prefixes will be included.")
 
-(defconst autoload-defs-autoload-max-size 5
+(defconst autoload-def-prefixes-max-entries 5
   "Target length of the list of definition prefixes per file.
 If set too small, the prefixes will be too generic (i.e. they'll use little
 memory, we'll end up looking in too many files when we need a particular
 prefix), and if set too large, they will be too specific (i.e. they will
 cost more memory use).")
 
-(defvar autoload-popular-prefixes nil)
+(defconst autoload-def-prefixes-max-length 12
+  "Target size of definition prefixes.
+Don't try to split prefixes that are already longer than that.")
+
+(require 'radix-tree)
 
 (defun autoload--make-defs-autoload (defs file)
-  ;; FIXME: avoid redundant entries.  E.g. opascal currently has
-  ;; "opascal-" "opascal--literal-start-re" "opascal--syntax-propertize"
-  ;; where only the first one should be kept.
-  ;; FIXME: Avoid keeping too-long-prefixes.  E.g. ob-scheme currently has
-  ;; "org-babel-scheme-" "org-babel-default-header-args:scheme"
-  ;; "org-babel-expand-body:scheme" "org-babel-execute:scheme".
 
   ;; Remove the defs that obey the rule that file foo.el (or
   ;; foo-mode.el) uses "foo-" as prefix.
@@ -548,39 +533,32 @@ cost more memory use).")
 
   ;; Then compute a small set of prefixes that cover all the
   ;; remaining definitions.
-  (let ((prefixes (autoload--split-prefixes-1 defs))
-        (again t))
-    ;; (message "Initial prefixes %s : %S" file (mapcar #'car prefixes))
-    (while again
-      (setq again nil)
-      (let ((newprefixes
-             (sort
-              (mapcar (lambda (cell)
-                        (cons cell
-                              (autoload--split-prefixes-1 (cdr cell))))
-                      prefixes)
-              (lambda (x y) (< (length (cdr x)) (length (cdr y)))))))
-        (setq prefixes nil)
-        (while newprefixes
-          (let ((x (pop newprefixes)))
-            (if (or (equal '("") (cdar x))
-                    (and (cddr x)
-                         (not (member (caar x)
-                                      autoload-popular-prefixes))
-                         (> (+ (length prefixes) (length newprefixes)
-                               (length (cdr x)))
-                            autoload-defs-autoload-max-size)))
-                ;; Nothing to split or would split too deep.
-                (push (car x) prefixes)
-              ;; (message "Expand %S to %S" (caar x) (cdr x))
-              (setq again t)
-              (setq prefixes
-                    (nconc (mapcar (lambda (cell)
-                                     (cons (concat (caar x)
-                                                   (car cell))
-                                           (cdr cell)))
-                                   (cdr x))
-                           prefixes)))))))
+  (let* ((tree (let ((tree radix-tree-empty))
+                 (dolist (def defs)
+                   (setq tree (radix-tree-insert tree def t)))
+                 tree))
+         (prefixes (list (cons "" tree))))
+    (while
+        (let ((newprefixes nil)
+              (changes nil))
+          (dolist (pair prefixes)
+            (let ((prefix (car pair)))
+              (if (or (> (length prefix) autoload-def-prefixes-max-length)
+                      (radix-tree-lookup (cdr pair) ""))
+                  ;; No point splitting it any further.
+                  (push pair newprefixes)
+                (setq changes t)
+                (radix-tree-iter-subtrees
+                 (cdr pair) (lambda (sprefix subtree)
+                              (push (cons (concat prefix sprefix) subtree)
+                                    newprefixes))))))
+          (and changes
+               (or (and (null (cdr prefixes)) (equal "" (caar prefixes)))
+                   (<= (length newprefixes)
+                       autoload-def-prefixes-max-entries))
+               (setq prefixes newprefixes)
+               (< (length prefixes) autoload-def-prefixes-max-entries))))
+
     ;; (message "Final prefixes %s : %S" file (mapcar #'car prefixes))
     (when prefixes
       `(if (fboundp 'register-definition-prefixes)
@@ -989,7 +967,7 @@ write its autoloads into the specified file instead."
                                                   t files-re))
                                dirs)))
          (done ())                      ;Files processed; to remove duplicates.
-         (changed nil)                  ;Non-nil if some change occured.
+         (changed nil)                  ;Non-nil if some change occurred.
          (last-time)
          ;; Files with no autoload cookies or whose autoloads go to other
          ;; files because of file-local autoload-generated-file settings.
diff --git a/lisp/emacs-lisp/pcase.el b/lisp/emacs-lisp/pcase.el
index 7e164c0fe5c..0b8dddfacc9 100644
--- a/lisp/emacs-lisp/pcase.el
+++ b/lisp/emacs-lisp/pcase.el
@@ -509,6 +509,7 @@ MATCH is the pattern that needs to be matched, of the form:
     (numberp . stringp)
     (numberp . byte-code-function-p)
     (consp . arrayp)
+    (consp . atom)
     (consp . vectorp)
     (consp . stringp)
     (consp . byte-code-function-p)
diff --git a/lisp/emacs-lisp/radix-tree.el b/lisp/emacs-lisp/radix-tree.el
new file mode 100644
index 00000000000..d4b5cd211e4
--- /dev/null
+++ b/lisp/emacs-lisp/radix-tree.el
@@ -0,0 +1,188 @@
+;;; radix-tree.el --- A simple library of radix trees  -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2016  Free Software Foundation, Inc.
+
+;; Author: Stefan Monnier <monnier@iro.umontreal.ca>
+;; Keywords:
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; There are many different options for how to represent radix trees
+;; in Elisp.  Here I chose a very simple one.  A radix-tree can be either:
+;; - a node, of the form ((PREFIX . PTREE) . RTREE) where PREFIX is a string
+;;   meaning that everything that starts with PREFIX is in PTREE,
+;;   and everything else in RTREE.  It also has the property that
+;;   everything that starts with the first letter of PREFIX but not with
+;;   that whole PREFIX is not in RTREE (i.e. is not in the tree at all).
+;; - anything else is taken as the value to associate with the empty string.
+;; So every node is basically an (improper) alist where each mapping applies
+;; to a different leading letter.
+;;
+;; The main downside of this representation is that the lookup operation
+;; is slower because each level of the tree is an alist rather than some kind
+;; of array, so every level's lookup is O(N) rather than O(1).  We could easily
+;; solve this by using char-tables instead of alists, but that would make every
+;; level take up a lot more memory, and it would make the resulting
+;; data structure harder to read (by a human) when printed out.
+
+;;; Code:
+
+(defun radix-tree--insert (tree key val i)
+  (pcase tree
+    (`((,prefix . ,ptree) . ,rtree)
+     (let* ((ni (+ i (length prefix)))
+            (cmp (compare-strings prefix nil nil key i ni)))
+       (if (eq t cmp)
+           (let ((nptree (radix-tree--insert ptree key val ni)))
+             `((,prefix . ,nptree) . ,rtree))
+         (let ((n (if (< cmp 0) (- -1 cmp) (- cmp 1))))
+           (if (zerop n)
+               (let ((nrtree (radix-tree--insert rtree key val i)))
+                 `((,prefix . ,ptree) . ,nrtree))
+             (let* ((nprefix (substring prefix 0 n))
+                    (kprefix (substring key (+ i n)))
+                    (pprefix (substring prefix n))
+                    (ktree (if (equal kprefix "") val
+                             `((,kprefix . ,val)))))
+               `((,nprefix
+                  . ((,pprefix . ,ptree) . ,ktree))
+                 . ,rtree)))))))
+    (_
+     (if (= (length key) i) val
+       (let ((prefix (substring key i)))
+         `((,prefix . ,val) . ,tree))))))
+
+(defun radix-tree--remove (tree key i)
+  (pcase tree
+    (`((,prefix . ,ptree) . ,rtree)
+     (let* ((ni (+ i (length prefix)))
+            (cmp (compare-strings prefix nil nil key i ni)))
+       (if (eq t cmp)
+           (pcase (radix-tree--remove ptree key ni)
+             (`nil rtree)
+             (`((,pprefix . ,pptree))
+              `((,(concat prefix pprefix) . ,pptree) . ,rtree))
+             (nptree `((,prefix . ,nptree) . ,rtree)))
+         (let ((n (if (< cmp 0) (- -1 cmp) (- cmp 1))))
+           (if (zerop n)
+               (let ((nrtree (radix-tree--remove rtree key i)))
+                 `((,prefix . ,ptree) . ,nrtree))
+             tree)))))
+    (_
+     (if (= (length key) i) nil tree))))
+
+
+(defun radix-tree--lookup (tree string i)
+  (pcase tree
+    (`((,prefix . ,ptree) . ,rtree)
+     (let* ((ni (+ i (length prefix)))
+            (cmp (compare-strings prefix nil nil string i ni)))
+       (if (eq t cmp)
+           (radix-tree--lookup ptree string ni)
+         (let ((n (if (< cmp 0) (- -1 cmp) (- cmp 1))))
+           (if (zerop n)
+               (radix-tree--lookup rtree string i)
+             (+ i n))))))
+    (val
+     (if (and val (equal (length string) i))
+         (if (integerp val) `(t . ,val) val)
+       i))))
+
+(defun radix-tree--subtree (tree string i)
+  (if (equal (length string) i) tree
+    (pcase tree
+      (`((,prefix . ,ptree) . ,rtree)
+       (let* ((ni (+ i (length prefix)))
+              (cmp (compare-strings prefix nil nil string i ni)))
+         (if (eq t cmp)
+             (radix-tree--subtree ptree string ni)
+           (let ((n (if (< cmp 0) (- -1 cmp) (- cmp 1))))
+             (cond
+              ((zerop n) (radix-tree--subtree rtree string i))
+              ((equal (+ n i) (length string))
+               (let ((nprefix (substring prefix n)))
+                 `((,nprefix . ,ptree))))
+              (t nil))))))
+      (_ nil))))
+
+;;; Entry points
+
+(defconst radix-tree-empty nil
+  "The empty radix-tree.")
+
+(defun radix-tree-insert (tree key val)
+  "Insert a mapping from KEY to VAL in radix TREE."
+  (when (consp val) (setq val `(t . ,val)))
+  (if val (radix-tree--insert tree key val 0)
+    (radix-tree--remove tree key 0)))
+
+(defun radix-tree-lookup (tree key)
+  "Return the value associated to KEY in radix TREE.
+If not found, return nil."
+  (pcase (radix-tree--lookup tree key 0)
+    (`(t . ,val) val)
+    ((pred numberp) nil)
+    (val val)))
+
+(defun radix-tree-subtree (tree string)
+  "Return the subtree of TREE rooted at the prefix STRING."
+  (radix-tree--subtree tree string 0))
+
+(eval-and-compile
+  (pcase-defmacro radix-tree-leaf (vpat)
+    ;; FIXME: We'd like to use a negative pattern (not consp), but pcase
+    ;; doesn't support it.  Using `atom' works but generates sub-optimal code.
+    `(or `(t . ,,vpat) (and (pred atom) ,vpat))))
+
+(defun radix-tree-iter-subtrees (tree fun)
+  "Apply FUN to every immediate subtree of radix TREE.
+FUN is called with two arguments: PREFIX and SUBTREE.
+You can test if SUBTREE is a leaf (and extract its value) with the
+pcase pattern (radix-tree-leaf PAT)."
+  (while tree
+    (pcase tree
+      (`((,prefix . ,ptree) . ,rtree)
+       (funcall fun prefix ptree)
+       (setq tree rtree))
+      (_ (funcall fun "" tree)
+         (setq tree nil)))))
+
+(defun radix-tree-iter-mappings (tree fun &optional prefix)
+  "Apply FUN to every mapping in TREE.
+FUN is called with two arguments: KEY and VAL.
+PREFIX is only used internally."
+  (radix-tree-iter-subtrees
+   tree
+   (lambda (p s)
+     (let ((nprefix (concat prefix p)))
+       (pcase s
+         ((radix-tree-leaf v) (funcall fun nprefix v))
+         (_ (radix-tree-iter-mappings s fun nprefix)))))))
+
+;; (defun radix-tree->alist (tree)
+;;   (let ((al nil))
+;;     (radix-tree-iter-mappings tree (lambda (p v) (push (cons p v) al)))
+;;     al))
+
+(defun radix-tree-count (tree)
+  (let ((i 0))
+    (radix-tree-iter-mappings tree (lambda (_ _) (setq i (1+ i))))
+    i))
+
+(provide 'radix-tree)
+;;; radix-tree.el ends here
diff --git a/lisp/gnus/message.el b/lisp/gnus/message.el
index 1ca7c5cafef..03ce789e9eb 100644
--- a/lisp/gnus/message.el
+++ b/lisp/gnus/message.el
@@ -4545,7 +4545,7 @@ This function could be useful in `message-setup-hook'."
           (setq message-options options)
           ;; Avoid copying text props (except hard newlines).
           (insert (with-current-buffer mailbuf
-                    (mml-buffer-substring-no-properties-except-hard-newlines
+                    (mml-buffer-substring-no-properties-except-some
                      (point-min) (point-max))))
           ;; Remove some headers.
           (message-encode-message-body)
@@ -4909,7 +4909,7 @@ Otherwise, generate and save a value for `canlock-password' first."
               ;; Avoid copying text props (except hard newlines).
               (insert
                (with-current-buffer messbuf
-                 (mml-buffer-substring-no-properties-except-hard-newlines
+                 (mml-buffer-substring-no-properties-except-some
                   (point-min) (point-max))))
               (message-encode-message-body)
               ;; Remove some headers.
@@ -8386,30 +8386,32 @@ Used in `message-simplify-recipients'."
 (defun message-toggle-image-thumbnails ()
   "For any included image files, insert a thumbnail of that image."
   (interactive)
-  (let ((overlays (overlays-in (point-min) (point-max)))
-        (displayed nil))
-    (while overlays
-      (let ((overlay (car overlays)))
-        (when (overlay-get overlay 'put-image)
-          (delete-overlay overlay)
-          (setq displayed t)))
-      (setq overlays (cdr overlays)))
+  (let ((displayed nil))
+    (save-excursion
+      (goto-char (point-min))
+      (while (not (eobp))
+        (when-let ((props (get-text-property (point) 'display)))
+          (when (and (consp props)
+                     (eq (car props) 'image))
+            (put-text-property (point) (1+ (point)) 'display nil)
+            (setq displayed t)))))
     (unless displayed
       (save-excursion
         (goto-char (point-min))
-        (while (re-search-forward "<img.*src=\"\\([^\"]+\\)" nil t)
-          (let ((file (match-string 1))
+        (while (re-search-forward "<img.*src=\"\\([^\"]+\\).*>" nil t)
+          (let ((string (match-string 0))
+                (file (match-string 1))
                 (edges (window-inside-pixel-edges
                         (get-buffer-window (current-buffer)))))
-            (put-image
+            (delete-region (match-beginning 0) (match-end 0))
+            (insert-image
              (create-image
               file 'imagemagick nil
               :max-width (truncate
                           (* 0.7 (- (nth 2 edges) (nth 0 edges))))
               :max-height (truncate
                            (* 0.5 (- (nth 3 edges) (nth 1 edges)))))
-             (match-beginning 0)
-             " ")))))))
+             string)))))))
 
 (provide 'message)
 
diff --git a/lisp/gnus/mml.el b/lisp/gnus/mml.el
index 97cc87d06e3..eae4c61be82 100644
--- a/lisp/gnus/mml.el
+++ b/lisp/gnus/mml.el
@@ -413,12 +413,21 @@ A message part needs to be split into %d charset parts.  Really send? "
     (setq contents (append (list (cons 'tag-location orig-point)) contents))
     (cons (intern name) (nreverse contents))))
 
-(defun mml-buffer-substring-no-properties-except-hard-newlines (start end)
+(defun mml-buffer-substring-no-properties-except-some (start end)
   (let ((str (buffer-substring-no-properties start end))
-        (bufstart start) tmp)
-    (while (setq tmp (text-property-any start end 'hard 't))
-      (set-text-properties (- tmp bufstart) (- tmp bufstart -1)
-                           '(hard t) str)
+        (bufstart start)
+        tmp)
+    ;; Copy over all hard newlines.
+    (while (setq tmp (text-property-any start end 'hard t))
+      (put-text-property (- tmp bufstart) (- tmp bufstart -1)
+                         'hard t str)
+      (setq start (1+ tmp)))
+    ;; Copy over all `display' properties (which are usually images).
+    (setq start bufstart)
+    (while (setq tmp (text-property-not-all start end 'display nil))
+      (put-text-property (- tmp bufstart) (- tmp bufstart -1)
+                         'display (get-text-property tmp 'display)
+                         str)
       (setq start (1+ tmp)))
     str))
 
@@ -435,21 +444,21 @@ If MML is non-nil, return the buffer up till the correspondent mml tag."
             (if (re-search-forward "<#\\(/\\)?mml." nil t)
                 (setq count (+ count (if (match-beginning 1) -1 1)))
               (goto-char (point-max))))
-          (mml-buffer-substring-no-properties-except-hard-newlines
+          (mml-buffer-substring-no-properties-except-some
            beg (if (> count 0)
                    (point)
                  (match-beginning 0))))
       (if (re-search-forward
            "<#\\(/\\)?\\(multipart\\|part\\|external\\|mml\\)." nil t)
           (prog1
-              (mml-buffer-substring-no-properties-except-hard-newlines
+              (mml-buffer-substring-no-properties-except-some
                beg (match-beginning 0))
             (if (or (not (match-beginning 1))
                     (equal (match-string 2) "multipart"))
                 (goto-char (match-beginning 0))
               (when (looking-at "[ \t]*\n")
                 (forward-line 1))))
-        (mml-buffer-substring-no-properties-except-hard-newlines
+        (mml-buffer-substring-no-properties-except-some
          beg (goto-char (point-max)))))))
 
 (defvar mml-boundary nil)
@@ -514,7 +523,9 @@ be \"related\" or \"alternate\"."
               (when (search-forward (url-filename parsed) end t)
                 (let ((cid (format "fsf.%d" cid)))
                   (replace-match (concat "cid:" cid) t t)
-                  (push (list cid (url-filename parsed)) new-parts))
+                  (push (list cid (url-filename parsed)
+                              (get-text-property start 'display))
+                        new-parts))
                 (setq cid (1+ cid)))))))
       ;; We have local images that we want to include.
       (if (not new-parts)
@@ -527,11 +538,41 @@ be \"related\" or \"alternate\"."
           (setq cont
                 (nconc cont
                        (list `(part (type . "image/png")
-                                    (filename . ,(nth 1 new-part))
+                                    ,@(mml--possibly-alter-image
+                                       (nth 1 new-part)
+                                       (nth 2 new-part))
                                     (id . ,(concat "<" (nth 0 new-part)
                                                    ">")))))))
         cont))))
 
+(defun mml--possibly-alter-image (file-name image)
+  (if (or (null image)
+          (not (consp image))
+          (not (eq (car image) 'image))
+          (not (image-property image :rotation))
+          (not (executable-find "exiftool")))
+      `((filename . ,file-name))
+    `((filename . ,file-name)
+      (buffer
+       .
+       ,(with-current-buffer (mml-generate-new-buffer " *mml rotation*")
+          (set-buffer-multibyte nil)
+          (call-process "exiftool"
+                        file-name
+                        (list (current-buffer) nil)
+                        nil
+                        (format "-Orientation#=%d"
+                                (cl-case (truncate
+                                          (image-property image :rotation))
+                                  (0 0)
+                                  (90 6)
+                                  (180 3)
+                                  (270 8)
+                                  (otherwise 0)))
+                        "-o" "-"
+                        "-")
+          (current-buffer))))))
+
 (defun mml-generate-mime-1 (cont)
   (let ((mm-use-ultra-safe-encoding
          (or mm-use-ultra-safe-encoding (assq 'sign cont))))
diff --git a/lisp/mail/rmail.el b/lisp/mail/rmail.el
index 734155e217d..e9882253c70 100644
--- a/lisp/mail/rmail.el
+++ b/lisp/mail/rmail.el
@@ -1818,9 +1818,21 @@ not be a new one).  It returns non-nil if it got any new messages."
         ;; Read in the contents of the inbox files, renaming them as
         ;; necessary, and adding to the list of files to delete
         ;; eventually.
-        (if file-name
-            (rmail-insert-inbox-text files nil)
-          (setq delete-files (rmail-insert-inbox-text files t)))
+        (unwind-protect
+            (progn
+              ;; Set modified now to lock the file, so that we don't
+              ;; encounter locking problems later in the middle of
+              ;; reading the mail.
+              (set-buffer-modified-p t)
+              (if file-name
+                  (rmail-insert-inbox-text files nil)
+                (setq delete-files (rmail-insert-inbox-text files t))))
+          ;; If there was no new mail, or we aborted before actually
+          ;; trying to get any, mark buffer unmodified.  Otherwise the
+          ;; buffer is correctly marked modified and the file locked
+          ;; until we save out the new mail.
+          (if (= (point-min) (point-max))
+              (set-buffer-modified-p nil)))
         ;; Scan the new text and convert each message to
         ;; Rmail/mbox format.
         (goto-char (point-min))
@@ -1969,11 +1981,6 @@ Value is the size of the newly read mail after conversion."
     size))
 
 (defun rmail-insert-inbox-text (files renamep)
-  ;; Detect a locked file now, so that we avoid moving mail
-  ;; out of the real inbox file.  (That could scare people.)
-  (or (memq (file-locked-p buffer-file-name) '(nil t))
-      (error "RMAIL file %s is locked"
-             (file-name-nondirectory buffer-file-name)))
   (let (file tofile delete-files popmail got-password password)
     (while files
       ;; Handle remote mailbox names specially; don't expand as filenames
diff --git a/lisp/net/tramp-adb.el b/lisp/net/tramp-adb.el
index 5940b713958..1281dbbd72d 100644
--- a/lisp/net/tramp-adb.el
+++ b/lisp/net/tramp-adb.el
@@ -535,7 +535,7 @@ Emacs dired can't find files."
   "Like `file-name-all-completions' for Tramp files."
   (all-completions
    filename
-   (with-parsed-tramp-file-name directory nil
+   (with-parsed-tramp-file-name (expand-file-name directory) nil
      (with-tramp-file-property v localname "file-name-all-completions"
        (save-match-data
          (tramp-adb-send-command
@@ -934,20 +934,22 @@ PRESERVE-UID-GID and PRESERVE-EXTENDED-ATTRIBUTES are completely ignored."
     (unless (stringp program)
       (tramp-error v 'file-error "PROGRAM must be a string"))
 
-    (let ((command
-           (format "cd %s; %s"
-                   (tramp-shell-quote-argument localname)
-                   (mapconcat 'tramp-shell-quote-argument
-                              (cons program args) " ")))
-          (tramp-process-connection-type
-           (or (null program) tramp-process-connection-type))
-          (bmp (and (buffer-live-p buffer) (buffer-modified-p buffer)))
-          (name1 name)
-          (i 0))
-
-      (unless buffer
-        ;; BUFFER can be nil.  We use a temporary buffer.
-        (setq buffer (generate-new-buffer tramp-temp-buffer-name)))
+    (let* ((buffer
+            (if buffer
+                (get-buffer-create buffer)
+              ;; BUFFER can be nil.  We use a temporary buffer.
+              (generate-new-buffer tramp-temp-buffer-name)))
+           (command
+            (format "cd %s; %s"
+                    (tramp-shell-quote-argument localname)
+                    (mapconcat 'tramp-shell-quote-argument
+                               (cons program args) " ")))
+           (tramp-process-connection-type
+            (or (null program) tramp-process-connection-type))
+           (bmp (and (buffer-live-p buffer) (buffer-modified-p buffer)))
+           (name1 name)
+           (i 0))
+
       (while (get-process name1)
         ;; NAME must be unique as process name.
         (setq i (1+ i)
diff --git a/lisp/net/tramp-gvfs.el b/lisp/net/tramp-gvfs.el
index 098d40e7cc0..ac390e5d5a6 100644
--- a/lisp/net/tramp-gvfs.el
+++ b/lisp/net/tramp-gvfs.el
@@ -407,6 +407,42 @@ Every entry is a list (NAME ADDRESS).")
 (defconst tramp-hal-interface-device "org.freedesktop.Hal.Device"
   "The device interface of the HAL daemon.")
 
+(defconst tramp-gvfs-file-attributes
+  '("type"
+    "standard::display-name"
+    ;; We don't need this one.  It is used as delimiter in case the
+    ;; display name contains spaces, which is hard to parse.
+    "standard::icon"
+    "standard::symlink-target"
+    "unix::nlink"
+    "unix::uid"
+    "owner::user"
+    "unix::gid"
+    "owner::group"
+    "time::access"
+    "time::modified"
+    "time::changed"
+    "standard::size"
+    "unix::mode"
+    "access::can-read"
+    "access::can-write"
+    "access::can-execute"
+    "unix::inode"
+    "unix::device")
+  "GVFS file attributes.")
+
+(defconst tramp-gvfs-file-attributes-with-gvfs-ls-regexp
+  (concat "[[:blank:]]"
+          (regexp-opt tramp-gvfs-file-attributes t)
+          "=\\([^[:blank:]]+\\)")
+  "Regexp to parse GVFS file attributes with `gvfs-ls'.")
+
+(defconst tramp-gvfs-file-attributes-with-gvfs-info-regexp
+  (concat "^[[:blank:]]*"
+          (regexp-opt tramp-gvfs-file-attributes t)
+          ":[[:blank:]]+\\(.*\\)$")
+  "Regexp to parse GVFS file attributes with `gvfs-info'.")
+
 
 ;; New handlers should be added here.
 (defconst tramp-gvfs-file-name-handler-alist
@@ -784,127 +820,185 @@ file names."
        (tramp-run-real-handler
         'expand-file-name (list localname))))))
 
-(defun tramp-gvfs-handle-file-attributes (filename &optional id-format)
-  "Like `file-attributes' for Tramp files."
-  (unless id-format (setq id-format 'integer))
+(defun tramp-gvfs-get-directory-attributes (directory)
+  "Return GVFS attributes association list of all files in DIRECTORY."
   (ignore-errors
     ;; Don't modify `last-coding-system-used' by accident.
     (let ((last-coding-system-used last-coding-system-used)
-          (process-environment (cons "LC_MESSAGES=C" process-environment))
-          dirp res-symlink-target res-numlinks res-uid res-gid res-access
-          res-mod res-change res-size res-filemodes res-inode res-device)
+          result)
+      (with-parsed-tramp-file-name directory nil
+        (with-tramp-file-property v localname "directory-gvfs-attributes"
+          (tramp-message v 5 "directory gvfs attributes: %s" localname)
+          ;; Send command.
+          (tramp-gvfs-send-command
+           v "gvfs-ls" "-h" "-n" "-a"
+           (mapconcat 'identity tramp-gvfs-file-attributes ",")
+           (tramp-gvfs-url-file-name directory))
+          ;; Parse output ...
+          (with-current-buffer (tramp-get-connection-buffer v)
+            (goto-char (point-min))
+            (while (re-search-forward
+                    (concat "^\\(.+\\)[[:blank:]]"
+                            "\\([[:digit:]]+\\)[[:blank:]]"
+                            "(\\(.+\\))[[:blank:]]"
+                            "standard::display-name=\\(.+\\)[[:blank:]]"
+                            "standard::icon=")
+                    (point-at-eol) t)
+              (let ((item (list (cons "standard::display-name" (match-string 4))
+                                (cons "type" (match-string 3))
+                                (cons "standard::size" (match-string 2))
+                                (match-string 1))))
+                (while (re-search-forward
+                        tramp-gvfs-file-attributes-with-gvfs-ls-regexp
+                        (point-at-eol) t)
+                  (push (cons (match-string 1) (match-string 2)) item))
+                (push (nreverse item) result))
+              (forward-line)))
+          result)))))
+
+(defun tramp-gvfs-get-root-attributes (filename)
+  "Return GVFS attributes association list of FILENAME."
+  (ignore-errors
+    ;; Don't modify `last-coding-system-used' by accident.
+    (let ((last-coding-system-used last-coding-system-used)
+          result)
       (with-parsed-tramp-file-name filename nil
-        (with-tramp-file-property
-            v localname (format "file-attributes-%s" id-format)
-          (tramp-message v 5 "file attributes: %s" localname)
+        (with-tramp-file-property v localname "file-gvfs-attributes"
+          (tramp-message v 5 "file gvfs attributes: %s" localname)
+          ;; Send command.
           (tramp-gvfs-send-command
            v "gvfs-info" (tramp-gvfs-url-file-name filename))
           ;; Parse output ...
           (with-current-buffer (tramp-get-connection-buffer v)
             (goto-char (point-min))
-            (when (re-search-forward "attributes:" nil t)
-              ;; ... directory or symlink
-              (goto-char (point-min))
-              (setq dirp (if (re-search-forward "type: directory" nil t) t))
-              (goto-char (point-min))
-              (setq res-symlink-target
-                    (if (re-search-forward
-                         "standard::symlink-target: \\(.+\\)$" nil t)
-                        (match-string 1)))
-              ;; ... number links
-              (goto-char (point-min))
-              (setq res-numlinks
-                    (if (re-search-forward "unix::nlink: \\([0-9]+\\)" nil t)
-                        (string-to-number (match-string 1)) 0))
-              ;; ... uid and gid
-              (goto-char (point-min))
-              (setq res-uid
-                    (if (eq id-format 'integer)
-                        (if (re-search-forward "unix::uid: \\([0-9]+\\)" nil t)
-                            (string-to-number (match-string 1))
-                          -1)
-                      (if (re-search-forward "owner::user: \\(.+\\)$" nil t)
-                          (match-string 1)
-                        "UNKNOWN")))
-              (setq res-gid
-                    (if (eq id-format 'integer)
-                        (if (re-search-forward "unix::gid: \\([0-9]+\\)" nil t)
-                            (string-to-number (match-string 1))
-                          -1)
-                      (if (re-search-forward "owner::group: \\(.+\\)$" nil t)
-                          (match-string 1)
-                        "UNKNOWN")))
-              ;; ... last access, modification and change time
-              (goto-char (point-min))
-              (setq res-access
-                    (if (re-search-forward "time::access: \\([0-9]+\\)" nil t)
-                        (seconds-to-time (string-to-number (match-string 1)))
-                      '(0 0)))
-              (goto-char (point-min))
-              (setq res-mod
-                    (if (re-search-forward "time::modified: \\([0-9]+\\)" nil t)
-                        (seconds-to-time (string-to-number (match-string 1)))
-                      '(0 0)))
-              (goto-char (point-min))
-              (setq res-change
-                    (if (re-search-forward "time::changed: \\([0-9]+\\)" nil t)
-                        (seconds-to-time (string-to-number (match-string 1)))
-                      '(0 0)))
-              ;; ... size
-              (goto-char (point-min))
-              (setq res-size
-                    (if (re-search-forward "standard::size: \\([0-9]+\\)" nil t)
-                        (string-to-number (match-string 1)) 0))
-              ;; ... file mode flags
-              (goto-char (point-min))
-              (setq res-filemodes
-                    (if (re-search-forward "unix::mode: \\([0-9]+\\)" nil t)
-                        (tramp-file-mode-from-int
-                         (string-to-number (match-string 1)))
-                      (if dirp "drwx------" "-rwx------")))
-              ;; ... inode and device
-              (goto-char (point-min))
-              (setq res-inode
-                    (if (re-search-forward "unix::inode: \\([0-9]+\\)" nil t)
-                        (string-to-number (match-string 1))
-                      (tramp-get-inode v)))
-              (goto-char (point-min))
-              (setq res-device
-                    (if (re-search-forward "unix::device: \\([0-9]+\\)" nil t)
-                        (string-to-number (match-string 1))
-                      (tramp-get-device v)))
-
-              ;; Return data gathered.
-              (list
-               ;; 0. t for directory, string (name linked to) for
-               ;; symbolic link, or nil.
-               (or dirp res-symlink-target)
-               ;; 1. Number of links to file.
-               res-numlinks
-               ;; 2. File uid.
-               res-uid
-               ;; 3. File gid.
-               res-gid
-               ;; 4. Last access time, as a list of integers.
-               ;; 5. Last modification time, likewise.
-               ;; 6. Last status change time, likewise.
-               res-access res-mod res-change
-               ;; 7. Size in bytes (-1, if number is out of range).
-               res-size
-               ;; 8. File modes.
-               res-filemodes
-               ;; 9. t if file's gid would change if file were deleted
-               ;; and recreated.
-               nil
-               ;; 10. Inode number.
-               res-inode
-               ;; 11. Device number.
-               res-device
-               ))))))))
+            (while (re-search-forward
+                    tramp-gvfs-file-attributes-with-gvfs-info-regexp nil t)
+              (push (cons (match-string 1) (match-string 2)) result))
+            result))))))
+
+(defun tramp-gvfs-get-file-attributes (filename)
+  "Return GVFS attributes association list of FILENAME."
+  (setq filename (directory-file-name (expand-file-name filename)))
+  (with-parsed-tramp-file-name filename nil
+    (if (or
+         (and (string-match "^\\(afp\\|smb\\)$" method)
+              (string-match "^/?\\([^/]+\\)$" localname))
+         (string-equal localname "/"))
+        (tramp-gvfs-get-root-attributes filename)
+      (assoc
+       (file-name-nondirectory filename)
+       (tramp-gvfs-get-directory-attributes (file-name-directory filename))))))
+
+(defun tramp-gvfs-handle-file-attributes (filename &optional id-format)
+  "Like `file-attributes' for Tramp files."
+  (unless id-format (setq id-format 'integer))
+  (ignore-errors
+    (let ((attributes (tramp-gvfs-get-file-attributes filename))
+          dirp res-symlink-target res-numlinks res-uid res-gid res-access
+          res-mod res-change res-size res-filemodes res-inode res-device)
+      (when attributes
+        ;; ... directory or symlink
+        (setq dirp (if (equal "directory" (cdr (assoc "type" attributes))) t))
+        (setq res-symlink-target
+              (cdr (assoc "standard::symlink-target" attributes)))
+        ;; ... number links
+        (setq res-numlinks
+              (string-to-number
+               (or (cdr (assoc "unix::nlink" attributes)) "0")))
+        ;; ... uid and gid
+        (setq res-uid
+              (if (eq id-format 'integer)
+                  (string-to-number
+                   (or (cdr (assoc "unix::uid" attributes))
+                       (format "%s" tramp-unknown-id-integer)))
+                (or (cdr (assoc "owner::user" attributes))
+                    (cdr (assoc "unix::uid" attributes))
+                    tramp-unknown-id-string)))
+        (setq res-gid
+              (if (eq id-format 'integer)
+                  (string-to-number
+                   (or (cdr (assoc "unix::gid" attributes))
+                       (format "%s" tramp-unknown-id-integer)))
+                (or (cdr (assoc "owner::group" attributes))
+                    (cdr (assoc "unix::gid" attributes))
+                    tramp-unknown-id-string)))
+        ;; ... last access, modification and change time
+        (setq res-access
+              (seconds-to-time
+               (string-to-number
+                (or (cdr (assoc "time::access" attributes)) "0"))))
+        (setq res-mod
+              (seconds-to-time
+               (string-to-number
+                (or (cdr (assoc "time::modified" attributes)) "0"))))
+        (setq res-change
+              (seconds-to-time
+               (string-to-number
+                (or (cdr (assoc "time::changed" attributes)) "0"))))
+        ;; ... size
+        (setq res-size
+              (string-to-number
+               (or (cdr (assoc "standard::size" attributes)) "0")))
+        ;; ... file mode flags
+        (setq res-filemodes
+              (let ((n (cdr (assoc "unix::mode" attributes))))
+                (if n
+                    (tramp-file-mode-from-int (string-to-number n))
+                  (format
+                   "%s%s%s%s------"
+                   (if dirp "d" "-")
+                   (if (equal (cdr (assoc "access::can-read" attributes))
+                              "FALSE")
+                       "-" "r")
+                   (if (equal (cdr (assoc "access::can-write" attributes))
+                              "FALSE")
+                       "-" "w")
+                   (if (equal (cdr (assoc "access::can-execute" attributes))
+                              "FALSE")
+                       "-" "x")))))
+        ;; ... inode and device
+        (setq res-inode
+              (let ((n (cdr (assoc "unix::inode" attributes))))
+                (if n
+                    (string-to-number n)
+                  (tramp-get-inode (tramp-dissect-file-name filename)))))
+        (setq res-device
+              (let ((n (cdr (assoc "unix::device" attributes))))
+                (if n
+                    (string-to-number n)
+                  (tramp-get-device (tramp-dissect-file-name filename)))))
+
+        ;; Return data gathered.
+        (list
+         ;; 0. t for directory, string (name linked to) for
+         ;; symbolic link, or nil.
+         (or dirp res-symlink-target)
+         ;; 1. Number of links to file.
+         res-numlinks
+         ;; 2. File uid.
+         res-uid
+         ;; 3. File gid.
+         res-gid
+         ;; 4. Last access time, as a list of integers.
+         ;; 5. Last modification time, likewise.
+         ;; 6. Last status change time, likewise.
+         res-access res-mod res-change
+         ;; 7. Size in bytes (-1, if number is out of range).
+         res-size
+         ;; 8. File modes.
+         res-filemodes
+         ;; 9. t if file's gid would change if file were deleted
+         ;; and recreated.
+         nil
+         ;; 10. Inode number.
+         res-inode
+         ;; 11. Device number.
+         res-device
+         )))))
 
 (defun tramp-gvfs-handle-file-directory-p (filename)
   "Like `file-directory-p' for Tramp files."
-  (eq t (car (file-attributes filename))))
+  (eq t (car (file-attributes (file-truename filename)))))
 
 (defun tramp-gvfs-handle-file-executable-p (filename)
   "Like `file-executable-p' for Tramp files."
@@ -926,73 +1020,21 @@ file names."
 (defun tramp-gvfs-handle-file-name-all-completions (filename directory)
   "Like `file-name-all-completions' for Tramp files."
   (unless (save-match-data (string-match "/" filename))
-    (with-parsed-tramp-file-name (expand-file-name directory) nil
-
-      (all-completions
-       filename
-       (mapcar
-        'list
-        (or
-         ;; Try cache entries for filename, filename with last
-         ;; character removed, filename with last two characters
-         ;; removed, ..., and finally the empty string - all
-         ;; concatenated to the local directory name.
-         (let ((remote-file-name-inhibit-cache
-                (or remote-file-name-inhibit-cache
-                    tramp-completion-reread-directory-timeout)))
-
-           ;; This is inefficient for very long filenames, pity
-           ;; `reduce' is not available...
-           (car
-            (apply
-             'append
-             (mapcar
-              (lambda (x)
-                (let ((cache-hit
-                       (tramp-get-file-property
-                        v
-                        (concat localname (substring filename 0 x))
-                        "file-name-all-completions"
-                        nil)))
-                  (when cache-hit (list cache-hit))))
-              ;; We cannot use a length of 0, because file properties
-              ;; for "foo" and "foo/" are identical.
-              (number-sequence (length filename) 1 -1)))))
-
-         ;; Cache expired or no matching cache entry found so we need
-         ;; to perform a remote operation.
-         (let ((result '("." ".."))
+    (all-completions
+     filename
+     (with-parsed-tramp-file-name (expand-file-name directory) nil
+       (with-tramp-file-property v localname "file-name-all-completions"
+         (let ((result '("./" "../"))
                entry)
            ;; Get a list of directories and files.
-           (tramp-gvfs-send-command
-            v "gvfs-ls" "-h" (tramp-gvfs-url-file-name directory))
-
-           ;; Now grab the output.
-           (with-temp-buffer
-             (insert-buffer-substring (tramp-get-connection-buffer v))
-             (goto-char (point-max))
-             (while (zerop (forward-line -1))
-               (setq entry (buffer-substring (point) (point-at-eol)))
-               (when (string-match filename entry)
-                 (if (file-directory-p (expand-file-name entry directory))
-                     (push (concat entry "/") result)
-                   (push entry result)))))
-
-           ;; Because the remote op went through OK we know the
-           ;; directory we `cd'-ed to exists.
-           (tramp-set-file-property v localname "file-exists-p" t)
-
-           ;; Because the remote op went through OK we know every
-           ;; file listed by `ls' exists.
-           (mapc (lambda (entry)
-                   (tramp-set-file-property
-                    v (concat localname entry) "file-exists-p" t))
-                 result)
-
-           ;; Store result in the cache.
-           (tramp-set-file-property
-            v (concat localname filename)
-            "file-name-all-completions" result))))))))
+           (dolist (item (tramp-gvfs-get-directory-attributes directory) result)
+             (setq entry
+                   (or ;; Use display-name if available (google-drive).
+                    ;(cdr (assoc "standard::display-name" item))
+                    (car item)))
+             (if (string-equal (cdr (assoc "type" item)) "directory")
+                 (push (file-name-as-directory entry) result)
+               (push entry result)))))))))
 
 (defun tramp-gvfs-handle-file-notify-add-watch (file-name flags _callback)
   "Like `file-notify-add-watch' for Tramp files."
@@ -1528,7 +1570,7 @@ connection if a previous connection has died for some reason."
     (let ((p (make-network-process
               :name (tramp-buffer-name vec)
               :buffer (tramp-get-connection-buffer vec)
-              :server t :host 'local :service t)))
+              :server t :host 'local :service t :noquery t)))
       (set-process-query-on-exit-flag p nil)))
 
   (unless (tramp-gvfs-connection-mounted-p vec)
@@ -1635,10 +1677,17 @@ connection if a previous connection has died for some reason."
   "Send the COMMAND with its ARGS to connection VEC.
 COMMAND is usually a command from the gvfs-* utilities.
 `call-process' is applied, and it returns t if the return code is zero."
-  (with-current-buffer (tramp-get-connection-buffer vec)
-    (tramp-gvfs-maybe-open-connection vec)
-    (erase-buffer)
-    (zerop (apply 'tramp-call-process vec command nil t nil args))))
+  (let* ((locale (tramp-get-local-locale vec))
+         (process-environment
+          (append
+           `(,(format "LANG=%s" locale)
+             ,(format "LANGUAGE=%s" locale)
+             ,(format "LC_ALL=%s" locale))
+           process-environment)))
+    (with-current-buffer (tramp-get-connection-buffer vec)
+      (tramp-gvfs-maybe-open-connection vec)
+      (erase-buffer)
+      (zerop (apply 'tramp-call-process vec command nil t nil args)))))
 
 
 ;; D-Bus BLUEZ functions.
@@ -1772,35 +1821,37 @@ This uses \"avahi-browse\" in case D-Bus is not enabled in Avahi."
 
 ;; Add completion functions for AFP, DAV, DAVS, SFTP and SMB methods.
 (when tramp-gvfs-enabled
-  (zeroconf-init tramp-gvfs-zeroconf-domain)
-  (if (zeroconf-list-service-types)
-      (progn
+  ;; Suppress D-Bus error messages.
+  (let (tramp-gvfs-dbus-event-vector)
+    (zeroconf-init tramp-gvfs-zeroconf-domain)
+    (if (zeroconf-list-service-types)
+        (progn
+          (tramp-set-completion-function
+           "afp" '((tramp-zeroconf-parse-device-names "_afpovertcp._tcp")))
+          (tramp-set-completion-function
+           "dav" '((tramp-zeroconf-parse-device-names "_webdav._tcp")))
+          (tramp-set-completion-function
+           "davs" '((tramp-zeroconf-parse-device-names "_webdav._tcp")))
+          (tramp-set-completion-function
+           "sftp" '((tramp-zeroconf-parse-device-names "_ssh._tcp")
+                    (tramp-zeroconf-parse-device-names "_workstation._tcp")))
+          (when (member "smb" tramp-gvfs-methods)
+            (tramp-set-completion-function
+             "smb" '((tramp-zeroconf-parse-device-names "_smb._tcp")))))
+
+      (when (executable-find "avahi-browse")
         (tramp-set-completion-function
-         "afp" '((tramp-zeroconf-parse-device-names "_afpovertcp._tcp")))
+         "afp" '((tramp-gvfs-parse-device-names "_afpovertcp._tcp")))
         (tramp-set-completion-function
-         "dav" '((tramp-zeroconf-parse-device-names "_webdav._tcp")))
+         "dav" '((tramp-gvfs-parse-device-names "_webdav._tcp")))
         (tramp-set-completion-function
-         "davs" '((tramp-zeroconf-parse-device-names "_webdav._tcp")))
+         "davs" '((tramp-gvfs-parse-device-names "_webdav._tcp")))
         (tramp-set-completion-function
-         "sftp" '((tramp-zeroconf-parse-device-names "_ssh._tcp")
-                  (tramp-zeroconf-parse-device-names "_workstation._tcp")))
+         "sftp" '((tramp-gvfs-parse-device-names "_ssh._tcp")
+                  (tramp-gvfs-parse-device-names "_workstation._tcp")))
         (when (member "smb" tramp-gvfs-methods)
           (tramp-set-completion-function
-           "smb" '((tramp-zeroconf-parse-device-names "_smb._tcp")))))
-
-    (when (executable-find "avahi-browse")
-      (tramp-set-completion-function
-       "afp" '((tramp-gvfs-parse-device-names "_afpovertcp._tcp")))
-      (tramp-set-completion-function
-       "dav" '((tramp-gvfs-parse-device-names "_webdav._tcp")))
-      (tramp-set-completion-function
-       "davs" '((tramp-gvfs-parse-device-names "_webdav._tcp")))
-      (tramp-set-completion-function
-       "sftp" '((tramp-gvfs-parse-device-names "_ssh._tcp")
-                (tramp-gvfs-parse-device-names "_workstation._tcp")))
-      (when (member "smb" tramp-gvfs-methods)
-        (tramp-set-completion-function
-         "smb" '((tramp-gvfs-parse-device-names "_smb._tcp")))))))
+           "smb" '((tramp-gvfs-parse-device-names "_smb._tcp"))))))))
 
 
 ;; D-Bus SYNCE functions.
diff --git a/lisp/net/tramp-sh.el b/lisp/net/tramp-sh.el
index 14c6f949853..e9f78b7d1ce 100644
--- a/lisp/net/tramp-sh.el
+++ b/lisp/net/tramp-sh.el
@@ -84,8 +84,12 @@ e.g. \"$HOME/.sh_history\"."
                  (string :tag "Redirect to a file")))
 
 ;;;###tramp-autoload
-(defconst tramp-color-escape-sequence-regexp "\e[[;0-9]+m"
-  "Escape sequences produced by the \"ls\" command.")
+(defconst tramp-display-escape-sequence-regexp "\e[[;0-9]+m"
+  "Terminal control escape sequences for display attributes.")
+
+;;;###tramp-autoload
+(defconst tramp-device-escape-sequence-regexp "\e[[0-9]+n"
+  "Terminal control escape sequences for device status.")
 
 ;; ksh on OpenBSD 4.5 requires that $PS1 contains a `#' character for
 ;; root users.  It uses the `$' character for other users.  In order
@@ -658,29 +662,19 @@ Escape sequence %s is replaced with name of Perl binary.
 This string is passed to `format', so percent characters need to be doubled.")
 
 (defconst tramp-perl-file-name-all-completions
-  "%s -e 'sub case {
- my $str = shift;
- if ($ARGV[2]) {
-  return lc($str);
- }
- else {
-  return $str;
- }
-}
+  "%s -e '
 opendir(d, $ARGV[0]) || die(\"$ARGV[0]: $!\\nfail\\n\");
 @files = readdir(d); closedir(d);
 foreach $f (@files) {
- if (case(substr($f, 0, length($ARGV[1]))) eq case($ARGV[1])) {
-  if (-d \"$ARGV[0]/$f\") {
-   print \"$f/\\n\";
-  }
-  else {
-   print \"$f\\n\";
-  }
+ if (-d \"$ARGV[0]/$f\") {
+  print \"$f/\\n\";
+ }
+ else {
+  print \"$f\\n\";
  }
 }
 print \"ok\\n\"
-' \"$1\" \"$2\" \"$3\" 2>/dev/null"
+' \"$1\" 2>/dev/null"
   "Perl script to produce output suitable for use with
 `file-name-all-completions' on the remote file system.  Escape
 sequence %s is replaced with name of Perl binary.  This string is
@@ -1339,8 +1333,10 @@ target of the symlink differ."
           (setq res-gid (read (current-buffer)))
           (if (eq id-format 'integer)
               (progn
-                (unless (numberp res-uid) (setq res-uid -1))
-                (unless (numberp res-gid) (setq res-gid -1)))
+                (unless (numberp res-uid)
+                  (setq res-uid tramp-unknown-id-integer))
+                (unless (numberp res-gid)
+                  (setq res-gid tramp-unknown-id-integer)))
             (progn
               (unless (stringp res-uid) (setq res-uid (symbol-name res-uid)))
               (unless (stringp res-gid) (setq res-gid (symbol-name res-gid)))))
@@ -1862,135 +1858,63 @@ be non-negative integers."
 (defun tramp-sh-handle-file-name-all-completions (filename directory)
   "Like `file-name-all-completions' for Tramp files."
   (unless (save-match-data (string-match "/" filename))
-    (with-parsed-tramp-file-name (expand-file-name directory) nil
+    (all-completions
+     filename
+     (with-parsed-tramp-file-name (expand-file-name directory) nil
+       (with-tramp-file-property v localname "file-name-all-completions"
+         (let (result)
+           ;; Get a list of directories and files, including reliably
+           ;; tagging the directories with a trailing "/".  Because I
+           ;; rock.  --daniel@danann.net
+           (tramp-send-command
+            v
+            (if (tramp-get-remote-perl v)
+                (progn
+                  (tramp-maybe-send-script
+                   v tramp-perl-file-name-all-completions
+                   "tramp_perl_file_name_all_completions")
+                  (format "tramp_perl_file_name_all_completions %s"
+                          (tramp-shell-quote-argument localname)))
+
+              (format (concat
+                       "(cd %s 2>&1 && %s -a 2>/dev/null"
+                       " | while IFS= read f; do"
+                       " if %s -d \"$f\" 2>/dev/null;"
+                       " then \\echo \"$f/\"; else \\echo \"$f\"; fi; done"
+                       " && \\echo ok) || \\echo fail")
+                      (tramp-shell-quote-argument localname)
+                      (tramp-get-ls-command v)
+                      (tramp-get-test-command v))))
 
-      (all-completions
-       filename
-       (mapcar
-        'list
-        (or
-         ;; Try cache entries for `filename', `filename' with last
-         ;; character removed, `filename' with last two characters
-         ;; removed, ..., and finally the empty string - all
-         ;; concatenated to the local directory name.
-         (let ((remote-file-name-inhibit-cache
-                (or remote-file-name-inhibit-cache
-                    tramp-completion-reread-directory-timeout)))
-
-           ;; This is inefficient for very long file names, pity
-           ;; `reduce' is not available...
-           (car
-            (apply
-             'append
-             (mapcar
-              (lambda (x)
-                (let ((cache-hit
-                       (tramp-get-file-property
-                        v
-                        (concat localname (substring filename 0 x))
-                        "file-name-all-completions"
-                        nil)))
-                  (when cache-hit (list cache-hit))))
-              ;; We cannot use a length of 0, because file properties
-              ;; for "foo" and "foo/" are identical.
-              (number-sequence (length filename) 1 -1)))))
-
-         ;; Cache expired or no matching cache entry found so we need
-         ;; to perform a remote operation.
-         (let (result)
-           ;; Get a list of directories and files, including reliably
-           ;; tagging the directories with a trailing '/'.  Because I
-           ;; rock.  --daniel@danann.net
-
-           ;; Changed to perform `cd' in the same remote op and only
-           ;; get entries starting with `filename'.  Capture any `cd'
-           ;; error messages.  Ensure any `cd' and `echo' aliases are
-           ;; ignored.
-           (tramp-send-command
-            v
-            (if (tramp-get-remote-perl v)
-                (progn
-                  (tramp-maybe-send-script
-                   v tramp-perl-file-name-all-completions
-                   "tramp_perl_file_name_all_completions")
-                  (format "tramp_perl_file_name_all_completions %s %s %d"
-                          (tramp-shell-quote-argument localname)
-                          (tramp-shell-quote-argument filename)
-                          (if read-file-name-completion-ignore-case 1 0)))
-
-              (format (concat
-                       "(cd %s 2>&1 && (%s -a %s 2>/dev/null"
-                       ;; `ls' with wildcard might fail with `Argument
-                       ;; list too long' error in some corner cases; if
-                       ;; `ls' fails after `cd' succeeded, chances are
-                       ;; that's the case, so let's retry without
-                       ;; wildcard.  This will return "too many" entries
-                       ;; but that isn't harmful.
-                       " || %s -a 2>/dev/null)"
-                       " | while IFS= read f; do"
-                       " if %s -d \"$f\" 2>/dev/null;"
-                       " then \\echo \"$f/\"; else \\echo \"$f\"; fi; done"
-                       " && \\echo ok) || \\echo fail")
-                      (tramp-shell-quote-argument localname)
-                      (tramp-get-ls-command v)
-                      ;; When `filename' is empty, just `ls' without
-                      ;; `filename' argument is more efficient than `ls *'
-                      ;; for very large directories and might avoid the
-                      ;; `Argument list too long' error.
-                      ;;
-                      ;; With and only with wildcard, we need to add
-                      ;; `-d' to prevent `ls' from descending into
-                      ;; sub-directories.
-                      (if (zerop (length filename))
-                          "."
-                        (format "-d %s*" (tramp-shell-quote-argument filename)))
-                      (tramp-get-ls-command v)
-                      (tramp-get-test-command v))))
-
-           ;; Now grab the output.
-           (with-current-buffer (tramp-get-buffer v)
-             (goto-char (point-max))
-
-             ;; Check result code, found in last line of output.
-             (forward-line -1)
-             (if (looking-at "^fail$")
-                 (progn
-                   ;; Grab error message from line before last line
-                   ;; (it was put there by `cd 2>&1').
-                   (forward-line -1)
-                   (tramp-error
-                    v 'file-error
-                    "tramp-sh-handle-file-name-all-completions: %s"
-                    (buffer-substring (point) (point-at-eol))))
-               ;; For peace of mind, if buffer doesn't end in `fail'
-               ;; then it should end in `ok'.  If neither are in the
-               ;; buffer something went seriously wrong on the remote
-               ;; side.
-               (unless (looking-at "^ok$")
-                 (tramp-error
-                  v 'file-error
-                  "\
+           ;; Now grab the output.
+           (with-current-buffer (tramp-get-buffer v)
+             (goto-char (point-max))
+
+             ;; Check result code, found in last line of output.
+             (forward-line -1)
+             (if (looking-at "^fail$")
+                 (progn
+                   ;; Grab error message from line before last line
+                   ;; (it was put there by `cd 2>&1').
+                   (forward-line -1)
+                   (tramp-error
+                    v 'file-error
+                    "tramp-sh-handle-file-name-all-completions: %s"
+                    (buffer-substring (point) (point-at-eol))))
+               ;; For peace of mind, if buffer doesn't end in `fail'
+               ;; then it should end in `ok'.  If neither are in the
+               ;; buffer something went seriously wrong on the remote
+               ;; side.
+               (unless (looking-at "^ok$")
+                 (tramp-error
+                  v 'file-error
+                  "\
 tramp-sh-handle-file-name-all-completions: internal error accessing `%s': `%s'"
-                  (tramp-shell-quote-argument localname) (buffer-string))))
-
-             (while (zerop (forward-line -1))
-               (push (buffer-substring (point) (point-at-eol)) result)))
-
-           ;; Because the remote op went through OK we know the
-           ;; directory we `cd'-ed to exists.
-           (tramp-set-file-property v localname "file-exists-p" t)
-
-           ;; Because the remote op went through OK we know every
-           ;; file listed by `ls' exists.
-           (mapc (lambda (entry)
-                   (tramp-set-file-property
-                    v (concat localname entry) "file-exists-p" t))
-                 result)
+                  (tramp-shell-quote-argument localname) (buffer-string))))
 
-           ;; Store result in the cache.
-           (tramp-set-file-property
-            v (concat localname filename)
-            "file-name-all-completions" result))))))))
+             (while (zerop (forward-line -1))
+               (push (buffer-substring (point) (point-at-eol)) result)))
+           result))))))
 
 ;; cp, mv and ln
 
@@ -2836,7 +2760,8 @@ The method used must be an out-of-band method."
           (unless
               (string-match "color" (tramp-get-connection-property v "ls" ""))
             (goto-char beg)
-            (while (re-search-forward tramp-color-escape-sequence-regexp nil t)
+            (while
+                (re-search-forward tramp-display-escape-sequence-regexp nil t)
               (replace-match "")))
 
           ;; Decode the output, it could be multibyte.
@@ -2934,7 +2859,12 @@ the result will be a local, non-Tramp, file name."
 (defun tramp-sh-handle-start-file-process (name buffer program &rest args)
   "Like `start-file-process' for Tramp files."
   (with-parsed-tramp-file-name (expand-file-name default-directory) nil
-    (let* (;; When PROGRAM matches "*sh", and the first arg is "-c",
+    (let* ((buffer
+            (if buffer
+                (get-buffer-create buffer)
+              ;; BUFFER can be nil.  We use a temporary buffer.
+              (generate-new-buffer tramp-temp-buffer-name)))
+           ;; When PROGRAM matches "*sh", and the first arg is "-c",
            ;; it might be that the arguments exceed the command line
            ;; length.  Therefore, we modify the command.
            (heredoc (and (stringp program)
@@ -2992,9 +2922,6 @@ the result will be a local, non-Tramp, file name."
            ;; `eshell' and friends.
            (tramp-current-connection nil))
 
-      (unless buffer
-        ;; BUFFER can be nil.  We use a temporary buffer.
-        (setq buffer (generate-new-buffer tramp-temp-buffer-name)))
       (while (get-process name1)
         ;; NAME must be unique as process name.
         (setq i (1+ i)
@@ -4030,7 +3957,7 @@ file exists and nonzero exit status otherwise."
           shell)
       (setq shell
             (with-tramp-connection-property vec "remote-shell"
-              ;; CCC: "root" does not exist always, see QNAP 459.
+              ;; CCC: "root" does not exist always, see my QNAP TS-459.
               ;; Which check could we apply instead?
               (tramp-send-command vec "echo ~root" t)
               (if (or (string-match "^~root$" (buffer-string))
@@ -4790,7 +4717,7 @@ connection if a previous connection has died for some reason."
                      (options (tramp-ssh-controlmaster-options vec))
                      (process-connection-type tramp-process-connection-type)
                      (process-adaptive-read-buffering nil)
-                     ;; There are unfortune settings for "cmdproxy" on
+                     ;; There are unfortunate settings for "cmdproxy" on
                      ;; W32 systems.
                      (process-coding-system-alist nil)
                      (coding-system-for-read nil)
@@ -5000,7 +4927,12 @@ function waits for output unless NOOUTPUT is set."
   (with-current-buffer (process-buffer proc)
     (let* (;; Initially, `tramp-end-of-output' is "#$ ".  There might
            ;; be leading escape sequences, which must be ignored.
-           (regexp (format "[^#$\n]*%s\r?$" (regexp-quote tramp-end-of-output)))
+           ;; Busyboxes built with the EDITING_ASK_TERMINAL config
+           ;; option send also escape sequences, which must be
+           ;; ignored.
+           (regexp (format "[^#$\n]*%s\\(%s\\)?\r?$"
+                           (regexp-quote tramp-end-of-output)
+                           tramp-device-escape-sequence-regexp))
            ;; Sometimes, the commands do not return a newline but a
            ;; null byte before the shell prompt, for example "git
            ;; ls-files -c -z ...".
@@ -5103,16 +5035,17 @@ Return ATTR."
   (when attr
     ;; Remove color escape sequences from symlink.
     (when (stringp (car attr))
-      (while (string-match tramp-color-escape-sequence-regexp (car attr))
+      (while (string-match tramp-display-escape-sequence-regexp (car attr))
         (setcar attr (replace-match "" nil nil (car attr)))))
-    ;; Convert uid and gid.  Use -1 as indication of unusable value.
+    ;; Convert uid and gid.  Use `tramp-unknown-id-integer' as
+    ;; indication of unusable value.
     (when (and (numberp (nth 2 attr)) (< (nth 2 attr) 0))
-      (setcar (nthcdr 2 attr) -1))
+      (setcar (nthcdr 2 attr) tramp-unknown-id-integer))
     (when (and (floatp (nth 2 attr))
                (<= (nth 2 attr) most-positive-fixnum))
       (setcar (nthcdr 2 attr) (round (nth 2 attr))))
     (when (and (numberp (nth 3 attr)) (< (nth 3 attr) 0))
-      (setcar (nthcdr 3 attr) -1))
+      (setcar (nthcdr 3 attr) tramp-unknown-id-integer))
     (when (and (floatp (nth 3 attr))
                (<= (nth 3 attr) most-positive-fixnum))
       (setcar (nthcdr 3 attr) (round (nth 3 attr))))
@@ -5556,8 +5489,10 @@ Return ATTR."
                (tramp-get-remote-uid-with-python vec id-format))))))
       ;; Ensure there is a valid result.
       (cond
-       ((and (equal id-format 'integer) (not (integerp res))) -1)
-       ((and (equal id-format 'string) (not (stringp res))) "UNKNOWN")
+       ((and (equal id-format 'integer) (not (integerp res)))
+        tramp-unknown-id-integer)
+       ((and (equal id-format 'string) (not (stringp res)))
+        tramp-unknown-id-string)
        (t res)))))
 
 (defun tramp-get-remote-gid-with-id (vec id-format)
@@ -5600,8 +5535,10 @@ Return ATTR."
                (tramp-get-remote-gid-with-python vec id-format))))))
       ;; Ensure there is a valid result.
       (cond
-       ((and (equal id-format 'integer) (not (integerp res))) -1)
-       ((and (equal id-format 'string) (not (stringp res))) "UNKNOWN")
+       ((and (equal id-format 'integer) (not (integerp res)))
+        tramp-unknown-id-integer)
+       ((and (equal id-format 'string) (not (stringp res)))
+        tramp-unknown-id-string)
        (t res)))))
 
 ;; Some predefined connection properties.
diff --git a/lisp/net/tramp-smb.el b/lisp/net/tramp-smb.el
index c4dde050c83..fbd7cd30008 100644
--- a/lisp/net/tramp-smb.el
+++ b/lisp/net/tramp-smb.el
@@ -663,8 +663,7 @@ PRESERVE-UID-GID and PRESERVE-EXTENDED-ATTRIBUTES are completely ignored."
              result)))
     ;; Sort them if necessary.
     (unless nosort (setq result (sort result 'string-lessp)))
-    ;; Remove double entries.
-    (delete-dups result)))
+    result))
 
 (defun tramp-smb-handle-expand-file-name (name &optional dir)
   "Like `expand-file-name' for Tramp files."
@@ -907,17 +906,17 @@ PRESERVE-UID-GID and PRESERVE-EXTENDED-ATTRIBUTES are completely ignored."
   "Like `file-name-all-completions' for Tramp files."
   (all-completions
    filename
-   (with-parsed-tramp-file-name directory nil
+   (with-parsed-tramp-file-name (expand-file-name directory) nil
      (with-tramp-file-property v localname "file-name-all-completions"
        (save-match-data
-         (let ((entries (tramp-smb-get-file-entries directory)))
-           (mapcar
-            (lambda (x)
-              (list
-               (if (string-match "d" (nth 1 x))
-                   (file-name-as-directory (nth 0 x))
-                 (nth 0 x))))
-            entries)))))))
+         (delete-dups
+          (mapcar
+           (lambda (x)
+             (list
+              (if (string-match "d" (nth 1 x))
+                  (file-name-as-directory (nth 0 x))
+                (nth 0 x))))
+           (tramp-smb-get-file-entries directory))))))))
 
 (defun tramp-smb-handle-file-writable-p (filename)
   "Like `file-writable-p' for Tramp files."
@@ -1389,16 +1388,18 @@ target of the symlink differ."
 (defun tramp-smb-handle-start-file-process (name buffer program &rest args)
   "Like `start-file-process' for Tramp files."
   (with-parsed-tramp-file-name default-directory nil
-    (let ((command (mapconcat 'identity (cons program args) " "))
-          (bmp (and (buffer-live-p buffer) (buffer-modified-p buffer)))
-          (name1 name)
-          (i 0))
+    (let* ((buffer
+            (if buffer
+                (get-buffer-create buffer)
+              ;; BUFFER can be nil.  We use a temporary buffer.
+              (generate-new-buffer tramp-temp-buffer-name)))
+           (command (mapconcat 'identity (cons program args) " "))
+           (bmp (and (buffer-live-p buffer) (buffer-modified-p buffer)))
+           (name1 name)
+           (i 0))
       (unwind-protect
           (save-excursion
             (save-restriction
-              (unless buffer
-                ;; BUFFER can be nil.  We use a temporary buffer.
-                (setq buffer (generate-new-buffer tramp-temp-buffer-name)))
               (while (get-process name1)
                 ;; NAME must be unique as process name.
                 (setq i (1+ i)
diff --git a/lisp/net/tramp.el b/lisp/net/tramp.el
index 28fc9c748bb..e3755533b9d 100644
--- a/lisp/net/tramp.el
+++ b/lisp/net/tramp.el
@@ -774,6 +774,12 @@ Derived from `tramp-postfix-host-format'.")
 (defconst tramp-localname-regexp ".*$"
   "Regexp matching localnames.")
 
+(defconst tramp-unknown-id-string "UNKNOWN"
+  "String used to denote an unknown user or group")
+
+(defconst tramp-unknown-id-integer -1
+  "Integer used to denote an unknown user or group")
+
 ;;; File name format:
 
 (defconst tramp-remote-file-name-spec-regexp
@@ -2861,11 +2867,21 @@ User is always nil."
     (error
      "tramp-handle-file-name-completion invoked on non-tramp directory `%s'"
      directory))
-  (try-completion
-   filename
-   (mapcar 'list (file-name-all-completions filename directory))
-   (when predicate
-     (lambda (x) (funcall predicate (expand-file-name (car x) directory))))))
+  (let (hits-ignored-extensions)
+    (or
+     (try-completion
+      filename (file-name-all-completions filename directory)
+      (lambda (x)
+        (when (funcall (or predicate 'identity) (expand-file-name x directory))
+          (not
+           (and
+            completion-ignored-extensions
+            (string-match
+             (concat (regexp-opt completion-ignored-extensions 'paren) "$") x)
+            ;; We remember the hit.
+            (push x hits-ignored-extensions))))))
+     ;; No match.  So we try again for ignored files.
+     (try-completion filename hits-ignored-extensions))))
 
 (defun tramp-handle-file-name-directory (file)
   "Like `file-name-directory' but aware of Tramp files."
@@ -3834,7 +3850,10 @@ be granted."
                 vec (concat "uid-" suffix) nil))
               (remote-gid
                (tramp-get-connection-property
-                vec (concat "gid-" suffix) nil)))
+                vec (concat "gid-" suffix) nil))
+              (unknown-id
+               (if (string-equal suffix "string")
+                   tramp-unknown-id-string tramp-unknown-id-integer)))
           (and
            file-attr
            (or
@@ -3847,12 +3866,14 @@ be granted."
             ;; User accessible and owned by user.
             (and
              (eq access (aref (nth 8 file-attr) offset))
-             (equal remote-uid (nth 2 file-attr)))
+             (or (equal remote-uid (nth 2 file-attr))
+                 (equal unknown-id (nth 2 file-attr))))
             ;; Group accessible and owned by user's
             ;; principal group.
             (and
              (eq access (aref (nth 8 file-attr) (+ offset 3)))
-             (equal remote-gid (nth 3 file-attr)))))))))))
+             (or (equal remote-gid (nth 3 file-attr))
+                 (equal unknown-id (nth 3 file-attr))))))))))))
 
 ;;;###tramp-autoload
 (defun tramp-local-host-p (vec)
diff --git a/lisp/progmodes/cc-engine.el b/lisp/progmodes/cc-engine.el
index 2450a5db8b9..4d6a1203c25 100644
--- a/lisp/progmodes/cc-engine.el
+++ b/lisp/progmodes/cc-engine.el
@@ -229,8 +229,12 @@
 ;; The starting position from where we determined `c-macro-cache'.
 (defvar c-macro-cache-syntactic nil)
 (make-variable-buffer-local 'c-macro-cache-syntactic)
-;; non-nil iff `c-macro-cache' has both elements set AND the cdr is at a
-;; syntactic end of macro, not merely an apparent one.
+;; Either nil, or the syntactic end of the macro currently represented by
+;; `c-macro-cache'.
+(defvar c-macro-cache-no-comment nil)
+(make-variable-buffer-local 'c-macro-cache-no-comment)
+;; Either nil, or the last character of the macro currently represented by
+;; `c-macro-cache' which isn't in a comment. */
 
 (defun c-invalidate-macro-cache (beg end)
   ;; Called from a before-change function.  If the change region is before or
@@ -242,12 +246,14 @@
    ((< beg (car c-macro-cache))
     (setq c-macro-cache nil
           c-macro-cache-start-pos nil
-          c-macro-cache-syntactic nil))
+          c-macro-cache-syntactic nil
+          c-macro-cache-no-comment nil))
    ((and (cdr c-macro-cache)
          (< beg (cdr c-macro-cache)))
     (setcdr c-macro-cache nil)
     (setq c-macro-cache-start-pos beg
-          c-macro-cache-syntactic nil))))
+          c-macro-cache-syntactic nil
+          c-macro-cache-no-comment nil))))
 
 (defun c-macro-is-genuine-p ()
   ;; Check that the ostensible CPP construct at point is a real one.  In
@@ -288,7 +294,8 @@ comment at the start of cc-engine.el for more info."
                    t))
         (setq c-macro-cache nil
               c-macro-cache-start-pos nil
-              c-macro-cache-syntactic nil)
+              c-macro-cache-syntactic nil
+              c-macro-cache-no-comment nil)
 
         (save-restriction
           (if lim (narrow-to-region lim (point-max)))
@@ -323,7 +330,8 @@ comment at the start of cc-engine.el for more info."
                   (>= (point) (car c-macro-cache)))
        (setq c-macro-cache nil
              c-macro-cache-start-pos nil
-             c-macro-cache-syntactic nil))
+             c-macro-cache-syntactic nil
+             c-macro-cache-no-comment nil))
      (while (progn
               (end-of-line)
               (when (and (eq (char-before) ?\\)
@@ -347,14 +355,38 @@ comment at the start of cc-engine.el for more info."
   (let* ((here (point))
          (there (progn (c-end-of-macro) (point)))
          s)
-    (unless c-macro-cache-syntactic
+    (if c-macro-cache-syntactic
+        (goto-char c-macro-cache-syntactic)
       (setq s (parse-partial-sexp here there))
       (while (and (or (nth 3 s)  ; in a string
                       (nth 4 s)) ; in a comment (maybe at end of line comment)
                   (> there here))       ; No infinite loops, please.
         (setq there (1- (nth 8 s)))
         (setq s (parse-partial-sexp here there)))
-      (setq c-macro-cache-syntactic (car c-macro-cache)))
+      (setq c-macro-cache-syntactic (point)))
+    (point)))
+
+(defun c-no-comment-end-of-macro ()
+  ;; Go to the end of a CPP directive, or a pos just before which isn't in a
+  ;; comment.  For this purpose, open strings are ignored.
+  ;;
+  ;; This function must only be called from the beginning of a CPP construct.
+  ;;
+  ;; Note that this function might do hidden buffer changes.  See the comment
+  ;; at the start of cc-engine.el for more info.
+  (let* ((here (point))
+         (there (progn (c-end-of-macro) (point)))
+         s)
+    (if c-macro-cache-no-comment
+        (goto-char c-macro-cache-no-comment)
+      (setq s (parse-partial-sexp here there))
+      (while (and (nth 3 s)      ; in a string
+                  (> there here))       ; No infinite loops, please.
+        (setq here (1+ (nth 8 s)))
+        (setq s (parse-partial-sexp here there)))
+      (when (nth 4 s)
+        (goto-char (1- (nth 8 s))))
+      (setq c-macro-cache-no-comment (point)))
     (point)))
 
 (defun c-forward-over-cpp-define-id ()
@@ -8899,6 +8931,22 @@ comment at the start of cc-engine.el for more info."
                   (c-syntactic-skip-backward c-block-prefix-charset limit t)
                   (eq (char-before) ?>))))))
 
+    ;; Skip back over noise clauses.
+    (while (and
+            c-opt-cpp-prefix
+            (eq (char-before) ?\))
+            (let ((after-paren (point)))
+              (if (and (c-go-list-backward)
+                       (progn (c-backward-syntactic-ws)
+                              (c-simple-skip-symbol-backward))
+                       (or (looking-at c-paren-nontype-key)
+                           (looking-at c-noise-macro-with-parens-name-re)))
+                  (progn
+                    (c-syntactic-skip-backward c-block-prefix-charset limit t)
+                    t)
+                (goto-char after-paren)
+                nil))))
+
     ;; Note: Can't get bogus hits inside template arglists below since they
     ;; have gotten paren syntax above.
     (when (and
diff --git a/lisp/progmodes/cc-langs.el b/lisp/progmodes/cc-langs.el
index 705f723d55d..6f4d1f16857 100644
--- a/lisp/progmodes/cc-langs.el
+++ b/lisp/progmodes/cc-langs.el
@@ -476,7 +476,8 @@ so that all identifiers are recognized as words.")
   c++ '(c-extend-region-for-CPP
         c-before-change-check-<>-operators
         c-invalidate-macro-cache)
-  (c objc) '(c-extend-region-for-CPP c-invalidate-macro-cache)
+  (c objc) '(c-extend-region-for-CPP
+             c-invalidate-macro-cache)
   ;; java 'c-before-change-check-<>-operators
   awk 'c-awk-record-region-clear-NL)
 (c-lang-defvar c-get-state-before-change-functions
@@ -505,9 +506,11 @@ parameters \(point-min) and \(point-max).")
   ;; For documentation see the following c-lang-defvar of the same name.
   ;; The value here may be a list of functions or a single function.
   t 'c-change-expand-fl-region
-  (c objc) '(c-neutralize-syntax-in-and-mark-CPP
+  (c objc) '(c-extend-font-lock-region-for-macros
+             c-neutralize-syntax-in-and-mark-CPP
              c-change-expand-fl-region)
-  c++ '(c-neutralize-syntax-in-and-mark-CPP
+  c++ '(c-extend-font-lock-region-for-macros
+        c-neutralize-syntax-in-and-mark-CPP
         c-restore-<>-properties
         c-change-expand-fl-region)
   java '(c-restore-<>-properties
@@ -2264,6 +2267,10 @@ contain type identifiers."
             ;; MSVC extension.
             "__declspec"))
 
+(c-lang-defconst c-paren-nontype-key
+  t (c-make-keywords-re t (c-lang-const c-paren-nontype-kwds)))
+(c-lang-defvar c-paren-nontype-key (c-lang-const c-paren-nontype-key))
+
 (c-lang-defconst c-paren-type-kwds
   "Keywords that may be followed by a parenthesis expression containing
 type identifiers separated by arbitrary tokens."
diff --git a/lisp/progmodes/cc-mode.el b/lisp/progmodes/cc-mode.el
index de903b80ade..9ab04808af6 100644
--- a/lisp/progmodes/cc-mode.el
+++ b/lisp/progmodes/cc-mode.el
@@ -865,14 +865,6 @@ Note that the style variables are always made local to the buffer."
 
 ;;; Change hooks, linking with Font Lock and electric-indent-mode.
 
-;; Buffer local variables recording Beginning/End-of-Macro position before a
-;; change, when a macro straddles, respectively, the BEG or END (or both) of
-;; the change region.  Otherwise these have the values BEG/END.
-(defvar c-old-BOM 0)
-(make-variable-buffer-local 'c-old-BOM)
-(defvar c-old-EOM 0)
-(make-variable-buffer-local 'c-old-EOM)
-
 (defun c-called-from-text-property-change-p ()
   ;; Is the primitive which invoked `before-change-functions' or
   ;; `after-change-functions' one which merely changes text properties?  This
@@ -886,8 +878,8 @@ Note that the style variables are always made local to the buffer."
         '(put-text-property remove-list-of-text-properties)))
 
 (defun c-extend-region-for-CPP (beg end)
-  ;; Set c-old-BOM or c-old-EOM respectively to BEG, END, each extended to the
-  ;; beginning/end of any preprocessor construct they may be in.
+  ;; Adjust `c-new-BEG', `c-new-END' respectively to the beginning and end of
+  ;; any preprocessor construct they may be in. 
   ;;
   ;; Point is undefined both before and after this function call; the buffer
   ;; has already been widened, and match-data saved.  The return value is
@@ -896,45 +888,33 @@ Note that the style variables are always made local to the buffer."
   ;; This function is in the C/C++/ObjC values of
   ;; `c-get-state-before-change-functions' and is called exclusively as a
   ;; before change function.
-  (goto-char beg)
+  (goto-char c-new-BEG)
   (c-beginning-of-macro)
-  (setq c-old-BOM (point))
+  (setq c-new-BEG (point))
 
-  (goto-char end)
+  (goto-char c-new-END)
   (when (c-beginning-of-macro)
     (c-end-of-macro)
     (or (eobp) (forward-char)))  ; Over the terminating NL which may be marked
                                  ; with a c-cpp-delimiter category property
-  (setq c-old-EOM (point)))
-
-(defun c-extend-font-lock-region-for-macros (begg endd &optional old-len)
-  ;; Extend the region (BEGG ENDD) to cover all (possibly changed)
-  ;; preprocessor macros; return the cons (new-BEG . new-END).  OLD-LEN should
-  ;; be either the old length parameter when called from an
-  ;; after-change-function, or nil otherwise.  This defun uses the variables
-  ;; c-old-BOM, c-new-BOM.
+  (setq c-new-END (point)))
+
+(defun c-extend-font-lock-region-for-macros (begg endd old-len)
+  ;; Extend the region (c-new-BEG c-new-END) to cover all (possibly changed)
+  ;; preprocessor macros; The return value has no significance.
   ;;
   ;; Point is undefined on both entry and exit to this function.  The buffer
   ;; will have been widened on entry.
-  (let (limits new-beg new-end)
-    (goto-char c-old-BOM)         ; already set to old start of macro or begg.
-    (setq new-beg
-          (min begg
-               (if (setq limits (c-state-literal-at (point)))
-                   (cdr limits)     ; go forward out of any string or comment.
-                 (point))))
-
-    (goto-char endd)
-    (if (setq limits (c-state-literal-at (point)))
-        (goto-char (car limits)))  ; go backward out of any string or comment.
-    (if (c-beginning-of-macro)
-        (c-end-of-macro))
-    (setq new-end (max endd
-                       (if old-len
-                           (+ (- c-old-EOM old-len) (- endd begg))
-                         c-old-EOM)
-                       (point)))
-    (cons new-beg new-end)))
+  ;;
+  ;; This function is in the C/C++/ObjC value of `c-before-font-lock-functions'.
+  (goto-char endd)
+  (if (c-beginning-of-macro)
+      (c-end-of-macro))
+  (setq c-new-END (max endd c-new-END (point)))
+  ;; Determine the region, (c-new-BEG c-new-END), which will get font
+  ;; locked.  This restricts the region should there be long macros.
+  (setq c-new-BEG (max c-new-BEG (c-determine-limit 500 begg))
+        c-new-END (min c-new-END (c-determine-+ve-limit 500 endd))))
 
 (defun c-neutralize-CPP-line (beg end)
   ;; BEG and END bound a region, typically a preprocessor line.  Put a
@@ -963,19 +943,14 @@ Note that the style variables are always made local to the buffer."
              (t nil)))))))
 
 (defun c-neutralize-syntax-in-and-mark-CPP (begg endd old-len)
-  ;; (i) Extend the font lock region to cover all changed preprocessor
-  ;; regions; it does this by setting the variables `c-new-BEG' and
-  ;; `c-new-END' to the new boundaries.
-  ;;
-  ;; (ii) "Neutralize" every preprocessor line wholly or partially in the
-  ;; extended changed region.  "Restore" lines which were CPP lines before the
-  ;; change and are no longer so; these can be located from the Buffer local
-  ;; variables `c-old-BOM' and `c-old-EOM'.
+  ;; (i) "Neutralize" every preprocessor line wholly or partially in the
+  ;; changed region.  "Restore" lines which were CPP lines before the change
+  ;; and are no longer so.
   ;;
-  ;; (iii) Mark every CPP construct by placing a `category' property value
+  ;; (ii) Mark each CPP construct by placing a `category' property value
   ;; `c-cpp-delimiter' at its start and end.  The marked characters are the
   ;; opening # and usually the terminating EOL, but sometimes the character
-  ;; before a comment/string delimiter.
+  ;; before a comment delimiter.
   ;;
   ;; That is, set syntax-table properties on characters that would otherwise
   ;; interact syntactically with those outside the CPP line(s).
@@ -992,15 +967,8 @@ Note that the style variables are always made local to the buffer."
   ;; Note: SPEED _MATTERS_ IN THIS FUNCTION!!!
   ;;
   ;; This function might make hidden buffer changes.
-  (c-save-buffer-state (new-bounds)
-    ;; First determine the region, (c-new-BEG c-new-END), which will get font
-    ;; locked.  It might need "neutralizing".  This region may not start
-    ;; inside a string, comment, or macro.
-    (setq new-bounds (c-extend-font-lock-region-for-macros
-                      c-new-BEG c-new-END old-len))
-    (setq c-new-BEG (max (car new-bounds) (c-determine-limit 500 begg))
-          c-new-END (min (cdr new-bounds) (c-determine-+ve-limit 500 endd)))
-    ;; Clear all old relevant properties.
+  (c-save-buffer-state (limits )
+    ;; Clear 'syntax-table properties "punctuation":
     (c-clear-char-property-with-value c-new-BEG c-new-END 'syntax-table '(1))
 
     ;; CPP "comment" markers:
@@ -1011,6 +979,8 @@ Note that the style variables are always made local to the buffer."
 
     ;; Add needed properties to each CPP construct in the region.
     (goto-char c-new-BEG)
+    (if (setq limits (c-literal-limits)) ; Go past any literal.
+        (goto-char (cdr limits)))
     (skip-chars-backward " \t")
     (let ((pps-position (point))  pps-state mbeg)
       (while (and (< (point) c-new-END)
@@ -1030,7 +1000,7 @@ Note that the style variables are always made local to the buffer."
                       (nth 4 pps-state)))) ; in a comment?
           (goto-char (match-beginning 1))
           (setq mbeg (point))
-          (if (> (c-syntactic-end-of-macro) mbeg)
+          (if (> (c-no-comment-end-of-macro) mbeg)
               (progn
                 (c-neutralize-CPP-line mbeg (point)) ; "punctuation" properties
                 (if (eval-when-compile
@@ -1256,10 +1226,15 @@ Note that the style variables are always made local to the buffer."
   ;;
   ;; This is called from an after-change-function, but the parameters BEG END
   ;; and OLD-LEN are not used.
-   (if font-lock-mode
-       (setq c-new-BEG
-             (or (c-fl-decl-start c-new-BEG) (c-point 'bol c-new-BEG))
-             c-new-END (c-point 'bonl c-new-END))))
+  (if font-lock-mode
+      (setq c-new-BEG
+            (or (c-fl-decl-start c-new-BEG) (c-point 'bol c-new-BEG))
+            c-new-END
+            (save-excursion
+              (goto-char c-new-END)
+              (if (bolp)
+                  (point)
+                (c-point 'bonl c-new-END))))))
 
 (defun c-context-expand-fl-region (beg end)
   ;; Return a cons (NEW-BEG . NEW-END), where NEW-BEG is the beginning of a
diff --git a/lisp/recentf.el b/lisp/recentf.el
index df7f3e2e565..3321f2fe101 100644
--- a/lisp/recentf.el
+++ b/lisp/recentf.el
@@ -1064,7 +1064,6 @@ Go to the beginning of buffer if not found."
     (define-key km "q" 'recentf-cancel-dialog)
     (define-key km "n" 'next-line)
     (define-key km "p" 'previous-line)
-    (define-key km [follow-link] "\C-m")
     km)
   "Keymap used in recentf dialogs.")
 
diff --git a/lisp/simple.el b/lisp/simple.el
index affc403dcdc..3d25ec19ab2 100644
--- a/lisp/simple.el
+++ b/lisp/simple.el
@@ -6054,7 +6054,13 @@ If NOERROR, don't signal an error if we can't move that many lines."
           (setq temporary-goal-column
                 (cons (/ (float x-pos)
                          (frame-char-width))
-                      hscroll))))))
+                      hscroll)))
+         (executing-kbd-macro
+          ;; When we move beyond the first/last character visible in
+          ;; the window, posn-at-point will return nil, so we need to
+          ;; approximate the goal column as below.
+          (setq temporary-goal-column
+                (mod (current-column) (window-text-width)))))))
     (if target-hscroll
         (set-window-hscroll (selected-window) target-hscroll))
     ;; vertical-motion can move more than it was asked to if it moves
diff --git a/lisp/wid-edit.el b/lisp/wid-edit.el
index 0a0f4582b32..9ede9a5633f 100644
--- a/lisp/wid-edit.el
+++ b/lisp/wid-edit.el
@@ -1789,7 +1789,13 @@ If END is omitted, it defaults to the length of LIST."
   "An embedded link."
   :button-prefix 'widget-link-prefix
   :button-suffix 'widget-link-suffix
-  :follow-link 'mouse-face
+  ;; The `follow-link' property should only be used in those contexts where the
+  ;; mouse-1 event normally doesn't follow the link, yet the `link' widget
+  ;; seems to almost always be used in contexts where (down-)mouse-1 is bound
+  ;; to `widget-button-click' and hence the "mouse-1 to mouse-2" remapping is
+  ;; not necessary (and can even be harmful).  So let's not add a :follow-link
+  ;; by default.  See (bug#22434).
+  ;; :follow-link 'mouse-face
   :help-echo "Follow the link."
   :format "%[%t%]")