Merge from origin/emacs-25

50650cb Doc fixes for fclist and grep
5e814e0 Minor doc fixes for quoting
3347a73 `nreverse' the marker pairs list
1a4127d Use save-excursion in xref-location-marker more
ab3ba91 shell-quote-argument DIR when appropriate
922c7a3 Rework xref-query-replace-in-results
3fe3510 * lisp/replace.el (query-replace-read-from): Use minibuffer-w...
0932b94 Fix todo-mode bug involving archived items (bug#23447)
e68ad1f ; * etc/NEWS: Tiny edit. (Bug#23432)
adc80b7 ; * test/automated/xref-tests.el: Add copyright and license.
4d8fd9c Handle "empty line" regexp in xref searches
f559b37 Add tests for xref-collect-matches
6428aa0 Use grep-find-ignored-directories instead of vc-directory-exc...
6f82d8e Clear buffer-undo-list when showing xrefs
c68a091 Note the quote translation in `message' in section "incompati...
52f86a7 * etc/NEWS: Mention (message "%s" (format ...)).
93703c5 (Common Keywords): Correct what missing :group means
79e5800 Improve documentation of Dired's 'A' and 'Q' commands
2ea2a2f Doc fixes for quoting
8544b98 posnp doc clarification
805204f Mention what a missing :group does
ec554d7 Fix documentation of dired-aux search/replace commands

11 files changed, 158 insertions, 101 deletions

diff --git a/lisp/calendar/todo-mode.el b/lisp/calendar/todo-mode.el
index 0529e970333..8e752586053 100644
--- a/lisp/calendar/todo-mode.el
+++ b/lisp/calendar/todo-mode.el
@@ -902,17 +902,19 @@ Categories mode."
       (todo-show)
     (let* ((archive (eq where 'archive))
            (cat (unless archive where))
+           (goto-archive (and cat
+                              todo-skip-archived-categories
+                              (zerop (todo-get-count 'todo cat))
+                              (zerop (todo-get-count 'done cat))
+                              (not (zerop (todo-get-count 'archived cat)))))
            (file0 (when cat             ; We're in Todo Categories mode.
-                    ;; With non-nil `todo-skip-archived-categories'
-                    ;; jump to archive file of a category with only
-                    ;; archived items.
-                    (if (and todo-skip-archived-categories
-                             (zerop (todo-get-count 'todo cat))
-                             (zerop (todo-get-count 'done cat))
-                             (not (zerop (todo-get-count 'archived cat))))
+                    (if goto-archive
+                        ;; If the category has only archived items and
+                        ;; `todo-skip-archived-categories' is non-nil, jump to
+                        ;; the archive category.
                         (concat (file-name-sans-extension
                                  todo-current-todo-file) ".toda")
-                      ;; Otherwise, jump to current todo file.
+                      ;; Otherwise, jump to the category in the todo file.
                       todo-current-todo-file)))
            (len (length todo-categories))
            (cat+file (unless cat
@@ -923,18 +925,15 @@ Categories mode."
            (category (or cat (car cat+file))))
       (unless cat (setq file0 (cdr cat+file)))
       (with-current-buffer (find-file-noselect file0 'nowarn)
-        (setq todo-current-todo-file file0)
-        ;; If called from Todo Categories mode, clean up before jumping.
-        (if (string= (buffer-name) todo-categories-buffer)
-            (kill-buffer))
-        (set-window-buffer (selected-window)
-                           (set-buffer (find-buffer-visiting file0)))
-        (unless todo-global-current-todo-file
-          (setq todo-global-current-todo-file todo-current-todo-file))
-        (todo-category-number category)
-        (todo-category-select)
-        (goto-char (point-min))
-        (when add-item (todo-insert-item--basic))))))
+        (when goto-archive (todo-archive-mode))
+        (set-window-buffer (selected-window)
+                           (set-buffer (find-buffer-visiting file0)))
+        (unless todo-global-current-todo-file
+          (setq todo-global-current-todo-file todo-current-todo-file))
+        (todo-category-number category)
+        (todo-category-select)
+        (goto-char (point-min))
+        (when add-item (todo-insert-item--basic))))))
 
 (defun todo-next-item (&optional count)
   "Move point down to the beginning of the next item.
diff --git a/lisp/cedet/cedet-cscope.el b/lisp/cedet/cedet-cscope.el
index 9a54d341f56..373149c16e3 100644
--- a/lisp/cedet/cedet-cscope.el
+++ b/lisp/cedet/cedet-cscope.el
@@ -52,7 +52,7 @@ SCOPE is the scope of the search, such as 'project or 'subdirs."
   ;; -0 = Find C symbol
   ;; -1 = Find global definition
   ;; -3 = Find references
-  ;; -6 = Find egrep pattern
+  ;; -6 = Find grep -E pattern
   ;; -7 = Find file
   (let ((idx (cond ((eq type 'file)
                     "-7")
diff --git a/lisp/cedet/semantic/symref.el b/lisp/cedet/semantic/symref.el
index 088740b2624..516a4f30414 100644
--- a/lisp/cedet/semantic/symref.el
+++ b/lisp/cedet/semantic/symref.el
@@ -266,7 +266,7 @@ Returns an object of class `semantic-symref-result'."
 ;;;###autoload
 (defun semantic-symref-find-text (text &optional scope)
   "Find a list of occurrences of TEXT in the current project.
-TEXT is a regexp formatted for use with egrep.
+TEXT is a regexp formatted for use with grep -E.
 Optional SCOPE specifies which file set to search.  Defaults to `project'.
 Refers to `semantic-symref-tool', to determine the reference tool to use
 for the current buffer.
diff --git a/lisp/dired-aux.el b/lisp/dired-aux.el
index b23b04a45fa..8d6f6656a9a 100644
--- a/lisp/dired-aux.el
+++ b/lisp/dired-aux.el
@@ -2747,14 +2747,21 @@ with the command \\[tags-loop-continue]."
 
 ;;;###autoload
 (defun dired-do-find-regexp (regexp)
-  "Find all matches for REGEXP in all marked files, recursively."
+  "Find all matches for REGEXP in all marked files.
+For any marked directory, all of its files are searched recursively.
+However, files matching `grep-find-ignored-files' and subdirectories
+matching `grep-find-ignored-directories' are skipped in the marked
+directories.
+
+REGEXP should use constructs supported by your local `grep' command."
   (interactive "sSearch marked files (regexp): ")
   (require 'grep)
   (defvar grep-find-ignored-files)
+  (defvar grep-find-ignored-directories)
   (let* ((files (dired-get-marked-files))
          (ignores (nconc (mapcar
                           (lambda (s) (concat s "/"))
-                          vc-directory-exclusion-list)
+                          grep-find-ignored-directories)
                          grep-find-ignored-files))
          (xrefs (cl-mapcan
                  (lambda (file)
@@ -2768,7 +2775,13 @@ with the command \\[tags-loop-continue]."
 
 ;;;###autoload
 (defun dired-do-find-regexp-and-replace (from to)
-  "Replace matches of FROM with TO, in all marked files, recursively."
+  "Replace matches of FROM with TO, in all marked files.
+For any marked directory, matches in all of its files are replaced,
+recursively.  However, files matching `grep-find-ignored-files'
+and subdirectories matching `grep-find-ignored-directories' are skipped
+in the marked directories.
+
+REGEXP should use constructs supported by your local `grep' command."
   (interactive
    (let ((common
           (query-replace-read-args
diff --git a/lisp/man.el b/lisp/man.el
index 2b2ee99a9be..5acf90baf28 100644
--- a/lisp/man.el
+++ b/lisp/man.el
@@ -964,7 +964,7 @@ otherwise look like a page name.
 
 An \"apropos\" query with -k gives a buffer of matching page
 names or descriptions.  The pattern argument is usually an
-\"egrep\" style regexp.
+\"grep -E\" style regexp.
 
     -k pattern"
 
diff --git a/lisp/progmodes/elisp-mode.el b/lisp/progmodes/elisp-mode.el
index 1c728484ab5..2ad22ddd0ff 100644
--- a/lisp/progmodes/elisp-mode.el
+++ b/lisp/progmodes/elisp-mode.el
@@ -823,8 +823,9 @@ non-nil result supercedes the xrefs produced by
   (pcase-let (((cl-struct xref-elisp-location symbol type file) l))
     (let ((buffer-point (find-function-search-for-symbol symbol type file)))
       (with-current-buffer (car buffer-point)
-        (goto-char (or (cdr buffer-point) (point-min)))
-        (point-marker)))))
+        (save-excursion
+          (goto-char (or (cdr buffer-point) (point-min)))
+          (point-marker))))))
 
 (cl-defmethod xref-location-group ((l xref-elisp-location))
   (xref-elisp-location-file l))
diff --git a/lisp/progmodes/etags.el b/lisp/progmodes/etags.el
index a2a0df2d6e1..890d55294cf 100644
--- a/lisp/progmodes/etags.el
+++ b/lisp/progmodes/etags.el
@@ -2146,8 +2146,9 @@ for \\[find-tag] (which see)."
   (with-slots (tag-info file) l
     (let ((buffer (find-file-noselect file)))
       (with-current-buffer buffer
-        (etags-goto-tag-location tag-info)
-        (point-marker)))))
+        (save-excursion
+          (etags-goto-tag-location tag-info)
+          (point-marker))))))
 
 (cl-defmethod xref-location-line ((l xref-etags-location))
   (with-slots (tag-info) l
diff --git a/lisp/progmodes/project.el b/lisp/progmodes/project.el
index 9c8a88c80fc..a51c383b93b 100644
--- a/lisp/progmodes/project.el
+++ b/lisp/progmodes/project.el
@@ -172,7 +172,8 @@ to find the list of ignores for each directory."
             (let ((command
                    (format "%s %s %s -type f -print0"
                            find-program
-                           dir
+                           (shell-quote-argument
+                            (expand-file-name dir))
                            (xref--find-ignores-arguments
                             (project-ignores project dir)
                             (expand-file-name dir)))))
diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index b5632af13ab..f651dc9cd18 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -521,58 +521,86 @@ references displayed in the current *xref* buffer."
    (let ((fr (read-regexp "Xref query-replace (regexp)" ".*")))
      (list fr
            (read-regexp (format "Xref query-replace (regexp) %s with: " fr)))))
-  (let ((reporter (make-progress-reporter (format "Saving search results...")
-                                          0 (line-number-at-pos (point-max))))
-        (counter 0)
-        pairs item)
+  (let* (item xrefs iter)
+    (save-excursion
+      (while (setq item (xref--search-property 'xref-item))
+        (when (xref-match-length item)
+          (push item xrefs))))
     (unwind-protect
         (progn
-          (save-excursion
-            (goto-char (point-min))
-            ;; TODO: This list should be computed on-demand instead.
-            ;; As long as the UI just iterates through matches one by
-            ;; one, there's no need to compute them all in advance.
-            ;; Then we can throw away the reporter.
-            (while (setq item (xref--search-property 'xref-item))
-              (when (xref-match-length item)
-                (save-excursion
-                  (let* ((loc (xref-item-location item))
-                         (beg (xref-location-marker loc))
-                         (end (move-marker (make-marker)
-                                           (+ beg (xref-match-length item))
-                                           (marker-buffer beg))))
-                    ;; Perform sanity check first.
-                    (xref--goto-location loc)
-                    ;; FIXME: The check should probably be a generic
-                    ;; function, instead of the assumption that all
-                    ;; matches contain the full line as summary.
-                    ;; TODO: Offer to re-scan otherwise.
-                    (unless (equal (buffer-substring-no-properties
-                                    (line-beginning-position)
-                                    (line-end-position))
-                                   (xref-item-summary item))
-                      (user-error "Search results out of date"))
-                    (progress-reporter-update reporter (cl-incf counter))
-                    (push (cons beg end) pairs)))))
-            (setq pairs (nreverse pairs)))
-          (unless pairs (user-error "No suitable matches here"))
-          (progress-reporter-done reporter)
-          (xref--query-replace-1 from to pairs))
-      (dolist (pair pairs)
-        (move-marker (car pair) nil)
-        (move-marker (cdr pair) nil)))))
+          (goto-char (point-min))
+          (setq iter (xref--buf-pairs-iterator (nreverse xrefs)))
+          (xref--query-replace-1 from to iter))
+      (funcall iter :cleanup))))
+
+(defun xref--buf-pairs-iterator (xrefs)
+  (let (chunk-done item next-pair file-buf pairs all-pairs)
+    (lambda (action)
+      (pcase action
+        (:next
+         (when (or xrefs next-pair)
+           (setq chunk-done nil)
+           (when next-pair
+             (setq file-buf (marker-buffer (car next-pair))
+                   pairs (list next-pair)
+                   next-pair nil))
+           (while (and (not chunk-done)
+                       (setq item (pop xrefs)))
+             (save-excursion
+               (let* ((loc (xref-item-location item))
+                      (beg (xref-location-marker loc))
+                      (end (move-marker (make-marker)
+                                        (+ beg (xref-match-length item))
+                                        (marker-buffer beg))))
+                 (let ((pair (cons beg end)))
+                   (push pair all-pairs)
+                   ;; Perform sanity check first.
+                   (xref--goto-location loc)
+                   (if (xref--outdated-p item
+                                         (buffer-substring-no-properties
+                                          (line-beginning-position)
+                                          (line-end-position)))
+                       (message "Search result out of date, skipping")
+                     (cond
+                      ((null file-buf)
+                       (setq file-buf (marker-buffer beg))
+                       (push pair pairs))
+                      ((equal file-buf (marker-buffer beg))
+                       (push pair pairs))
+                      (t
+                       (setq chunk-done t
+                             next-pair pair))))))))
+           (cons file-buf (nreverse pairs))))
+        (:cleanup
+         (dolist (pair all-pairs)
+           (move-marker (car pair) nil)
+           (move-marker (cdr pair) nil)))))))
+
+(defun xref--outdated-p (item line-text)
+  ;; FIXME: The check should probably be a generic function instead of
+  ;; the assumption that all matches contain the full line as summary.
+  (let ((summary (xref-item-summary item))
+        (strip (lambda (s) (if (string-match "\r\\'" s)
+                          (substring-no-properties s 0 -1)
+                        s))))
+    (not
+     ;; Sometimes buffer contents include ^M, and sometimes Grep
+     ;; output includes it, and they don't always match.
+     (equal (funcall strip line-text)
+            (funcall strip summary)))))
 
 ;; FIXME: Write a nicer UI.
-(defun xref--query-replace-1 (from to pairs)
+(defun xref--query-replace-1 (from to iter)
   (let* ((query-replace-lazy-highlight nil)
-         current-beg current-end current-buf
+         (continue t)
+         did-it-once buf-pairs pairs
+         current-beg current-end
          ;; Counteract the "do the next match now" hack in
          ;; `perform-replace'.  And still, it'll report that those
          ;; matches were "filtered out" at the end.
          (isearch-filter-predicate
           (lambda (beg end)
             (and current-beg
-                 (eq (current-buffer) current-buf)
                  (>= beg current-beg)
                  (<= end current-end))))
          (replace-re-search-function
@@ -581,19 +609,22 @@ references displayed in the current *xref* buffer."
               (while (and (not found) pairs)
                 (setq pair (pop pairs)
                       current-beg (car pair)
-                      current-end (cdr pair)
-                      current-buf (marker-buffer current-beg))
-                (xref--with-dedicated-window
-                 (pop-to-buffer current-buf))
+                      current-end (cdr pair))
                 (goto-char current-beg)
                 (when (re-search-forward from current-end noerror)
                   (setq found t)))
               found))))
-    ;; FIXME: Despite this being a multi-buffer replacement, `N'
-    ;; doesn't work, because we're not using
-    ;; `multi-query-replace-map', and it would expect the below
-    ;; function to be called once per buffer.
-    (perform-replace from to t t nil)))
+    (while (and continue (setq buf-pairs (funcall iter :next)))
+      (if did-it-once
+          ;; Reuse the same window for subsequent buffers.
+          (switch-to-buffer (car buf-pairs))
+        (xref--with-dedicated-window
+         (pop-to-buffer (car buf-pairs)))
+        (setq did-it-once t))
+      (setq pairs (cdr buf-pairs))
+      (setq continue
+            (perform-replace from to t t nil nil multi-query-replace-map)))
+    (unless did-it-once (user-error "No suitable matches here"))))
 
 (defvar xref--xref-buffer-mode-map
   (let ((map (make-sparse-keymap)))
@@ -687,7 +718,9 @@ Return an alist of the form ((FILENAME . (XREF ...)) ...)."
 (defun xref--show-xref-buffer (xrefs alist)
   (let ((xref-alist (xref--analyze xrefs)))
     (with-current-buffer (get-buffer-create xref-buffer-name)
-      (let ((inhibit-read-only t))
+      (setq buffer-undo-list nil)
+      (let ((inhibit-read-only t)
+            (buffer-undo-list t))
         (erase-buffer)
         (xref--insert-xrefs xref-alist)
         (xref--xref-buffer-mode)
@@ -908,6 +941,8 @@ IGNORES is a list of glob patterns."
   (require 'find-dired)      ; for `find-name-arg'
   (defvar grep-find-template)
   (defvar find-name-arg)
+  ;; `shell-quote-argument' quotes the tilde as well.
+  (cl-assert (not (string-match-p "\\`~" dir)))
   (grep-expand-template
    grep-find-template
    regexp
@@ -919,14 +954,13 @@ IGNORES is a list of glob patterns."
             (concat " -o " find-name-arg " "))
            " "
            (shell-quote-argument ")"))
-   dir
+   (shell-quote-argument dir)
    (xref--find-ignores-arguments ignores dir)))
 
 (defun xref--find-ignores-arguments (ignores dir)
   "Convert IGNORES and DIR to a list of arguments for 'find'.
 IGNORES is a list of glob patterns.  DIR is an absolute
 directory, used as the root of the ignore globs."
-  ;; `shell-quote-argument' quotes the tilde as well.
   (cl-assert (not (string-match-p "\\`~" dir)))
   (when ignores
     (concat
@@ -1014,7 +1048,11 @@ directory, used as the root of the ignore globs."
     (syntax-propertize line-end)
     ;; FIXME: This results in several lines with the same
     ;; summary. Solve with composite pattern?
-    (while (re-search-forward regexp line-end t)
+    (while (and
+            ;; REGEXP might match an empty string.  Or line.
+            (or (null matches)
+                (> (point) line-beg))
+            (re-search-forward regexp line-end t))
       (let* ((beg-column (- (match-beginning 0) line-beg))
              (end-column (- (match-end 0) line-beg))
              (loc (xref-make-file-location file line beg-column))
diff --git a/lisp/replace.el b/lisp/replace.el
index 169fc3f4701..0b25200be57 100644
--- a/lisp/replace.el
+++ b/lisp/replace.el
@@ -191,18 +191,15 @@ wants to replace FROM with TO."
             ;; a region in order to specify the minibuffer input.
             ;; That should not clobber the region for the query-replace itself.
             (save-excursion
-              ;; The `with-current-buffer' ensures that the binding
-              ;; for `text-property-default-nonsticky' isn't a buffer
-              ;; local binding in the current buffer, which
-              ;; `read-from-minibuffer' wouldn't see.
-              (with-current-buffer (window-buffer (minibuffer-window))
-                (let ((text-property-default-nonsticky
-                       (cons '(separator . t) text-property-default-nonsticky)))
-                  (if regexp-flag
-                      (read-regexp prompt nil 'query-replace-from-to-history)
-                    (read-from-minibuffer
-                     prompt nil nil nil 'query-replace-from-to-history
-                     (car (if regexp-flag regexp-search-ring search-ring)) t))))))
+              (minibuffer-with-setup-hook
+                  (lambda ()
+                    (setq-local text-property-default-nonsticky
+                                (cons '(separator . t) text-property-default-nonsticky)))
+                (if regexp-flag
+                    (read-regexp prompt nil 'query-replace-from-to-history)
+                  (read-from-minibuffer
+                   prompt nil nil nil 'query-replace-from-to-history
+                   (car (if regexp-flag regexp-search-ring search-ring)) t)))))
            (to))
       (if (and (zerop (length from)) query-replace-defaults)
           (cons (caar query-replace-defaults)
diff --git a/lisp/subr.el b/lisp/subr.el
index f67f70f85c9..094710b026c 100644
--- a/lisp/subr.el
+++ b/lisp/subr.el
@@ -291,21 +291,27 @@ This function accepts any number of arguments, but ignores them."
 
 ;; Signal a compile-error if the first arg is missing.
 (defun error (&rest args)
-  "Signal an error, making error message by passing all args to `format'.
+  "Signal an error, making a message by passing args to `format-message'.
 In Emacs, the convention is that error messages start with a capital
 letter but *do not* end with a period.  Please follow this convention
-for the sake of consistency."
+for the sake of consistency.
+
+Note: (error \"%s\" VALUE) makes the message VALUE without
+interpreting format characters like `%', `\\=`', and `\\=''."
   (declare (advertised-calling-convention (string &rest args) "23.1"))
   (signal 'error (list (apply #'format-message args))))
 
 (defun user-error (format &rest args)
-  "Signal a pilot error, making error message by passing all args to `format'.
+  "Signal a pilot error, making a message by passing args to `format-message'.
 In Emacs, the convention is that error messages start with a capital
 letter but *do not* end with a period.  Please follow this convention
 for the sake of consistency.
 This is just like `error' except that `user-error's are expected to be the
 result of an incorrect manipulation on the part of the user, rather than the
-result of an actual problem."
+result of an actual problem.
+
+Note: (user-error \"%s\" VALUE) makes the message VALUE without
+interpreting format characters like `%', `\\=`', and `\\=''."
   (signal 'user-error (list (apply #'format-message format args))))
 
 (defun define-error (name message &optional parent)
@@ -1123,6 +1129,7 @@ The return value is a positive integer."
 
 (defun posnp (obj)
   "Return non-nil if OBJ appears to be a valid `posn' object specifying a window.
+A `posn' object is returned from functions such as `event-start'.
 If OBJ is a valid `posn' object, but specifies a frame rather
 than a window, return nil."
   ;; FIXME: Correct the behavior of this function so that all valid