Minimize entry parsing in auth-source-pass

Prior to this commit, while searching for the most applicable entry
password-store entries were decrypted and parsed to ensure they were
valid.  The entries were parsed in the order they were found on the
filesystem and all applicable entries would be decrypted and parsed,
which varied based on the contents of the password-store and the entry
to be found.

This is fine when the GPG key is cached and each entry can be
decrypted without user interaction.  However, for security some people
have their GPG on a hardware token like a Yubikey setup so that they
have to touch a sensor on the toke for every cryptographic operation,
in which case it becomes inconvenient as each attempt to find an entry
requires a variable number of touches of the hardware token.

The implementation already assumes that names which contain more of
the information in the search key should be preferred so there is an
ordering of preference of applicable entries.  If the decrypt and
parsing is removed from the initial identification of applicable
entries in the store then in most cases a single decrypt and parse of
the most preferred entry will suffice, improving the experience for
hardware token users that require interaction with the token.

This commit implements that strategy.  It is in spirit a refactor of
the existing code.

* lisp/auth-source-pass.el (auth-source-pass--matching-entries): New
function, generate an ordered list of regular expression matchers for
all possible names that could be in the password-store for the entry
to be found and then makes a pass over the password-store entry names
accumulating the matching entries in a list after the regexp that
matched.  This implementation ensures the password-store entry list
still only has to be scanned once.
(auth-source-pass--find-match-unambiguous): Use it to obtain candidate
entries and then parse them one by one until an entry containing the
desired information is located.  When complete, return the parsed data
of the entry instead of the entry name so that the information can be
used directly to construct the auth-source response.
(auth-source-pass--build-result): Update accordingly.
(auth-source-pass--find-match): Update docstring accordingly.
(auth-source-pass--select-one-entry)
(auth-source-pass--entry-valid-p)
(auth-source-pass--find-all-by-entry-name)
(auth-source-pass--find-one-by-entry-name): Remove.
(auth-source-pass--select-from-entries)
(auth-source-pass--accumulate-matches)
(auth-source-pass--entry-reducer)
(auth-source-pass--generate-entry-suffixes)
(auth-source-pass--domains)
(auth-source-pass--name-port-user-suffixes): New functions.

* test/lisp/auth-source-pass-tests.el: One test case was added to the
test suite to verify that only the minimal number of entries are
parsed in common cases.  The
auth-source-pass-only-return-entries-that-can-be-open test case had to
be re-implemented because the function it was used eliminated as the
functionality is provided elsewhere.  All the other fairly substantial
changes to the test suite are the result of mechanical changes that
were required to adapt to auth-source-pass--find-match returning the
data from a parsed password-store entry instead of the entry name.

1 files changed, 160 insertions, 104 deletions

diff --git a/test/lisp/auth-source-pass-tests.el b/test/lisp/auth-source-pass-tests.el
index 1539d9611f6..2c28f799453 100644
--- a/test/lisp/auth-source-pass-tests.el
+++ b/test/lisp/auth-source-pass-tests.el
@@ -63,14 +63,19 @@
 This function is intended to be set to `auth-source-debug`."
   (add-to-list 'auth-source-pass--debug-log (apply #'format msg) t))
 
+(defvar auth-source-pass--parse-log nil)
+
 (defmacro auth-source-pass--with-store (store &rest body)
   "Use STORE as password-store while executing BODY."
   (declare (indent 1))
-  `(cl-letf (((symbol-function 'auth-source-pass-parse-entry) (lambda (entry) (cdr (cl-find entry ,store :key #'car :test #'string=))) )
-             ((symbol-function 'auth-source-pass-entries) (lambda () (mapcar #'car ,store)))
-             ((symbol-function 'auth-source-pass--entry-valid-p) (lambda (_entry) t)))
+  `(cl-letf (((symbol-function 'auth-source-pass-parse-entry)
+              (lambda (entry)
+                (add-to-list 'auth-source-pass--parse-log entry)
+                (cdr (cl-find entry ,store :key #'car :test #'string=))))
+             ((symbol-function 'auth-source-pass-entries) (lambda () (mapcar #'car ,store))))
      (let ((auth-source-debug #'auth-source-pass--debug)
-           (auth-source-pass--debug-log nil))
+           (auth-source-pass--debug-log nil)
+           (auth-source-pass--parse-log nil))
        ,@body)))
 
 (ert-deftest auth-source-pass-any-host ()
@@ -88,125 +93,184 @@ This function is intended to be set to `auth-source-debug`."
                                   ("bar"))
     (should-not (auth-source-pass-search :host "baz"))))
 
+(ert-deftest auth-source-pass-find-match-minimal-parsing ()
+  (let ((store-contents
+         '(("baz" ("secret" . "baz password"))
+           ("baz:123" ("secret" . "baz:123 password"))
+           ("baz/foo" ("secret" . "baz/foo password"))
+           ("foo@baz" ("secret" . "foo@baz password"))
+           ("baz:123/foo" ("secret" . "baz:123/foo password"))
+           ("foo@baz:123" ("secret" . "foo@baz:123 password"))
+           ("bar.baz" ("secret" . "bar.baz password"))
+           ("bar.baz:123" ("secret" . "bar.baz:123 password"))
+           ("bar.baz/foo" ("secret" . "bar.baz/foo password"))
+           ("foo@bar.baz" ("secret" . "foo@bar.baz password"))
+           ("bar.baz:123/foo" ("secret" . "bar.baz:123/foo password"))
+           ("foo@bar.baz:123" ("secret" . "foo@bar.baz:123 password")))))
+    (auth-source-pass--with-store store-contents
+      (auth-source-pass--find-match "bar.baz" "foo" "123")
+      (should (equal auth-source-pass--parse-log '("foo@bar.baz:123"))))
+    (auth-source-pass--with-store store-contents
+      (auth-source-pass--find-match "bar.baz" "foo" nil)
+      (should (equal auth-source-pass--parse-log '("foo@bar.baz"))))
+    (auth-source-pass--with-store store-contents
+      (auth-source-pass--find-match "bar.baz" nil "123")
+      (should (equal auth-source-pass--parse-log '("bar.baz:123"))))
+    (auth-source-pass--with-store store-contents
+      (auth-source-pass--find-match "bar.baz" nil nil)
+      (should (equal auth-source-pass--parse-log '("bar.baz"))))
+    (auth-source-pass--with-store store-contents
+      (auth-source-pass--find-match "baz" nil nil)
+      (should (equal auth-source-pass--parse-log '("baz"))))))
 
 (ert-deftest auth-source-pass-find-match-matching-at-entry-name ()
-  (auth-source-pass--with-store '(("foo"))
-    (should (equal (auth-source-pass--find-match "foo" nil nil)
-                   "foo"))))
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password")))
+    (let ((result (auth-source-pass--find-match "foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "foo password")))))
 
 (ert-deftest auth-source-pass-find-match-matching-at-entry-name-part ()
-  (auth-source-pass--with-store '(("foo"))
-    (should (equal (auth-source-pass--find-match "https://foo" nil nil)
-                   "foo"))))
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password")))
+    (let ((result (auth-source-pass--find-match "https://foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "foo password")))))
 
 (ert-deftest auth-source-pass-find-match-matching-at-entry-name-ignoring-user ()
-  (auth-source-pass--with-store '(("foo"))
-    (should (equal (auth-source-pass--find-match "https://SomeUser@foo" nil nil)
-                   "foo"))))
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password")))
+    (let ((result (auth-source-pass--find-match "https://SomeUser@foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "foo password")))))
 
 (ert-deftest auth-source-pass-find-match-matching-at-entry-name-with-user ()
-  (auth-source-pass--with-store '(("SomeUser@foo"))
-    (should (equal (auth-source-pass--find-match "https://SomeUser@foo" nil nil)
-                   "SomeUser@foo"))))
+  (auth-source-pass--with-store
+      '(("SomeUser@foo" ("secret" . "SomeUser@foo password")))
+    (let ((result (auth-source-pass--find-match "https://SomeUser@foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "SomeUser@foo password")))))
 
 (ert-deftest auth-source-pass-find-match-matching-at-entry-name-prefer-full ()
-  (auth-source-pass--with-store '(("SomeUser@foo") ("foo"))
-    (should (equal (auth-source-pass--find-match "https://SomeUser@foo" nil nil)
-                   "SomeUser@foo"))))
+  (auth-source-pass--with-store
+      '(("SomeUser@foo" ("secret" . "SomeUser@foo password"))
+        ("foo" ("secret" . "foo password")))
+    (let ((result (auth-source-pass--find-match "https://SomeUser@foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "SomeUser@foo password")))))
 
 (ert-deftest auth-source-pass-find-match-matching-at-entry-name-prefer-full-reversed ()
-  (auth-source-pass--with-store '(("foo") ("SomeUser@foo"))
-    (should (equal (auth-source-pass--find-match "https://SomeUser@foo" nil nil)
-                   "SomeUser@foo"))))
-
-(ert-deftest auth-source-pass-find-match-matching-at-entry-name-without-subdomain ()
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password"))
+        ("SomeUser@foo" ("secret" . "SomeUser@foo password")))
+    (let ((result (auth-source-pass--find-match "https://SomeUser@foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "SomeUser@foo password")))))
+
+(ert-deftest auth-source-pass-matching-entries-name-without-subdomain ()
   (auth-source-pass--with-store '(("bar.com"))
-    (should (equal (auth-source-pass--find-match "foo.bar.com" nil nil)
-                   "bar.com"))))
+    (should (equal (auth-source-pass--matching-entries "foo.bar.com" nil nil)
+                   '(nil ("bar.com") nil)))))
 
-(ert-deftest auth-source-pass-find-match-matching-at-entry-name-without-subdomain-with-user ()
+(ert-deftest auth-source-pass-matching-entries-name-without-subdomain-with-user ()
   (auth-source-pass--with-store '(("someone@bar.com"))
-    (should (equal (auth-source-pass--find-match "foo.bar.com" "someone" nil)
-                   "someone@bar.com"))))
+    (should (equal (auth-source-pass--matching-entries "foo.bar.com" "someone" nil)
+                   '(nil nil nil ("someone@bar.com") nil nil nil nil nil)))))
 
-(ert-deftest auth-source-pass-find-match-matching-at-entry-name-without-subdomain-with-bad-user ()
+(ert-deftest auth-source-pass-matching-entries-name-without-subdomain-with-bad-user ()
   (auth-source-pass--with-store '(("someoneelse@bar.com"))
-    (should (equal (auth-source-pass--find-match "foo.bar.com" "someone" nil)
-                   nil))))
+    (should (equal (auth-source-pass--matching-entries "foo.bar.com" "someone" nil)
+                   '(nil nil nil nil nil nil nil nil nil)))))
 
-(ert-deftest auth-source-pass-find-match-matching-at-entry-name-without-subdomain-prefer-full ()
+(ert-deftest auth-source-pass-matching-entries-name-without-subdomain-prefer-full ()
   (auth-source-pass--with-store '(("bar.com") ("foo.bar.com"))
-    (should (equal (auth-source-pass--find-match "foo.bar.com" nil nil)
-                   "foo.bar.com"))))
+    (should (equal (auth-source-pass--matching-entries "foo.bar.com" nil nil)
+                   '(("foo.bar.com") ("bar.com") nil)))))
 
 (ert-deftest auth-source-pass-dont-match-at-folder-name ()
   (auth-source-pass--with-store '(("foo.bar.com/foo"))
-    (should (equal (auth-source-pass--find-match "foo.bar.com" nil nil)
-                   nil))))
+    (should (equal (auth-source-pass--matching-entries "foo.bar.com" nil nil)
+                   '(nil nil nil)))))
 
-(ert-deftest auth-source-pass-find-match-matching-host-port-and-subdir-user ()
+(ert-deftest auth-source-pass-matching-entries-host-port-and-subdir-user ()
   (auth-source-pass--with-store '(("bar.com:443/someone"))
-    (should (equal (auth-source-pass--find-match "bar.com" "someone" "443")
-                   "bar.com:443/someone"))))
+    (should (equal (auth-source-pass--matching-entries "bar.com" "someone" "443")
+                   '(nil ("bar.com:443/someone") nil nil nil nil
+                         nil nil nil nil nil nil)))))
 
-(ert-deftest auth-source-pass-find-match-matching-host-port-and-subdir-user-with-custom-separator ()
+(ert-deftest auth-source-pass-matching-entries-host-port-and-subdir-user-with-custom-separator ()
   (let ((auth-source-pass-port-separator "#"))
     (auth-source-pass--with-store '(("bar.com#443/someone"))
-      (should (equal (auth-source-pass--find-match "bar.com" "someone" "443")
-                     "bar.com#443/someone")))))
-
-(ert-deftest auth-source-pass-find-match-matching-extracting-user-from-host ()
-  (auth-source-pass--with-store '(("foo.com/bar"))
-    (should (equal (auth-source-pass--find-match "https://bar@foo.com" nil nil)
-                   "foo.com/bar"))))
-
-(ert-deftest auth-source-pass-search-with-user-first ()
+      (should (equal (auth-source-pass--matching-entries "bar.com" "someone" "443")
+                     '(nil ("bar.com#443/someone") nil nil nil nil
+                           nil nil nil nil nil nil))))))
+
+(ert-deftest auth-source-pass-matching-entries-extracting-user-from-host ()
+  (auth-source-pass--with-store
+      '(("foo.com/bar" ("secret" . "foo.com/bar password")))
+    (let ((result (auth-source-pass--find-match "https://bar@foo.com" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "foo.com/bar password")))))
+
+(ert-deftest auth-source-pass-matching-entries-with-user-first ()
   (auth-source-pass--with-store '(("foo") ("user@foo"))
-    (should (equal (auth-source-pass--find-match "foo" "user" nil)
-                   "user@foo"))
-    (auth-source-pass--should-have-message-containing "Found 1 match")))
+    (should (equal (auth-source-pass--matching-entries "foo" "user" nil)
+                   '(("user@foo") nil ("foo"))))
+    (auth-source-pass--should-have-message-containing "found: (\"user@foo\" \"foo\"")))
 
 (ert-deftest auth-source-pass-give-priority-to-desired-user ()
-  (auth-source-pass--with-store '(("foo") ("subdir/foo" ("user" . "someone")))
-    (should (equal (auth-source-pass--find-match "foo" "someone" nil)
-                   "subdir/foo"))
-    (auth-source-pass--should-have-message-containing "Found 2 matches")
-    (auth-source-pass--should-have-message-containing "matching user field")))
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password"))
+        ("subdir/foo" ("secret" . "subdir/foo password") ("user" . "someone")))
+    (let ((result (auth-source-pass--find-match "foo" "someone" nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "subdir/foo password"))
+      (should (equal (auth-source-pass--get-attr "user" result)
+                     "someone")))
+    (auth-source-pass--should-have-message-containing "found: (\"foo\" \"subdir/foo\"")))
 
 (ert-deftest auth-source-pass-give-priority-to-desired-user-reversed ()
-  (auth-source-pass--with-store '(("foo" ("user" . "someone")) ("subdir/foo"))
-    (should (equal (auth-source-pass--find-match "foo" "someone" nil)
-                   "foo"))
-    (auth-source-pass--should-have-message-containing "Found 2 matches")
-    (auth-source-pass--should-have-message-containing "matching user field")))
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password") ("user" . "someone"))
+        ("subdir/foo" ("secret" . "subdir/foo password")))
+    (let ((result (auth-source-pass--find-match "foo" "someone" nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "foo password")))
+    (auth-source-pass--should-have-message-containing "found: (\"foo\" \"subdir/foo\"")))
 
 (ert-deftest auth-source-pass-return-first-when-several-matches ()
-  (auth-source-pass--with-store '(("foo") ("subdir/foo"))
-    (should (equal (auth-source-pass--find-match "foo" nil nil)
-                   "foo"))
-    (auth-source-pass--should-have-message-containing "Found 2 matches")
-    (auth-source-pass--should-have-message-containing "the first one")))
-
-(ert-deftest auth-source-pass-make-divansantana-happy ()
+  (auth-source-pass--with-store
+      '(("foo" ("secret" . "foo password"))
+        ("subdir/foo" ("secret" . "subdir/foo password")))
+    (let ((result (auth-source-pass--find-match "foo" nil nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "foo password")))
+    (auth-source-pass--should-have-message-containing "found: (\"foo\" \"subdir/foo\"")))
+
+(ert-deftest auth-source-pass-matching-entries-make-divansantana-happy ()
   (auth-source-pass--with-store '(("host.com"))
-    (should (equal (auth-source-pass--find-match "smtp.host.com" "myusername@host.co.za" nil)
-                   "host.com"))))
+    (should (equal (auth-source-pass--matching-entries "smtp.host.com" "myusername@host.co.za" nil)
+                   '(nil nil nil nil nil ("host.com") nil nil nil)))))
 
 (ert-deftest auth-source-pass-find-host-without-port ()
-  (auth-source-pass--with-store '(("host.com"))
-    (should (equal (auth-source-pass--find-match "host.com:8888" "someuser" nil)
-                   "host.com"))))
+  (auth-source-pass--with-store
+      '(("host.com" ("secret" . "host.com password")))
+    (let ((result (auth-source-pass--find-match "host.com:8888" "someuser" nil)))
+      (should (equal (auth-source-pass--get-attr "secret" result)
+                     "host.com password")))))
 
-(ert-deftest auth-source-pass-find-host-with-port ()
+(ert-deftest auth-source-pass-matching-entries-host-with-port ()
   (auth-source-pass--with-store '(("host.com:443"))
-    (should (equal (auth-source-pass--find-match "host.com" "someuser" "443")
-                   "host.com:443"))))
+    (should (equal (auth-source-pass--matching-entries "host.com" "someuser" "443")
+                   '(nil nil nil nil ("host.com:443") nil
+                         nil nil nil nil nil nil)))))
 
-(ert-deftest auth-source-pass-find-host-with-custom-port-separator ()
+(ert-deftest auth-source-pass-matching-entries-with-custom-port-separator ()
   (let ((auth-source-pass-port-separator "#"))
     (auth-source-pass--with-store '(("host.com#443"))
-      (should (equal (auth-source-pass--find-match "host.com" "someuser" "443")
-                     "host.com#443")))))
+      (should (equal (auth-source-pass--matching-entries "host.com" "someuser" "443")
+                     '(nil nil nil nil ("host.com#443") nil
+                           nil nil nil nil nil nil))))))
 
 (defmacro auth-source-pass--with-store-find-foo (store &rest body)
   "Use STORE while executing BODY.  \"foo\" is the matched entry."
@@ -218,7 +282,8 @@ This function is intended to be set to `auth-source-debug`."
        ,@body)))
 
 (ert-deftest auth-source-pass-build-result-return-parameters ()
-  (auth-source-pass--with-store-find-foo '(("foo"))
+  (auth-source-pass--with-store-find-foo
+      '(("foo" ("secret" . "foo password")))
     (let ((result (auth-source-pass--build-result "foo" 512 "user")))
       (should (equal (plist-get result :port) 512))
       (should (equal (plist-get result :user) "user")))))
@@ -238,7 +303,9 @@ This function is intended to be set to `auth-source-debug`."
 (ert-deftest auth-source-pass-build-result-passes-full-host-to-find-match ()
   (let (passed-host)
     (cl-letf (((symbol-function 'auth-source-pass--find-match)
-               (lambda (host _user _port) (setq passed-host host))))
+               (lambda (host _user _port)
+                 (setq passed-host host)
+                 nil)))
       (auth-source-pass--build-result "https://user@host.com:123" nil nil)
       (should (equal passed-host "https://user@host.com:123"))
       (auth-source-pass--build-result "https://user@host.com" nil nil)
@@ -249,27 +316,16 @@ This function is intended to be set to `auth-source-debug`."
       (should (equal passed-host "user@host.com:443")))))
 
 (ert-deftest auth-source-pass-only-return-entries-that-can-be-open ()
-  (cl-letf (((symbol-function 'auth-source-pass-entries)
-             (lambda () '("foo.site.com" "bar.site.com" "mail/baz.site.com/scott")))
-            ((symbol-function 'auth-source-pass--entry-valid-p)
-             ;; only foo.site.com and "mail/baz.site.com/scott" are valid
-             (lambda (entry) (member entry '("foo.site.com" "mail/baz.site.com/scott")))))
-    (should (equal (auth-source-pass--find-all-by-entry-name "foo.site.com" "someuser")
-                   '("foo.site.com")))
-    (should (equal (auth-source-pass--find-all-by-entry-name "bar.site.com" "someuser")
-                   '()))
-    (should (equal (auth-source-pass--find-all-by-entry-name "baz.site.com" "scott")
-                   '("mail/baz.site.com/scott")))))
-
-(ert-deftest auth-source-pass-entry-is-not-valid-when-unreadable ()
-  (cl-letf (((symbol-function 'auth-source-pass--read-entry)
-             (lambda (entry)
-               ;; only foo is a valid entry
-               (if (string-equal entry "foo")
-                   "password"
-                 nil))))
-    (should (auth-source-pass--entry-valid-p "foo"))
-    (should-not (auth-source-pass--entry-valid-p "bar"))))
+  (auth-source-pass--with-store
+      '(("foo.site.com" ("secret" . "foo.site.com password"))
+        ("bar.site.com") ; An entry name with no data is invalid
+        ("mail/baz.site.com/scott" ("secret" . "mail/baz.site.com/scott password")))
+    (should (equal (auth-source-pass--find-match "foo.site.com" "someuser" nil)
+                   '(("secret" . "foo.site.com password"))))
+    (should (equal (auth-source-pass--find-match "bar.site.com" "someuser" nil)
+                   nil))
+    (should (equal (auth-source-pass--find-match "baz.site.com" "scott" nil)
+                   '(("secret" . "mail/baz.site.com/scott password"))))))
 
 (ert-deftest auth-source-pass-can-start-from-auth-source-search ()
   (auth-source-pass--with-store '(("gitlab.com" ("user" . "someone")))