Flymake backends can report multiple times per check

Rewrote a significant part of the Flymake backend API.  Flymake now
ignores the return value of backend functions: a function can either
returns or errors.  If it doesn't error, a backend is no longer
constrained to call REPORT-FN exactly once.  It may do so any number
of times, cumulatively reporting diagnostics.  Flymake keeps track of
outdated REPORT-FN instances and disconsiders obsolete reports.
Backends should avoid reporting obsolete data by cancelling any
ongoing processing at every renewed call to the backend function.

Consolidated flymake.el internal data structures to require less
buffer-local variables.  Adjusted Flymake's mode-line indicator to the
new semantics.

Adapted and simplified the implementation of elisp and legacy
backends, fixing potential race conditions when calling backends in
rapid succession.

Added a new test for a backend that calls REPORT-FN multiple
times.  Simplify test infrastructure.

* lisp/progmodes/flymake-elisp.el (flymake-elisp-checkdoc)
(flymake-elisp-byte-compile): Error instead of returning nil
if not in emacs-lisp-mode.
(flymake-elisp--byte-compile-process): New buffer-local variable.
(flymake-elisp-byte-compile): Mark (and kill) previous process
obsolete process before starting a new one.  Don't report if
obsolete process.

* lisp/progmodes/flymake-proc.el
(flymake-proc--current-process): New buffer-local variable.
(flymake-proc--processes): Remove.
(flymake-proc--process-filter): Don't bind
flymake-proc--report-fn.
(flymake-proc--process-sentinel): Rewrite.  Don't report if
obsolete process.
(flymake-proc-legacy-flymake): Rewrite.  Mark (and kill)
previous process obsolete process before starting a new
one.  Integrate flymake-proc--start-syntax-check-process
helper.
(flymake-proc--start-syntax-check-process): Delete.
(flymake-proc-stop-all-syntax-checks): Don't use
flymake-proc--processes, iterate buffers.
(flymake-proc-compile):

* lisp/progmodes/flymake.el (subr-x): Require it
explicitly.
(flymake-diagnostic-functions): Reword docstring.
(flymake--running-backends, flymake--disabled-backends)
(flymake--diagnostics-table): Delete.
(flymake--backend-state): New buffer-local variable and new defstruct.
(flymake--with-backend-state, flymake--collect)
(flymake-running-backends, flymake-disabled-backends)
(flymake-reporting-backends): New helpers.
(flymake-is-running): Use flymake-running-backends.
(flymake--handle-report): Rewrite.
(flymake-make-report-fn): Ensure REPORT-FN runs in the correct
buffer or not at all.
(flymake--disable-backend, flymake--run-backend): Rewrite.
(flymake-start): Rewrite.
(flymake-mode): Set flymake--backend-state.
(flymake--mode-line-format): Rewrite.

* test/lisp/progmodes/flymake-tests.el
(flymake-tests--wait-for-backends): New helper.
(flymake-tests--call-with-fixture): Use it.
(included-c-header-files): Fix whitespace.
(flymake-tests--diagnose-words): New helper.
(dummy-backends): Rewrite for new semantics.  Use cl-letf.
(flymake-tests--assert-set): Use quote.
(recurrent-backend): New test.

1 files changed, 147 insertions, 99 deletions

diff --git a/test/lisp/progmodes/flymake-tests.el b/test/lisp/progmodes/flymake-tests.el
index 222c8f11848..5e042f2b082 100644
--- a/test/lisp/progmodes/flymake-tests.el
+++ b/test/lisp/progmodes/flymake-tests.el
@@ -36,6 +36,26 @@
 
 ;;
 ;;
+(defun flymake-tests--wait-for-backends ()
+  ;; Weirdness here...  http://debbugs.gnu.org/17647#25
+  ;; ... meaning `sleep-for', and even
+  ;; `accept-process-output', won't suffice as ways to get
+  ;; process filters and sentinels to run, though they do work
+  ;; fine in a non-interactive batch session. The only thing
+  ;; that will indeed unblock pending process output is
+  ;; reading an input event, so, as a workaround, use a dummy
+  ;; `read-event' with a very short timeout.
+  (unless noninteractive (read-event "" nil 0.1))
+  (cl-loop repeat 5
+           for notdone = (cl-set-difference (flymake-running-backends)
+                                            (flymake-reporting-backends))
+           while notdone
+           unless noninteractive do (read-event "" nil 0.1)
+           do (sleep-for (+ 0.5 flymake-no-changes-timeout))
+           finally (when notdone (ert-fail
+                                  (format "Some backends not reporting yet %s"
+                                          notdone)))))
+
 (cl-defun flymake-tests--call-with-fixture (fn file
                                                &key (severity-predicate
                                                      nil sev-pred-supplied-p))
@@ -46,7 +66,6 @@ SEVERITY-PREDICATE is used to setup
          (visiting (find-buffer-visiting file))
          (buffer (or visiting (find-file-noselect file)))
          (process-environment (cons "LC_ALL=C" process-environment))
-         (i 0)
          (warning-minimum-log-level :error))
     (unwind-protect
         (with-current-buffer buffer
@@ -55,18 +74,7 @@ SEVERITY-PREDICATE is used to setup
               (setq-local flymake-proc-diagnostic-type-pred severity-predicate))
             (goto-char (point-min))
             (unless flymake-mode (flymake-mode 1))
-            ;; Weirdness here...  http://debbugs.gnu.org/17647#25
-            ;; ... meaning `sleep-for', and even
-            ;; `accept-process-output', won't suffice as ways to get
-            ;; process filters and sentinels to run, though they do work
-            ;; fine in a non-interactive batch session. The only thing
-            ;; that will indeed unblock pending process output is
-            ;; reading an input event, so, as a workaround, use a dummy
-            ;; `read-event' with a very short timeout.
-            (unless noninteractive (read-event "" nil 0.1))
-            (while (and (flymake-is-running) (< (setq i (1+ i)) 10))
-              (unless noninteractive (read-event "" nil 0.1))
-              (sleep-for (+ 0.5 flymake-no-changes-timeout)))
+            (flymake-tests--wait-for-backends)
             (funcall fn)))
       (and buffer
            (not visiting)
@@ -119,38 +127,37 @@ SEVERITY-PREDICATE is used to setup
 (ert-deftest different-diagnostic-types ()
   "Test GCC warning via function predicate."
   (skip-unless (and (executable-find "gcc") (executable-find "make")))
-  (flymake-tests--with-flymake
-      ("errors-and-warnings.c")
-    (flymake-goto-next-error)
-    (should (eq 'flymake-error (face-at-point)))
-    (flymake-goto-next-error)
-    (should (eq 'flymake-note (face-at-point)))
-    (flymake-goto-next-error)
-    (should (eq 'flymake-warning (face-at-point)))
-    (flymake-goto-next-error)
-    (should (eq 'flymake-error (face-at-point)))
-    (flymake-goto-next-error)
-    (should (eq 'flymake-warning (face-at-point)))
-    (flymake-goto-next-error)
-    (should (eq 'flymake-warning (face-at-point)))
-    (let ((flymake-wrap-around nil))
-      (should-error (flymake-goto-next-error nil nil t))) ))
+  (let ((flymake-wrap-around nil))
+    (flymake-tests--with-flymake
+        ("errors-and-warnings.c")
+      (flymake-goto-next-error)
+      (should (eq 'flymake-error (face-at-point)))
+      (flymake-goto-next-error)
+      (should (eq 'flymake-note (face-at-point)))
+      (flymake-goto-next-error)
+      (should (eq 'flymake-warning (face-at-point)))
+      (flymake-goto-next-error)
+      (should (eq 'flymake-error (face-at-point)))
+      (flymake-goto-next-error)
+      (should (eq 'flymake-warning (face-at-point)))
+      (flymake-goto-next-error)
+      (should (eq 'flymake-warning (face-at-point)))
+      (should-error (flymake-goto-next-error nil nil t)))))
 
 (ert-deftest included-c-header-files ()
   "Test inclusion of .h header files."
   (skip-unless (and (executable-find "gcc") (executable-find "make")))
-  (flymake-tests--with-flymake
-      ("some-problems.h")
-    (flymake-goto-next-error)
-    (should (eq 'flymake-warning (face-at-point)))
-    (flymake-goto-next-error)
-    (should (eq 'flymake-error (face-at-point)))
-    (let ((flymake-wrap-around nil))
-      (should-error (flymake-goto-next-error nil nil t))) )
-  (flymake-tests--with-flymake
-      ("no-problems.h")
-    (let ((flymake-wrap-around nil))
-      (should-error (flymake-goto-next-error nil nil t))) ))
+  (let ((flymake-wrap-around nil))
+    (flymake-tests--with-flymake
+        ("some-problems.h")
+      (flymake-goto-next-error)
+      (should (eq 'flymake-warning (face-at-point)))
+      (flymake-goto-next-error)
+      (should (eq 'flymake-error (face-at-point)))
+      (should-error (flymake-goto-next-error nil nil t)))
+    (flymake-tests--with-flymake
+        ("no-problems.h")
+      (should-error (flymake-goto-next-error nil nil t)))))
 
 (defmacro flymake-tests--assert-set (set
                                      should
@@ -159,19 +166,15 @@ SEVERITY-PREDICATE is used to setup
   `(progn
      ,@(cl-loop
         for s in should
-        collect `(should (memq ,s ,set)))
+        collect `(should (memq (quote ,s) ,set)))
      ,@(cl-loop
         for s in should-not
-        collect `(should-not (memq ,s ,set)))))
+        collect `(should-not (memq (quote ,s) ,set)))))
 
-(ert-deftest dummy-backends ()
-  "Test GCC warning via function predicate."
-  (with-temp-buffer
-    (cl-labels
-        ((diagnose
-          (report-fn type words)
-          (funcall
-           report-fn
+(defun flymake-tests--diagnose-words
+    (report-fn type words)
+  "Helper. Call REPORT-FN with diagnostics for WORDS in buffer."
+  (funcall report-fn
            (cl-loop
             for word in words
             append
@@ -184,32 +187,34 @@ SEVERITY-PREDICATE is used to setup
                                 (match-end 0)
                                 type
                                 (concat word " is wrong")))))))
-         (error-backend
-          (report-fn)
-          (run-with-timer
-           0.5 nil
-           #'diagnose report-fn :error '("manha" "prognata")))
-         (warning-backend
-          (report-fn)
-          (run-with-timer
-           0.5 nil
-           #'diagnose report-fn :warning '("ut" "dolor")))
-         (sync-backend
-          (report-fn)
-          (diagnose report-fn :note '("quis" "commodo")))
-         (refusing-backend
-          (_report-fn)
-          nil)
-         (panicking-backend
-          (report-fn)
-          (run-with-timer
-           0.5 nil
-           report-fn :panic :explanation "The spanish inquisition!"))
-         (crashing-backend
-          (_report-fn)
-          ;; HACK: Shoosh log during tests
-          (setq-local warning-minimum-log-level :emergency)
-          (error "crashed")))
+
+(ert-deftest dummy-backends ()
+  "Test many different kinds of backends."
+  (with-temp-buffer
+    (cl-letf
+        (((symbol-function 'error-backend)
+          (lambda (report-fn)
+            (run-with-timer
+             0.5 nil
+             #'flymake-tests--diagnose-words report-fn :error '("manha" "prognata"))))
+         ((symbol-function 'warning-backend)
+          (lambda (report-fn)
+            (run-with-timer
+             0.5 nil
+             #'flymake-tests--diagnose-words report-fn :warning '("ut" "dolor"))))
+         ((symbol-function 'sync-backend)
+          (lambda (report-fn)
+            (flymake-tests--diagnose-words report-fn :note '("quis" "commodo"))))
+         ((symbol-function 'panicking-backend)
+          (lambda (report-fn)
+            (run-with-timer
+             0.5 nil
+             report-fn :panic :explanation "The spanish inquisition!")))
+         ((symbol-function 'crashing-backend)
+          (lambda (_report-fn)
+            ;; HACK: Shoosh log during tests
+            (setq-local warning-minimum-log-level :emergency)
+            (error "crashed"))))
       (insert "Lorem ipsum dolor sit amet, consectetur adipiscing
     elit, sed do eiusmod tempor incididunt ut labore et dolore
     manha aliqua. Ut enim ad minim veniam, quis nostrud
@@ -220,31 +225,27 @@ SEVERITY-PREDICATE is used to setup
     sunt in culpa qui officia deserunt mollit anim id est
     laborum.")
       (let ((flymake-diagnostic-functions
-             (list #'error-backend #'warning-backend #'sync-backend
-                   #'refusing-backend #'panicking-backend
-                   #'crashing-backend
-                   )))
+             (list 'error-backend 'warning-backend 'sync-backend
+                   'panicking-backend
+                   'crashing-backend
+                   ))
+            (flymake-wrap-around nil))
         (flymake-mode)
-        ;; FIXME: accessing some flymake-ui's internals here...
-        (flymake-tests--assert-set flymake--running-backends
-          (#'error-backend #'warning-backend #'panicking-backend)
-          (#'sync-backend #'crashing-backend #'refusing-backend))
 
-        (flymake-tests--assert-set flymake--disabled-backends
-          (#'crashing-backend)
-          (#'error-backend #'warning-backend #'sync-backend
-                           #'panicking-backend #'refusing-backend))
+        (flymake-tests--assert-set (flymake-running-backends)
+          (error-backend warning-backend panicking-backend)
+          (crashing-backend))
 
-        (cl-loop repeat 10 while (flymake-is-running)
-                 unless noninteractive do (read-event "" nil 0.1)
-                 do (sleep-for (+ 0.5 flymake-no-changes-timeout)))
+        (flymake-tests--assert-set (flymake-disabled-backends)
+          (crashing-backend)
+          (error-backend warning-backend sync-backend
+                         panicking-backend))
 
-        (should (eq flymake--running-backends '()))
+        (flymake-tests--wait-for-backends)
 
-        (flymake-tests--assert-set flymake--disabled-backends
-          (#'crashing-backend #'panicking-backend)
-          (#'error-backend #'warning-backend #'sync-backend
-                           #'refusing-backend))
+        (flymake-tests--assert-set (flymake-disabled-backends)
+          (crashing-backend panicking-backend)
+          (error-backend warning-backend sync-backend))
 
         (goto-char (point-min))
         (flymake-goto-next-error)
@@ -265,8 +266,55 @@ SEVERITY-PREDICATE is used to setup
         (should (eq 'flymake-warning (face-at-point))) ; dolor
         (flymake-goto-next-error)
         (should (eq 'flymake-error (face-at-point))) ; prognata
-        (let ((flymake-wrap-around nil))
-          (should-error (flymake-goto-next-error nil nil t)))))))
+        (should-error (flymake-goto-next-error nil nil t))))))
+
+(ert-deftest recurrent-backend ()
+  "Test a backend that calls REPORT-FN multiple times"
+  (with-temp-buffer
+    (let (tick)
+      (cl-letf
+          (((symbol-function 'eager-backend)
+            (lambda (report-fn)
+              (funcall report-fn nil :explanation "very eager but no diagnostics")
+              (display-buffer (current-buffer))
+              (run-with-timer
+               0.5 nil
+               (lambda ()
+                 (flymake-tests--diagnose-words report-fn :warning '("consectetur"))
+                 (setq tick t)
+                 (run-with-timer
+                  0.5 nil
+                  (lambda ()
+                    (flymake-tests--diagnose-words report-fn :error '("fugiat"))
+                    (setq tick t))))))))
+        (insert "Lorem ipsum dolor sit amet, consectetur adipiscing
+    elit, sed do eiusmod tempor incididunt ut labore et dolore
+    manha aliqua. Ut enim ad minim veniam, quis nostrud
+    exercitation ullamco laboris nisi ut aliquip ex ea commodo
+    consequat. Duis aute irure dolor in reprehenderit in
+    voluptate velit esse cillum dolore eu fugiat nulla
+    pariatur. Excepteur sint occaecat cupidatat non prognata
+    sunt in culpa qui officia deserunt mollit anim id est
+    laborum.")
+        (let ((flymake-diagnostic-functions
+               (list 'eager-backend))
+              (flymake-wrap-around nil))
+          (flymake-mode)
+          (flymake-tests--assert-set (flymake-running-backends)
+            (eager-backend) ())
+          (cl-loop until tick repeat 4 do (sleep-for 0.2))
+          (setq tick nil)
+          (goto-char (point-max))
+          (flymake-goto-prev-error)
+          (should (eq 'flymake-warning (face-at-point))) ; consectetur
+          (should-error (flymake-goto-prev-error nil nil t))
+          (cl-loop until tick repeat 4 do (sleep-for 0.2))
+          (flymake-goto-next-error)
+          (should (eq 'flymake-error (face-at-point))) ; fugiat
+          (flymake-goto-prev-error)
+          (should (eq 'flymake-warning (face-at-point))) ; back at consectetur
+          (should-error (flymake-goto-prev-error nil nil t))
+          )))))
 
 (provide 'flymake-tests)