Simplify and parallize test/automated Makefile

* Makefile.in (mostlyclean, clean): Maybe clean test/automated.

* lisp/emacs-lisp/ert.el (ert-summarize-tests-batch-and-exit): New.

* test/automated/Makefile.in: Simplify and parallelize. 
(XARGS_LIMIT, BYTE_COMPILE_EXTRA_FLAGS)
(setwins, compile-targets, compile-main, compile-clean): Remove.
(GREP_OPTIONS): Unexport.
(.el.elc): Replace with pattern rule.
(%.elc, %.log): New pattern rules.
(ELFILES, LOGFILES): New variables.
(check): Depend on LOGFILES.  Call ert-summarize-tests-batch-and-exit.
(clean, mostlyclean): New rules.
(bootstrap-clean): Simplify.
(bootstrap-clean, distclean): Depend on clean.

* .bzrignore: Ignore test/automated/*.log.

Fixes: debbugs:15991

6 files changed, 141 insertions, 73 deletions

diff --git a/ChangeLog b/ChangeLog
index 32fde8b0c4c..397117eb077 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2014-06-26  Glenn Morris  <rgm@gnu.org>
+
+        * Makefile.in (mostlyclean, clean): Maybe clean test/automated.
+
 2014-06-21  Paul Eggert  <eggert@cs.ucla.edu>
 
         * configure.ac: Warn about --enable-link-time-optimization's issues
diff --git a/Makefile.in b/Makefile.in
index 1a7acef0c5a..ce44aa5de84 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -798,7 +798,9 @@ mostlyclean_dirs = src oldXMenu lwlib lib lib-src nt doc/emacs doc/misc \
 $(foreach dir,$(mostlyclean_dirs),$(eval $(call submake_template,$(dir),mostlyclean)))
 
 mostlyclean: $(mostlyclean_dirs:=_mostlyclean)
-
+        for dir in test/automated; do \
+          [ ! -d $$dir ] || $(MAKE) -C $$dir mostlyclean; \
+        done
 
 ### `clean'
 ###      Delete all files from the current directory that are normally
@@ -813,6 +815,9 @@ clean_dirs = $(mostlyclean_dirs) nextstep
 $(foreach dir,$(clean_dirs),$(eval $(call submake_template,$(dir),clean)))
 
 clean: $(clean_dirs:=_clean)
+        for dir in test/automated; do \
+          [ ! -d $$dir ] || $(MAKE) -C $$dir clean; \
+        done
         -rm -f etc/emacs.tmpdesktop
 
 ### `bootclean'
diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index 658825d2ba8..9b4a91a9ea8 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,3 +1,7 @@
+2014-06-26  Glenn Morris  <rgm@gnu.org>
+
+        * emacs-lisp/ert.el (ert-summarize-tests-batch-and-exit): New.
+
 2014-06-25  Glenn Morris  <rgm@gnu.org>
 
         * Makefile.in ($(lisp)/progmodes/cc-defs.elc)
diff --git a/lisp/emacs-lisp/ert.el b/lisp/emacs-lisp/ert.el
index 6ecb218091a..024110b93e0 100644
--- a/lisp/emacs-lisp/ert.el
+++ b/lisp/emacs-lisp/ert.el
@@ -1463,6 +1463,65 @@ the tests)."
       (kill-emacs 2))))
 
 
+(defun ert-summarize-tests-batch-and-exit ()
+  "Summarize the results of testing.
+Expects to be called in batch mode, with logfiles as command-line arguments.
+The logfiles should have the `ert-run-tests-batch' format.  When finished,
+this exits Emacs, with status as per `ert-run-tests-batch-and-exit'."
+  (or noninteractive
+      (user-error "This function is only for use in batch mode"))
+  (let ((nlogs (length command-line-args-left))
+        (ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0)
+        nnotrun logfile notests badtests unexpected)
+    (with-temp-buffer
+      (while (setq logfile (pop command-line-args-left))
+        (erase-buffer)
+        (insert-file-contents logfile)
+        (if (not (re-search-forward "^Running \\([0-9]+\\) tests" nil t))
+            (push logfile notests)
+          (setq ntests (+ ntests (string-to-number (match-string 1))))
+          (if (not (re-search-forward "^\\(Aborted: \\)?\
+Ran \\([0-9]+\\) tests, \\([0-9]+\\) results as expected\
+\\(?:, \\([0-9]+\\) unexpected\\)?\
+\\(?:, \\([0-9]+\\) skipped\\)?" nil t))
+              (push logfile badtests)
+            (if (match-string 1) (push logfile badtests))
+            (setq nrun (+ nrun (string-to-number (match-string 2)))
+                  nexpected (+ nexpected (string-to-number (match-string 3))))
+            (when (match-string 4)
+              (push logfile unexpected)
+              (setq nunexpected (+ nunexpected
+                                   (string-to-number (match-string 4)))))
+            (if (match-string 5)
+                (setq nskipped (+ nskipped
+                                  (string-to-number (match-string 5)))))))))
+    (setq nnotrun (- ntests nrun))
+    (message "\nSUMMARY OF TEST RESULTS")
+    (message "-----------------------")
+    (message "Files examined: %d" nlogs)
+    (message "Ran %d tests%s, %d results as expected%s%s"
+             nrun
+             (if (zerop nnotrun) "" (format ", %d failed to run" nnotrun))
+             nexpected
+             (if (zerop nunexpected)
+                 ""
+               (format ", %d unexpected" nunexpected))
+             (if (zerop nskipped)
+                 ""
+               (format ", %d skipped" nskipped)))
+    (when notests
+      (message "%d files did not contain any tests:" (length notests))
+      (mapc (lambda (l) (message "  %s" l)) notests))
+    (when badtests
+      (message "%d files did not finish:" (length badtests))
+      (mapc (lambda (l) (message "  %s" l)) badtests))
+    (when unexpected
+      (message "%d files contained unexpected results:" (length unexpected))
+      (mapc (lambda (l) (message "  %s" l)) unexpected))
+    (kill-emacs (cond ((or notests badtests (not (zerop nnotrun))) 2)
+                      (unexpected 1)
+                      (t 0)))))
+
 ;;; Utility functions for load/unload actions.
 
 (defun ert--activate-font-lock-keywords ()
diff --git a/test/ChangeLog b/test/ChangeLog
index 3f982ac2809..e3c748c8deb 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,3 +1,17 @@
+2014-06-26  Glenn Morris  <rgm@gnu.org>
+
+        * automated/Makefile.in: Simplify and parallelize.  (Bug#15991)
+        (XARGS_LIMIT, BYTE_COMPILE_EXTRA_FLAGS)
+        (setwins, compile-targets, compile-main, compile-clean): Remove.
+        (GREP_OPTIONS): Unexport.
+        (.el.elc): Replace with pattern rule.
+        (%.elc, %.log): New pattern rules.
+        (ELFILES, LOGFILES): New variables.
+        (check): Depend on LOGFILES.  Call ert-summarize-tests-batch-and-exit.
+        (clean, mostlyclean): New rules.
+        (bootstrap-clean): Simplify.
+        (bootstrap-clean, distclean): Depend on clean.
+
 2014-06-25  Glenn Morris  <rgm@gnu.org>
 
         * automated/flymake-tests.el (flymake-tests--current-face):
diff --git a/test/automated/Makefile.in b/test/automated/Makefile.in
index e0800f09eb0..053812919ad 100644
--- a/test/automated/Makefile.in
+++ b/test/automated/Makefile.in
@@ -24,10 +24,6 @@ VPATH = $(srcdir)
 
 SEPCHAR = @SEPCHAR@
 
-# Empty for all systems except MinGW, where xargs needs an explicit
-# limitation.
-XARGS_LIMIT = @XARGS_LIMIT@
-
 # We never change directory before running Emacs, so a relative file
 # name is fine, and makes life easier.  If we need to change
 # directory, we can use emacs --chdir.
@@ -38,87 +34,73 @@ EMACS = ../../src/emacs
 # but we might as well be explicit.
 EMACSOPT = -batch --no-site-file --no-site-lisp -L "$(SEPCHAR)$(srcdir)"
 
-# Extra flags to pass to the byte compiler.
-BYTE_COMPILE_EXTRA_FLAGS =
-
 # Prevent any settings in the user environment causing problems.
-unexport EMACSDATA EMACSDOC EMACSPATH
+unexport EMACSDATA EMACSDOC EMACSPATH GREP_OPTIONS
 
 # The actual Emacs command run in the targets below.
 # Prevent any setting of EMACSLOADPATH in user environment causing problems.
 emacs = EMACSLOADPATH= LC_ALL=C EMACS_TEST_DIRECTORY=$(srcdir) "$(EMACS)" $(EMACSOPT)
 
-# Common command to find subdirectories
-setwins=for file in `find $(srcdir) -type d -print`; do \
-           case $$file in $(srcdir)*/data* | $(srcdir)*/flymake* ) ;; \
-                *) wins="$$wins$${wins:+ }$$file" ;; \
-           esac; \
-        done
-
 .PHONY: all check
 
 all: check
 
-# The compilation stuff is copied from lisp/Makefile - see comments there.
-
-.SUFFIXES: .elc .el
-
-.el.elc:
+%.elc: %.el
         @echo Compiling $<
-        @$(emacs) $(BYTE_COMPILE_EXTRA_FLAGS) -f batch-byte-compile $<
-
-
-.PHONY: compile-targets compile-main compile-clean
-
-# TARGETS is set dynamically in the recursive call from `compile-main'.
-compile-targets: $(TARGETS)
-
-# Compile all the Elisp files that need it.  Beware: it approximates
-# `no-byte-compile', so watch out for false-positives!
-compile-main: compile-clean
-        @$(setwins); \
-        els=`echo "$$wins " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| |/*.el |g'`; \
-        for el in $$els; do \
-          test -f $$el || continue; \
-          test ! -f $${el}c && GREP_OPTIONS= grep '^;.*no-byte-compile: t' $$el > /dev/null && continue; \
-          echo "$${el}c"; \
-        done | xargs $(XARGS_LIMIT) echo | \
-        while read chunk; do \
-          $(MAKE) compile-targets EMACS="$(EMACS)" TARGETS="$$chunk"; \
-        done
-
-# Erase left-over .elc files that do not have a corresponding .el file.
-compile-clean:
-        @$(setwins); \
-        elcs=`echo "$$wins " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| |/*.elc |g'`; \
-        for el in $$(echo $$elcs | sed -e 's/\.elc/\.el/g'); do \
-          if test -f "$$el" -o \! -f "$${el}c"; then :; else \
-            echo rm "$${el}c"; \
-            rm "$${el}c"; \
-          fi \
-        done
-
-
-.PHONY: bootstrap-clean distclean maintainer-clean
-
-bootstrap-clean:
-        -cd $(srcdir) && rm -f *.elc */*.elc */*/*.elc */*/*/*.elc
-
-distclean:
+        @$(emacs) -f batch-byte-compile $<
+
+## Ignore any test errors so we can continue to test other files.
+## (It would be nice if we could get an error when running an
+## individual test, but not when running check.)
+## But compilation errors are always fatal.
+##
+## I'd prefer to use -emacs -f ert-run-tests-batch-and-exit rather
+## than || true, since the former makes problems more obvious.
+## I'd also prefer to @-hide the grep part and not the
+## ert-run-tests-batch-and-exit part.
+##
+## We need to use $loadfile because:
+## i) -L :$srcdir -l basename does not work, because we have files whose
+## basename duplicates a file in lisp/ (eg eshell.el).
+## ii) Although -l basename will automatically load .el or .elc,
+## -l ./basename treats basename as a literal file (it would be nice
+## to change this).
+##
+## Beware: it approximates `no-byte-compile', so watch out for false-positives!
+%.log: ${srcdir}/%.el
+        @if grep '^;.*no-byte-compile: t' $< > /dev/null; then \
+          loadfile=$<; \
+        else \
+          loadfile=$<c; \
+          ${MAKE} $$loadfile; \
+        fi; \
+        echo Testing $$loadfile; \
+        stat=OK ; \
+        $(emacs) -l ert -l $$loadfile \
+          -f ert-run-tests-batch-and-exit >& $@ || stat=ERROR; \
+        echo $$stat: $@
+
+ELFILES = $(wildcard ${srcdir}/*.el)
+LOGFILES = $(patsubst %.el,%.log,$(notdir ${ELFILES}))
+
+## If we have to interrupt a hanging test, preserve the log so we can
+## see what the problem was.
+.PRECIOUS: %.log
+
+check: ${LOGFILES}
+        $(emacs) -l ert -f ert-summarize-tests-batch-and-exit $^
+
+.PHONY: mostlyclean clean bootstrap-clean distclean maintainer-clean
+
+clean mostlyclean:
+        -rm -f *.log
+
+bootstrap-clean: clean
+        -rm -f ${srcdir}/*.elc
+
+distclean: clean
         rm -f Makefile
 
 maintainer-clean: distclean bootstrap-clean
 
-
-check: compile-main
-        @$(setwins); \
-        pattern=`echo "$$wins " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| |/*.el |g'`; \
-        for el in $$pattern; do \
-          test -f $$el || continue; \
-          args="$$args -l $$el"; \
-          els="$$els $$el"; \
-        done; \
-        echo Testing $$els; \
-        $(emacs) $$args -f ert-run-tests-batch-and-exit
-
 # Makefile ends here.