ERT can generate JUnit test reports

* .gitignore: Add test/**/*.xml.

* admin/notes/emba: Mention JUnit test report.

* etc/NEWS: ERT can generate JUnit test reports.

* lisp/emacs-lisp/ert.el (xml-escape-string): Autoload.
(ert-write-junit-test-report)
(ert-write-junit-test-summary-report): New defuns.
(ert-run-tests-batch, ert-summarize-tests-batch-and-exit): Call them.

* test/Makefile.in (clean): Remove *.xml.

* test/README: Mention $EMACS_TEST_JUNIT_REPORT environment variable.

* test/infra/Makefile.in ($(FILE)): Generate header commentary.
(clean): Remove.

* test/infra/gitlab-ci.yml (variables): Set EMACS_TEST_JUNIT_REPORT.
(.job-template): Use it in script and after_script.
(.build-template, .gnustep-template, .filenotify-gio-template)
(.native-comp-template): Adapt rules.
(.test-template): Trigger JUnit test report.

* test/infra/test-jobs.yml: Regenerate.

9 files changed, 159 insertions, 20 deletions

diff --git a/.gitignore b/.gitignore
index f1abb2ab687..7baee47b0a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -159,6 +159,7 @@ test/manual/etags/CTAGS
 test/manual/indent/*.new
 test/lisp/gnus/mml-sec-resources/random_seed
 test/lisp/play/fortune-resources/fortunes.dat
+test/**/*.xml
 
 # ctags, etags.
 TAGS
diff --git a/admin/notes/emba b/admin/notes/emba
index f1b52b2cde0..2135c7a97cc 100644
--- a/admin/notes/emba
+++ b/admin/notes/emba
@@ -63,6 +63,10 @@ They can be downloaded from the server, visiting the URL
 <https://emba.gnu.org/emacs/emacs/-/pipelines>, and selecting the job
 in question.
 
+Every pipeline generates a JUnit test report for the respective test
+jobs, which can be inspected on the pipeline web page.  This test
+report counts completed ERT tests, aborted tests are not counted.
+
 * Emba configuration
 
 The emba configuration files are hosted on
diff --git a/etc/NEWS b/etc/NEWS
index b55b30665be..8d83b2a7e36 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -163,6 +163,12 @@ the previous definition to be discarded, which was probably not
 intended when this occurs in batch mode.  To remedy the error, rename
 tests so that they all have unique names.
 
++++
+*** ERT can generate JUnit test reports.
+When environment variable 'EMACS_TEST_JUNIT_REPORT' is set, ERT
+generates a JUnit test report under this file name.  This is useful
+for Emacs integration into CI/CD test environments.
+
 ** Emoji
 
 +++
@@ -1143,6 +1149,7 @@ This variable is bound to t during the preparation of a "*Help*" buffer.
 ** 'date-to-time' now assumes earliest values if its argument lacks
 month, day, or time.  For example, (date-to-time "2021-12-04") now
 assumes a time of 00:00 instead of signaling an error.
+
 
 * Changes in Emacs 29.1 on Non-Free Operating Systems
 
diff --git a/lisp/emacs-lisp/ert.el b/lisp/emacs-lisp/ert.el
index 946193e40dc..981e23931c2 100644
--- a/lisp/emacs-lisp/ert.el
+++ b/lisp/emacs-lisp/ert.el
@@ -65,6 +65,8 @@
 (require 'pp)
 (require 'map)
 
+(autoload 'xml-escape-string "xml.el")
+
 ;;; UI customization options.
 
 (defgroup ert ()
@@ -247,7 +249,6 @@ in batch mode, an error is signalled.
           "%s\\(\\s-\\|$\\)")
   "The regexp the `find-function' mechanisms use for finding test definitions.")
 
-
 (define-error 'ert-test-failed "Test failed")
 (define-error 'ert-test-skipped "Test skipped")
 
@@ -677,7 +678,6 @@ and is displayed in front of the value of MESSAGE-FORM."
      ,@body))
 
 
-
 ;;; Facilities for running a single test.
 
 (defvar ert-debug-on-error nil
@@ -1437,7 +1437,9 @@ Returns the stats object."
                                   (if (getenv "EMACS_TEST_VERBOSE")
                                       (ert-reason-for-test-result result)
                                     ""))))
-              (message "%s" "")))))
+              (message "%s" ""))
+            (when (getenv "EMACS_TEST_JUNIT_REPORT")
+              (ert-write-junit-test-report stats)))))
        (test-started)
        (test-ended
         (cl-destructuring-bind (stats test result) event-args
@@ -1525,6 +1527,128 @@ the tests)."
           (backtrace))
       (kill-emacs 2))))
 
+(defun ert-write-junit-test-report (stats)
+  "Write a JUnit test report, generated from STATS."
+  ;; https://www.ibm.com/docs/de/developer-for-zos/14.1.0?topic=formats-junit-xml-format
+  ;; https://llg.cubic.org/docs/junit/
+  (unless (zerop (length (ert--stats-tests stats)))
+    (when-let ((test-file
+                (symbol-file
+                 (ert-test-name (aref (ert--stats-tests stats) 0)) 'ert--test)))
+      (with-temp-file (file-name-with-extension test-file "xml")
+        (insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
+        (insert (format "<testsuites name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
+                        (file-name-nondirectory test-file)
+                        (ert-stats-total stats)
+                        (ert-stats-completed-unexpected stats)
+                        (ert-stats-skipped stats)
+                        (float-time
+                         (time-subtract
+                          (ert--stats-end-time stats)
+                          (ert--stats-start-time stats)))))
+        (insert (format "  <testsuite id=\"0\" name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\" timestamp=\"%s\">\n"
+                        (file-name-nondirectory test-file)
+                        (ert-stats-total stats)
+                        (ert-stats-completed-unexpected stats)
+                        (ert-stats-skipped stats)
+                        (float-time
+                         (time-subtract
+                          (ert--stats-end-time stats)
+                          (ert--stats-start-time stats)))
+                        (ert--format-time-iso8601 (ert--stats-end-time stats))))
+        (insert "    <properties>\n"
+                (format "      <property name=\"selector\" value=\"%s\"/>\n"
+                        (ert--stats-selector stats))
+                "    </properties>\n")
+        (cl-loop for test across (ert--stats-tests stats)
+                 for result = (ert-test-most-recent-result test) do
+                 (insert (format "    <testcase name=\"%s\" status=\"%s\" time=\"%s\""
+                                 (xml-escape-string
+                                  (symbol-name (ert-test-name test)))
+                                 (ert-string-for-test-result
+                                  result
+                                  (ert-test-result-expected-p test result))
+                                 (ert-test-result-duration result)))
+                 (if (and (ert-test-result-expected-p test result)
+                          (not (ert-test-skipped-p result))
+                          (zerop (length (ert-test-result-messages result))))
+                     (insert "/>\n")
+                   (insert ">\n")
+                   (if (ert-test-skipped-p result)
+                       (insert (format "      <skipped message=\"%s\" type=\"%s\">\n"
+                                       (xml-escape-string
+                                        (string-trim
+                                         (ert-reason-for-test-result result)))
+                                       (ert-string-for-test-result
+                                        result
+                                        (ert-test-result-expected-p
+                                         test result)))
+                               (xml-escape-string
+                                (string-trim
+                                 (ert-reason-for-test-result result)))
+                               "\n"
+                               "      </skipped>\n")
+                     (unless
+                         (ert-test-result-type-p
+                          result (ert-test-expected-result-type test))
+                       (insert (format "      <failure message=\"%s\" type=\"%s\">\n"
+                                       (xml-escape-string
+                                        (string-trim
+                                         (ert-reason-for-test-result result)))
+                                       (ert-string-for-test-result
+                                        result
+                                        (ert-test-result-expected-p
+                                         test result)))
+                               (xml-escape-string
+                                (string-trim
+                                 (ert-reason-for-test-result result)))
+                               "\n"
+                               "      </failure>\n")))
+                   (unless (zerop (length (ert-test-result-messages result)))
+                     (insert "      <system-out>\n"
+                             (xml-escape-string
+                              (ert-test-result-messages result))
+                             "      </system-out>\n"))
+                   (insert "    </testcase>\n")))
+        (insert "  </testsuite>\n")
+        (insert "</testsuites>\n")))))
+
+(defun ert-write-junit-test-summary-report (&rest logfiles)
+  "Write a JUnit summary test report, generated from LOGFILES."
+  (let ((report (file-name-with-extension
+                 (getenv "EMACS_TEST_JUNIT_REPORT") "xml"))
+        (tests 0) (failures 0) (skipped 0) (time 0) (id 0))
+    (with-temp-file report
+      (dolist (logfile logfiles)
+        (let ((test-file (file-name-with-extension logfile "xml")))
+          (when (file-readable-p test-file)
+            (insert-file-contents-literally test-file)
+            (when (looking-at-p
+                   (regexp-quote "<?xml version=\"1.0\" encoding=\"utf-8\"?>"))
+              (delete-region (point) (line-beginning-position 2)))
+            (when (looking-at
+                   "<testsuites name=\".+\" tests=\"\\(.+\\)\" failures=\"\\(.+\\)\" skipped=\"\\(.+\\)\" time=\"\\(.+\\)\">")
+              (cl-incf tests (string-to-number (match-string 1)))
+              (cl-incf failures  (string-to-number (match-string 2)))
+              (cl-incf skipped (string-to-number (match-string 3)))
+              (cl-incf time (string-to-number (match-string 4)))
+              (delete-region (point) (line-beginning-position 2)))
+            (when (looking-at "  <testsuite id=\"\\(0\\)\"")
+              (replace-match (number-to-string id) nil nil nil 1)
+              (cl-incf id 1))
+            (goto-char (point-max))
+            (beginning-of-line 0)
+            (when (looking-at-p "</testsuites>")
+              (delete-region (point) (line-beginning-position 2)))
+            (narrow-to-region (point-max) (point-max)))))
+
+      (insert "</testsuites>\n")
+      (widen)
+      (goto-char (point-min))
+      (insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
+      (insert (format "<testsuites name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
+                      (file-name-nondirectory report)
+                      tests failures skipped time)))))
 
 (defun ert-summarize-tests-batch-and-exit (&optional high)
   "Summarize the results of testing.
@@ -1540,6 +1664,8 @@ If HIGH is a natural number, the HIGH long lasting tests are summarized."
   ;; behavior.
   (setq attempt-stack-overflow-recovery nil
         attempt-orderly-shutdown-on-fatal-signal nil)
+  (when (getenv "EMACS_TEST_JUNIT_REPORT")
+    (apply #'ert-write-junit-test-summary-report command-line-args-left))
   (let ((nlogs (length command-line-args-left))
         (ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0)
         nnotrun logfile notests badtests unexpected skipped tests)
@@ -1855,7 +1981,6 @@ Also sets `ert--results-progress-bar-button-begin'."
      ;; should test it again.)
      "\n")))
 
-
 (defvar ert-test-run-redisplay-interval-secs .1
   "How many seconds ERT should wait between redisplays while running tests.
 
@@ -2037,7 +2162,6 @@ STATS is the stats object; LISTENER is the results listener."
           (goto-char (1- (point-max)))
           buffer)))))
 
-
 (defvar ert--selector-history nil
   "List of recent test selectors read from terminal.")
 
diff --git a/test/Makefile.in b/test/Makefile.in
index f2c49584e7f..eeda2918fa3 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -353,6 +353,7 @@ mostlyclean:
 
 clean:
         find . '(' -name '*.log' -o -name '*.log~' ')' $(FIND_DELETE)
+        find . '(' -name '*.xml' -a ! -path '*resources*' ')' $(FIND_DELETE)
         rm -f ${srcdir}/lisp/gnus/mml-sec-resources/random_seed
         rm -f $(test_module_dir)/*.o $(test_module_dir)/*.so \
           $(test_module_dir)/*.dll
diff --git a/test/README b/test/README
index 4d447c9bf15..2bd84b5f9b3 100644
--- a/test/README
+++ b/test/README
@@ -114,6 +114,9 @@ mode--only the names of the failed tests are listed.  If the
 $EMACS_TEST_VERBOSE environment variable is set, the failure summaries
 will also include the data from the failing test.
 
+If the $EMACS_TEST_JUNIT_REPORT environment variable is set to a file
+name, a JUnit test report is generated under this name.
+
 Some of the tests require a remote temporary directory
 (autorevert-tests.el, filenotify-tests.el, shadowfile-tests.el and
 tramp-tests.el).  Per default, a mock-up connection method is used
diff --git a/test/infra/Makefile.in b/test/infra/Makefile.in
index fd11d367983..e4f99743e09 100644
--- a/test/infra/Makefile.in
+++ b/test/infra/Makefile.in
@@ -93,10 +93,8 @@ all: generate-test-jobs
 
 .PHONY: generate-test-jobs $(FILE) $(SUBDIR_TARGETS)
 
-generate-test-jobs: clean $(FILE) $(SUBDIR_TARGETS)
+generate-test-jobs: $(FILE) $(SUBDIR_TARGETS)
 
 $(FILE):
         $(AM_V_GEN)
-
-clean:
-        @rm -f $(FILE)
+        @echo "# Generated by \"make generate-test-jobs\", don't edit." >$(FILE)
diff --git a/test/infra/gitlab-ci.yml b/test/infra/gitlab-ci.yml
index b0ea6813b30..3903642c792 100644
--- a/test/infra/gitlab-ci.yml
+++ b/test/infra/gitlab-ci.yml
@@ -44,6 +44,7 @@ workflow:
 variables:
   GIT_STRATEGY: fetch
   EMACS_EMBA_CI: 1
+  EMACS_TEST_JUNIT_REPORT: junit-test-report.xml
   EMACS_TEST_TIMEOUT: 3600
   EMACS_TEST_VERBOSE: 1
   # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
@@ -85,7 +86,7 @@ default:
     # TODO: with make -j4 several of the tests were failing, for
     # example shadowfile-tests, but passed without it.
     - 'export PWD=$(pwd)'
-    - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"'
+    - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_JUNIT_REPORT=${EMACS_TEST_JUNIT_REPORT} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"'
   after_script:
     # - docker ps -a
     # - printenv
@@ -93,7 +94,8 @@ default:
     # Prepare test artifacts.
     - test -n "$(docker ps -aq -f name=${test_name})" && docker cp ${test_name}:checkout/test ${test_name}
     - test -n "$(docker ps -aq -f name=${test_name})" && docker rm ${test_name}
-    - find ${test_name} ! -name "*.log" -type f -delete
+    - find ${test_name} ! \( -name "*.log" -o -name ${EMACS_TEST_JUNIT_REPORT} \) -type f -delete
+    # BusyBox find does not know -empty.
     - find ${test_name} -type d -depth -exec rmdir {} + 2>/dev/null
 
 .build-template:
@@ -103,7 +105,6 @@ default:
       when: always
     - changes:
         - "**.in"
-        - "**.yml"
         - GNUmakefile
         - aclocal.m4
         - autogen.sh
@@ -112,7 +113,7 @@ default:
         - lib/malloc/*.{h,c}
         - lisp/emacs-lisp/*.el
         - src/*.{h,c}
-        - test/infra/Dockerfile.emba
+        - test/infra/*
     - changes:
         # gfilemonitor, kqueue
         - src/gfilenotify.c
@@ -133,9 +134,11 @@ default:
     name: ${test_name}
     public: true
     expire_in: 1 week
+    when: always
     paths:
       - ${test_name}/
-    when: always
+    reports:
+      junit: ${test_name}/${EMACS_TEST_JUNIT_REPORT}
 
 .gnustep-template:
   rules:
@@ -143,12 +146,11 @@ default:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       changes:
         - "**.in"
-        - "**.yml"
         - src/ns*.{h,m}
         - src/macfont.{h,m}
         - lisp/term/ns-win.el
         - nextstep/**
-        - test/infra/Dockerfile.emba
+        - test/infra/*
 
 .filenotify-gio-template:
   rules:
@@ -156,12 +158,11 @@ default:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       changes:
         - "**.in"
-        - "**.yml"
         - lisp/autorevert.el
         - lisp/filenotify.el
         - lisp/net/tramp-sh.el
         - src/gfilenotify.c
-        - test/infra/Dockerfile.emba
+        - test/infra/*
         - test/lisp/autorevert-tests.el
         - test/lisp/filenotify-tests.el
 
@@ -171,11 +172,10 @@ default:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       changes:
         - "**.in"
-        - "**.yml"
         - lisp/emacs-lisp/comp.el
         - lisp/emacs-lisp/comp-cstr.el
         - src/comp.{h,m}
-        - test/infra/Dockerfile.emba
+        - test/infra/*
         - test/src/comp-resources/*.el
         - test/src/comp-tests.el
   timeout: 8 hours
diff --git a/test/infra/test-jobs.yml b/test/infra/test-jobs.yml
index bad8575b5c5..63b052bf8c7 100644
--- a/test/infra/test-jobs.yml
+++ b/test/infra/test-jobs.yml
@@ -1,3 +1,4 @@
+# Generated by "make generate-test-jobs", don't edit.
 
 test-lib-src-inotify:
   stage: normal