Automate upload of Emacs manuals to gnu.org

* admin/make-manuals, admin/upload-manuals: New scripts.
* admin/admin.el (make-manuals, make-manuals-dist): Handle batch mode.
* admin/make-tarball.txt: Update web-page details.

4 files changed, 604 insertions, 20 deletions

diff --git a/admin/admin.el b/admin/admin.el
index dab61bb9166..3cb5dbc2d92 100644
--- a/admin/admin.el
+++ b/admin/admin.el
@@ -261,8 +261,12 @@ ROOT should be the root of an Emacs source tree."
 ROOT should be the root of an Emacs source tree.
 Interactively with a prefix argument, prompt for TYPE.
 Optional argument TYPE is type of output (nil means all)."
-  (interactive (let ((root (read-directory-name "Emacs root directory: "
-                                                source-directory nil t)))
+  (interactive (let ((root
+                      (if noninteractive
+                          (or (pop command-line-args-left)
+                              default-directory)
+                        (read-directory-name "Emacs root directory: "
+                                             source-directory nil t))))
                  (list root
                        (if current-prefix-arg
                            (completing-read
@@ -717,8 +721,12 @@ style=\"text-align:left\">")
 ROOT should be the root of an Emacs source tree.
 Interactively with a prefix argument, prompt for TYPE.
 Optional argument TYPE is type of output (nil means all)."
-  (interactive (let ((root (read-directory-name "Emacs root directory: "
-                                                source-directory nil t)))
+  (interactive (let ((root
+                      (if noninteractive
+                          (or (pop command-line-args-left)
+                              default-directory)
+                        (read-directory-name "Emacs root directory: "
+                                             source-directory nil t))))
                  (list root
                        (if current-prefix-arg
                            (completing-read
diff --git a/admin/make-manuals b/admin/make-manuals
new file mode 100755
index 00000000000..7b9f6a28718
--- /dev/null
+++ b/admin/make-manuals
@@ -0,0 +1,214 @@
+#!/bin/bash
+### make-manuals - create the Emacs manuals to upload to the gnu.org website
+
+## Copyright 2018 Free Software Foundation, Inc.
+
+## Author: Glenn Morris <rgm@gnu.org>
+
+## This file is part of GNU Emacs.
+
+## GNU Emacs is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+
+## GNU Emacs is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+### Commentary:
+
+## This is a helper script to create the Emacs manuals as used on the
+## gnu.org website.  After this, use upload-manuals to upload them.
+##
+## Usage:
+## Call from the top-level directory of an Emacs source tree.
+## This should normally be a release.
+## The info files should exist.
+
+### Code:
+
+die ()                          # write error to stderr and exit
+{
+    [ $# -gt 0 ] && echo "$PN: $@" >&2
+    exit 1
+}
+
+PN=${0##*/}                     # basename of script
+
+usage ()
+{
+    cat 1>&2 <<EOF
+Usage: ${PN} [-c] [-e emacs]
+Create the Emacs manuals for the gnu.org website.
+Call this script from the top-level of the Emacs source tree that
+contains the manuals you want to use (normally a release).
+The admin/ directory is required.
+Options:
+-c: do not delete any pre-existing $outdir/ directory
+-e: Emacs executable to use (default $emacs)
+EOF
+    exit 1
+}
+
+
+## Defaults.
+continue=
+emacs=emacs
+
+## Parameters.
+outdir=manual
+gzip="gzip --best --no-name"
+tar="tar --numeric-owner --owner=0 --group=0 --mode=go+u,go-w"
+## Requires GNU tar >= 1.28 so that the tarballs are more reproducible.
+## (Personally I think this is way OOT.  I'm not even sure if anyone
+## uses these tarfiles, let alone cares whether they are reproducible.)
+tar --help | grep -- '--sort.*name' >& /dev/null && tar="$tar --sort=name"
+
+while getopts ":hce:" option ; do
+    case $option in
+        (h) usage ;;
+
+        (c) continue=t ;;
+
+        (e) emacs=$OPTARG ;;
+
+        (\?) die "Bad option -$OPTARG" ;;
+
+        (:) die "Option -$OPTARG requires an argument" ;;
+
+        (*) die "getopts error" ;;
+    esac
+done
+shift $(( --OPTIND ))
+OPTIND=1
+
+[ $# -eq 0 ] || usage
+
+
+[ -e admin/admin.el ] || die "admin/admin.el not found"
+
+
+tempfile=/tmp/$PN.$$
+trap "rm -f $tempfile 2> /dev/null" EXIT
+
+
+[ "$continue" ] || rm -rf $outdir
+
+if [ -e $outdir ]; then
+    ## Speed up repeat invocation.
+    echo "Re-using existing $outdir/ directory"
+
+else
+
+    ## This creates the manuals in a manual/ directory.
+    ## Note makeinfo >= 5 is much slower than makeinfo 4.
+    echo "Making manuals (slow)..."
+    $emacs --batch -Q -l admin/admin.el -f make-manuals \
+              >| $tempfile 2>&1 || {
+        cat $tempfile 1>&2
+
+        die "error running make-manuals"
+    }
+fi
+
+find $outdir -name '*~' -exec rm {} +
+
+
+echo "Adding compressed html files..."
+for f in emacs elisp; do
+    $tar -C $outdir/html_node -cf - $f | $gzip \
+       > $outdir/$f.html_node.tar.gz || die "error for $f"
+done
+
+
+echo "Making manual tarfiles..."
+$emacs --batch -Q -l admin/admin.el -f make-manuals-dist \
+          >| $tempfile || {
+
+    cat $tempfile 1>&2
+
+    die "error running make-manuals-dist"
+}
+
+o=$outdir/texi
+mkdir -p $o
+
+for f in $outdir/*.tar; do
+    of=${f##*/}
+    of=${of#emacs-}
+    of=${of%%-[0-9]*}.texi.tar
+    of=${of/lispintro/eintr}
+    of=${of/lispref/elisp}
+    of=${of/manual/emacs}
+    of=$o/$of
+    mv $f $of
+    $gzip $of || die "error compressing $f"
+done
+
+
+echo "Making refcards..."
+make -C etc/refcards dist >| $tempfile 2>&1 || {
+    cat $tempfile 1>&2
+    die "failed make dist"
+}
+
+## This may hang if eg german.sty is missing.
+make -k -C etc/refcards pdf >| $tempfile 2>&1 || {
+    cat $tempfile 1>&2
+    echo "Warning: ignored failure(s) from make pdf"
+}
+
+## Newer Texlive only provide mex (used by pl refcards) for pdf, AFAICS.
+make -k -C etc/refcards ps >| $tempfile 2>&1 || {
+    cat $tempfile 1>&2
+    echo "Warning: ignored failure(s) from make ps"
+}
+
+## Note that in the website, refcards/ is not a subdirectory of manual/.
+refdir=$outdir/refcards
+
+mkdir -p $refdir
+
+mv etc/refcards/emacs-refcards.tar $refdir
+$gzip $refdir/emacs-refcards.tar
+
+for fmt in pdf ps; do
+
+    o=$refdir/$fmt
+
+    mkdir -p $o
+
+    [ $fmt = pdf ] && {
+        cp etc/refcards/*.$fmt $o
+        rm $o/gnus-logo.pdf
+        continue
+    }
+
+    for f in  etc/refcards/*.$fmt; do
+        $gzip < $f > $o/${f##*/}.gz
+    done
+done
+
+make -C etc/refcards extraclean > /dev/null
+
+
+echo "Adding compressed info files..."
+
+o=$outdir/info
+mkdir -p $o
+
+for f in eintr.info elisp.info emacs.info; do
+
+    $gzip < info/$f > $o/$f.gz || die "error for $f"
+done
+
+
+echo "Finished OK, you might want to run upload-manuals now"
+
+
+exit 0
diff --git a/admin/make-tarball.txt b/admin/make-tarball.txt
index 6d6312c9b1b..092027d1e2a 100644
--- a/admin/make-tarball.txt
+++ b/admin/make-tarball.txt
@@ -203,22 +203,8 @@ For every new release, a banner is displayed on top of the emacs.html
 page.  Uncomment and the release banner in emacs.html.  Keep it on the
 page for about a month, then comment it again.
 
-Use M-x make-manuals from admin/admin.el to regenerate the html
-manuals in manual/.  If there are new manuals, add appropriate index
-pages in manual/ and add them to manual/index.html.  In the
-manual/html_node directory, delete any old manual pages that are no
-longer present.
-
-Tar up the generated html_node/emacs/ and elisp/ directories and update
-the files manual/elisp.html_node.tar.gz and emacs.html_node.tar.gz.
-
-Use M-x make-manuals-dist from admin/admin.el to update the
-manual/texi/ tarfiles.
-
-Add compressed copies of the main info pages from the tarfile to manual/info/.
-
-Update the refcards/pdf/ and ps/ directories, and also
-refcards/emacs-refcards.tar.gz (use make -C etc/refcards pdf ps dist).
+Regenerate the various manuals in manual/.
+The scripts admin/make-manuals and admin/upload-manuals summarize the process.
 
 Browsing <https://web.cvs.savannah.gnu.org/viewvc/?root=emacs> is one
 way to check for any files that still need updating.
diff --git a/admin/upload-manuals b/admin/upload-manuals
new file mode 100755
index 00000000000..1aa7d8be32d
--- /dev/null
+++ b/admin/upload-manuals
@@ -0,0 +1,376 @@
+#!/bin/bash
+
+### upload-manuals - upload the Emacs manuals to the gnu.org website
+
+## Copyright 2018 Free Software Foundation, Inc.
+
+## Author: Glenn Morris <rgm@gnu.org>
+
+## This file is part of GNU Emacs.
+
+## GNU Emacs is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+
+## GNU Emacs is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+### Commentary:
+
+## Run this on the output of make-manuals.
+
+## We assume you have already checked out a local copy of the website
+## following the instructions at
+## https://savannah.gnu.org/cvs/?group=emacs
+
+## Usage:
+## Call from the manual/ directory created by make-manual.
+## upload-manuals /path/to/cvs/checkout
+
+### Code:
+
+
+die ()                          # write error to stderr and exit
+{
+    [ $# -gt 0 ] && echo "$PN: $@" >&2
+    exit 1
+}
+
+PN=${0##*/}                     # basename of script
+
+usage ()
+{
+    cat 1>&2 <<EOF
+Usage: ${PN} [-m message] [-n] /path/to/cvs/checkout
+Upload the Emacs manuals to the gnu.org website.
+Call this script from the manual/ directory created by make-manuals.
+This script destructively modifies the source directory.
+Options:
+-m: commit message to use (default "$message")
+-n: dry-run (do not commit files)
+EOF
+    exit 1
+}
+
+
+## Parameters
+version=$(gunzip -c info/emacs.info.gz 2> /dev/null | sed -n '0,/updated for Emacs version/s/.*updated for Emacs version \([0-9.]*\).*\.$/\1/p')
+
+## Defaults
+cvs=cvs
+message="Regenerate manuals for Emacs $version"
+umessage=
+
+while getopts ":hm:n" option ; do
+    case $option in
+        (h) usage ;;
+
+        (m) umessage=t ; message="$OPTARG" ;;
+
+        (n) cvs="echo $cvs" ;;
+
+        (\?) die "Bad option -$OPTARG" ;;
+
+        (:) die "Option -$OPTARG requires an argument" ;;
+
+        (*) die "getopts error" ;;
+    esac
+done
+shift $(( --OPTIND ))
+OPTIND=1
+
+[ $# -eq 1 ] || usage
+
+[ "$version$umessage" ] || \
+    die "Could not get version to use for commit message"
+
+webdir=$1
+
+[ -e $webdir/CVS/Entries ] && [ -e $webdir/refcards/pdf/refcard.pdf ] || \
+    die "$webdir does not look like a checkout of the Emacs webpages"
+
+[ -e html_mono/emacs.html ] && [ -e html_node/emacs/index.html ] || \
+    die "Current directory does not like the manual/ directory"
+
+
+echo "Doing refcards..."
+
+mv refcards/emacs-refcards.tar.gz $webdir/refcards/
+(
+    cd $webdir/refcards
+    $cvs commit -m "$message" emacs-refcards.tar.gz || die "commit error"
+)
+
+## For refcards, we assume a missing file is due to a tex failure,
+## rather than a refcard that should be deleted.
+for fmt in pdf ps.gz; do
+
+    clist=
+
+    for f in $webdir/refcards/${fmt%.gz}/*.$fmt; do
+
+        s=${f#$webdir/}
+
+        if [ -e $s ]; then
+            mv $s $f
+            clist="$clist ${f##*/}"
+        else
+            echo "$s seems to be missing"
+        fi
+    done
+
+
+    ## Check for new files.
+    new=
+    for f in refcards/${fmt%.gz}/*.$fmt; do
+        [ -e $f ] || break
+        new="$new $f"
+        clist="$clist ${f##*/}"
+    done
+
+    [ "$new" ] && mv $new $webdir/refcards/${fmt%.gz}/
+
+    [ "$clist" ] && (
+        cd $webdir
+        [ "$new" ] && {
+            echo "Adding new files: $new"
+            $cvs add -kb $new || die "add error"
+            echo "Remember to add new refcards to refcards/index.html"
+        }
+        cd refcards/${fmt%.gz}
+        $cvs commit -m "$message" $clist || die "commit error"
+    )
+
+done                            # $fmt
+
+
+echo "Doing non-html manuals..."
+
+for fmt in info pdf ps texi; do
+
+    clist=
+
+    for f in $webdir/manual/$fmt/*; do
+
+        [ ${f##*/} = CVS ] && continue
+
+        s=$fmt/${f##*/}
+
+        if [ -e $s ]; then
+            mv $s $f
+            clist="$clist ${f##*/}"
+        else
+            case ${f##*/} in
+                *_7x9*.pdf) continue ;;
+            esac
+
+            echo "$s seems to be missing"
+        fi
+    done
+
+    ## Check for new files.
+    new=
+    for f in $fmt/*.$fmt*; do
+        [ -e $f ] || break
+        new="$new $f"
+        clist="$clist ${f##*/}"
+    done
+
+    [ "$new" ] && mv $new $webdir/manual/$fmt/
+
+    [ "$clist" ] && (
+        cd $webdir/manual
+        [ "$new" ] && {
+            echo "Adding new files: $new"
+            $cvs add $new || die "add error"
+            echo "Remember to add new files to the appropriate index pages"
+        }
+        cd $fmt
+        $cvs commit -m "$message" $clist || die "commit error"
+    )
+
+done
+
+
+echo "Doing tarred html..."
+
+clist=
+
+for f in $webdir/manual/*html*.tar*; do
+
+    s=${f##*/}
+
+    if [ -e $s ]; then
+        mv $s $f
+        clist="$clist ${f##*/}"
+    else
+        echo "$s seems to be missing"
+    fi
+done
+
+## Check for new files.
+new=
+for f in *html*.tar*; do
+    [ -e $f ] || break
+    new="$new $f"
+    clist="$clist ${f##*/}"
+done
+
+[ "$new" ] && mv $new $webdir/manual
+
+[ "$clist" ] && (
+    cd $webdir/manual
+    [ "$new" ] && {
+        echo "Adding new files: $new"
+        $cvs add -kb $new || die "add error"
+        echo "Remember to add new files to the appropriate index pages"
+    }
+    $cvs commit -m "$message" $clist || die "commit error"
+)
+
+
+## This happens so rarely it would be less effort to do by hand...
+new_manual () {
+    local t=eww
+    local i=$webdir/manual/$t.html # template
+
+    [ -r $i ] || die "Cannot read template $i"
+
+    local name o mono title
+
+    for name; do
+
+        name=${name##*/}
+        name=${name%.html}
+
+        o=$webdir/manual/$name.html
+
+        [ -e $o ] && die "$o already exists"
+
+        mono=$webdir/manual/html_mono/$name.html
+
+        [ -r $mono ] || die "Cannot read $mono"
+
+        title=$(sed -n 's|^<title>\(.*\)</title>|\1|p' $mono)
+
+        : ${title:?}
+
+        echo "$title" | grep -qi "Emacs" || title="Emacs $title"
+        echo "$title" | grep -qi "manual" || title="$title Manual"
+
+        ## It is a pain to extract and insert a good "documenting...".
+        ## Improve it by hand if you care.
+        sed -e "s|^<title>.*\( - GNU Project\)|<title>$title\1|" \
+            -e "s|^<h2>.*|<h2>$title</h2>|" \
+            -e "s/^documenting.*/documenting \"$title\"./" \
+            -e "s/$t/$name/" \
+            -e "s/&copy;.* Free/\&copy; $(date +%Y) Free/" $i > $o
+
+        (
+            cd $webdir/manual
+            $cvs add ${o##*/} || die "add error for $o"
+        )
+    done
+
+    return 0
+}
+
+
+echo "Doing html_mono..."
+
+clist=
+
+for f in $webdir/manual/html_mono/*.html; do
+
+    s=${f##*manual/}
+
+    if [ -e $s ]; then
+        mv $s $f
+        clist="$clist ${f##*/}"
+    else
+        echo "$s seems to be missing"
+    fi
+done
+
+## Check for new files.
+new=
+for f in html_mono/*.html; do
+    [ -e $f ] || break
+    new="$new $f"
+    clist="$clist ${f##*/}"
+done
+
+[ "$new" ] && mv $new $webdir/manual/html_mono/
+
+## TODO: check for removed manuals.
+
+[ "$clist" ] && (
+    cd $webdir/manual/html_mono
+    [ "$new" ] && {
+        echo "Adding new files: $new"
+        $cvs add $new || die "add error"
+        new_manual $new || die
+        echo "Remember to add new entries to manual/index.html"
+    }
+    $cvs commit -m "$message" $clist || die "commit error"
+)
+
+
+echo "Doing html_node..."
+
+for d in html_node/*; do
+
+    [ -e $d ] || break
+
+    echo "Doing $d..."
+
+    [ -e $webdir/manual/$d ] || {
+        echo "New directory: $d"
+        mkdir $webdir/manual/$d
+        $cvs add $webdir/manual/$d || die "add error"
+    }
+
+    new=
+    for f in $d/*.html; do
+        [ -e $webdir/manual/$f ] || new="$new ${f##*/}"
+    done
+
+    stale=
+    for f in $webdir/manual/$d/*.html; do
+        [ -e ${f#$webdir/manual/} ] || stale="$stale ${f##*/}"
+    done
+
+    mv $d/*.html $webdir/manual/$d/
+
+    (
+        cd $webdir/manual/$d
+        [ "$new" ] && {
+            echo "Adding new files: $new"
+            $cvs add $new || die "add error"
+        }
+
+        [ "$stale" ] && {
+            echo "Removing stale files: $stale"
+            $cvs remove -f $stale || die "remove error"
+        }
+
+        ## -f: create a new revision even if no change.
+        $cvs commit -f -m "$message" *.html $stale || die "commit error"
+    )
+
+done
+
+
+echo "Checking for stray files..."
+find -type f
+
+
+echo "Finished"
+
+exit 0