* admin/nt/dist-build/build-dep-zips.py: (deps src) handle zst

This patch fixes various problems preventing this script from
working. The job of this script is to collect Emacs' dependanicies and
the (MSYS2/MinGW64) source archives for dependancies (including 2nd
etc. order dependancies) into zip archives.  GPL requires distributing such
archives (along with Emacs' own sources) when distributing GPL binaries.

Most principle of the changes is support for MSYS2 "w64" source archives
in zst format; previously this script assumed all sources were available
as tar.gz files.   Additionaly several DLLs are added to list of
required, including sqllite3 and tree-sitter.

In general this is a maintaince fix to ensure others have access to the
code used to create pretest (and, presumably release) version of Emacs
30.  Additional changes are required to progmatically identify changes
in Emacs' dependancies (e.g. after building a new Emacs binary for
distribution).  There is commented out code to this effect in the
changes I've installed which did work, for a brief period in time; I
suspect some changes in MSYS2 made it more difficult (or impossible?) to
invoke the MSYS2 shell (under which this python script must run) from
the MINGW64 (under which Emacs is built).  I may resort to a powershell
script as a way to orchistrate/invoke both MSYS2 and MING64 but I'm
currently still messing with a Makefile based approach (which would run
under MINGW64), in any event I believe we're all agreed this script
must interigate Emacs to create the "starting list" of dependancies to
include as a way to resolve this issue (missing/incorrect deps are
frequent source of broken binary releases).  (bug#65206)

1 files changed, 213 insertions, 67 deletions

diff --git a/admin/nt/dist-build/build-dep-zips.py b/admin/nt/dist-build/build-dep-zips.py
index 0b1cc4d8695..c145cd3fc00 100755
--- a/admin/nt/dist-build/build-dep-zips.py
+++ b/admin/nt/dist-build/build-dep-zips.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-## Copyright (C) 2017-2024 Free Software Foundation, Inc.
+## Copyright (C) 2017-2023 Free Software Foundation, Inc.
 
 ## This file is part of GNU Emacs.
 
@@ -20,41 +20,56 @@ import argparse
 import os
 import shutil
 import re
-import functools
-import operator
+import subprocess
 
 from subprocess import check_output
 
 ## Constants
-EMACS_MAJOR_VERSION="28"
+EMACS_MAJOR_VERSION= os.getenv('EMACS_MAJOR_VERSION') or "30"
 
-# This list derives from the features we want Emacs to compile with.
+# Base URI for the package sources mapped in PKG_REQ
+SRC_REPO="https://repo.msys2.org/mingw/sources"
+
+# Map items in `dynamic-library-alist' to source pakages
 PKG_REQ='''mingw-w64-x86_64-giflib
 mingw-w64-x86_64-gnutls
 mingw-w64-x86_64-harfbuzz
+mingw-w64-x86_64-jansson
 mingw-w64-x86_64-lcms2
 mingw-w64-x86_64-libjpeg-turbo
 mingw-w64-x86_64-libpng
 mingw-w64-x86_64-librsvg
+mingw-w64-x86_64-libwebp
 mingw-w64-x86_64-libtiff
 mingw-w64-x86_64-libxml2
-mingw-w64-x86_64-xpm-nox'''.split()
-
-DLL_REQ='''libgif
-libgnutls
-libharfbuzz
-liblcms2
-libturbojpeg
-libpng
-librsvg
-libtiff
-libxml
-libXpm'''.split()
-
+mingw-w64-x86_64-gmp
+mingw-w64-x86_64-xpm-nox
+mingw-w64-x86_64-tree-sitter
+mingw-w64-x86_64-sqlite3'''.split()
+
+# Emacs style path to dependancy DLLs on build system
+DLL_SRC="c:/msys64/mingw64/bin"
+
+# libraries we never include
+DLL_SKIP=["libgccjit-0.dll"]
+
+# Report first existing file for entries in dynamic-library-alist
+# ELISP_PROG="""
+# (message "%s" (mapconcat 'identity (remove nil
+#       (mapcar (lambda(lib)
+#                 (seq-find
+#                  (lambda(file)
+#                    (file-exists-p
+#                     (file-name-concat "{}"
+#                                       file)))
+#                  (cdr lib)))
+#               dynamic-library-alist)
+#       ) "\\n"))
+# """.format(DLL_SRC)
 
 ## Options
 DRY_RUN=False
-
+# NEW_EMACS="bin/emacs.exe"
 
 def check_output_maybe(*args,**kwargs):
     if(DRY_RUN):
@@ -62,31 +77,76 @@ def check_output_maybe(*args,**kwargs):
     else:
         return check_output(*args,**kwargs)
 
+####################
 ## DLL Capture
+
+# entry point
 def gather_deps():
 
     os.mkdir("x86_64")
     os.chdir("x86_64")
 
-    for dep in full_dll_dependency():
-        check_output_maybe(["cp /mingw64/bin/{}*.dll .".format(dep)],
-                           shell=True)
-
-    print("Zipping")
-    check_output_maybe("zip -9r ../emacs-{}-{}deps.zip *"
-                       .format(EMACS_MAJOR_VERSION, DATE),
-                       shell=True)
+    #full=full_dll_dependency(init_deps())
+    #filtered=filter(lambda x: x not in DLL_SKIP, full)
+    #print("full:",full.len(), " filtered:",filtered.len())
+    #exit
+
+    for dep in full_dll_dependency(init_deps()):
+        if dep not in DLL_SKIP:
+            if args.l != True:
+                print("Adding dep", dep)
+            check_output_maybe(["cp /mingw64/bin/{} .".format(dep)], shell=True)
+        else:
+            if args.l != True:
+                print("Skipping dep", dep)
+
+    zipfile="../emacs-{}-{}deps.zip".format(EMACS_MAJOR_VERSION, DATE)
+    tmpfile="{}.tmp".format(zipfile)
+    print("Zipping deps in", os.getcwd(), "as", tmpfile)
+    check_output_maybe("zip -9vr {} *.dll".format(tmpfile), shell=True)
+    if os.path.isfile(zipfile):
+        os.remove(zipfile)
+    os.rename(tmpfile, zipfile)
+    print("Deps updated in", os.getcwd(), "as", zipfile)
     os.chdir("../")
 
-## Return all Emacs dependencies
-def full_dll_dependency():
-    deps = [dll_dependency(dep) for dep in DLL_REQ]
-    return set(sum(deps, []) + DLL_REQ)
-
-## Dependencies for a given DLL
+# Return dependancies listed in Emacs
+def init_deps():
+    return '''libXpm-nox4.dll
+libpng16-16.dll
+libjpeg-8.dll
+libgif-7.dll
+librsvg-2-2.dll
+libwebp-7.dll
+libwebpdemux-2.dll
+libsqlite3-0.dll
+libgdk_pixbuf-2.0-0.dll
+libglib-2.0-0.dll
+libgio-2.0-0.dll
+libgobject-2.0-0.dll
+libgnutls-30.dll
+libxml2-2.dll
+zlib1.dll
+liblcms2-2.dll
+libgccjit-0.dll
+libtree-sitter.dll'''.split()
+    # job_args=[NEW_EMACS, "--batch", "--eval", ELISP_PROG]
+    # #print("args: ", job_args)
+    # return subprocess.check_output(job_args, stderr=subprocess.STDOUT
+    #                                ).decode('utf-8').splitlines()
+
+# Return all second order dependencies
+def full_dll_dependency(dlls):
+    deps = [dll_dependency(dep) for dep in dlls]
+    return set(sum(deps, []) + dlls)
+
+#xs = filter(lambda x: x.attribute == value, xs)
+
+# Dependencies for a given DLL
 def dll_dependency(dll):
     output = check_output(["/mingw64/bin/ntldd", "--recursive",
-                           "/mingw64/bin/{}*.dll".format(dll)]).decode("utf-8")
+                           "/mingw64/bin/{}".format(dll)]
+                          ).decode("utf-8")
     ## munge output
     return ntldd_munge(output)
 
@@ -101,9 +161,8 @@ def ntldd_munge(out):
 
         ## if it's the former, we want it, if its the later we don't
         splt = dep.split()
-        if len(splt) > 2 and "msys64" in splt[2]:
-            print("Adding dep", splt[0])
-            rtn.append(splt[0].split(".")[0])
+        if len(splt) > 2 and "mingw64" in splt[2]:
+            rtn.append(splt[0])
 
     return rtn
 
@@ -112,26 +171,92 @@ def ntldd_munge(out):
 ## Packages to fiddle with
 ## Source for gcc-libs is part of gcc
 SKIP_SRC_PKGS=["mingw-w64-gcc-libs"]
-SKIP_DEP_PKGS=frozenset(["mingw-w64-x86_64-glib2"])
-MUNGE_SRC_PKGS={"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git"}
+SKIP_DEP_PKGS=["mingw-w64-glib2", "mingw-w64-ca-certificates-20211016-3"]
+MUNGE_SRC_PKGS={
+    "mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git",
+    "mingw-w64-gettext-runtime":"mingw-w64-gettext"
+}
 MUNGE_DEP_PKGS={
     "mingw-w64-x86_64-libwinpthread":"mingw-w64-x86_64-libwinpthread-git",
     "mingw-w64-x86_64-libtre": "mingw-w64-x86_64-libtre-git",
 }
+SRC_EXT={
+    "mingw-w64-freetype": ".src.tar.zst",
+    "mingw-w64-fribidi": ".src.tar.zst",
+    "mingw-w64-glib2": ".src.tar.zst",
+    "mingw-w64-harfbuzz": ".src.tar.zst",
+    "mingw-w64-libunistring": ".src.tar.zst",
+    "mingw-w64-winpthreads-git": ".src.tar.zst",
+    "mingw-w64-ca-certificates": ".src.tar.zst",
+    "mingw-w64-libxml2": ".src.tar.zst",
+    "mingw-w64-ncurses": ".src.tar.zst",
+    "mingw-w64-openssl": ".src.tar.zst",
+    "mingw-w64-pango": ".src.tar.zst",
+    "mingw-w64-python": ".src.tar.zst",
+    "mingw-w64-sqlite3": ".src.tar.zst",
+    "mingw-w64-xpm-nox": ".src.tar.zst",
+    "mingw-w64-xz": ".src.tar.zst",
+    "mingw-w64-bzip2": ".src.tar.zst",
+    "mingw-w64-cairo": ".src.tar.zst",
+    "mingw-w64-expat": ".src.tar.zst",
+    "mingw-w64-fontconfig":  ".src.tar.zst",
+    "mingw-w64-gdk-pixbuf2":  ".src.tar.zst",
+    "mingw-w64-giflib":  ".src.tar.zst",
+    "mingw-w64-gmp":  ".src.tar.zst",
+    "mingw-w64-gnutls":  ".src.tar.zst",
+    "mingw-w64-graphite2":  ".src.tar.zst",
+    "mingw-w64-jbigkit":  ".src.tar.zst",
+    "mingw-w64-lcms2":  ".src.tar.zst",
+    "mingw-w64-lerc":  ".src.tar.zst",
+    "mingw-w64-libdatrie":  ".src.tar.zst",
+    "mingw-w64-libffi":  ".src.tar.zst",
+    "mingw-w64-libiconv":  ".src.tar.zst",
+    "mingw-w64-libiconv":  ".src.tar.zst",
+    "mingw-w64-libpng":  ".src.tar.zst",
+    "mingw-w64-librsvg": ".src.tar.zst",
+    "mingw-w64-libsystre": ".src.tar.zst",
+    "mingw-w64-libtasn": ".src.tar.zst",
+    "mingw-w64-libthai": ".src.tar.zst",
+    "mingw-w64-libtiff": ".src.tar.zst",
+    "mingw-w64-libtre-git": ".src.tar.zst",
+    "mingw-w64-libwebp": ".src.tar.zst",
+    "mingw-w64-mpdecimal": ".src.tar.zst",
+    "mingw-w64-nettle": ".src.tar.zst",
+    "mingw-w64-p11-kit": ".src.tar.zst",
+    "mingw-w64-pcre": ".src.tar.zst",
+    "mingw-w64-pixman": ".src.tar.zst",
+    "mingw-w64-python-packaging": ".src.tar.zst",
+    "mingw-w64-readline": ".src.tar.zst",
+    "mingw-w64-tcl": ".src.tar.zst",
+    "mingw-w64-termcap": ".src.tar.zst",
+    "mingw-w64-tk": ".src.tar.zst",
+    "mingw-w64-tree-sitter": ".src.tar.zst",
+    "mingw-w64-tzdata": ".src.tar.zst",
+    "mingw-w64-wineditline": ".src.tar.zst",
+    "mingw-w64-zlib": ".src.tar.zst",
+    "mingw-w64-zstd": ".src.tar.zst",
+    "mingw-w64-brotli": ".src.tar.zst",
+    "mingw-w64-gettext": ".src.tar.zst",
+    "mingw-w64-libdeflate": ".src.tar.zst",
+    "mingw-w64-libidn2": ".src.tar.zst",
+    "mingw-w64-libjpeg-turbo": ".src.tar.zst",
+    "mingw-w64-libtasn1": ".src.tar.zst",
+    "mingw-w64-pcre2": ".src.tar.zst",
+}
 
 ## Currently no packages seem to require this!
 ARCH_PKGS=[]
-SRC_REPO="https://repo.msys2.org/mingw/sources"
 
+def immediate_deps(pkg):
+    package_info = check_output(["pacman", "-Si", pkg]).decode("utf-8").split("\n")
 
-def immediate_deps(pkgs):
-    package_info = check_output(["pacman", "-Si"] + pkgs).decode("utf-8").splitlines()
+    ## Extract the "Depends On" line
+    depends_on = [x for x in package_info if x.startswith("Depends On")][0]
+    ## Remove "Depends On" prefix
+    dependencies = depends_on.split(":")[1]
 
-    ## Extract the packages listed for "Depends On:" lines.
-    dependencies = [line.split(":")[1].split() for line in package_info
-                    if line.startswith("Depends On")]
-    ## Flatten dependency lists from multiple packages into one list.
-    dependencies = functools.reduce(operator.iconcat, dependencies, [])
+    ## Split into dependencies
+    dependencies = dependencies.strip().split(" ")
 
     ## Remove > signs TODO can we get any other punctuation here?
     dependencies = [d.split(">")[0] for d in dependencies if d]
@@ -147,18 +272,16 @@ def extract_deps():
     print( "Extracting deps" )
 
     # Get a list of all dependencies needed for packages mentioned above.
-    pkgs = set(PKG_REQ)
-    newdeps = pkgs
-    print("adding...")
-    while True:
-        subdeps = frozenset(immediate_deps(list(newdeps)))
-        newdeps = subdeps - SKIP_DEP_PKGS - pkgs
-        if not newdeps:
-            break
-        print('\n'.join(newdeps))
-        pkgs |= newdeps
+    pkgs = PKG_REQ[:]
+    n = 0
+    while n < len(pkgs):
+        subdeps = immediate_deps(pkgs[n])
+        for p in subdeps:
+            if not (p in pkgs or p in SKIP_DEP_PKGS):
+                pkgs.append(p)
+        n = n + 1
 
-    return list(pkgs)
+    return sorted(pkgs)
 
 
 def download_source(tarball):
@@ -206,14 +329,24 @@ def gather_source(deps):
         ## Switch names if necessary
         pkg_name = MUNGE_SRC_PKGS.get(pkg_name,pkg_name)
 
-        tarball = "{}-{}.src.tar.gz".format(pkg_name,pkg_version)
+        ## src archive is usually a .tar.gz
+        if pkg_name in SRC_EXT.keys():
+            src_ext = SRC_EXT[pkg_name]
+        else:
+            src_ext = ".src.tar.gz"
+
+        tarball = "{}-{}{}".format(pkg_name,pkg_version,src_ext)
 
         download_source(tarball)
 
-    print("Zipping")
-    check_output_maybe("zip -9 ../emacs-{}-{}deps-mingw-w64-src.zip *"
-                       .format(EMACS_MAJOR_VERSION,DATE),
-                       shell=True)
+    srczip="../emacs-{}-{}deps-mingw-w64-src.zip".format(EMACS_MAJOR_VERSION,DATE)
+    tmpzip="{}.tmp".format(srczip)
+    print("Zipping Dsrc in", os.getcwd(), "as", tmpzip)
+    check_output_maybe("zip -9 {} *".format(tmpzip), shell=True)
+    if os.path.isfile(srczip):
+        os.remove(srczip)
+    os.rename(tmpzip, srczip)
+    print("Dsrc updated in", os.getcwd(), "as", srczip)
 
     os.chdir("..")
 
@@ -231,6 +364,9 @@ if(os.environ["MSYSTEM"] != "MSYS"):
 
 
 parser = argparse.ArgumentParser()
+
+#parser.add_argument("emacs", help="emacs executable")
+
 parser.add_argument("-s", help="snapshot build",
                     action="store_true")
 
@@ -243,19 +379,29 @@ parser.add_argument("-c", help="clean only",
 parser.add_argument("-d", help="dry run",
                     action="store_true")
 
-parser.add_argument("-l", help="list dependencies only",
+parser.add_argument("-l", help="list dependencies",
+                    action="store_true")
+
+parser.add_argument("-e", help="extract direct dependancies",
                     action="store_true")
 
 args = parser.parse_args()
 do_all=not (args.c or args.r)
 
-
+#NEW_EMACS=args.emacs
 
 DRY_RUN=args.d
 
+if( args.e ):
+    print("\n".join(init_deps()))
+
 if( args.l ):
-    print("List of dependencies")
-    print( deps )
+    print("List of dependencies:")
+    print(full_dll_dependency(init_deps()))
+    print("List of source packages:")
+    print( extract_deps() )
+
+if( args.e or args.l ):
     exit(0)
 
 if args.s: