Workaround dumping problem on PaX-enabled Linux kernels

* configure.in (PAXCTL): Check for paxctl.

* src/Makefile.in (PAXCTL): Define.
(temacs$(EXEEXT)): Disable memory randomization for the temacs
binary via PaX flags if the paxctl utility is available.
(emacs$(EXEEXT), bootstrap-emacs$(EXEEXT)):
Restore PaX flags to their default.

Fixes: debbugs:11398

1 files changed, 10 insertions, 0 deletions

diff --git a/src/Makefile.in b/src/Makefile.in
index 27199578fc0..47c7d5cfbc2 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -113,6 +113,12 @@ TEMACS_LDFLAGS = $(LD_SWITCH_SYSTEM) $(LD_SWITCH_SYSTEM_TEMACS)
 ## $LDFLAGS or empty if NS_IMPL_GNUSTEP (for some reason).
 TEMACS_LDFLAGS2 = @TEMACS_LDFLAGS2@
 
+## If available, the full path to the paxctl program.
+## On grsecurity/PaX systems, unexec will fail due to a gap between
+## the bss section and the heap.  This can be prevented by disabling
+## memory randomization in temacs with "paxctl -r".  See bug#11398.
+PAXCTL = @PAXCTL@
+
 ## Some systems define this to request special libraries.
 LIBS_SYSTEM=@LIBS_SYSTEM@
 
@@ -403,6 +409,7 @@ emacs$(EXEEXT): temacs$(EXEEXT) $(etc)/DOC $(lisp) $(leimdir)/leim-list.el
             -f list-load-path-shadows || true; \
         else \
           LC_ALL=C $(RUN_TEMACS) -batch -l loadup dump || exit 1; \
+          test "X$(PAXCTL)" = X || $(PAXCTL) -zex emacs$(EXEEXT); \
           ln -f emacs$(EXEEXT) bootstrap-emacs$(EXEEXT); \
           ./emacs -batch -f list-load-path-shadows || true; \
         fi
@@ -450,6 +457,8 @@ $(lib)/libgnu.a: $(config_h)
 temacs$(EXEEXT): $(START_FILES) stamp-oldxmenu $(obj) $(otherobj) $(lib)/libgnu.a
         $(CC) $(LD_FIRSTFLAG) $(ALL_CFLAGS) $(TEMACS_LDFLAGS) $(TEMACS_LDFLAGS2) \
           -o temacs $(START_FILES) $(obj) $(otherobj) $(lib)/libgnu.a $(LIBES)
+        test "$(CANNOT_DUMP)" = "yes" || \
+          test "X$(PAXCTL)" = X || $(PAXCTL) -r temacs$(EXEEXT)
 
 ## The following oldxmenu-related rules are only (possibly) used if
 ## HAVE_X11 && !USE_GTK, but there is no harm in always defining them
@@ -597,6 +606,7 @@ bootstrap-emacs$(EXEEXT): temacs$(EXEEXT)
           ln -f temacs$(EXEEXT) bootstrap-emacs$(EXEEXT); \
         else \
           $(RUN_TEMACS) --batch --load loadup bootstrap || exit 1; \
+          test "X$(PAXCTL)" = X || $(PAXCTL) -zex emacs$(EXEEXT); \
           mv -f emacs$(EXEEXT) bootstrap-emacs$(EXEEXT); \
         fi
         @: Compile some files earlier to speed up further compilation.