Clean up memory allocation and unexec support on MS-Windows

* src/w32heap.c (report_temacs_memory_usage): Condition on
!CANNOT_DUMP, in addition to ENABLE_CHECKING.
(init_heap): Accept an argument, which tells us what heap
allocation method to use.
(DUMPED_HEAP_SIZE) [CANNOT_DUMP]: Define to a small value, as
we don't use dumped_data[] in this case.
* src/w32heap.h (init_heap): Adjust prototype.
<using_dynamic_heap>: Remove declaration.
* src/emacs.c (main) [WINDOWSNT]: Determine heap allocation
method based on whether we are in temacs and whether unexec
will be used to dump Emacs.  Pass the heap allocation method
to init_heap, which is now called after parsing the
--temacs=METHOD option.
* src/unexw32.c (unexec): Don't fiddle with using_dynamic_heap.
<using_dynamic_heap>: Remove definition.
* src/w32proc.c (malloc_before_init, realloc_before_init)
(free_before_init): New functions, to catch memory allocation
before heap allocation method is set up.

5 files changed, 78 insertions, 40 deletions

diff --git a/src/emacs.c b/src/emacs.c
index 834f55b6f32..b9f50397241 100644
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -871,8 +871,25 @@ main (int argc, char **argv)
 #endif
 
 #if defined WINDOWSNT || defined HAVE_NTGUI
+  /* Grab our malloc arena space now, before anything important
+     happens.  This relies on the static heap being needed only in
+     temacs and only if we are going to dump with unexec.  */
+  bool use_dynamic_heap = false;
+  if (strstr (argv[0], "temacs") != NULL)
+    {
+      eassert (temacs);
+      /* Note that gflags are set at this point only if we have been
+         called with the --temacs=METHOD option.  We assume here that
+         temacs is always called that way, otherwise the functions
+         that rely on gflags, like will_dump_with_pdumper_p below,
+         will not do their job.  */
+      use_dynamic_heap = will_dump_with_pdumper_p ();
+    }
+  else
+    use_dynamic_heap = true;
+  init_heap (use_dynamic_heap);
   /* Set global variables used to detect Windows version.  Do this as
-     early as possible.  (unexw32.c calls this function as well, but
+     early as possible.  (w32proc.c calls this function as well, but
      the additional call here is harmless.) */
   cache_system_info ();
 #ifdef WINDOWSNT
diff --git a/src/unexw32.c b/src/unexw32.c
index 6fa0fa055a6..59feaa74b9f 100644
--- a/src/unexw32.c
+++ b/src/unexw32.c
@@ -45,9 +45,6 @@ extern char *my_begbss_static;
 
 #include "w32heap.h"
 
-/* Basically, our "initialized" flag.  */
-BOOL using_dynamic_heap = FALSE;
-
 void get_section_info (file_data *p_file);
 void copy_executable_and_dump_data (file_data *, file_data *);
 void dump_bss_and_heap (file_data *p_infile, file_data *p_outfile);
@@ -649,15 +646,8 @@ unexec (const char *new_name, const char *old_name)
       exit (1);
     }
 
-  /* Set the flag (before dumping).  */
-  using_dynamic_heap = TRUE;
-
   copy_executable_and_dump_data (&in_file, &out_file);
 
-  /* Unset it because it is plain wrong to keep it after dumping.
-     Malloc can still occur!  */
-  using_dynamic_heap = FALSE;
-
   /* Patch up header fields; profiler is picky about this. */
   {
     PIMAGE_DOS_HEADER dos_header;
diff --git a/src/w32heap.c b/src/w32heap.c
index 3de8f245ccc..8a2c1b5877b 100644
--- a/src/w32heap.c
+++ b/src/w32heap.c
@@ -28,7 +28,7 @@
   Memory allocation scheme for w32/w64:
 
   - Buffers are mmap'ed using a very simple emulation of mmap/munmap
-  - During the temacs phase:
+  - During the temacs phase, if unexec is to be used:
     * we use a private heap declared to be stored into the `dumped_data'
     * unfortunately, this heap cannot be made growable, so the size of
       blocks it can allocate is limited to (0x80000 - pagesize)
@@ -37,7 +37,7 @@
       We use a very simple first-fit scheme to reuse those blocks.
     * we check that the private heap does not cross the area used
       by the bigger chunks.
-  - During the emacs phase:
+  - During the emacs phase, or always if pdumper is used:
     * we create a private heap for new memory blocks
     * we make sure that we never free a block that has been dumped.
       Freeing a dumped block could work in principle, but may prove
@@ -115,10 +115,16 @@ typedef struct _RTL_HEAP_PARAMETERS {
    than half of the size stated below.  It would be nice to find a way
    to build only the first bootstrap-emacs.exe with the large size,
    and reset that to a lower value afterwards.  */
-#if defined _WIN64 || defined WIDE_EMACS_INT
-# define DUMPED_HEAP_SIZE (23*1024*1024)
+#ifdef CANNOT_DUMP
+/* We don't use dumped_data[] when CANNOT_DUMP, so define to a small
+   size that won't matter.  */
+# define DUMPED_HEAP_SIZE 10
 #else
-# define DUMPED_HEAP_SIZE (13*1024*1024)
+# if defined _WIN64 || defined WIDE_EMACS_INT
+#  define DUMPED_HEAP_SIZE (23*1024*1024)
+# else
+#  define DUMPED_HEAP_SIZE (13*1024*1024)
+# endif
 #endif
 
 static unsigned char dumped_data[DUMPED_HEAP_SIZE];
@@ -173,8 +179,8 @@ static DWORD          blocks_number = 0;
 static unsigned char *bc_limit;
 
 /* Handle for the private heap:
-    - inside the dumped_data[] array before dump,
-    - outside of it after dump.
+    - inside the dumped_data[] array before dump with unexec,
+    - outside of it after dump, or always if pdumper is used.
 */
 HANDLE heap = NULL;
 
@@ -188,8 +194,8 @@ free_fn the_free_fn;
      http://stackoverflow.com/questions/307060/what-is-the-purpose-of-allocating-pages-in-the-pagefile-with-createfilemapping  */
 
 /* This is the function to commit memory when the heap allocator
-   claims for new memory.  Before dumping, we allocate space
-   from the fixed size dumped_data[] array.
+   claims for new memory.  Before dumping with unexec, we allocate
+   space from the fixed size dumped_data[] array.
 */
 static NTSTATUS NTAPI
 dumped_data_commit (PVOID Base, PVOID *CommitAddress, PSIZE_T CommitSize)
@@ -223,22 +229,14 @@ typedef enum _HEAP_INFORMATION_CLASS {
 typedef WINBASEAPI BOOL (WINAPI * HeapSetInformation_Proc)(HANDLE,HEAP_INFORMATION_CLASS,PVOID,SIZE_T);
 #endif
 
-#ifdef HAVE_PDUMPER
-BOOL using_dynamic_heap = FALSE;
-#endif
-
 void
-init_heap (void)
+init_heap (bool use_dynamic_heap)
 {
-#ifdef HAVE_PDUMPER
-  using_dynamic_heap = TRUE;
-#endif
-  if (using_dynamic_heap)
+  /* FIXME: Remove the condition, the 'else' branch below, and all the
+     related definitions and code, including dumped_data[], when unexec
+     support is removed from Emacs.  */
+  if (use_dynamic_heap)
     {
-#ifndef MINGW_W64
-      unsigned long enable_lfh = 2;
-#endif
-
       /* After dumping, use a new private heap.  We explicitly enable
          the low fragmentation heap (LFH) here, for the sake of pre
          Vista versions.  Note: this will harmlessly fail on Vista and
@@ -255,6 +253,7 @@ init_heap (void)
       heap = HeapCreate (0, 0, 0);
 
 #ifndef MINGW_W64
+      unsigned long enable_lfh = 2;
       /* Set the low-fragmentation heap for OS before Vista.  */
       HMODULE hm_kernel32dll = LoadLibrary ("kernel32.dll");
       HeapSetInformation_Proc s_pfn_Heap_Set_Information =
@@ -283,7 +282,7 @@ init_heap (void)
           the_free_fn = free_after_dump;
         }
     }
-  else
+  else  /* Before dumping with unexec: use static heap.  */
     {
       /* Find the RtlCreateHeap function.  Headers for this function
          are provided with the w32 DDK, but the function is available
@@ -362,6 +361,8 @@ malloc_after_dump (size_t size)
   return p;
 }
 
+/* FIXME: The *_before_dump functions should be removed when pdumper
+   becomes the only dumping method.  */
 void *
 malloc_before_dump (size_t size)
 {
@@ -596,7 +597,7 @@ free_after_dump_9x (void *ptr)
     }
 }
 
-#ifdef ENABLE_CHECKING
+#if !defined (CANNOT_DUMP) && defined (ENABLE_CHECKING)
 void
 report_temacs_memory_usage (void)
 {
diff --git a/src/w32heap.h b/src/w32heap.h
index 6b9dca38a3b..13f7a6325b2 100644
--- a/src/w32heap.h
+++ b/src/w32heap.h
@@ -31,7 +31,6 @@ along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
 extern unsigned char *get_data_start (void);
 extern unsigned char *get_data_end (void);
 extern size_t         reserved_heap_size;
-extern BOOL           using_dynamic_heap;
 
 extern void *mmap_realloc (void **, size_t);
 extern void  mmap_free (void **);
@@ -43,7 +42,7 @@ extern void report_temacs_memory_usage (void);
 extern void *sbrk (ptrdiff_t size);
 
 /* Initialize heap structures for sbrk on startup.  */
-extern void init_heap (void);
+extern void init_heap (bool);
 
 /* ----------------------------------------------------------------- */
 /* Useful routines for manipulating memory-mapped files.  */
diff --git a/src/w32proc.c b/src/w32proc.c
index a5d08f60117..05e6c46b336 100644
--- a/src/w32proc.c
+++ b/src/w32proc.c
@@ -81,6 +81,36 @@ static sigset_t sig_mask;
 
 static CRITICAL_SECTION crit_sig;
 
+/* Catch memory allocation before the heap allocation scheme is set
+   up.  These functions should never be called, unless code is added
+   early on in 'main' that runs before init_heap is called.  */
+_Noreturn void * malloc_before_init (size_t);
+_Noreturn void * realloc_before_init (void *, size_t);
+_Noreturn void   free_before_init (void *);
+
+_Noreturn void *
+malloc_before_init (size_t size)
+{
+  fprintf (stderr,
+           "error: 'malloc' called before setting up heap allocation; exiting.\n");
+  exit (-1);
+}
+
+_Noreturn void *
+realloc_before_init (void *ptr, size_t size)
+{
+  fprintf (stderr,
+           "error: 'realloc' called before setting up heap allocation; exiting.\n");
+  exit (-1);
+}
+
+_Noreturn void
+free_before_init (void *ptr)
+{
+  fprintf (stderr,
+           "error: 'free' called before setting up heap allocation; exiting.\n");
+  exit (-1);
+}
 
 extern BOOL ctrl_c_handler (unsigned long type);
 
@@ -110,12 +140,13 @@ _start (void)
     DebugBreak ();
 #endif
 
+  the_malloc_fn = malloc_before_init;
+  the_realloc_fn = realloc_before_init;
+  the_free_fn = free_before_init;
+
   /* Cache system info, e.g., the NT page size.  */
   cache_system_info ();
 
-  /* Grab our malloc arena space now, before CRT starts up. */
-  init_heap ();
-
   /* This prevents ctrl-c's in shells running while we're suspended from
      having us exit.  */
   SetConsoleCtrlHandler ((PHANDLER_ROUTINE) ctrl_c_handler, TRUE);