Port memory-full checking to GnuTLS 3.3

Instead of using gnutls_global_set_mem_functions, check every call
to a GnuTLS function that might return an indication of memory
exhaustion.  Suggested by Dmitry Antipov in:
http://lists.gnu.org/archive/html/emacs-devel/2014-12/msg02056.html
* src/gnutls.c (gnutls_global_set_mem_functions) [WINDOWSNT]: Remove.
(init_gnutls_functions): Do not load gnutls_global_set_mem_functions.
(fn_gnutls_global_set_mem_functions) [!WINDOWSNT]: Remove.
All uses removed.
(check_memory_full): New function.
(emacs_gnutls_handshake, emacs_gnutls_handle_error)
(gnutls_make_error, Fgnutls_boot): Use it.
(emacs_gnutls_global_init): Avoid gnutls_global_set_mem_functions.

2 files changed, 41 insertions, 15 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 2df83082fbd..3ea60571a15 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,19 @@
+2014-12-28  Paul Eggert  <eggert@Penguin.CS.UCLA.EDU>
+
+        Port memory-full checking to GnuTLS 3.3
+        Instead of using gnutls_global_set_mem_functions, check every call
+        to a GnuTLS function that might return an indication of memory
+        exhaustion.  Suggested by Dmitry Antipov in:
+        http://lists.gnu.org/archive/html/emacs-devel/2014-12/msg02056.html
+        * gnutls.c (gnutls_global_set_mem_functions) [WINDOWSNT]: Remove.
+        (init_gnutls_functions): Do not load gnutls_global_set_mem_functions.
+        (fn_gnutls_global_set_mem_functions) [!WINDOWSNT]: Remove.
+        All uses removed.
+        (check_memory_full): New function.
+        (emacs_gnutls_handshake, emacs_gnutls_handle_error)
+        (gnutls_make_error, Fgnutls_boot): Use it.
+        (emacs_gnutls_global_init): Avoid gnutls_global_set_mem_functions.
+
 2014-12-25  Eli Zaretskii  <eliz@gnu.org>
 
         * xdisp.c (set_iterator_to_next) <GET_FROM_STRING>: Limit search in
diff --git a/src/gnutls.c b/src/gnutls.c
index ffa3c982572..f093568bb54 100644
--- a/src/gnutls.c
+++ b/src/gnutls.c
@@ -116,10 +116,6 @@ DEF_GNUTLS_FN (void, gnutls_global_set_log_function, (gnutls_log_func));
 DEF_GNUTLS_FN (void, gnutls_global_set_audit_log_function, (gnutls_audit_log_func));
 #endif
 DEF_GNUTLS_FN (void, gnutls_global_set_log_level, (int));
-DEF_GNUTLS_FN (void, gnutls_global_set_mem_functions,
-               (gnutls_alloc_function, gnutls_alloc_function,
-                gnutls_is_secure_function, gnutls_realloc_function,
-                gnutls_free_function));
 DEF_GNUTLS_FN (int, gnutls_handshake, (gnutls_session_t));
 DEF_GNUTLS_FN (int, gnutls_init, (gnutls_session_t *, unsigned int));
 DEF_GNUTLS_FN (int, gnutls_priority_set_direct,
@@ -184,7 +180,6 @@ init_gnutls_functions (void)
   LOAD_GNUTLS_FN (library, gnutls_global_set_audit_log_function);
 #endif
   LOAD_GNUTLS_FN (library, gnutls_global_set_log_level);
-  LOAD_GNUTLS_FN (library, gnutls_global_set_mem_functions);
   LOAD_GNUTLS_FN (library, gnutls_handshake);
   LOAD_GNUTLS_FN (library, gnutls_init);
   LOAD_GNUTLS_FN (library, gnutls_priority_set_direct);
@@ -244,7 +239,6 @@ init_gnutls_functions (void)
 #define fn_gnutls_global_set_audit_log_function gnutls_global_set_audit_log_function
 #endif
 #define fn_gnutls_global_set_log_level          gnutls_global_set_log_level
-#define fn_gnutls_global_set_mem_functions      gnutls_global_set_mem_functions
 #define fn_gnutls_handshake                     gnutls_handshake
 #define fn_gnutls_init                          gnutls_init
 #define fn_gnutls_priority_set_direct           gnutls_priority_set_direct
@@ -264,6 +258,17 @@ init_gnutls_functions (void)
 #endif /* !WINDOWSNT */
 
 
+/* Report memory exhaustion if ERR is an out-of-memory indication.  */
+static void
+check_memory_full (int err)
+{
+  /* When GnuTLS exhausts memory, it doesn't say how much memory it
+     asked for, so tell the Emacs allocator that GnuTLS asked for no
+     bytes.  This isn't accurate, but it's good enough.  */
+  if (err == GNUTLS_E_MEMORY_ERROR)
+    memory_full (0);
+}
+
 #ifdef HAVE_GNUTLS3
 /* Function to log a simple audit message.  */
 static void
@@ -360,7 +365,7 @@ emacs_gnutls_handshake (struct Lisp_Process *proc)
     }
   else
     {
-      fn_gnutls_alert_send_appropriate (state, ret);
+      check_memory_full (fn_gnutls_alert_send_appropriate (state, ret));
     }
   return ret;
 }
@@ -477,6 +482,8 @@ emacs_gnutls_handle_error (gnutls_session_t session, int err)
   if (err >= 0)
     return 1;
 
+  check_memory_full (err);
+
   max_log_level = global_gnutls_log_level;
 
   /* TODO: use gnutls-error-fatalp and gnutls-error-string.  */
@@ -542,6 +549,7 @@ gnutls_make_error (int err)
       return Qgnutls_e_invalid_session;
     }
 
+  check_memory_full (err);
   return make_number (err);
 }
 
@@ -682,11 +690,8 @@ emacs_gnutls_global_init (void)
   int ret = GNUTLS_E_SUCCESS;
 
   if (!gnutls_global_initialized)
-    {
-      fn_gnutls_global_set_mem_functions (xmalloc, xmalloc, NULL,
-                                          xrealloc, xfree);
-      ret = fn_gnutls_global_init ();
-    }
+    ret = fn_gnutls_global_init ();
+
   gnutls_global_initialized = 1;
 
   return gnutls_make_error (ret);
@@ -854,7 +859,8 @@ one trustfile (usually a CA bundle).  */)
       unsigned int gnutls_verify_flags = GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT;
 
       GNUTLS_LOG (2, max_log_level, "allocating x509 credentials");
-      fn_gnutls_certificate_allocate_credentials (&x509_cred);
+      check_memory_full ((fn_gnutls_certificate_allocate_credentials
+                          (&x509_cred)));
       XPROCESS (proc)->gnutls_x509_cred = x509_cred;
 
       verify_flags = Fplist_get (proplist, QCgnutls_bootprop_verify_flags);
@@ -873,7 +879,8 @@ one trustfile (usually a CA bundle).  */)
   else /* Qgnutls_anon: */
     {
       GNUTLS_LOG (2, max_log_level, "allocating anon credentials");
-      fn_gnutls_anon_allocate_client_credentials (&anon_cred);
+      check_memory_full ((fn_gnutls_anon_allocate_client_credentials
+                          (&anon_cred)));
       XPROCESS (proc)->gnutls_anon_cred = anon_cred;
     }
 
@@ -1105,7 +1112,10 @@ one trustfile (usually a CA bundle).  */)
           return gnutls_make_error (ret);
         }
 
-      if (!fn_gnutls_x509_crt_check_hostname (gnutls_verify_cert, c_hostname))
+      int err
+        = fn_gnutls_x509_crt_check_hostname (gnutls_verify_cert, c_hostname);
+      check_memory_full (err);
+      if (!err)
         {
           if (verify_error_all
               || !NILP (Fmember (QCgnutls_bootprop_hostname, verify_error)))