net/ssl-bear: fix OOM crashes and pre-existing leaks in x509 cert parsing

Three interlocking bugs in the BearSSL cert-parsing path, all
triggered by OOM during trust-anchor load:

=== Bug 1: blobdup had no NULL check ===

    static uint8_t* blobdup(const void * src, size_t len)
    {
       uint8_t *ret = (uint8_t*)malloc(len);
       memcpy(ret, src, len);   /* NULL-deref on OOM */
       return ret;
    }

Called three times from append_cert_x509 to copy the RSA .n / .e
or EC .q public-key blobs into the trust-anchor slot.  On malloc
failure the memcpy segfaulted before blobdup could return.

Fix: NULL-check the malloc and return NULL.  Each call site now
propagates the NULL back out of append_cert_x509.

=== Bug 2: vdn_append did realloc-to-self ===

    current_vdn = (uint8_t*)realloc(current_vdn, current_vdn_size + len);
    memcpy(current_vdn + current_vdn_size, src, len);

This is the classic 'x = realloc(x, n)' pattern: on failure,
current_vdn is overwritten with NULL and the original buffer is
leaked, then the next line memcpy's into NULL.

vdn_append is a BearSSL x509-decoder callback with a void return
type, so there is no way to signal failure mid-decode.  Latched a
file-scope 'current_vdn_failed' flag: on realloc failure set the
flag, skip the memcpy, and short-circuit subsequent invocations.
append_cert_x509 then checks the flag after br_x509_decoder_push
returns and bails if it is set.

=== Bug 3: append_cert_x509 had pre-existing leaks of current_vdn ===

Two early-return paths (the '!pk || !isCA' check at line 71 and
the default case of the key_type switch) returned 'false' without
freeing current_vdn.  Ownership had not yet been transferred to
ta->dn.data when those paths fired, so the buffer was orphaned
every time a non-CA or unsupported-key-type certificate appeared
in the PEM bundle.

/etc/ssl/certs/ca-certificates.crt on a typical Linux system
contains ~140 certificates and is loaded once at SSL-startup, so
this was a bounded leak in practice, but still a leak.  Added
'free(current_vdn)' on both paths.

=== Bug 4: partial-blobdup failure would crash TLS handshakes ===

    ta->pkey.key.rsa.n = blobdup(...);
    ta->pkey.key.rsa.e = blobdup(...);
    break;   /* no check; TAs_NUM++ and proceed */

With Bug 1 fixed, blobdup now returns NULL cleanly on OOM.  But
append_cert_x509 still committed the half-built trust anchor
regardless.  BearSSL dereferences ta->pkey.key.rsa.n / .e during
every TLS handshake against a certificate chain that terminates
at this TA; a NULL field there causes a crash in the handshake
path, well away from the original OOM.

Added explicit checks after each blobdup call.  On any NULL,
free any siblings that did succeed (free(NULL) is a no-op for
the already-NULL one), free current_vdn, zero out the pointer
fields so debugging is cleaner, and return false without
incrementing TAs_NUM.

=== Not fixed in this commit ===

ssl_socket_init in this same file has a separate no-NULL-check
calloc bug that will be addressed in a follow-up.  The mbedtls
variant (net_socket_ssl_mbed.c) has the same ssl_socket_init
NULL-deref plus an mbedtls-sub-object leak on its 'error:' label;
also follow-up material.

Thread-safety: unchanged.  append_cert_x509 and vdn_append run
only from ssl_socket_init -> initialize() at startup.  The file
already carries a TODO on 'initialize()' noting it is not thread
safe; the new current_vdn_failed flag is no worse than the
existing current_vdn / current_vdn_size statics it lives
alongside.

Reachability: every SSL-enabled connection at startup, via
initialize() -> append_certs_pem_x509 -> append_cert_x509 ->
vdn_append / blobdup.  Both malloc and realloc here are for
small allocations (DN bytes are ~100 bytes, public-key blobs
are ~250-550 bytes for RSA-2048), so OOM is unlikely but
possible under extreme memory pressure at startup.

Author: LibretroAdmin

libretro-common/net/net_socket_ssl_bear.c

@@ -42,16 +42,43 @@ static size_t TAs_NUM = 0;
 
 static uint8_t* current_vdn;
 static size_t current_vdn_size;
+/* Set by vdn_append when a realloc() fails during DN accumulation.
+ * BearSSL's x509 decoder drives vdn_append as a void-returning
+ * callback, so we have no failure channel to hand back mid-decode;
+ * instead we latch this flag, skip further appends, and let
+ * append_cert_x509 observe it after the decoder returns. */
+static bool current_vdn_failed;
 
 static uint8_t* blobdup(const void * src, size_t len)
 {
    uint8_t *ret = (uint8_t*)malloc(len);
+   /* Pre-patch: no NULL check, so the next line memcpy'd into NULL
+    * on OOM and crashed.  Return NULL and make it the caller's
+    * problem - all three call sites below now propagate the NULL
+    * back out of append_cert_x509. */
+   if (!ret)
+      return NULL;
    memcpy(ret, src, len);
    return ret;
 }
 static void vdn_append(void* dest_ctx, const void * src, size_t len)
 {
-   current_vdn = (uint8_t*)realloc(current_vdn, current_vdn_size + len);
+   uint8_t *tmp;
+   if (current_vdn_failed)
+      return;
+   /* Use a tmp pointer: the pre-patch form
+    *     current_vdn = (uint8_t*)realloc(current_vdn, ...)
+    * overwrote current_vdn with NULL on realloc failure, leaking
+    * the original buffer, and the subsequent memcpy then
+    * dereferenced NULL.  The callback signature is 'void' so we
+    * cannot signal to the decoder to stop - we just latch the
+    * flag and short-circuit subsequent invocations. */
+   if (!(tmp = (uint8_t*)realloc(current_vdn, current_vdn_size + len)))
+   {
+      current_vdn_failed = true;
+      return;
+   }
+   current_vdn = tmp;
    memcpy(current_vdn + current_vdn_size, src, len);
    current_vdn_size += len;
 }
@@ -64,12 +91,29 @@ static bool append_cert_x509(void* x509, size_t len)
 
    current_vdn              = NULL;
    current_vdn_size         = 0;
+   current_vdn_failed       = false;
 
    br_x509_decoder_init(&dc, vdn_append, NULL);
    br_x509_decoder_push(&dc, x509, len);
+
+   /* If any vdn_append realloc failed during decoding, whatever
+    * bytes we did accumulate in current_vdn are incomplete; drop
+    * the DN buffer and abandon this trust anchor. */
+   if (current_vdn_failed)
+   {
+      free(current_vdn);
+      return false;
+   }
+
    pk                       = br_x509_decoder_get_pkey(&dc);
    if (!pk || !br_x509_decoder_isCA(&dc))
+   {
+      /* Pre-existing leak: the original code returned here without
+       * freeing current_vdn.  Ownership had not yet been transferred
+       * to ta->dn.data, so the buffer was orphaned. */
+      free(current_vdn);
       return false;
+   }
 
    ta->dn.len               = current_vdn_size;
    ta->dn.data              = current_vdn;
@@ -83,14 +127,38 @@ static bool append_cert_x509(void* x509, size_t len)
          ta->pkey.key.rsa.n    = blobdup(pk->key.rsa.n, pk->key.rsa.nlen);
          ta->pkey.key.rsa.elen = pk->key.rsa.elen;
          ta->pkey.key.rsa.e    = blobdup(pk->key.rsa.e, pk->key.rsa.elen);
+         /* If either blobdup OOM'd, we cannot install a half-built
+          * trust anchor - BearSSL dereferences ta->pkey during
+          * every handshake against a cert that chains to this TA,
+          * and a NULL .n or .e is a crash. */
+         if (!ta->pkey.key.rsa.n || !ta->pkey.key.rsa.e)
+         {
+            free(ta->pkey.key.rsa.n);
+            free(ta->pkey.key.rsa.e);
+            free(current_vdn);
+            ta->pkey.key.rsa.n = NULL;
+            ta->pkey.key.rsa.e = NULL;
+            ta->dn.data        = NULL;
+            return false;
+         }
          break;
       case BR_KEYTYPE_EC:
          ta->pkey.key_type     = BR_KEYTYPE_EC;
          ta->pkey.key.ec.curve = pk->key.ec.curve;
          ta->pkey.key.ec.qlen  = pk->key.ec.qlen;
          ta->pkey.key.ec.q     = blobdup(pk->key.ec.q, pk->key.ec.qlen);
+         if (!ta->pkey.key.ec.q)
+         {
+            free(current_vdn);
+            ta->dn.data = NULL;
+            return false;
+         }
          break;
       default:
+         /* Pre-existing leak: same pattern as the '!pk || !isCA'
+          * early return above.  current_vdn was orphaned. */
+         free(current_vdn);
+         ta->dn.data = NULL;
          return false;
    }