buffer: Change limits for array_mult_safe

- Lower the limit to 1GB on 32bit systems.
  The limit of 4GB-1 makes no sense on systems that usually
  don't allow a single process to allocate anything near to
  this limit.
- Increate the limit from 4GB-1 to 4GB on other systems. It
  makes no difference in protection but makes it much easier
  to use the limit in other contexts, e.g. if dividing it.

Change-Id: I4f95edd7ce2098180aa620a231727217f333a12d
Signed-off-by: Frank Lichtenheld <[email protected]>
Acked-by: Gert Doering <[email protected]>
Gerrit URL: https://gerrit.openvpn.net/c/openvpn/+/1436
Message-Id: <[email protected]>
URL: https://www.mail-archive.com/[email protected]/msg35044.html
Signed-off-by: Gert Doering <[email protected]>

Author: flichtenheld

src/openvpn/buffer.c

@@ -39,7 +39,7 @@
 size_t
 array_mult_safe(const size_t m1, const size_t m2, const size_t extra)
 {
-    const size_t limit = 0xFFFFFFFF;
+    const size_t limit = ALLOC_SIZE_MAX;
     unsigned long long res =
         (unsigned long long)m1 * (unsigned long long)m2 + (unsigned long long)extra;
     if (unlikely(m1 > limit) || unlikely(m2 > limit) || unlikely(extra > limit)

src/openvpn/buffer.h

@@ -1044,6 +1044,18 @@ gc_reset(struct gc_arena *a)
  * Allocate memory to hold a structure
  */
 
+/* When allocating arrays make sure we do not use a excessive amount
+ * of memory.
+ */
+#if UINTPTR_MAX <= UINT32_MAX
+/* 1 GB on 32bit systems, they usually can only allocate 2 GB for the
+ * whole process.
+ */
+#define ALLOC_SIZE_MAX (1u << 30)
+#else
+#define ALLOC_SIZE_MAX ((size_t)1 << 32) /* 4 GB */
+#endif
+
 #define ALLOC_OBJ(dptr, type)                                       \
     {                                                               \
         check_malloc_return((dptr) = (type *)malloc(sizeof(type))); \