Bug 697985: bounds check the array allocations methods

The clump allocator has four allocation functions that use 'number of elements'
and 'size of elements' parameters (rather than a simple 'number of bytes').

Those need specific bounds checking.

1 files changed, 28 insertions, 14 deletions

diff --git a/base/gsalloc.c b/base/gsalloc.c
index 741ba008f..10c04dddd 100644
--- a/base/gsalloc.c
+++ b/base/gsalloc.c
@@ -1248,19 +1248,32 @@ i_alloc_struct_immovable(gs_memory_t * mem, gs_memory_type_ptr_t pstype,
     alloc_trace("|+<.", imem, cname, pstype, size, obj);
     return obj;
 }
+
+static inline bool
+alloc_array_check_size(ulong num_elements, ulong elt_size, ulong *lsize)
+{
+    int64_t s = (int64_t)num_elements * elt_size;
+    if (s > max_uint) {
+        return false;
+    }
+    *lsize = (ulong)s;
+    return true;
+}
+
 static byte *
 i_alloc_byte_array(gs_memory_t * mem, uint num_elements, uint elt_size,
                    client_name_t cname)
 {
     gs_ref_memory_t * const imem = (gs_ref_memory_t *)mem;
     obj_header_t *obj;
-
+    ulong lsize;
 #ifdef MEMENTO
     if (Memento_failThisEvent())
         return NULL;
 #endif
-
-    obj = alloc_obj(imem, (ulong) num_elements * elt_size,
+    if (alloc_array_check_size(num_elements, elt_size, &lsize) == false)
+        return NULL;
+    obj = alloc_obj(imem, lsize,
                     &st_bytes, ALLOC_DIRECT, cname);
 
     if_debug6m('A', mem, "[a%d:+b.]%s -bytes-*(%lu=%u*%u) = 0x%lx\n",
@@ -1275,13 +1288,14 @@ i_alloc_byte_array_immovable(gs_memory_t * mem, uint num_elements,
 {
     gs_ref_memory_t * const imem = (gs_ref_memory_t *)mem;
     obj_header_t *obj;
-
+    ulong lsize;
 #ifdef MEMENTO
     if (Memento_failThisEvent())
         return NULL;
 #endif
-
-    obj = alloc_obj(imem, (ulong) num_elements * elt_size,
+    if (alloc_array_check_size(num_elements, elt_size, &lsize) == false)
+        return NULL;
+    obj = alloc_obj(imem, lsize,
                     &st_bytes, ALLOC_IMMOVABLE | ALLOC_DIRECT,
                     cname);
 
@@ -1297,7 +1311,7 @@ i_alloc_struct_array(gs_memory_t * mem, uint num_elements,
 {
     gs_ref_memory_t * const imem = (gs_ref_memory_t *)mem;
     obj_header_t *obj;
-
+    ulong lsize;
 #ifdef MEMENTO
     if (Memento_failThisEvent())
         return NULL;
@@ -1311,9 +1325,9 @@ i_alloc_struct_array(gs_memory_t * mem, uint num_elements,
         return NULL;            /* fail */
     }
 #endif
-    obj = alloc_obj(imem,
-                    (ulong) num_elements * pstype->ssize,
-                    pstype, ALLOC_DIRECT, cname);
+    if (alloc_array_check_size(num_elements, pstype->ssize, &lsize) == false)
+        return NULL;
+    obj = alloc_obj(imem, lsize, pstype, ALLOC_DIRECT, cname);
     if_debug7m('A', mem, "[a%d:+<.]%s %s*(%lu=%u*%u) = 0x%lx\n",
                alloc_trace_space(imem), client_name_string(cname),
                struct_type_name_string(pstype),
@@ -1327,16 +1341,16 @@ i_alloc_struct_array_immovable(gs_memory_t * mem, uint num_elements,
 {
     gs_ref_memory_t * const imem = (gs_ref_memory_t *)mem;
     obj_header_t *obj;
-
+    ulong lsize;
 #ifdef MEMENTO
     if (Memento_failThisEvent())
         return NULL;
 #endif
 
     ALLOC_CHECK_SIZE(mem,pstype);
-    obj = alloc_obj(imem,
-                    (ulong) num_elements * pstype->ssize,
-                    pstype, ALLOC_IMMOVABLE | ALLOC_DIRECT, cname);
+    if (alloc_array_check_size(num_elements, pstype->ssize, &lsize) == false)
+        return NULL;
+    obj = alloc_obj(imem, lsize, pstype, ALLOC_IMMOVABLE | ALLOC_DIRECT, cname);
     if_debug7m('A', mem, "[a%d|+<.]%s %s*(%lu=%u*%u) = 0x%lx\n",
                alloc_trace_space(imem), client_name_string(cname),
                struct_type_name_string(pstype),