implement in-place growth of large allocations

malloc.c

@@ -844,11 +844,11 @@ EXPORT void *h_realloc(void *old, size_t size) {
         }
         mutex_unlock(&regions_lock);
 
+        size_t old_rounded_size = PAGE_CEILING(old_size);
+        size_t rounded_size = PAGE_CEILING(size);
+
         // in-place shrink
         if (size < old_size && size > max_slab_size_class) {
-            size_t rounded_size = PAGE_CEILING(size);
-            size_t old_rounded_size = PAGE_CEILING(old_size);
-
             void *new_end = (char *)old + rounded_size;
             if (memory_map_fixed(new_end, old_guard_size)) {
                 return NULL;
@@ -867,6 +867,27 @@ EXPORT void *h_realloc(void *old, size_t size) {
             return old;
         }
 
+        // in-place growth
+        if (size > old_size) {
+            void *guard_end = (char *)old + old_rounded_size + old_guard_size;
+            size_t extra = rounded_size - old_rounded_size;
+            if (!memory_remap((char *)old + old_rounded_size, old_guard_size, old_guard_size + extra)) {
+                if (memory_protect_rw((char *)old + old_rounded_size, extra)) {
+                    memory_unmap(guard_end, extra);
+                } else {
+                    mutex_lock(&regions_lock);
+                    struct region_info *region = regions_find(old);
+                    if (region == NULL) {
+                        fatal_error("invalid realloc");
+                    }
+                    region->size = size;
+                    mutex_unlock(&regions_lock);
+
+                    return old;
+                }
+            }
+        }
+
         size_t copy_size = size < old_size ? size : old_size;
         if (copy_size >= mremap_threshold) {
             void *new = allocate(size);

memory.c

@@ -51,6 +51,17 @@ int memory_protect_ro(void *ptr, size_t size) {
     return memory_protect_prot(ptr, size, PROT_READ);
 }
 
+int memory_remap(void *old, size_t old_size, size_t new_size) {
+    void *ptr = mremap(old, old_size, new_size, 0);
+    if (unlikely(ptr == MAP_FAILED)) {
+        if (errno != ENOMEM) {
+            fatal_error("non-ENOMEM mremap failure");
+        }
+        return 1;
+    }
+    return 0;
+}
+
 int memory_remap_fixed(void *old, size_t old_size, void *new, size_t new_size) {
     void *ptr = mremap(old, old_size, new_size, MREMAP_MAYMOVE|MREMAP_FIXED, new);
     if (unlikely(ptr == MAP_FAILED)) {

memory.h

@@ -8,6 +8,7 @@ int memory_map_fixed(void *ptr, size_t size);
 int memory_unmap(void *ptr, size_t size);
 int memory_protect_rw(void *ptr, size_t size);
 int memory_protect_ro(void *ptr, size_t size);
+int memory_remap(void *old, size_t old_size, size_t new_size);
 int memory_remap_fixed(void *old, size_t old_size, void *new, size_t new_size);
 
 #endif