2018-08-21 21:23:22 +02:00
|
|
|
#include <assert.h>
|
|
|
|
#include <errno.h>
|
2019-04-19 22:54:43 +02:00
|
|
|
#include <inttypes.h>
|
2018-08-21 21:23:22 +02:00
|
|
|
#include <stdatomic.h>
|
|
|
|
#include <stdbool.h>
|
2019-08-18 08:39:55 +02:00
|
|
|
#include <stdio.h>
|
2018-08-21 21:23:22 +02:00
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
2019-06-06 14:10:57 +02:00
|
|
|
|
|
|
|
#if __has_include(<threads.h>)
|
2019-03-25 19:49:50 +01:00
|
|
|
#include <threads.h>
|
2019-06-06 14:10:57 +02:00
|
|
|
#else
|
|
|
|
// glibc < 2.28
|
|
|
|
#define thread_local _Thread_local
|
2019-03-26 06:28:27 +01:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-12-05 08:22:10 +01:00
|
|
|
#include <malloc.h>
|
2018-08-21 21:23:22 +02:00
|
|
|
#include <pthread.h>
|
2018-10-20 03:29:40 +02:00
|
|
|
#include <sys/mman.h>
|
2019-03-20 15:55:29 +01:00
|
|
|
#include <sys/utsname.h>
|
2018-10-19 01:35:21 +02:00
|
|
|
#include <unistd.h>
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-08-30 09:03:19 +02:00
|
|
|
#include "third_party/libdivide.h"
|
2018-08-27 12:57:44 +02:00
|
|
|
|
2018-12-05 08:21:28 +01:00
|
|
|
#include "h_malloc.h"
|
2018-08-29 06:53:12 +02:00
|
|
|
#include "memory.h"
|
2019-08-18 08:39:55 +02:00
|
|
|
#include "mutex.h"
|
2018-09-02 08:03:27 +02:00
|
|
|
#include "pages.h"
|
2018-08-21 21:23:22 +02:00
|
|
|
#include "random.h"
|
|
|
|
#include "util.h"
|
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
#define SLAB_QUARANTINE (SLAB_QUARANTINE_RANDOM_LENGTH > 0 || SLAB_QUARANTINE_QUEUE_LENGTH > 0)
|
2019-01-06 06:52:25 +01:00
|
|
|
#define MREMAP_MOVE_THRESHOLD (32 * 1024 * 1024)
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
static_assert(sizeof(void *) == 8, "64-bit only");
|
|
|
|
|
2018-09-07 06:33:51 +02:00
|
|
|
static_assert(!WRITE_AFTER_FREE_CHECK || ZERO_ON_FREE, "WRITE_AFTER_FREE_CHECK depends on ZERO_ON_FREE");
|
|
|
|
|
2019-01-06 06:52:25 +01:00
|
|
|
static_assert(SLAB_QUARANTINE_RANDOM_LENGTH >= 0 && SLAB_QUARANTINE_RANDOM_LENGTH <= 65536,
|
|
|
|
"invalid slab quarantine random length");
|
|
|
|
static_assert(SLAB_QUARANTINE_QUEUE_LENGTH >= 0 && SLAB_QUARANTINE_QUEUE_LENGTH <= 65536,
|
|
|
|
"invalid slab quarantine queue length");
|
|
|
|
static_assert(REGION_QUARANTINE_RANDOM_LENGTH >= 0 && REGION_QUARANTINE_RANDOM_LENGTH <= 65536,
|
|
|
|
"invalid region quarantine random length");
|
|
|
|
static_assert(REGION_QUARANTINE_QUEUE_LENGTH >= 0 && REGION_QUARANTINE_QUEUE_LENGTH <= 65536,
|
|
|
|
"invalid region quarantine queue length");
|
|
|
|
static_assert(FREE_SLABS_QUARANTINE_RANDOM_LENGTH >= 0 && FREE_SLABS_QUARANTINE_RANDOM_LENGTH <= 65536,
|
|
|
|
"invalid free slabs quarantine random length");
|
|
|
|
|
2019-07-11 23:07:36 +02:00
|
|
|
static_assert(GUARD_SLABS_INTERVAL >= 1, "invalid guard slabs interval (minimum 1)");
|
|
|
|
static_assert(GUARD_SIZE_DIVISOR >= 1, "invalid guard size divisor (minimum 1)");
|
|
|
|
static_assert(CONFIG_CLASS_REGION_SIZE >= 1048576, "invalid class region size (minimum 1048576)");
|
|
|
|
static_assert(CONFIG_CLASS_REGION_SIZE <= 1099511627776, "invalid class region size (maximum 1099511627776)");
|
2019-01-06 06:52:25 +01:00
|
|
|
static_assert(REGION_QUARANTINE_SKIP_THRESHOLD >= 0,
|
2019-07-11 23:07:36 +02:00
|
|
|
"invalid region quarantine skip threshold (minimum 0)");
|
2019-01-06 06:52:25 +01:00
|
|
|
static_assert(MREMAP_MOVE_THRESHOLD >= REGION_QUARANTINE_SKIP_THRESHOLD,
|
|
|
|
"mremap move threshold must be above region quarantine limit");
|
|
|
|
|
2018-10-04 20:25:16 +02:00
|
|
|
// either sizeof(u64) or 0
|
|
|
|
static const size_t canary_size = SLAB_CANARY ? sizeof(u64) : 0;
|
2018-09-02 14:36:48 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
static_assert(N_ARENA >= 1, "must have at least 1 arena");
|
2019-07-12 09:43:19 +02:00
|
|
|
static_assert(N_ARENA <= 256, "maximum number of arenas is currently 256");
|
2018-09-01 05:10:26 +02:00
|
|
|
#define CACHELINE_SIZE 64
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
#if N_ARENA > 1
|
|
|
|
__attribute__((tls_model("initial-exec")))
|
|
|
|
static thread_local unsigned thread_arena = N_ARENA;
|
2019-04-15 01:57:22 +02:00
|
|
|
static atomic_uint thread_arena_counter = 0;
|
2019-03-25 19:49:50 +01:00
|
|
|
#else
|
|
|
|
static const unsigned thread_arena = 0;
|
|
|
|
#endif
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
static union {
|
|
|
|
struct {
|
2018-11-05 08:19:04 +01:00
|
|
|
void *_Atomic slab_region_start;
|
2018-08-21 21:23:22 +02:00
|
|
|
void *slab_region_end;
|
2019-03-25 19:49:50 +01:00
|
|
|
struct size_class *size_class_metadata[N_ARENA];
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *region_allocator;
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *regions[2];
|
2018-10-20 03:29:40 +02:00
|
|
|
#ifdef USE_PKEY
|
|
|
|
int metadata_pkey;
|
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
};
|
|
|
|
char padding[PAGE_SIZE];
|
2018-10-13 22:49:06 +02:00
|
|
|
} ro __attribute__((aligned(PAGE_SIZE)));
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
static inline void *get_slab_region_start() {
|
|
|
|
return atomic_load_explicit(&ro.slab_region_start, memory_order_acquire);
|
|
|
|
}
|
|
|
|
|
2018-11-01 06:00:59 +01:00
|
|
|
#define SLAB_METADATA_COUNT
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
struct slab_metadata {
|
2018-10-07 19:34:52 +02:00
|
|
|
u64 bitmap[4];
|
2018-08-21 21:23:22 +02:00
|
|
|
struct slab_metadata *next;
|
|
|
|
struct slab_metadata *prev;
|
2018-10-04 20:25:16 +02:00
|
|
|
u64 canary_value;
|
2018-11-01 06:00:59 +01:00
|
|
|
#ifdef SLAB_METADATA_COUNT
|
|
|
|
u16 count;
|
|
|
|
#endif
|
2018-11-05 22:19:50 +01:00
|
|
|
#if SLAB_QUARANTINE
|
2019-03-25 01:24:40 +01:00
|
|
|
u64 quarantine_bitmap[4];
|
2018-11-05 22:19:50 +01:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
};
|
|
|
|
|
2018-09-10 23:42:58 +02:00
|
|
|
static const size_t min_align = 16;
|
2019-01-02 20:22:28 +01:00
|
|
|
#define MIN_SLAB_SIZE_CLASS_SHIFT 4
|
2018-09-16 19:26:59 +02:00
|
|
|
|
2020-05-24 15:36:43 +02:00
|
|
|
// set slab cache size based on the size of the largest slab
|
2018-09-16 19:26:59 +02:00
|
|
|
#if !CONFIG_EXTENDED_SIZE_CLASSES
|
2020-05-24 15:31:02 +02:00
|
|
|
static const size_t MAX_SLAB_SIZE_CLASS = 65536;
|
2019-01-02 20:22:28 +01:00
|
|
|
#define MAX_SLAB_SIZE_CLASS_SHIFT 14
|
2018-09-16 19:26:59 +02:00
|
|
|
#else
|
2020-05-13 07:05:37 +02:00
|
|
|
static const size_t MAX_SLAB_SIZE_CLASS = 131072;
|
2019-04-10 22:54:11 +02:00
|
|
|
#define MAX_SLAB_SIZE_CLASS_SHIFT 17
|
2018-09-16 19:26:59 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-16 19:26:59 +02:00
|
|
|
static const u32 size_classes[] = {
|
2018-08-24 09:22:52 +02:00
|
|
|
/* 0 */ 0,
|
2018-08-21 21:23:22 +02:00
|
|
|
/* 16 */ 16, 32, 48, 64, 80, 96, 112, 128,
|
|
|
|
/* 32 */ 160, 192, 224, 256,
|
|
|
|
/* 64 */ 320, 384, 448, 512,
|
|
|
|
/* 128 */ 640, 768, 896, 1024,
|
|
|
|
/* 256 */ 1280, 1536, 1792, 2048,
|
|
|
|
/* 512 */ 2560, 3072, 3584, 4096,
|
|
|
|
/* 1024 */ 5120, 6144, 7168, 8192,
|
2018-09-16 19:26:59 +02:00
|
|
|
/* 2048 */ 10240, 12288, 14336, 16384,
|
|
|
|
#if CONFIG_EXTENDED_SIZE_CLASSES
|
|
|
|
/* 4096 */ 20480, 24576, 28672, 32768,
|
|
|
|
/* 8192 */ 40960, 49152, 57344, 65536,
|
2019-04-10 22:32:24 +02:00
|
|
|
/* 16384 */ 81920, 98304, 114688, 131072,
|
2018-09-16 19:26:59 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
};
|
|
|
|
|
2018-10-04 20:25:16 +02:00
|
|
|
static const u16 size_class_slots[] = {
|
2018-08-24 09:22:52 +02:00
|
|
|
/* 0 */ 256,
|
2018-08-23 23:53:10 +02:00
|
|
|
/* 16 */ 256, 128, 85, 64, 51, 42, 36, 64,
|
2018-08-23 23:42:17 +02:00
|
|
|
/* 32 */ 51, 64, 54, 64,
|
2018-08-21 21:23:22 +02:00
|
|
|
/* 64 */ 64, 64, 64, 64,
|
|
|
|
/* 128 */ 64, 64, 64, 64,
|
|
|
|
/* 256 */ 16, 16, 16, 16,
|
|
|
|
/* 512 */ 8, 8, 8, 8,
|
|
|
|
/* 1024 */ 8, 8, 8, 8,
|
2018-09-16 19:26:59 +02:00
|
|
|
/* 2048 */ 6, 5, 4, 4,
|
|
|
|
#if CONFIG_EXTENDED_SIZE_CLASSES
|
|
|
|
/* 4096 */ 2, 2, 2, 2,
|
2019-04-10 22:26:39 +02:00
|
|
|
/* 8192 */ 1, 1, 1, 1,
|
2019-04-10 22:32:24 +02:00
|
|
|
/* 16384 */ 1, 1, 1, 1,
|
2018-09-16 19:26:59 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
};
|
|
|
|
|
2018-12-11 18:37:37 +01:00
|
|
|
static const char *const size_class_labels[] = {
|
|
|
|
/* 0 */ "malloc 0",
|
2019-04-08 00:02:56 +02:00
|
|
|
/* 16 */ "malloc 16", "malloc 32", "malloc 48", "malloc 64",
|
|
|
|
/* 16 */ "malloc 80", "malloc 96", "malloc 112", "malloc 128",
|
2018-12-11 18:37:37 +01:00
|
|
|
/* 32 */ "malloc 160", "malloc 192", "malloc 224", "malloc 256",
|
|
|
|
/* 64 */ "malloc 320", "malloc 384", "malloc 448", "malloc 512",
|
|
|
|
/* 128 */ "malloc 640", "malloc 768", "malloc 896", "malloc 1024",
|
|
|
|
/* 256 */ "malloc 1280", "malloc 1536", "malloc 1792", "malloc 2048",
|
|
|
|
/* 512 */ "malloc 2560", "malloc 3072", "malloc 3584", "malloc 4096",
|
|
|
|
/* 1024 */ "malloc 5120", "malloc 6144", "malloc 7168", "malloc 8192",
|
2018-09-16 19:26:59 +02:00
|
|
|
/* 2048 */ "malloc 10240", "malloc 12288", "malloc 14336", "malloc 16384",
|
|
|
|
#if CONFIG_EXTENDED_SIZE_CLASSES
|
|
|
|
/* 4096 */ "malloc 20480", "malloc 24576", "malloc 28672", "malloc 32768",
|
|
|
|
/* 8192 */ "malloc 40960", "malloc 49152", "malloc 57344", "malloc 65536",
|
2019-04-10 22:32:24 +02:00
|
|
|
/* 16384 */ "malloc 81920", "malloc 98304", "malloc 114688", "malloc 131072",
|
2018-09-16 19:26:59 +02:00
|
|
|
#endif
|
2018-12-11 18:37:37 +01:00
|
|
|
};
|
|
|
|
|
2019-04-08 00:02:56 +02:00
|
|
|
static void label_slab(void *slab, size_t slab_size, unsigned class) {
|
|
|
|
memory_set_name(slab, slab_size, size_class_labels[class]);
|
|
|
|
}
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
#define N_SIZE_CLASSES (sizeof(size_classes) / sizeof(size_classes[0]))
|
|
|
|
|
|
|
|
struct size_info {
|
|
|
|
size_t size;
|
|
|
|
size_t class;
|
|
|
|
};
|
|
|
|
|
2018-08-29 21:13:53 +02:00
|
|
|
static inline struct size_info get_size_info(size_t size) {
|
2018-08-24 09:22:52 +02:00
|
|
|
if (size == 0) {
|
2018-09-11 20:51:36 +02:00
|
|
|
return (struct size_info){0, 0};
|
2018-08-24 09:22:52 +02:00
|
|
|
}
|
2018-09-11 20:44:15 +02:00
|
|
|
if (size <= 128) {
|
|
|
|
return (struct size_info){(size + 15) & ~15, ((size - 1) >> 4) + 1};
|
|
|
|
}
|
|
|
|
for (unsigned class = 9; class < N_SIZE_CLASSES; class++) {
|
2018-08-30 13:13:18 +02:00
|
|
|
size_t real_size = size_classes[class];
|
2018-08-21 21:23:22 +02:00
|
|
|
if (size <= real_size) {
|
2018-08-30 13:13:18 +02:00
|
|
|
return (struct size_info){real_size, class};
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
fatal_error("invalid size for slabs");
|
|
|
|
}
|
|
|
|
|
2018-09-10 23:42:58 +02:00
|
|
|
// alignment must be a power of 2 <= PAGE_SIZE since slabs are only page aligned
|
|
|
|
static inline struct size_info get_size_info_align(size_t size, size_t alignment) {
|
|
|
|
for (unsigned class = 1; class < N_SIZE_CLASSES; class++) {
|
|
|
|
size_t real_size = size_classes[class];
|
|
|
|
if (size <= real_size && !(real_size & (alignment - 1))) {
|
|
|
|
return (struct size_info){real_size, class};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fatal_error("invalid size for slabs");
|
|
|
|
}
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
static size_t get_slab_size(size_t slots, size_t size) {
|
|
|
|
return PAGE_CEILING(slots * size);
|
|
|
|
}
|
|
|
|
|
2018-09-01 04:42:34 +02:00
|
|
|
// limit on the number of cached empty slabs before attempting purging instead
|
2020-05-13 07:05:37 +02:00
|
|
|
static const size_t max_empty_slabs_total = MAX_SLAB_SIZE_CLASS;
|
2018-08-30 12:44:58 +02:00
|
|
|
|
2018-10-15 04:06:16 +02:00
|
|
|
struct __attribute__((aligned(CACHELINE_SIZE))) size_class {
|
2018-09-07 07:08:51 +02:00
|
|
|
struct mutex lock;
|
2018-10-14 21:09:55 +02:00
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
void *class_region_start;
|
2018-08-27 11:16:47 +02:00
|
|
|
struct slab_metadata *slab_info;
|
2018-10-14 21:09:55 +02:00
|
|
|
struct libdivide_u32_t size_divisor;
|
|
|
|
struct libdivide_u64_t slab_size_divisor;
|
2018-09-01 04:42:34 +02:00
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
#if SLAB_QUARANTINE_RANDOM_LENGTH > 0
|
2019-01-02 20:22:28 +01:00
|
|
|
void *quarantine_random[SLAB_QUARANTINE_RANDOM_LENGTH << (MAX_SLAB_SIZE_CLASS_SHIFT - MIN_SLAB_SIZE_CLASS_SHIFT)];
|
2018-11-05 22:19:50 +01:00
|
|
|
#endif
|
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
#if SLAB_QUARANTINE_QUEUE_LENGTH > 0
|
2019-01-02 20:22:28 +01:00
|
|
|
void *quarantine_queue[SLAB_QUARANTINE_QUEUE_LENGTH << (MAX_SLAB_SIZE_CLASS_SHIFT - MIN_SLAB_SIZE_CLASS_SHIFT)];
|
2018-11-05 22:19:50 +01:00
|
|
|
size_t quarantine_queue_index;
|
|
|
|
#endif
|
|
|
|
|
2018-09-01 04:42:34 +02:00
|
|
|
// slabs with at least one allocated slot and at least one free slot
|
|
|
|
//
|
|
|
|
// LIFO doubly-linked list
|
2018-08-21 21:23:22 +02:00
|
|
|
struct slab_metadata *partial_slabs;
|
2018-09-01 04:42:34 +02:00
|
|
|
|
|
|
|
// slabs without allocated slots that are cached for near-term usage
|
|
|
|
//
|
|
|
|
// LIFO singly-linked list
|
2018-08-30 10:55:39 +02:00
|
|
|
struct slab_metadata *empty_slabs;
|
2018-09-01 04:42:34 +02:00
|
|
|
size_t empty_slabs_total; // length * slab_size
|
|
|
|
|
|
|
|
// slabs without allocated slots that are purged and memory protected
|
|
|
|
//
|
|
|
|
// FIFO singly-linked list
|
|
|
|
struct slab_metadata *free_slabs_head;
|
|
|
|
struct slab_metadata *free_slabs_tail;
|
2019-01-02 19:40:02 +01:00
|
|
|
struct slab_metadata *free_slabs_quarantine[FREE_SLABS_QUARANTINE_RANDOM_LENGTH];
|
2018-09-01 04:42:34 +02:00
|
|
|
|
2019-04-07 15:06:03 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-07 05:06:47 +02:00
|
|
|
u64 nmalloc; // may wrap (per jemalloc API)
|
|
|
|
u64 ndalloc; // may wrap (per jemalloc API)
|
2019-04-07 02:49:52 +02:00
|
|
|
size_t allocated;
|
|
|
|
size_t slab_allocated;
|
|
|
|
#endif
|
|
|
|
|
2018-08-25 06:21:02 +02:00
|
|
|
struct random_state rng;
|
2018-08-27 11:16:47 +02:00
|
|
|
size_t metadata_allocated;
|
|
|
|
size_t metadata_count;
|
2018-10-06 21:36:03 +02:00
|
|
|
size_t metadata_count_unguarded;
|
2018-10-15 04:06:16 +02:00
|
|
|
};
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-12-05 15:23:05 +01:00
|
|
|
#define CLASS_REGION_SIZE (size_t)CONFIG_CLASS_REGION_SIZE
|
2018-10-12 00:12:20 +02:00
|
|
|
#define REAL_CLASS_REGION_SIZE (CLASS_REGION_SIZE * 2)
|
2019-03-25 19:49:50 +01:00
|
|
|
#define ARENA_SIZE (REAL_CLASS_REGION_SIZE * N_SIZE_CLASSES)
|
|
|
|
static const size_t slab_region_size = ARENA_SIZE * N_ARENA;
|
2018-08-21 21:23:22 +02:00
|
|
|
static_assert(PAGE_SIZE == 4096, "bitmap handling will need adjustment for other page sizes");
|
|
|
|
|
2018-09-07 00:41:00 +02:00
|
|
|
static void *get_slab(struct size_class *c, size_t slab_size, struct slab_metadata *metadata) {
|
|
|
|
size_t index = metadata - c->slab_info;
|
|
|
|
return (char *)c->class_region_start + (index * slab_size);
|
|
|
|
}
|
|
|
|
|
2018-10-18 22:01:04 +02:00
|
|
|
#define MAX_METADATA_MAX (CLASS_REGION_SIZE / PAGE_SIZE)
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
static size_t get_metadata_max(size_t slab_size) {
|
2018-10-12 00:12:20 +02:00
|
|
|
return CLASS_REGION_SIZE / slab_size;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-07 00:41:00 +02:00
|
|
|
static struct slab_metadata *alloc_metadata(struct size_class *c, size_t slab_size, bool non_zero_size) {
|
2018-09-07 00:53:06 +02:00
|
|
|
if (unlikely(c->metadata_count >= c->metadata_allocated)) {
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t metadata_max = get_metadata_max(slab_size);
|
2018-09-07 00:53:06 +02:00
|
|
|
if (c->metadata_count >= metadata_max) {
|
2018-08-29 04:46:20 +02:00
|
|
|
errno = ENOMEM;
|
2018-08-21 21:23:22 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2018-10-18 21:19:54 +02:00
|
|
|
size_t allocate = max(c->metadata_allocated * 2, PAGE_SIZE / sizeof(struct slab_metadata));
|
2018-08-21 21:23:22 +02:00
|
|
|
if (allocate > metadata_max) {
|
|
|
|
allocate = metadata_max;
|
|
|
|
}
|
2018-10-20 03:29:40 +02:00
|
|
|
if (memory_protect_rw_metadata(c->slab_info, allocate * sizeof(struct slab_metadata))) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
c->metadata_allocated = allocate;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct slab_metadata *metadata = c->slab_info + c->metadata_count;
|
2018-09-07 00:41:00 +02:00
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
|
|
|
if (non_zero_size && memory_protect_rw(slab, slab_size)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
c->metadata_count++;
|
2018-10-06 21:36:03 +02:00
|
|
|
c->metadata_count_unguarded++;
|
|
|
|
if (c->metadata_count_unguarded >= GUARD_SLABS_INTERVAL) {
|
2018-09-07 00:53:06 +02:00
|
|
|
c->metadata_count++;
|
2018-10-06 21:36:03 +02:00
|
|
|
c->metadata_count_unguarded = 0;
|
2018-09-07 00:53:06 +02:00
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
return metadata;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void set_slot(struct slab_metadata *metadata, size_t index) {
|
2018-10-15 00:57:59 +02:00
|
|
|
size_t bucket = index / 64;
|
|
|
|
metadata->bitmap[bucket] |= 1UL << (index - bucket * 64);
|
2018-11-01 06:00:59 +01:00
|
|
|
#ifdef SLAB_METADATA_COUNT
|
|
|
|
metadata->count++;
|
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void clear_slot(struct slab_metadata *metadata, size_t index) {
|
2018-10-15 00:57:59 +02:00
|
|
|
size_t bucket = index / 64;
|
|
|
|
metadata->bitmap[bucket] &= ~(1UL << (index - bucket * 64));
|
2018-11-01 06:00:59 +01:00
|
|
|
#ifdef SLAB_METADATA_COUNT
|
|
|
|
metadata->count--;
|
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool get_slot(struct slab_metadata *metadata, size_t index) {
|
2018-10-15 00:57:59 +02:00
|
|
|
size_t bucket = index / 64;
|
|
|
|
return (metadata->bitmap[bucket] >> (index - bucket * 64)) & 1UL;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-11-05 22:19:50 +01:00
|
|
|
#if SLAB_QUARANTINE
|
|
|
|
static void set_quarantine(struct slab_metadata *metadata, size_t index) {
|
|
|
|
size_t bucket = index / 64;
|
2019-03-25 01:24:40 +01:00
|
|
|
metadata->quarantine_bitmap[bucket] |= 1UL << (index - bucket * 64);
|
2018-11-05 22:19:50 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static void clear_quarantine(struct slab_metadata *metadata, size_t index) {
|
|
|
|
size_t bucket = index / 64;
|
2019-03-25 01:24:40 +01:00
|
|
|
metadata->quarantine_bitmap[bucket] &= ~(1UL << (index - bucket * 64));
|
2018-11-05 22:19:50 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool get_quarantine(struct slab_metadata *metadata, size_t index) {
|
|
|
|
size_t bucket = index / 64;
|
2019-03-25 01:24:40 +01:00
|
|
|
return (metadata->quarantine_bitmap[bucket] >> (index - bucket * 64)) & 1UL;
|
2018-11-05 22:19:50 +01:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2018-10-04 20:25:16 +02:00
|
|
|
static u64 get_mask(size_t slots) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return slots < 64 ? ~0UL << slots : 0;
|
|
|
|
}
|
|
|
|
|
2018-08-25 09:02:39 +02:00
|
|
|
static size_t get_free_slot(struct random_state *rng, size_t slots, struct slab_metadata *metadata) {
|
2018-09-07 06:25:02 +02:00
|
|
|
if (SLOT_RANDOMIZE) {
|
|
|
|
// randomize start location for linear search (uniform random choice is too slow)
|
2018-10-07 19:34:52 +02:00
|
|
|
unsigned random_index = get_random_u16_uniform(rng, slots);
|
|
|
|
unsigned first_bitmap = random_index / 64;
|
|
|
|
u64 random_split = ~(~0UL << (random_index - first_bitmap * 64));
|
|
|
|
|
2018-10-30 23:09:07 +01:00
|
|
|
unsigned i = first_bitmap;
|
|
|
|
u64 masked = metadata->bitmap[i];
|
|
|
|
masked |= random_split;
|
|
|
|
for (;;) {
|
2018-10-07 19:34:52 +02:00
|
|
|
if (i == slots / 64) {
|
|
|
|
masked |= get_mask(slots - i * 64);
|
|
|
|
}
|
|
|
|
|
2018-10-30 21:37:23 +01:00
|
|
|
if (masked != ~0UL) {
|
|
|
|
return ffzl(masked) - 1 + i * 64;
|
2018-10-07 19:34:52 +02:00
|
|
|
}
|
|
|
|
|
2018-10-30 23:09:07 +01:00
|
|
|
i = i == (slots - 1) / 64 ? 0 : i + 1;
|
|
|
|
masked = metadata->bitmap[i];
|
2018-10-07 19:34:52 +02:00
|
|
|
}
|
2018-10-30 23:09:07 +01:00
|
|
|
} else {
|
|
|
|
for (unsigned i = 0; i <= (slots - 1) / 64; i++) {
|
|
|
|
u64 masked = metadata->bitmap[i];
|
|
|
|
if (i == (slots - 1) / 64) {
|
|
|
|
masked |= get_mask(slots - i * 64);
|
|
|
|
}
|
2018-10-07 19:34:52 +02:00
|
|
|
|
2018-10-30 23:09:07 +01:00
|
|
|
if (masked != ~0UL) {
|
|
|
|
return ffzl(masked) - 1 + i * 64;
|
|
|
|
}
|
2018-10-07 19:34:52 +02:00
|
|
|
}
|
2018-08-25 09:02:39 +02:00
|
|
|
}
|
2018-09-07 06:25:02 +02:00
|
|
|
|
2018-10-07 19:34:52 +02:00
|
|
|
fatal_error("no zero bits");
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool has_free_slots(size_t slots, struct slab_metadata *metadata) {
|
2018-11-01 06:00:59 +01:00
|
|
|
#ifdef SLAB_METADATA_COUNT
|
|
|
|
return metadata->count < slots;
|
|
|
|
#else
|
2018-10-07 19:34:52 +02:00
|
|
|
if (slots <= 64) {
|
|
|
|
u64 masked = metadata->bitmap[0] | get_mask(slots);
|
|
|
|
return masked != ~0UL;
|
2018-11-16 21:12:10 +01:00
|
|
|
}
|
|
|
|
if (slots <= 128) {
|
2018-10-07 19:34:52 +02:00
|
|
|
u64 masked = metadata->bitmap[1] | get_mask(slots - 64);
|
|
|
|
return metadata->bitmap[0] != ~0UL || masked != ~0UL;
|
2018-11-16 21:12:10 +01:00
|
|
|
}
|
|
|
|
if (slots <= 192) {
|
2018-10-07 19:34:52 +02:00
|
|
|
u64 masked = metadata->bitmap[2] | get_mask(slots - 128);
|
|
|
|
return metadata->bitmap[0] != ~0UL || metadata->bitmap[1] != ~0UL || masked != ~0UL;
|
2018-08-25 09:02:39 +02:00
|
|
|
}
|
2018-10-07 19:34:52 +02:00
|
|
|
u64 masked = metadata->bitmap[3] | get_mask(slots - 192);
|
|
|
|
return metadata->bitmap[0] != ~0UL || metadata->bitmap[1] != ~0UL || metadata->bitmap[2] != ~0UL || masked != ~0UL;
|
2018-11-01 06:00:59 +01:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool is_free_slab(struct slab_metadata *metadata) {
|
2018-11-01 06:00:59 +01:00
|
|
|
#ifdef SLAB_METADATA_COUNT
|
|
|
|
return !metadata->count;
|
|
|
|
#else
|
2018-10-07 19:34:52 +02:00
|
|
|
return !metadata->bitmap[0] && !metadata->bitmap[1] && !metadata->bitmap[2] &&
|
|
|
|
!metadata->bitmap[3];
|
2018-11-01 06:00:59 +01:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2020-04-30 22:06:32 +02:00
|
|
|
static struct slab_metadata *get_metadata(struct size_class *c, const void *p) {
|
|
|
|
size_t offset = (const char *)p - (const char *)c->class_region_start;
|
2018-08-27 12:57:44 +02:00
|
|
|
size_t index = libdivide_u64_do(offset, &c->slab_size_divisor);
|
2018-08-24 11:18:20 +02:00
|
|
|
// still caught without this check either as a read access violation or "double free"
|
|
|
|
if (index >= c->metadata_allocated) {
|
|
|
|
fatal_error("invalid free within a slab yet to be used");
|
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
return c->slab_info + index;
|
|
|
|
}
|
|
|
|
|
2018-08-24 08:55:53 +02:00
|
|
|
static void *slot_pointer(size_t size, void *slab, size_t slot) {
|
|
|
|
return (char *)slab + slot * size;
|
|
|
|
}
|
|
|
|
|
2018-12-05 08:45:24 +01:00
|
|
|
static void write_after_free_check(const char *p, size_t size) {
|
2018-09-07 06:17:22 +02:00
|
|
|
if (!WRITE_AFTER_FREE_CHECK) {
|
2018-09-07 06:00:32 +02:00
|
|
|
return;
|
2018-09-02 14:36:48 +02:00
|
|
|
}
|
2018-09-07 06:00:32 +02:00
|
|
|
|
2018-10-04 20:25:16 +02:00
|
|
|
for (size_t i = 0; i < size; i += sizeof(u64)) {
|
2018-12-05 08:45:24 +01:00
|
|
|
if (*(const u64 *)(const void *)(p + i)) {
|
2018-09-07 06:17:22 +02:00
|
|
|
fatal_error("detected write after free");
|
2018-09-07 06:00:32 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-04 20:25:16 +02:00
|
|
|
static const u64 canary_mask = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ?
|
2018-10-03 23:17:20 +02:00
|
|
|
0xffffffffffffff00UL :
|
|
|
|
0x00ffffffffffffffUL;
|
2018-10-03 23:09:57 +02:00
|
|
|
|
2018-09-07 06:00:32 +02:00
|
|
|
static void set_canary(struct slab_metadata *metadata, void *p, size_t size) {
|
|
|
|
memcpy((char *)p + size - canary_size, &metadata->canary_value, canary_size);
|
2018-09-02 14:36:48 +02:00
|
|
|
}
|
|
|
|
|
2018-10-31 00:22:56 +01:00
|
|
|
static u64 get_random_canary(struct random_state *rng) {
|
|
|
|
return get_random_u64(rng) & canary_mask;
|
|
|
|
}
|
|
|
|
|
2019-04-09 09:57:44 +02:00
|
|
|
static inline void stats_small_allocate(UNUSED struct size_class *c, UNUSED size_t size) {
|
2019-04-11 06:49:45 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-09 09:57:44 +02:00
|
|
|
c->allocated += size;
|
|
|
|
c->nmalloc++;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void stats_small_deallocate(UNUSED struct size_class *c, UNUSED size_t size) {
|
2019-04-11 06:49:45 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-09 09:57:44 +02:00
|
|
|
c->allocated -= size;
|
|
|
|
c->ndalloc++;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void stats_slab_allocate(UNUSED struct size_class *c, UNUSED size_t slab_size) {
|
2019-04-11 06:49:45 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-09 09:57:44 +02:00
|
|
|
c->slab_allocated += slab_size;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2019-04-11 06:49:45 +02:00
|
|
|
static inline void stats_slab_deallocate(UNUSED struct size_class *c, UNUSED size_t slab_size) {
|
|
|
|
#if CONFIG_STATS
|
2019-04-09 09:57:44 +02:00
|
|
|
c->slab_allocated -= slab_size;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
static inline void *allocate_small(unsigned arena, size_t requested_size) {
|
2018-08-21 21:23:22 +02:00
|
|
|
struct size_info info = get_size_info(requested_size);
|
2018-09-11 20:51:36 +02:00
|
|
|
size_t size = info.size ? info.size : 16;
|
2019-03-25 19:49:50 +01:00
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][info.class];
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t slots = size_class_slots[info.class];
|
|
|
|
size_t slab_size = get_slab_size(slots, size);
|
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_lock(&c->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
|
|
|
if (c->partial_slabs == NULL) {
|
2018-08-30 10:55:39 +02:00
|
|
|
if (c->empty_slabs != NULL) {
|
|
|
|
struct slab_metadata *metadata = c->empty_slabs;
|
|
|
|
c->empty_slabs = c->empty_slabs->next;
|
2018-08-30 12:44:58 +02:00
|
|
|
c->empty_slabs_total -= slab_size;
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-08-30 11:42:45 +02:00
|
|
|
metadata->next = NULL;
|
2018-08-21 21:23:22 +02:00
|
|
|
metadata->prev = NULL;
|
|
|
|
|
2019-04-10 22:26:39 +02:00
|
|
|
c->partial_slabs = slots > 1 ? metadata : NULL;
|
2018-08-21 21:23:22 +02:00
|
|
|
|
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
2018-08-26 00:02:13 +02:00
|
|
|
size_t slot = get_free_slot(&c->rng, slots, metadata);
|
|
|
|
set_slot(metadata, slot);
|
|
|
|
void *p = slot_pointer(size, slab, slot);
|
2018-09-07 06:00:32 +02:00
|
|
|
if (requested_size) {
|
|
|
|
write_after_free_check(p, size - canary_size);
|
|
|
|
set_canary(metadata, p, size);
|
|
|
|
}
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_small_allocate(c, size);
|
2018-08-26 00:02:13 +02:00
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-30 11:18:38 +02:00
|
|
|
return p;
|
2018-11-16 21:12:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (c->free_slabs_head != NULL) {
|
2018-09-01 04:42:34 +02:00
|
|
|
struct slab_metadata *metadata = c->free_slabs_head;
|
2018-10-31 00:22:56 +01:00
|
|
|
metadata->canary_value = get_random_canary(&c->rng);
|
2018-08-30 11:18:38 +02:00
|
|
|
|
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
2018-09-07 06:00:32 +02:00
|
|
|
if (requested_size && memory_protect_rw(slab, slab_size)) {
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-30 11:18:38 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-09-01 04:42:34 +02:00
|
|
|
c->free_slabs_head = c->free_slabs_head->next;
|
|
|
|
if (c->free_slabs_head == NULL) {
|
|
|
|
c->free_slabs_tail = NULL;
|
|
|
|
}
|
2018-08-30 11:18:38 +02:00
|
|
|
|
|
|
|
metadata->next = NULL;
|
|
|
|
metadata->prev = NULL;
|
|
|
|
|
2019-04-10 22:26:39 +02:00
|
|
|
c->partial_slabs = slots > 1 ? metadata : NULL;
|
2018-08-30 11:18:38 +02:00
|
|
|
|
|
|
|
size_t slot = get_free_slot(&c->rng, slots, metadata);
|
|
|
|
set_slot(metadata, slot);
|
|
|
|
void *p = slot_pointer(size, slab, slot);
|
2018-09-07 06:00:32 +02:00
|
|
|
if (requested_size) {
|
|
|
|
set_canary(metadata, p, size);
|
|
|
|
}
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_slab_allocate(c, slab_size);
|
|
|
|
stats_small_allocate(c, size);
|
2018-08-30 11:18:38 +02:00
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-26 00:02:13 +02:00
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-07 00:41:00 +02:00
|
|
|
struct slab_metadata *metadata = alloc_metadata(c, slab_size, requested_size);
|
|
|
|
if (unlikely(metadata == NULL)) {
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2018-10-31 00:22:56 +01:00
|
|
|
metadata->canary_value = get_random_canary(&c->rng);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-04-10 22:26:39 +02:00
|
|
|
c->partial_slabs = slots > 1 ? metadata : NULL;
|
2018-09-07 00:41:00 +02:00
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
2018-08-26 00:02:13 +02:00
|
|
|
size_t slot = get_free_slot(&c->rng, slots, metadata);
|
|
|
|
set_slot(metadata, slot);
|
|
|
|
void *p = slot_pointer(size, slab, slot);
|
2018-09-07 06:00:32 +02:00
|
|
|
if (requested_size) {
|
|
|
|
set_canary(metadata, p, size);
|
|
|
|
}
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_slab_allocate(c, slab_size);
|
|
|
|
stats_small_allocate(c, size);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-26 00:02:13 +02:00
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
struct slab_metadata *metadata = c->partial_slabs;
|
2018-08-25 09:02:39 +02:00
|
|
|
size_t slot = get_free_slot(&c->rng, slots, metadata);
|
2018-08-21 21:23:22 +02:00
|
|
|
set_slot(metadata, slot);
|
|
|
|
|
|
|
|
if (!has_free_slots(slots, metadata)) {
|
|
|
|
c->partial_slabs = c->partial_slabs->next;
|
|
|
|
if (c->partial_slabs) {
|
|
|
|
c->partial_slabs->prev = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
2018-08-24 08:55:53 +02:00
|
|
|
void *p = slot_pointer(size, slab, slot);
|
2018-09-07 06:00:32 +02:00
|
|
|
if (requested_size) {
|
|
|
|
write_after_free_check(p, size - canary_size);
|
|
|
|
set_canary(metadata, p, size);
|
|
|
|
}
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_small_allocate(c, size);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
struct slab_size_class_info {
|
|
|
|
unsigned arena;
|
|
|
|
size_t class;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct slab_size_class_info slab_size_class(const void *p) {
|
2018-12-05 09:27:09 +01:00
|
|
|
size_t offset = (const char *)p - (const char *)ro.slab_region_start;
|
2019-03-25 19:49:50 +01:00
|
|
|
unsigned arena = 0;
|
|
|
|
if (N_ARENA > 1) {
|
|
|
|
arena = offset / ARENA_SIZE;
|
|
|
|
offset -= arena * ARENA_SIZE;
|
|
|
|
}
|
|
|
|
return (struct slab_size_class_info){arena, offset / REAL_CLASS_REGION_SIZE};
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-12-05 09:27:09 +01:00
|
|
|
static size_t slab_usable_size(const void *p) {
|
2019-03-25 19:49:50 +01:00
|
|
|
return size_classes[slab_size_class(p).class];
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-01 04:42:34 +02:00
|
|
|
static void enqueue_free_slab(struct size_class *c, struct slab_metadata *metadata) {
|
|
|
|
metadata->next = NULL;
|
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
static_assert(FREE_SLABS_QUARANTINE_RANDOM_LENGTH < (u16)-1, "free slabs quarantine too large");
|
|
|
|
size_t index = get_random_u16_uniform(&c->rng, FREE_SLABS_QUARANTINE_RANDOM_LENGTH);
|
2018-10-13 19:55:44 +02:00
|
|
|
struct slab_metadata *substitute = c->free_slabs_quarantine[index];
|
|
|
|
c->free_slabs_quarantine[index] = metadata;
|
|
|
|
|
|
|
|
if (substitute == NULL) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-09-01 04:42:34 +02:00
|
|
|
if (c->free_slabs_tail != NULL) {
|
2018-10-13 19:55:44 +02:00
|
|
|
c->free_slabs_tail->next = substitute;
|
2018-09-01 04:42:34 +02:00
|
|
|
} else {
|
2018-10-13 19:55:44 +02:00
|
|
|
c->free_slabs_head = substitute;
|
2018-09-01 04:42:34 +02:00
|
|
|
}
|
2018-10-13 19:55:44 +02:00
|
|
|
c->free_slabs_tail = substitute;
|
2018-09-01 04:42:34 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 00:28:01 +02:00
|
|
|
static inline void deallocate_small(void *p, const size_t *expected_size) {
|
2019-03-25 19:49:50 +01:00
|
|
|
struct slab_size_class_info size_class_info = slab_size_class(p);
|
|
|
|
size_t class = size_class_info.class;
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
struct size_class *c = &ro.size_class_metadata[size_class_info.arena][class];
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t size = size_classes[class];
|
2018-09-18 23:28:52 +02:00
|
|
|
if (expected_size && size != *expected_size) {
|
2018-10-05 07:05:40 +02:00
|
|
|
fatal_error("sized deallocation mismatch (small)");
|
2018-09-18 23:28:52 +02:00
|
|
|
}
|
2018-08-25 08:48:47 +02:00
|
|
|
bool is_zero_size = size == 0;
|
|
|
|
if (is_zero_size) {
|
|
|
|
size = 16;
|
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t slots = size_class_slots[class];
|
|
|
|
size_t slab_size = get_slab_size(slots, size);
|
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_lock(&c->lock);
|
2019-04-09 09:57:44 +02:00
|
|
|
|
|
|
|
stats_small_deallocate(c, size);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-08-27 12:57:44 +02:00
|
|
|
struct slab_metadata *metadata = get_metadata(c, p);
|
2018-08-24 08:55:53 +02:00
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
2018-08-27 12:57:44 +02:00
|
|
|
size_t slot = libdivide_u32_do((char *)p - (char *)slab, &c->size_divisor);
|
2018-08-24 08:55:53 +02:00
|
|
|
|
|
|
|
if (slot_pointer(size, slab, slot) != p) {
|
|
|
|
fatal_error("invalid unaligned free");
|
|
|
|
}
|
|
|
|
|
2018-08-24 10:51:52 +02:00
|
|
|
if (!get_slot(metadata, slot)) {
|
|
|
|
fatal_error("double free");
|
|
|
|
}
|
|
|
|
|
2018-09-02 14:36:48 +02:00
|
|
|
if (!is_zero_size) {
|
|
|
|
if (canary_size) {
|
2018-10-04 20:25:16 +02:00
|
|
|
u64 canary_value;
|
2018-09-02 14:36:48 +02:00
|
|
|
memcpy(&canary_value, (char *)p + size - canary_size, canary_size);
|
|
|
|
if (unlikely(canary_value != metadata->canary_value)) {
|
|
|
|
fatal_error("canary corrupted");
|
|
|
|
}
|
|
|
|
}
|
2018-10-29 02:07:35 +01:00
|
|
|
|
|
|
|
if (ZERO_ON_FREE) {
|
|
|
|
memset(p, 0, size - canary_size);
|
|
|
|
}
|
2018-09-02 14:36:48 +02:00
|
|
|
}
|
|
|
|
|
2018-11-05 22:19:50 +01:00
|
|
|
#if SLAB_QUARANTINE
|
|
|
|
if (get_quarantine(metadata, slot)) {
|
|
|
|
fatal_error("double free (quarantine)");
|
|
|
|
}
|
|
|
|
|
|
|
|
set_quarantine(metadata, slot);
|
|
|
|
|
2019-01-02 20:22:28 +01:00
|
|
|
size_t quarantine_shift = __builtin_clzl(size) - (63 - MAX_SLAB_SIZE_CLASS_SHIFT);
|
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
#if SLAB_QUARANTINE_RANDOM_LENGTH > 0
|
2019-01-02 20:22:28 +01:00
|
|
|
size_t slab_quarantine_random_length = SLAB_QUARANTINE_RANDOM_LENGTH << quarantine_shift;
|
|
|
|
|
|
|
|
size_t random_index = get_random_u16_uniform(&c->rng, slab_quarantine_random_length);
|
2019-01-02 19:23:49 +01:00
|
|
|
void *random_substitute = c->quarantine_random[random_index];
|
2018-11-05 22:19:50 +01:00
|
|
|
c->quarantine_random[random_index] = p;
|
|
|
|
|
2019-01-02 19:23:49 +01:00
|
|
|
if (random_substitute == NULL) {
|
2018-11-05 22:19:50 +01:00
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-01-02 19:23:49 +01:00
|
|
|
p = random_substitute;
|
2018-11-05 22:19:50 +01:00
|
|
|
#endif
|
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
#if SLAB_QUARANTINE_QUEUE_LENGTH > 0
|
2019-01-02 20:22:28 +01:00
|
|
|
size_t slab_quarantine_queue_length = SLAB_QUARANTINE_QUEUE_LENGTH << quarantine_shift;
|
|
|
|
|
2019-01-02 19:23:49 +01:00
|
|
|
void *queue_substitute = c->quarantine_queue[c->quarantine_queue_index];
|
2018-11-05 22:19:50 +01:00
|
|
|
c->quarantine_queue[c->quarantine_queue_index] = p;
|
2019-01-02 20:22:28 +01:00
|
|
|
c->quarantine_queue_index = (c->quarantine_queue_index + 1) % slab_quarantine_queue_length;
|
2018-11-05 22:19:50 +01:00
|
|
|
|
2019-01-02 19:23:49 +01:00
|
|
|
if (queue_substitute == NULL) {
|
2018-11-05 22:19:50 +01:00
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-01-02 19:23:49 +01:00
|
|
|
p = queue_substitute;
|
2018-11-05 22:19:50 +01:00
|
|
|
#endif
|
|
|
|
|
|
|
|
metadata = get_metadata(c, p);
|
|
|
|
slab = get_slab(c, slab_size, metadata);
|
|
|
|
slot = libdivide_u32_do((char *)p - (char *)slab, &c->size_divisor);
|
|
|
|
|
|
|
|
clear_quarantine(metadata, slot);
|
|
|
|
#endif
|
|
|
|
|
2019-04-10 22:26:39 +02:00
|
|
|
// triggered even for slots == 1 and then undone below
|
2018-08-21 21:23:22 +02:00
|
|
|
if (!has_free_slots(slots, metadata)) {
|
|
|
|
metadata->next = c->partial_slabs;
|
|
|
|
metadata->prev = NULL;
|
|
|
|
|
|
|
|
if (c->partial_slabs) {
|
|
|
|
c->partial_slabs->prev = metadata;
|
|
|
|
}
|
|
|
|
c->partial_slabs = metadata;
|
|
|
|
}
|
|
|
|
|
|
|
|
clear_slot(metadata, slot);
|
|
|
|
|
|
|
|
if (is_free_slab(metadata)) {
|
|
|
|
if (metadata->prev) {
|
|
|
|
metadata->prev->next = metadata->next;
|
|
|
|
} else {
|
|
|
|
c->partial_slabs = metadata->next;
|
|
|
|
}
|
|
|
|
if (metadata->next) {
|
|
|
|
metadata->next->prev = metadata->prev;
|
|
|
|
}
|
|
|
|
|
|
|
|
metadata->prev = NULL;
|
2018-08-30 12:44:58 +02:00
|
|
|
|
|
|
|
if (c->empty_slabs_total + slab_size > max_empty_slabs_total) {
|
|
|
|
if (!memory_map_fixed(slab, slab_size)) {
|
2019-04-08 00:02:56 +02:00
|
|
|
label_slab(slab, slab_size, class);
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_slab_deallocate(c, slab_size);
|
2018-09-01 04:42:34 +02:00
|
|
|
enqueue_free_slab(c, metadata);
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-30 12:44:58 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
// handle out-of-memory by just putting it into the empty slabs list
|
|
|
|
}
|
|
|
|
|
|
|
|
metadata->next = c->empty_slabs;
|
2018-08-30 10:55:39 +02:00
|
|
|
c->empty_slabs = metadata;
|
2018-08-30 12:44:58 +02:00
|
|
|
c->empty_slabs_total += slab_size;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&c->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata {
|
2018-08-21 21:23:22 +02:00
|
|
|
void *p;
|
|
|
|
size_t size;
|
2018-08-26 10:42:01 +02:00
|
|
|
size_t guard_size;
|
2018-08-21 21:23:22 +02:00
|
|
|
};
|
|
|
|
|
2018-10-09 20:08:36 +02:00
|
|
|
struct quarantine_info {
|
|
|
|
void *p;
|
|
|
|
size_t size;
|
|
|
|
};
|
|
|
|
|
2019-04-15 06:04:00 +02:00
|
|
|
#define INITIAL_REGION_TABLE_SIZE 128
|
2018-10-15 21:07:59 +02:00
|
|
|
#define MAX_REGION_TABLE_SIZE (CLASS_REGION_SIZE / PAGE_SIZE / sizeof(struct region_metadata))
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator {
|
|
|
|
struct mutex lock;
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *regions;
|
2018-10-15 03:12:37 +02:00
|
|
|
size_t total;
|
|
|
|
size_t free;
|
2019-04-07 15:06:03 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-07 02:49:52 +02:00
|
|
|
size_t allocated;
|
|
|
|
#endif
|
2019-01-02 19:40:02 +01:00
|
|
|
struct quarantine_info quarantine_random[REGION_QUARANTINE_RANDOM_LENGTH];
|
|
|
|
struct quarantine_info quarantine_queue[REGION_QUARANTINE_QUEUE_LENGTH];
|
2018-10-15 03:12:37 +02:00
|
|
|
size_t quarantine_queue_index;
|
|
|
|
struct random_state rng;
|
|
|
|
};
|
2018-10-08 21:50:31 +02:00
|
|
|
|
2019-04-11 06:49:45 +02:00
|
|
|
static inline void stats_large_allocate(UNUSED struct region_allocator *ra, UNUSED size_t size) {
|
|
|
|
#if CONFIG_STATS
|
|
|
|
ra->allocated += size;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void stats_large_deallocate(UNUSED struct region_allocator *ra, UNUSED size_t size) {
|
|
|
|
#if CONFIG_STATS
|
|
|
|
ra->allocated -= size;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2018-10-18 22:01:04 +02:00
|
|
|
struct __attribute__((aligned(PAGE_SIZE))) slab_info_mapping {
|
|
|
|
struct slab_metadata slab_info[MAX_METADATA_MAX];
|
|
|
|
};
|
|
|
|
|
2018-10-16 01:08:20 +02:00
|
|
|
struct __attribute__((aligned(PAGE_SIZE))) allocator_state {
|
2019-03-25 19:49:50 +01:00
|
|
|
struct size_class size_class_metadata[N_ARENA][N_SIZE_CLASSES];
|
2018-10-15 20:46:57 +02:00
|
|
|
struct region_allocator region_allocator;
|
2018-10-16 01:08:20 +02:00
|
|
|
// padding until next page boundary for mprotect
|
2018-10-15 21:07:59 +02:00
|
|
|
struct region_metadata regions_a[MAX_REGION_TABLE_SIZE] __attribute__((aligned(PAGE_SIZE)));
|
2018-10-16 01:08:20 +02:00
|
|
|
// padding until next page boundary for mprotect
|
2018-10-15 21:07:59 +02:00
|
|
|
struct region_metadata regions_b[MAX_REGION_TABLE_SIZE] __attribute__((aligned(PAGE_SIZE)));
|
2018-10-16 01:08:20 +02:00
|
|
|
// padding until next page boundary for mprotect
|
2019-03-25 19:49:50 +01:00
|
|
|
struct slab_info_mapping slab_info_mapping[N_ARENA][N_SIZE_CLASSES];
|
2018-10-18 22:01:04 +02:00
|
|
|
// padding until next page boundary for mprotect
|
2018-10-15 20:46:57 +02:00
|
|
|
};
|
|
|
|
|
2018-10-08 21:50:31 +02:00
|
|
|
static void regions_quarantine_deallocate_pages(void *p, size_t size, size_t guard_size) {
|
|
|
|
if (size >= REGION_QUARANTINE_SKIP_THRESHOLD) {
|
|
|
|
deallocate_pages(p, size, guard_size);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlikely(memory_map_fixed(p, size))) {
|
|
|
|
deallocate_pages(p, size, guard_size);
|
|
|
|
return;
|
|
|
|
}
|
2019-03-24 03:57:19 +01:00
|
|
|
memory_set_name(p, size, "malloc large");
|
2018-10-08 21:50:31 +02:00
|
|
|
|
2018-10-12 21:03:59 +02:00
|
|
|
struct quarantine_info a =
|
2018-10-09 20:08:36 +02:00
|
|
|
(struct quarantine_info){(char *)p - guard_size, size + guard_size * 2};
|
2018-10-12 21:03:59 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
mutex_lock(&ra->lock);
|
2018-10-12 21:03:59 +02:00
|
|
|
|
2019-01-02 19:40:02 +01:00
|
|
|
size_t index = get_random_u64_uniform(&ra->rng, REGION_QUARANTINE_RANDOM_LENGTH);
|
2018-10-15 03:12:37 +02:00
|
|
|
struct quarantine_info b = ra->quarantine_random[index];
|
|
|
|
ra->quarantine_random[index] = a;
|
2018-10-12 21:03:59 +02:00
|
|
|
if (b.p == NULL) {
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-10-12 21:03:59 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
a = ra->quarantine_queue[ra->quarantine_queue_index];
|
|
|
|
ra->quarantine_queue[ra->quarantine_queue_index] = b;
|
2019-01-02 19:40:02 +01:00
|
|
|
ra->quarantine_queue_index = (ra->quarantine_queue_index + 1) % REGION_QUARANTINE_QUEUE_LENGTH;
|
2018-10-12 21:03:59 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-10-12 21:06:51 +02:00
|
|
|
|
|
|
|
if (a.p != NULL) {
|
|
|
|
memory_unmap(a.p, a.size);
|
|
|
|
}
|
2018-10-08 21:50:31 +02:00
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
|
|
|
|
static int regions_grow(void) {
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
2018-10-15 03:54:23 +02:00
|
|
|
if (ra->total > SIZE_MAX / sizeof(struct region_metadata) / 2) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
size_t newtotal = ra->total * 2;
|
2018-10-15 03:54:23 +02:00
|
|
|
size_t newsize = newtotal * sizeof(struct region_metadata);
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t mask = newtotal - 1;
|
|
|
|
|
2018-10-15 21:07:59 +02:00
|
|
|
if (newtotal > MAX_REGION_TABLE_SIZE) {
|
2018-09-01 16:20:23 +02:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *p = ra->regions == ro.regions[0] ?
|
2018-09-01 16:20:23 +02:00
|
|
|
ro.regions[1] : ro.regions[0];
|
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
if (memory_protect_rw_metadata(p, newsize)) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
for (size_t i = 0; i < ra->total; i++) {
|
|
|
|
void *q = ra->regions[i].p;
|
2018-08-21 21:23:22 +02:00
|
|
|
if (q != NULL) {
|
|
|
|
size_t index = hash_page(q) & mask;
|
|
|
|
while (p[index].p != NULL) {
|
|
|
|
index = (index - 1) & mask;
|
|
|
|
}
|
2018-10-15 03:12:37 +02:00
|
|
|
p[index] = ra->regions[i];
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:54:23 +02:00
|
|
|
memory_map_fixed(ra->regions, ra->total * sizeof(struct region_metadata));
|
2019-03-20 20:18:34 +01:00
|
|
|
memory_set_name(ra->regions, ra->total * sizeof(struct region_metadata), "malloc allocator_state");
|
2018-10-15 03:12:37 +02:00
|
|
|
ra->free = ra->free + ra->total;
|
|
|
|
ra->total = newtotal;
|
|
|
|
ra->regions = p;
|
2018-08-21 21:23:22 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-08-26 10:42:01 +02:00
|
|
|
static int regions_insert(void *p, size_t size, size_t guard_size) {
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
if (ra->free * 4 < ra->total) {
|
2018-08-21 21:23:22 +02:00
|
|
|
if (regions_grow()) {
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
size_t mask = ra->total - 1;
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t index = hash_page(p) & mask;
|
2018-10-15 03:12:37 +02:00
|
|
|
void *q = ra->regions[index].p;
|
2018-08-21 21:23:22 +02:00
|
|
|
while (q != NULL) {
|
|
|
|
index = (index - 1) & mask;
|
2018-10-15 03:12:37 +02:00
|
|
|
q = ra->regions[index].p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
2018-10-15 03:12:37 +02:00
|
|
|
ra->regions[index].p = p;
|
|
|
|
ra->regions[index].size = size;
|
|
|
|
ra->regions[index].guard_size = guard_size;
|
|
|
|
ra->free--;
|
2018-08-21 21:23:22 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-12-05 09:27:09 +01:00
|
|
|
static struct region_metadata *regions_find(const void *p) {
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
size_t mask = ra->total - 1;
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t index = hash_page(p) & mask;
|
2018-10-15 03:12:37 +02:00
|
|
|
void *r = ra->regions[index].p;
|
2018-08-21 21:23:22 +02:00
|
|
|
while (r != p && r != NULL) {
|
|
|
|
index = (index - 1) & mask;
|
2018-10-15 03:12:37 +02:00
|
|
|
r = ra->regions[index].p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
2018-10-15 03:12:37 +02:00
|
|
|
return (r == p && r != NULL) ? &ra->regions[index] : NULL;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 03:54:23 +02:00
|
|
|
static void regions_delete(struct region_metadata *region) {
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
size_t mask = ra->total - 1;
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
ra->free++;
|
|
|
|
|
|
|
|
size_t i = region - ra->regions;
|
2018-08-21 21:23:22 +02:00
|
|
|
for (;;) {
|
2018-10-15 03:12:37 +02:00
|
|
|
ra->regions[i].p = NULL;
|
|
|
|
ra->regions[i].size = 0;
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t j = i;
|
|
|
|
for (;;) {
|
|
|
|
i = (i - 1) & mask;
|
2018-10-15 03:12:37 +02:00
|
|
|
if (ra->regions[i].p == NULL) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return;
|
|
|
|
}
|
2018-10-15 03:12:37 +02:00
|
|
|
size_t r = hash_page(ra->regions[i].p) & mask;
|
2018-08-21 21:23:22 +02:00
|
|
|
if ((i <= r && r < j) || (r < j && j < i) || (j < i && i <= r)) {
|
|
|
|
continue;
|
|
|
|
}
|
2018-10-15 03:12:37 +02:00
|
|
|
ra->regions[j] = ra->regions[i];
|
2018-08-21 21:23:22 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-07 06:50:26 +02:00
|
|
|
int get_metadata_key(void) {
|
|
|
|
#ifdef USE_PKEY
|
|
|
|
return ro.metadata_pkey;
|
|
|
|
#else
|
|
|
|
return -1;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2020-08-04 23:22:03 +02:00
|
|
|
static inline void thread_set_metadata_access(UNUSED unsigned access) {
|
2019-04-07 06:50:26 +02:00
|
|
|
#ifdef USE_PKEY
|
|
|
|
if (ro.metadata_pkey == -1) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
pkey_set(ro.metadata_pkey, access);
|
|
|
|
#endif
|
2020-08-04 23:22:03 +02:00
|
|
|
}
|
2019-04-07 06:50:26 +02:00
|
|
|
|
|
|
|
static inline void thread_unseal_metadata(void) {
|
|
|
|
thread_set_metadata_access(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void thread_seal_metadata(void) {
|
|
|
|
#ifdef USE_PKEY
|
|
|
|
thread_set_metadata_access(PKEY_DISABLE_ACCESS);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2018-09-06 20:17:06 +02:00
|
|
|
static void full_lock(void) {
|
2018-10-23 22:28:54 +02:00
|
|
|
thread_unseal_metadata();
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_lock(&ro.region_allocator->lock);
|
2019-03-25 19:49:50 +01:00
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
|
|
|
mutex_lock(&ro.size_class_metadata[arena][class].lock);
|
|
|
|
}
|
2018-08-23 23:15:50 +02:00
|
|
|
}
|
2018-10-23 22:28:54 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-23 23:15:50 +02:00
|
|
|
}
|
|
|
|
|
2018-09-06 20:17:06 +02:00
|
|
|
static void full_unlock(void) {
|
2018-10-23 22:28:54 +02:00
|
|
|
thread_unseal_metadata();
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ro.region_allocator->lock);
|
2019-03-25 19:49:50 +01:00
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
|
|
|
mutex_unlock(&ro.size_class_metadata[arena][class].lock);
|
|
|
|
}
|
2018-08-23 23:15:50 +02:00
|
|
|
}
|
2018-10-23 22:28:54 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-23 23:15:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void post_fork_child(void) {
|
2018-10-23 22:28:54 +02:00
|
|
|
thread_unseal_metadata();
|
2018-10-24 01:07:41 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_init(&ro.region_allocator->lock);
|
|
|
|
random_state_init(&ro.region_allocator->rng);
|
2019-03-25 19:49:50 +01:00
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][class];
|
|
|
|
mutex_init(&c->lock);
|
|
|
|
random_state_init(&c->rng);
|
|
|
|
}
|
2018-08-23 23:15:50 +02:00
|
|
|
}
|
2018-10-23 22:28:54 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-23 23:15:50 +02:00
|
|
|
}
|
|
|
|
|
2018-09-03 01:58:57 +02:00
|
|
|
static inline bool is_init(void) {
|
2018-11-05 08:19:04 +01:00
|
|
|
return get_slab_region_start() != NULL;
|
2018-09-03 01:58:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void enforce_init(void) {
|
|
|
|
if (!is_init()) {
|
|
|
|
fatal_error("invalid uninitialized allocator usage");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
COLD static void init_slow_path(void) {
|
2018-09-07 07:08:51 +02:00
|
|
|
static struct mutex lock = MUTEX_INITIALIZER;
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_lock(&lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-03 01:58:57 +02:00
|
|
|
if (is_init()) {
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
#ifdef USE_PKEY
|
|
|
|
ro.metadata_pkey = pkey_alloc(0, 0);
|
|
|
|
#endif
|
|
|
|
|
2018-08-29 16:59:37 +02:00
|
|
|
if (sysconf(_SC_PAGESIZE) != PAGE_SIZE) {
|
|
|
|
fatal_error("page size mismatch");
|
|
|
|
}
|
|
|
|
|
2019-03-23 04:17:26 +01:00
|
|
|
struct random_state *rng = allocate_pages(sizeof(struct random_state), PAGE_SIZE, true, "malloc init rng");
|
2018-10-15 04:41:17 +02:00
|
|
|
if (rng == NULL) {
|
|
|
|
fatal_error("failed to allocate init rng");
|
|
|
|
}
|
|
|
|
random_state_init(rng);
|
|
|
|
|
|
|
|
size_t metadata_guard_size =
|
|
|
|
(get_random_u64_uniform(rng, REAL_CLASS_REGION_SIZE / PAGE_SIZE) + 1) * PAGE_SIZE;
|
|
|
|
|
2018-10-15 20:46:57 +02:00
|
|
|
struct allocator_state *allocator_state =
|
2019-03-23 04:17:26 +01:00
|
|
|
allocate_pages(sizeof(struct allocator_state), metadata_guard_size, false, "malloc allocator_state");
|
2018-10-15 20:46:57 +02:00
|
|
|
if (allocator_state == NULL) {
|
|
|
|
fatal_error("failed to reserve allocator state");
|
2018-10-15 04:06:16 +02:00
|
|
|
}
|
2018-10-20 03:29:40 +02:00
|
|
|
if (memory_protect_rw_metadata(allocator_state, offsetof(struct allocator_state, regions_a))) {
|
2018-10-15 21:07:59 +02:00
|
|
|
fatal_error("failed to unprotect allocator state");
|
|
|
|
}
|
2018-10-15 20:46:57 +02:00
|
|
|
|
|
|
|
ro.region_allocator = &allocator_state->region_allocator;
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
mutex_init(&ra->lock);
|
2019-04-15 12:29:57 +02:00
|
|
|
random_state_init_from_random_state(&ra->rng, rng);
|
2018-10-15 21:07:59 +02:00
|
|
|
ro.regions[0] = allocator_state->regions_a;
|
|
|
|
ro.regions[1] = allocator_state->regions_b;
|
2018-10-15 03:12:37 +02:00
|
|
|
ra->regions = ro.regions[0];
|
|
|
|
ra->total = INITIAL_REGION_TABLE_SIZE;
|
2018-10-15 21:09:41 +02:00
|
|
|
ra->free = INITIAL_REGION_TABLE_SIZE;
|
2018-10-20 03:29:40 +02:00
|
|
|
if (memory_protect_rw_metadata(ra->regions, ra->total * sizeof(struct region_metadata))) {
|
2018-09-01 16:20:23 +02:00
|
|
|
fatal_error("failed to unprotect memory for regions table");
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
void *slab_region_start = memory_map(slab_region_size);
|
|
|
|
if (slab_region_start == NULL) {
|
2018-08-21 21:23:22 +02:00
|
|
|
fatal_error("failed to allocate slab region");
|
|
|
|
}
|
2018-11-05 08:19:04 +01:00
|
|
|
ro.slab_region_end = (char *)slab_region_start + slab_region_size;
|
2019-03-24 03:54:21 +01:00
|
|
|
memory_set_name(slab_region_start, slab_region_size, "malloc slab region gap");
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
|
|
|
ro.size_class_metadata[arena] = allocator_state->size_class_metadata[arena];
|
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][class];
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
mutex_init(&c->lock);
|
2019-04-15 12:29:57 +02:00
|
|
|
random_state_init_from_random_state(&c->rng, rng);
|
2018-08-25 06:21:02 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
size_t bound = (REAL_CLASS_REGION_SIZE - CLASS_REGION_SIZE) / PAGE_SIZE - 1;
|
|
|
|
size_t gap = (get_random_u64_uniform(rng, bound) + 1) * PAGE_SIZE;
|
|
|
|
c->class_region_start = (char *)slab_region_start + ARENA_SIZE * arena + REAL_CLASS_REGION_SIZE * class + gap;
|
2019-04-08 00:02:56 +02:00
|
|
|
label_slab(c->class_region_start, CLASS_REGION_SIZE, class);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
size_t size = size_classes[class];
|
|
|
|
if (size == 0) {
|
|
|
|
size = 16;
|
|
|
|
}
|
|
|
|
c->size_divisor = libdivide_u32_gen(size);
|
|
|
|
size_t slab_size = get_slab_size(size_class_slots[class], size);
|
|
|
|
c->slab_size_divisor = libdivide_u64_gen(slab_size);
|
|
|
|
c->slab_info = allocator_state->slab_info_mapping[arena][class].slab_info;
|
2018-08-24 09:22:52 +02:00
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 04:41:17 +02:00
|
|
|
deallocate_pages(rng, sizeof(struct random_state), PAGE_SIZE);
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
atomic_store_explicit(&ro.slab_region_start, slab_region_start, memory_order_release);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-08-29 16:52:10 +02:00
|
|
|
if (memory_protect_ro(&ro, sizeof(ro))) {
|
2018-08-21 21:23:22 +02:00
|
|
|
fatal_error("failed to protect allocator data");
|
|
|
|
}
|
2019-03-26 01:34:10 +01:00
|
|
|
memory_set_name(&ro, sizeof(ro), "malloc read-only after init");
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-07 07:08:51 +02:00
|
|
|
mutex_unlock(&lock);
|
2018-09-07 01:43:18 +02:00
|
|
|
|
|
|
|
// may allocate, so wait until the allocator is initialized to avoid deadlocking
|
2020-03-29 13:39:23 +02:00
|
|
|
if (pthread_atfork(full_lock, full_unlock, post_fork_child)) {
|
2018-09-07 01:43:18 +02:00
|
|
|
fatal_error("pthread_atfork failed");
|
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
static inline unsigned init(void) {
|
|
|
|
unsigned arena = thread_arena;
|
2019-04-10 14:06:56 +02:00
|
|
|
#if N_ARENA > 1
|
2019-04-15 01:57:22 +02:00
|
|
|
if (likely(arena < N_ARENA)) {
|
|
|
|
return arena;
|
2019-04-10 14:06:56 +02:00
|
|
|
}
|
2019-04-15 01:57:22 +02:00
|
|
|
thread_arena = arena = thread_arena_counter++ % N_ARENA;
|
|
|
|
#endif
|
2018-09-11 20:12:27 +02:00
|
|
|
if (unlikely(!is_init())) {
|
|
|
|
init_slow_path();
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
2019-04-15 01:57:22 +02:00
|
|
|
return arena;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-03 01:41:26 +02:00
|
|
|
// trigger early initialization to set up pthread_atfork and protect state as soon as possible
|
|
|
|
COLD __attribute__((constructor(101))) static void trigger_early_init(void) {
|
|
|
|
// avoid calling init directly to skip it if this isn't the malloc implementation
|
|
|
|
h_free(h_malloc(16));
|
2018-08-29 19:43:35 +02:00
|
|
|
}
|
|
|
|
|
2019-04-07 14:04:06 +02:00
|
|
|
// Returns 0 on overflow.
|
|
|
|
static size_t get_large_size_class(size_t size) {
|
|
|
|
if (CONFIG_LARGE_SIZE_CLASSES) {
|
|
|
|
// Continue small size class growth pattern of power of 2 spacing classes:
|
|
|
|
//
|
|
|
|
// 4 KiB [20 KiB, 24 KiB, 28 KiB, 32 KiB]
|
|
|
|
// 8 KiB [40 KiB, 48 KiB, 54 KiB, 64 KiB]
|
|
|
|
// 16 KiB [80 KiB, 96 KiB, 112 KiB, 128 KiB]
|
|
|
|
// 32 KiB [160 KiB, 192 KiB, 224 KiB, 256 KiB]
|
|
|
|
// 512 KiB [2560 KiB, 3 MiB, 3584 KiB, 4 MiB]
|
|
|
|
// 1 MiB [5 MiB, 6 MiB, 7 MiB, 8 MiB]
|
|
|
|
// etc.
|
|
|
|
size_t spacing_shift = 64 - __builtin_clzl(size - 1) - 3;
|
|
|
|
size_t spacing_class = 1ULL << spacing_shift;
|
|
|
|
return (size + (spacing_class - 1)) & ~(spacing_class - 1);
|
|
|
|
}
|
2019-04-09 06:06:17 +02:00
|
|
|
return PAGE_CEILING(size);
|
2019-04-07 14:04:06 +02:00
|
|
|
}
|
|
|
|
|
2018-08-26 10:42:01 +02:00
|
|
|
static size_t get_guard_size(struct random_state *state, size_t size) {
|
2018-10-06 21:17:55 +02:00
|
|
|
return (get_random_u64_uniform(state, size / PAGE_SIZE / GUARD_SIZE_DIVISOR) + 1) * PAGE_SIZE;
|
2018-08-26 10:42:01 +02:00
|
|
|
}
|
|
|
|
|
2019-04-07 11:30:07 +02:00
|
|
|
static void *allocate_large(size_t size) {
|
2019-04-07 14:04:06 +02:00
|
|
|
size = get_large_size_class(size);
|
|
|
|
if (unlikely(!size)) {
|
|
|
|
errno = ENOMEM;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
size_t guard_size = get_guard_size(&ra->rng, size);
|
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-26 10:42:01 +02:00
|
|
|
|
2019-03-23 04:17:26 +01:00
|
|
|
void *p = allocate_pages(size, guard_size, true, "malloc large");
|
2018-08-21 21:23:22 +02:00
|
|
|
if (p == NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_lock(&ra->lock);
|
2018-08-26 10:42:01 +02:00
|
|
|
if (regions_insert(p, size, guard_size)) {
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-26 10:42:01 +02:00
|
|
|
deallocate_pages(p, size, guard_size);
|
2018-08-21 21:23:22 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_large_allocate(ra, size);
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
static inline void *allocate(unsigned arena, size_t size) {
|
2020-05-13 07:05:37 +02:00
|
|
|
return size <= MAX_SLAB_SIZE_CLASS ? allocate_small(arena, size) : allocate_large(size);
|
2019-04-07 11:30:07 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 00:28:01 +02:00
|
|
|
static void deallocate_large(void *p, const size_t *expected_size) {
|
2018-09-11 20:13:18 +02:00
|
|
|
enforce_init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
mutex_lock(&ra->lock);
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *region = regions_find(p);
|
2018-08-21 21:23:22 +02:00
|
|
|
if (region == NULL) {
|
|
|
|
fatal_error("invalid free");
|
|
|
|
}
|
|
|
|
size_t size = region->size;
|
2018-09-18 23:28:52 +02:00
|
|
|
if (expected_size && size != *expected_size) {
|
2018-10-05 07:05:40 +02:00
|
|
|
fatal_error("sized deallocation mismatch (large)");
|
2018-09-18 23:28:52 +02:00
|
|
|
}
|
2018-08-26 10:42:01 +02:00
|
|
|
size_t guard_size = region->guard_size;
|
2018-08-21 21:23:22 +02:00
|
|
|
regions_delete(region);
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_large_deallocate(ra, size);
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-10-08 21:50:31 +02:00
|
|
|
regions_quarantine_deallocate_pages(p, size, guard_size);
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
static int alloc_aligned(unsigned arena, void **memptr, size_t alignment, size_t size, size_t min_alignment) {
|
2019-04-07 06:36:53 +02:00
|
|
|
if ((alignment - 1) & alignment || alignment < min_alignment) {
|
|
|
|
return EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (alignment <= PAGE_SIZE) {
|
2020-05-13 07:05:37 +02:00
|
|
|
if (size <= MAX_SLAB_SIZE_CLASS && alignment > min_align) {
|
2019-04-07 06:36:53 +02:00
|
|
|
size = get_size_info_align(size, alignment).size;
|
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
void *p = allocate(arena, size);
|
2019-04-07 06:36:53 +02:00
|
|
|
if (p == NULL) {
|
|
|
|
return ENOMEM;
|
|
|
|
}
|
|
|
|
*memptr = p;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-04-07 14:04:06 +02:00
|
|
|
size = get_large_size_class(size);
|
|
|
|
if (unlikely(!size)) {
|
|
|
|
return ENOMEM;
|
|
|
|
}
|
|
|
|
|
2019-04-07 06:36:53 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
size_t guard_size = get_guard_size(&ra->rng, size);
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
|
|
|
|
void *p = allocate_pages_aligned(size, alignment, guard_size, "malloc large");
|
|
|
|
if (p == NULL) {
|
|
|
|
return ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
if (regions_insert(p, size, guard_size)) {
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
deallocate_pages(p, size, guard_size);
|
|
|
|
return ENOMEM;
|
|
|
|
}
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
|
|
|
|
*memptr = p;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
static void *alloc_aligned_simple(unsigned arena, size_t alignment, size_t size) {
|
2019-04-07 06:36:53 +02:00
|
|
|
void *ptr;
|
2019-04-15 01:57:22 +02:00
|
|
|
int ret = alloc_aligned(arena, &ptr, alignment, size, 1);
|
2019-04-07 06:36:53 +02:00
|
|
|
if (ret) {
|
|
|
|
errno = ret;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
2019-04-07 11:48:10 +02:00
|
|
|
static size_t adjust_size_for_canaries(size_t size) {
|
2020-05-13 07:05:37 +02:00
|
|
|
if (size > 0 && size <= MAX_SLAB_SIZE_CLASS) {
|
2019-04-07 11:48:10 +02:00
|
|
|
return size + canary_size;
|
|
|
|
}
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void *alloc(size_t size) {
|
2019-04-15 01:57:22 +02:00
|
|
|
unsigned arena = init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-02 14:36:48 +02:00
|
|
|
size = adjust_size_for_canaries(size);
|
2019-04-15 01:57:22 +02:00
|
|
|
void *p = allocate(arena, size);
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2019-04-07 11:48:10 +02:00
|
|
|
EXPORT void *h_malloc(size_t size) {
|
|
|
|
return alloc(size);
|
|
|
|
}
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
EXPORT void *h_calloc(size_t nmemb, size_t size) {
|
|
|
|
size_t total_size;
|
2018-08-29 04:46:20 +02:00
|
|
|
if (unlikely(__builtin_mul_overflow(nmemb, size, &total_size))) {
|
2018-08-21 21:23:22 +02:00
|
|
|
errno = ENOMEM;
|
|
|
|
return NULL;
|
|
|
|
}
|
2019-04-15 01:57:22 +02:00
|
|
|
unsigned arena = init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-02 14:36:48 +02:00
|
|
|
total_size = adjust_size_for_canaries(total_size);
|
2019-04-15 01:57:22 +02:00
|
|
|
void *p = allocate(arena, total_size);
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2020-05-13 07:05:37 +02:00
|
|
|
if (!ZERO_ON_FREE && likely(p != NULL) && total_size && total_size <= MAX_SLAB_SIZE_CLASS) {
|
2018-09-07 06:33:51 +02:00
|
|
|
memset(p, 0, total_size - canary_size);
|
|
|
|
}
|
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT void *h_realloc(void *old, size_t size) {
|
|
|
|
if (old == NULL) {
|
2019-04-07 11:48:10 +02:00
|
|
|
return alloc(size);
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-02 14:36:48 +02:00
|
|
|
size = adjust_size_for_canaries(size);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2020-05-13 07:05:37 +02:00
|
|
|
if (size > MAX_SLAB_SIZE_CLASS) {
|
2019-04-07 14:04:06 +02:00
|
|
|
size = get_large_size_class(size);
|
|
|
|
if (unlikely(!size)) {
|
|
|
|
errno = ENOMEM;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
size_t old_size;
|
2018-11-05 08:19:04 +01:00
|
|
|
if (old >= get_slab_region_start() && old < ro.slab_region_end) {
|
2018-08-21 21:23:22 +02:00
|
|
|
old_size = slab_usable_size(old);
|
2020-05-13 07:05:37 +02:00
|
|
|
if (size <= MAX_SLAB_SIZE_CLASS && get_size_info(size).size == old_size) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return old;
|
|
|
|
}
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
} else {
|
2018-09-11 20:13:18 +02:00
|
|
|
enforce_init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-11 20:13:18 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
|
|
|
|
mutex_lock(&ra->lock);
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *region = regions_find(old);
|
2018-08-21 21:23:22 +02:00
|
|
|
if (region == NULL) {
|
|
|
|
fatal_error("invalid realloc");
|
|
|
|
}
|
|
|
|
old_size = region->size;
|
2018-09-05 11:17:40 +02:00
|
|
|
size_t old_guard_size = region->guard_size;
|
2019-04-07 14:04:06 +02:00
|
|
|
if (old_size == size) {
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
return old;
|
|
|
|
}
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-28 15:49:18 +02:00
|
|
|
|
2020-05-13 07:05:37 +02:00
|
|
|
if (size > MAX_SLAB_SIZE_CLASS) {
|
2018-10-06 19:31:16 +02:00
|
|
|
// in-place shrink
|
|
|
|
if (size < old_size) {
|
2019-04-07 14:04:06 +02:00
|
|
|
void *new_end = (char *)old + size;
|
2018-10-06 19:31:16 +02:00
|
|
|
if (memory_map_fixed(new_end, old_guard_size)) {
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-10-06 19:31:16 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2019-03-24 04:01:12 +01:00
|
|
|
memory_set_name(new_end, old_guard_size, "malloc large");
|
2018-10-06 19:31:16 +02:00
|
|
|
void *new_guard_end = (char *)new_end + old_guard_size;
|
2019-04-07 14:04:06 +02:00
|
|
|
regions_quarantine_deallocate_pages(new_guard_end, old_size - size, 0);
|
2018-09-05 11:17:40 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_lock(&ra->lock);
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *region = regions_find(old);
|
2018-10-06 19:31:16 +02:00
|
|
|
if (region == NULL) {
|
|
|
|
fatal_error("invalid realloc");
|
|
|
|
}
|
|
|
|
region->size = size;
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-09-05 11:17:40 +02:00
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-10-06 19:31:16 +02:00
|
|
|
return old;
|
|
|
|
}
|
2018-09-05 11:17:40 +02:00
|
|
|
|
2019-07-06 03:59:44 +02:00
|
|
|
#ifdef HAVE_COMPATIBLE_MREMAP
|
2019-04-10 01:15:43 +02:00
|
|
|
static const bool vma_merging_reliable = false;
|
|
|
|
if (vma_merging_reliable) {
|
|
|
|
// in-place growth
|
|
|
|
void *guard_end = (char *)old + old_size + old_guard_size;
|
|
|
|
size_t extra = size - old_size;
|
|
|
|
if (!memory_remap((char *)old + old_size, old_guard_size, old_guard_size + extra)) {
|
|
|
|
if (memory_protect_rw((char *)old + old_size, extra)) {
|
|
|
|
memory_unmap(guard_end, extra);
|
|
|
|
} else {
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
struct region_metadata *region = regions_find(old);
|
|
|
|
if (region == NULL) {
|
|
|
|
fatal_error("invalid realloc");
|
|
|
|
}
|
|
|
|
region->size = size;
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
|
|
|
|
thread_seal_metadata();
|
|
|
|
return old;
|
2018-10-06 16:40:55 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-18 21:25:14 +02:00
|
|
|
size_t copy_size = min(size, old_size);
|
2018-10-08 23:09:57 +02:00
|
|
|
if (copy_size >= MREMAP_MOVE_THRESHOLD) {
|
2019-04-15 01:46:22 +02:00
|
|
|
void *new = allocate_large(size);
|
2018-10-06 19:31:16 +02:00
|
|
|
if (new == NULL) {
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-10-06 19:31:16 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2018-08-28 15:49:18 +02:00
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_lock(&ra->lock);
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *region = regions_find(old);
|
2018-10-06 19:31:16 +02:00
|
|
|
if (region == NULL) {
|
|
|
|
fatal_error("invalid realloc");
|
|
|
|
}
|
|
|
|
regions_delete(region);
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-28 15:49:18 +02:00
|
|
|
|
2018-10-06 19:31:16 +02:00
|
|
|
if (memory_remap_fixed(old, old_size, new, size)) {
|
|
|
|
memcpy(new, old, copy_size);
|
|
|
|
deallocate_pages(old, old_size, old_guard_size);
|
|
|
|
} else {
|
|
|
|
memory_unmap((char *)old - old_guard_size, old_guard_size);
|
|
|
|
memory_unmap((char *)old + PAGE_CEILING(old_size), old_guard_size);
|
|
|
|
}
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-10-06 19:31:16 +02:00
|
|
|
return new;
|
2018-08-28 15:49:18 +02:00
|
|
|
}
|
2019-07-06 03:59:44 +02:00
|
|
|
#endif
|
2018-08-28 15:49:18 +02:00
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2019-04-15 01:57:22 +02:00
|
|
|
void *new = allocate(thread_arena, size);
|
2018-08-21 21:23:22 +02:00
|
|
|
if (new == NULL) {
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2018-10-18 21:25:14 +02:00
|
|
|
size_t copy_size = min(size, old_size);
|
2020-05-13 07:05:37 +02:00
|
|
|
if (copy_size > 0 && copy_size <= MAX_SLAB_SIZE_CLASS) {
|
2018-09-02 14:36:48 +02:00
|
|
|
copy_size -= canary_size;
|
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
memcpy(new, old, copy_size);
|
2020-05-13 07:05:37 +02:00
|
|
|
if (old_size <= MAX_SLAB_SIZE_CLASS) {
|
2018-09-18 23:28:52 +02:00
|
|
|
deallocate_small(old, NULL);
|
2018-09-11 20:13:18 +02:00
|
|
|
} else {
|
2018-09-18 23:28:52 +02:00
|
|
|
deallocate_large(old, NULL);
|
2018-09-11 20:13:18 +02:00
|
|
|
}
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
return new;
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT int h_posix_memalign(void **memptr, size_t alignment, size_t size) {
|
2019-04-15 01:57:22 +02:00
|
|
|
unsigned arena = init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-02 14:36:48 +02:00
|
|
|
size = adjust_size_for_canaries(size);
|
2019-04-15 01:57:22 +02:00
|
|
|
int ret = alloc_aligned(arena, memptr, alignment, size, sizeof(void *));
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
|
|
|
return ret;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT void *h_aligned_alloc(size_t alignment, size_t size) {
|
2019-04-15 01:57:22 +02:00
|
|
|
unsigned arena = init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-02 14:36:48 +02:00
|
|
|
size = adjust_size_for_canaries(size);
|
2019-04-15 01:57:22 +02:00
|
|
|
void *p = alloc_aligned_simple(arena, alignment, size);
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-09-06 22:30:22 +02:00
|
|
|
EXPORT void *h_memalign(size_t alignment, size_t size) ALIAS(h_aligned_alloc);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-04-07 06:30:47 +02:00
|
|
|
#ifndef __ANDROID__
|
2018-08-21 21:23:22 +02:00
|
|
|
EXPORT void *h_valloc(size_t size) {
|
2019-04-15 01:57:22 +02:00
|
|
|
unsigned arena = init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-02 14:36:48 +02:00
|
|
|
size = adjust_size_for_canaries(size);
|
2019-04-15 01:57:22 +02:00
|
|
|
void *p = alloc_aligned_simple(arena, PAGE_SIZE, size);
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT void *h_pvalloc(size_t size) {
|
2018-10-14 23:43:03 +02:00
|
|
|
size = PAGE_CEILING(size);
|
|
|
|
if (!size) {
|
2018-08-21 21:23:22 +02:00
|
|
|
errno = ENOMEM;
|
|
|
|
return NULL;
|
|
|
|
}
|
2019-04-15 01:57:22 +02:00
|
|
|
unsigned arena = init();
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-02 14:36:48 +02:00
|
|
|
size = adjust_size_for_canaries(size);
|
2019-04-15 01:57:22 +02:00
|
|
|
void *p = alloc_aligned_simple(arena, PAGE_SIZE, size);
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
|
|
|
return p;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
2019-04-07 06:30:47 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
|
|
|
|
EXPORT void h_free(void *p) {
|
|
|
|
if (p == NULL) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
if (p >= get_slab_region_start() && p < ro.slab_region_end) {
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-18 23:28:52 +02:00
|
|
|
deallocate_small(p, NULL);
|
2018-10-29 04:23:54 +01:00
|
|
|
thread_seal_metadata();
|
2018-09-11 20:13:18 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-09-18 23:28:52 +02:00
|
|
|
deallocate_large(p, NULL);
|
2018-10-20 03:29:40 +02:00
|
|
|
|
|
|
|
thread_seal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2019-04-07 06:24:23 +02:00
|
|
|
#ifdef __GLIBC__
|
2018-08-29 21:06:49 +02:00
|
|
|
EXPORT void h_cfree(void *ptr) ALIAS(h_free);
|
2019-04-07 06:24:23 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-09-18 23:28:52 +02:00
|
|
|
EXPORT void h_free_sized(void *p, size_t expected_size) {
|
|
|
|
if (p == NULL) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
if (p >= get_slab_region_start() && p < ro.slab_region_end) {
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
2018-09-18 23:28:52 +02:00
|
|
|
expected_size = get_size_info(adjust_size_for_canaries(expected_size)).size;
|
|
|
|
deallocate_small(p, &expected_size);
|
2018-10-29 04:23:54 +01:00
|
|
|
thread_seal_metadata();
|
2018-09-18 23:28:52 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
deallocate_large(p, &expected_size);
|
2018-10-20 03:29:40 +02:00
|
|
|
|
|
|
|
thread_seal_metadata();
|
2018-09-18 23:28:52 +02:00
|
|
|
}
|
|
|
|
|
2020-04-30 22:06:32 +02:00
|
|
|
static inline void memory_corruption_check_small(const void *p) {
|
2020-04-16 06:09:50 +02:00
|
|
|
struct slab_size_class_info size_class_info = slab_size_class(p);
|
|
|
|
size_t class = size_class_info.class;
|
|
|
|
struct size_class *c = &ro.size_class_metadata[size_class_info.arena][class];
|
|
|
|
size_t size = size_classes[class];
|
|
|
|
bool is_zero_size = size == 0;
|
|
|
|
if (is_zero_size) {
|
|
|
|
size = 16;
|
|
|
|
}
|
|
|
|
size_t slab_size = get_slab_size(size_class_slots[class], size);
|
2020-04-30 17:47:24 +02:00
|
|
|
|
2020-04-16 06:09:50 +02:00
|
|
|
mutex_lock(&c->lock);
|
|
|
|
|
|
|
|
struct slab_metadata *metadata = get_metadata(c, p);
|
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
2020-04-30 22:06:32 +02:00
|
|
|
size_t slot = libdivide_u32_do((const char *)p - (const char *)slab, &c->size_divisor);
|
2020-04-16 06:09:50 +02:00
|
|
|
|
|
|
|
if (slot_pointer(size, slab, slot) != p) {
|
|
|
|
fatal_error("invalid unaligned malloc_usable_size");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!get_slot(metadata, slot)) {
|
|
|
|
fatal_error("invalid malloc_usable_size");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!is_zero_size && canary_size) {
|
|
|
|
u64 canary_value;
|
2020-04-30 22:06:32 +02:00
|
|
|
memcpy(&canary_value, (const char *)p + size - canary_size, canary_size);
|
2020-04-16 06:09:50 +02:00
|
|
|
if (unlikely(canary_value != metadata->canary_value)) {
|
|
|
|
fatal_error("canary corrupted");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#if SLAB_QUARANTINE
|
|
|
|
if (get_quarantine(metadata, slot)) {
|
|
|
|
fatal_error("invalid malloc_usable_size (quarantine)");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
}
|
|
|
|
|
2018-12-05 09:27:09 +01:00
|
|
|
EXPORT size_t h_malloc_usable_size(H_MALLOC_USABLE_SIZE_CONST void *p) {
|
2018-08-21 21:23:22 +02:00
|
|
|
if (p == NULL) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-04-16 06:09:50 +02:00
|
|
|
enforce_init();
|
|
|
|
thread_unseal_metadata();
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
if (p >= get_slab_region_start() && p < ro.slab_region_end) {
|
2020-04-16 06:09:50 +02:00
|
|
|
memory_corruption_check_small(p);
|
|
|
|
thread_seal_metadata();
|
2020-04-30 17:47:24 +02:00
|
|
|
|
2018-09-02 14:36:48 +02:00
|
|
|
size_t size = slab_usable_size(p);
|
|
|
|
return size ? size - canary_size : 0;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
mutex_lock(&ra->lock);
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *region = regions_find(p);
|
2018-08-21 21:23:22 +02:00
|
|
|
if (p == NULL) {
|
|
|
|
fatal_error("invalid malloc_usable_size");
|
|
|
|
}
|
|
|
|
size_t size = region->size;
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-21 21:23:22 +02:00
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
2018-08-29 19:43:35 +02:00
|
|
|
EXPORT size_t h_malloc_object_size(void *p) {
|
2018-09-11 20:13:18 +02:00
|
|
|
if (p == NULL) {
|
2018-08-29 19:43:35 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-06-09 04:17:53 +02:00
|
|
|
thread_unseal_metadata();
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
void *slab_region_start = get_slab_region_start();
|
|
|
|
if (p >= slab_region_start && p < ro.slab_region_end) {
|
2020-06-09 04:17:53 +02:00
|
|
|
struct slab_size_class_info size_class_info = slab_size_class(p);
|
|
|
|
size_t class = size_class_info.class;
|
|
|
|
size_t size_class = size_classes[class];
|
|
|
|
struct size_class *c = &ro.size_class_metadata[size_class_info.arena][class];
|
|
|
|
|
|
|
|
mutex_lock(&c->lock);
|
|
|
|
|
|
|
|
struct slab_metadata *metadata = get_metadata(c, p);
|
|
|
|
size_t slab_size = get_slab_size(size_class_slots[class], size_class);
|
|
|
|
void *slab = get_slab(c, slab_size, metadata);
|
|
|
|
size_t slot = libdivide_u32_do((const char *)p - (const char *)slab, &c->size_divisor);
|
2020-06-18 04:00:32 +02:00
|
|
|
|
|
|
|
if (!get_slot(metadata, slot)) {
|
|
|
|
fatal_error("invalid malloc_object_size");
|
|
|
|
}
|
|
|
|
|
|
|
|
#if SLAB_QUARANTINE
|
|
|
|
if (get_quarantine(metadata, slot)) {
|
|
|
|
fatal_error("invalid malloc_object_size (quarantine)");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2020-06-09 04:17:53 +02:00
|
|
|
void *start = slot_pointer(size_class, slab, slot);
|
|
|
|
size_t offset = (const char *)p - (const char *)start;
|
|
|
|
|
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
thread_seal_metadata();
|
2020-06-13 15:59:50 +02:00
|
|
|
|
2018-09-02 14:36:48 +02:00
|
|
|
size_t size = slab_usable_size(p);
|
2020-06-09 04:17:53 +02:00
|
|
|
return size ? size - canary_size - offset : 0;
|
2018-08-29 19:43:35 +02:00
|
|
|
}
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
if (unlikely(slab_region_start == NULL)) {
|
2019-04-15 01:01:25 +02:00
|
|
|
return SIZE_MAX;
|
2018-09-11 20:13:18 +02:00
|
|
|
}
|
|
|
|
|
2018-10-15 03:12:37 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
mutex_lock(&ra->lock);
|
2018-10-15 03:54:23 +02:00
|
|
|
struct region_metadata *region = regions_find(p);
|
2018-08-29 19:43:35 +02:00
|
|
|
size_t size = p == NULL ? SIZE_MAX : region->size;
|
2018-10-15 03:12:37 +02:00
|
|
|
mutex_unlock(&ra->lock);
|
2018-08-29 19:43:35 +02:00
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
2018-08-29 19:43:35 +02:00
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT size_t h_malloc_object_size_fast(void *p) {
|
2018-09-11 20:13:18 +02:00
|
|
|
if (p == NULL) {
|
2018-08-29 19:43:35 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
void *slab_region_start = get_slab_region_start();
|
|
|
|
if (p >= slab_region_start && p < ro.slab_region_end) {
|
2018-09-02 14:36:48 +02:00
|
|
|
size_t size = slab_usable_size(p);
|
|
|
|
return size ? size - canary_size : 0;
|
2018-08-29 19:43:35 +02:00
|
|
|
}
|
|
|
|
|
2018-11-05 08:19:04 +01:00
|
|
|
if (unlikely(slab_region_start == NULL)) {
|
2018-09-11 20:13:18 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-08-29 19:43:35 +02:00
|
|
|
return SIZE_MAX;
|
|
|
|
}
|
|
|
|
|
2018-08-21 21:23:22 +02:00
|
|
|
EXPORT int h_mallopt(UNUSED int param, UNUSED int value) {
|
2019-04-07 09:35:26 +02:00
|
|
|
#ifdef __ANDROID__
|
|
|
|
if (param == M_PURGE) {
|
|
|
|
h_malloc_trim(0);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-08-30 15:29:15 +02:00
|
|
|
EXPORT int h_malloc_trim(UNUSED size_t pad) {
|
2018-09-07 08:53:15 +02:00
|
|
|
if (unlikely(!is_init())) {
|
2018-08-21 21:23:22 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_unseal_metadata();
|
|
|
|
|
2018-08-30 11:18:38 +02:00
|
|
|
bool is_trimmed = false;
|
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
|
|
|
// skip zero byte size class since there's nothing to change
|
|
|
|
for (unsigned class = 1; class < N_SIZE_CLASSES; class++) {
|
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][class];
|
|
|
|
size_t slab_size = get_slab_size(size_class_slots[class], size_classes[class]);
|
|
|
|
|
|
|
|
mutex_lock(&c->lock);
|
|
|
|
struct slab_metadata *iterator = c->empty_slabs;
|
|
|
|
while (iterator) {
|
|
|
|
void *slab = get_slab(c, slab_size, iterator);
|
|
|
|
if (memory_map_fixed(slab, slab_size)) {
|
|
|
|
break;
|
|
|
|
}
|
2019-04-08 00:02:56 +02:00
|
|
|
label_slab(slab, slab_size, class);
|
2019-04-09 09:57:44 +02:00
|
|
|
stats_slab_deallocate(c, slab_size);
|
2018-08-30 11:18:38 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
struct slab_metadata *trimmed = iterator;
|
|
|
|
iterator = iterator->next;
|
|
|
|
c->empty_slabs_total -= slab_size;
|
2018-08-30 11:18:38 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
enqueue_free_slab(c, trimmed);
|
2018-08-30 11:18:38 +02:00
|
|
|
|
2019-03-25 19:49:50 +01:00
|
|
|
is_trimmed = true;
|
|
|
|
}
|
|
|
|
c->empty_slabs = iterator;
|
|
|
|
mutex_unlock(&c->lock);
|
2018-08-30 11:18:38 +02:00
|
|
|
}
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
2018-10-20 03:29:40 +02:00
|
|
|
thread_seal_metadata();
|
|
|
|
|
2018-08-30 11:18:38 +02:00
|
|
|
return is_trimmed;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT void h_malloc_stats(void) {}
|
|
|
|
|
2018-09-06 20:35:08 +02:00
|
|
|
#if defined(__GLIBC__) || defined(__ANDROID__)
|
2018-08-21 21:23:22 +02:00
|
|
|
EXPORT struct mallinfo h_mallinfo(void) {
|
2019-04-07 02:49:52 +02:00
|
|
|
struct mallinfo info = {0};
|
|
|
|
|
|
|
|
// glibc mallinfo type definition and implementation are both broken
|
2019-04-07 15:06:03 +02:00
|
|
|
#if CONFIG_STATS && !defined(__GLIBC__)
|
2019-04-15 01:12:38 +02:00
|
|
|
if (!is_init()) {
|
|
|
|
return info;
|
|
|
|
}
|
|
|
|
|
2019-04-07 02:49:52 +02:00
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
info.hblkhd += ra->allocated;
|
|
|
|
info.uordblks += ra->allocated;
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
|
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
2019-04-07 04:43:56 +02:00
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
2019-04-07 02:49:52 +02:00
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][class];
|
|
|
|
|
|
|
|
mutex_lock(&c->lock);
|
|
|
|
info.hblkhd += c->slab_allocated;
|
|
|
|
info.uordblks += c->allocated;
|
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
info.fordblks = info.hblkhd - info.uordblks;
|
|
|
|
info.usmblks = info.hblkhd;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return info;
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
2018-09-06 20:35:08 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2019-04-19 22:54:43 +02:00
|
|
|
#ifndef __ANDROID__
|
2019-04-19 22:23:14 +02:00
|
|
|
EXPORT int h_malloc_info(int options, UNUSED FILE *fp) {
|
|
|
|
if (options) {
|
|
|
|
errno = EINVAL;
|
|
|
|
return -1;
|
|
|
|
}
|
2019-04-19 22:54:43 +02:00
|
|
|
|
|
|
|
#if CONFIG_STATS
|
|
|
|
fputs("<malloc version=\"hardened_malloc-1\">", fp);
|
|
|
|
for (unsigned arena = 0; arena < N_ARENA; arena++) {
|
|
|
|
fprintf(fp, "<heap nr=\"%u\">", arena);
|
|
|
|
|
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][class];
|
|
|
|
|
|
|
|
u64 nmalloc;
|
|
|
|
u64 ndalloc;
|
|
|
|
size_t slab_allocated;
|
|
|
|
size_t allocated;
|
|
|
|
|
|
|
|
mutex_lock(&c->lock);
|
|
|
|
nmalloc = c->nmalloc;
|
|
|
|
ndalloc = c->ndalloc;
|
|
|
|
slab_allocated = c->slab_allocated;
|
|
|
|
allocated = c->allocated;
|
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
|
|
|
|
if (nmalloc || ndalloc || slab_allocated || allocated) {
|
|
|
|
fprintf(fp, "<bin nr=\"%u\" size=\"%" PRIu32 "\">", class, size_classes[class]);
|
|
|
|
fprintf(fp, "<nmalloc>%" PRIu64 "</nmalloc>", nmalloc);
|
|
|
|
fprintf(fp, "<ndalloc>%" PRIu64 "</ndalloc>", ndalloc);
|
|
|
|
fprintf(fp, "<slab_allocated>%zu</slab_allocated>", slab_allocated);
|
|
|
|
fprintf(fp, "<allocated>%zu</allocated>", allocated);
|
|
|
|
fputs("</bin>", fp);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fputs("</heap>", fp);
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t region_allocated;
|
|
|
|
|
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
region_allocated = ra->allocated;
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
|
|
|
|
fprintf(fp, "<heap nr=\"%u\">", N_ARENA);
|
|
|
|
fprintf(fp, "<allocated_large>%zu</allocated_large>", region_allocated);
|
|
|
|
fputs("</heap>", fp);
|
|
|
|
|
|
|
|
fputs("</malloc>", fp);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
#else
|
2018-08-21 21:23:22 +02:00
|
|
|
errno = ENOSYS;
|
|
|
|
return -1;
|
2019-04-19 22:54:43 +02:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
}
|
2020-02-03 14:38:53 +01:00
|
|
|
#endif
|
2018-08-21 21:23:22 +02:00
|
|
|
|
2020-02-03 14:38:53 +01:00
|
|
|
#ifdef __GLIBC__
|
2018-08-21 21:23:22 +02:00
|
|
|
COLD EXPORT void *h_malloc_get_state(void) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
COLD EXPORT int h_malloc_set_state(UNUSED void *state) {
|
|
|
|
return -2;
|
|
|
|
}
|
2019-04-07 06:24:23 +02:00
|
|
|
#endif
|
2018-09-02 11:08:45 +02:00
|
|
|
|
|
|
|
#ifdef __ANDROID__
|
2019-09-08 00:30:04 +02:00
|
|
|
EXPORT size_t h_mallinfo_narenas(void) {
|
2019-04-07 04:21:11 +02:00
|
|
|
// Consider region allocator to be an arena with index N_ARENA.
|
|
|
|
return N_ARENA + 1;
|
2018-09-02 11:08:45 +02:00
|
|
|
}
|
|
|
|
|
2019-09-08 00:30:04 +02:00
|
|
|
EXPORT size_t h_mallinfo_nbins(void) {
|
2019-04-07 04:21:11 +02:00
|
|
|
return N_SIZE_CLASSES;
|
2018-09-02 11:08:45 +02:00
|
|
|
}
|
|
|
|
|
2019-04-07 04:21:11 +02:00
|
|
|
// This internal Android API uses mallinfo in a non-standard way to implement malloc_info:
|
|
|
|
//
|
|
|
|
// hblkhd: total mapped memory as usual
|
|
|
|
// ordblks: large allocations
|
|
|
|
// uordblks: huge allocations
|
|
|
|
// fsmblks: small allocations
|
|
|
|
// (other fields are unused)
|
2019-09-08 00:30:04 +02:00
|
|
|
EXPORT struct mallinfo h_mallinfo_arena_info(UNUSED size_t arena) {
|
2019-04-07 04:21:11 +02:00
|
|
|
struct mallinfo info = {0};
|
|
|
|
|
2019-04-07 15:06:03 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-07 04:21:11 +02:00
|
|
|
if (arena < N_ARENA) {
|
2019-04-07 04:43:56 +02:00
|
|
|
for (unsigned class = 0; class < N_SIZE_CLASSES; class++) {
|
2019-04-07 04:21:11 +02:00
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][class];
|
|
|
|
|
|
|
|
mutex_lock(&c->lock);
|
|
|
|
info.hblkhd += c->slab_allocated;
|
|
|
|
info.fsmblks += c->allocated;
|
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
}
|
|
|
|
} else if (arena == N_ARENA) {
|
|
|
|
struct region_allocator *ra = ro.region_allocator;
|
|
|
|
mutex_lock(&ra->lock);
|
|
|
|
info.hblkhd = ra->allocated;
|
|
|
|
// our large allocations are roughly comparable to jemalloc huge allocations
|
|
|
|
info.uordblks = ra->allocated;
|
|
|
|
mutex_unlock(&ra->lock);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return info;
|
2018-09-02 11:08:45 +02:00
|
|
|
}
|
|
|
|
|
2019-04-07 04:21:11 +02:00
|
|
|
// This internal Android API uses mallinfo in a non-standard way to implement malloc_info:
|
|
|
|
//
|
|
|
|
// ordblks: total allocated space
|
2019-04-07 05:06:47 +02:00
|
|
|
// uordblks: nmalloc
|
|
|
|
// fordblks: ndalloc
|
2019-04-07 04:21:11 +02:00
|
|
|
// (other fields are unused)
|
2019-09-08 00:30:04 +02:00
|
|
|
EXPORT struct mallinfo h_mallinfo_bin_info(UNUSED size_t arena, UNUSED size_t bin) {
|
2019-04-07 04:21:11 +02:00
|
|
|
struct mallinfo info = {0};
|
|
|
|
|
2019-04-07 15:06:03 +02:00
|
|
|
#if CONFIG_STATS
|
2019-04-15 01:12:38 +02:00
|
|
|
if (!is_init()) {
|
|
|
|
return info;
|
|
|
|
}
|
|
|
|
|
2019-04-07 04:43:56 +02:00
|
|
|
if (arena < N_ARENA && bin < N_SIZE_CLASSES) {
|
2019-04-07 04:21:11 +02:00
|
|
|
struct size_class *c = &ro.size_class_metadata[arena][bin];
|
|
|
|
|
|
|
|
mutex_lock(&c->lock);
|
|
|
|
info.ordblks = c->allocated;
|
2019-04-07 05:06:47 +02:00
|
|
|
info.uordblks = c->nmalloc;
|
|
|
|
info.fordblks = c->ndalloc;
|
2019-04-07 04:21:11 +02:00
|
|
|
mutex_unlock(&c->lock);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return info;
|
2018-09-02 11:08:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
COLD EXPORT int h_iterate(UNUSED uintptr_t base, UNUSED size_t size,
|
|
|
|
UNUSED void (*callback)(uintptr_t ptr, size_t size, void *arg),
|
|
|
|
UNUSED void *arg) {
|
|
|
|
fatal_error("not implemented");
|
|
|
|
}
|
|
|
|
|
|
|
|
COLD EXPORT void h_malloc_disable(void) {
|
2018-10-20 03:46:36 +02:00
|
|
|
init();
|
2018-09-06 20:17:06 +02:00
|
|
|
full_lock();
|
2018-09-02 11:08:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
COLD EXPORT void h_malloc_enable(void) {
|
2018-11-12 11:37:20 +01:00
|
|
|
enforce_init();
|
2018-09-06 20:17:06 +02:00
|
|
|
full_unlock();
|
2018-09-02 11:08:45 +02:00
|
|
|
}
|
|
|
|
#endif
|