fluencelabs/musl
1
0
Fork 0
mirror of https://github.com/fluencelabs/musl synced 2025-04-25 07:12:15 +00:00
Code Issues Projects Releases Wiki Activity
musl/src/malloc/malloc.c

533 lines
12 KiB
C
Raw Normal View History

namespace fixes for sys/mman.h
2011-04-20 15:55:58 -04:00
#define _GNU_SOURCE
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <stdint.h>
#include <errno.h>
#include <sys/mman.h>
#include "libc.h"
#include "atomic.h"
#include "pthread_impl.h"
workaround gcc got-register-reload performance problems in malloc with this patch, the malloc in libc.so built with -Os is nearly the same speed as the one built with -O3. thus it solves the performance regression that resulted from removing the forced -O3 when building libc.so; now libc.so can be both small and fast.
2012-09-14 23:52:51 -04:00
#if defined(__GNUC__) && defined(__PIC__)
#define inline inline __attribute__((always_inline))
#endif
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
uintptr_t __brk(uintptr_t);
void *__mmap(void *, size_t, int, int, int, off_t);
int __munmap(void *, size_t);
void *__mremap(void *, size_t, size_t, int, ...);
int __madvise(void *, size_t, int);
struct chunk {
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
size_t psize, csize;
struct chunk *next, *prev;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
};
struct bin {
make all objects used with atomic operations volatile the memory model we use internally for atomics permits plain loads of values which may be subject to concurrent modification without requiring that a special load function be used. since a compiler is free to make transformations that alter the number of loads or the way in which loads are performed, the compiler is theoretically free to break this usage. the most obvious concern is with atomic cas constructs: something of the form tmp=*p;a_cas(p,tmp,f(tmp)); could be transformed to a_cas(p,*p,f(*p)); where the latter is intended to show multiple loads of *p whose resulting values might fail to be equal; this would break the atomicity of the whole operation. but even more fundamental breakage is possible. with the changes being made now, objects that may be modified by atomics are modeled as volatile, and the atomic operations performed on them by other threads are modeled as asynchronous stores by hardware which happens to be acting on the request of another thread. such modeling of course does not itself address memory synchronization between cores/cpus, but that aspect was already handled. this all seems less than ideal, but it's the best we can do without mandating a C11 compiler and using the C11 model for atomics. in the case of pthread_once_t, the ABI type of the underlying object is not volatile-qualified. so we are assuming that accessing the object through a volatile-qualified lvalue via casts yields volatile access semantics. the language of the C standard is somewhat unclear on this matter, but this is an assumption the linux kernel also makes, and seems to be the correct interpretation of the standard.
2015-03-03 22:50:02 -05:00
volatile int lock[2];
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
struct chunk *head;
struct chunk *tail;
};
static struct {
uintptr_t brk;
size_t *heap;
make all objects used with atomic operations volatile the memory model we use internally for atomics permits plain loads of values which may be subject to concurrent modification without requiring that a special load function be used. since a compiler is free to make transformations that alter the number of loads or the way in which loads are performed, the compiler is theoretically free to break this usage. the most obvious concern is with atomic cas constructs: something of the form tmp=*p;a_cas(p,tmp,f(tmp)); could be transformed to a_cas(p,*p,f(*p)); where the latter is intended to show multiple loads of *p whose resulting values might fail to be equal; this would break the atomicity of the whole operation. but even more fundamental breakage is possible. with the changes being made now, objects that may be modified by atomics are modeled as volatile, and the atomic operations performed on them by other threads are modeled as asynchronous stores by hardware which happens to be acting on the request of another thread. such modeling of course does not itself address memory synchronization between cores/cpus, but that aspect was already handled. this all seems less than ideal, but it's the best we can do without mandating a C11 compiler and using the C11 model for atomics. in the case of pthread_once_t, the ABI type of the underlying object is not volatile-qualified. so we are assuming that accessing the object through a volatile-qualified lvalue via casts yields volatile access semantics. the language of the C standard is somewhat unclear on this matter, but this is an assumption the linux kernel also makes, and seems to be the correct interpretation of the standard.
2015-03-03 22:50:02 -05:00
volatile uint64_t binmap;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
struct bin bins[64];
make all objects used with atomic operations volatile the memory model we use internally for atomics permits plain loads of values which may be subject to concurrent modification without requiring that a special load function be used. since a compiler is free to make transformations that alter the number of loads or the way in which loads are performed, the compiler is theoretically free to break this usage. the most obvious concern is with atomic cas constructs: something of the form tmp=*p;a_cas(p,tmp,f(tmp)); could be transformed to a_cas(p,*p,f(*p)); where the latter is intended to show multiple loads of *p whose resulting values might fail to be equal; this would break the atomicity of the whole operation. but even more fundamental breakage is possible. with the changes being made now, objects that may be modified by atomics are modeled as volatile, and the atomic operations performed on them by other threads are modeled as asynchronous stores by hardware which happens to be acting on the request of another thread. such modeling of course does not itself address memory synchronization between cores/cpus, but that aspect was already handled. this all seems less than ideal, but it's the best we can do without mandating a C11 compiler and using the C11 model for atomics. in the case of pthread_once_t, the ABI type of the underlying object is not volatile-qualified. so we are assuming that accessing the object through a volatile-qualified lvalue via casts yields volatile access semantics. the language of the C standard is somewhat unclear on this matter, but this is an assumption the linux kernel also makes, and seems to be the correct interpretation of the standard.
2015-03-03 22:50:02 -05:00
volatile int brk_lock[2];
volatile int free_lock[2];
avoid malloc failure for small requests when brk can't be extended this issue mainly affects PIE binaries and execution of programs via direct invocation of the dynamic linker binary: depending on kernel behavior, in these cases the initial brk may be placed at at location where it cannot be extended, due to conflicting adjacent maps. when brk fails, mmap is used instead to expand the heap. in order to avoid expensive bookkeeping for managing fragmentation by merging these new heap regions, the minimum size for new heap regions increases exponentially in the number of regions. this limits the number of regions, and thereby the number of fixed fragmentation points, to a quantity which is logarithmic with respect to the size of virtual address space and thus negligible. the exponential growth is tuned so as to avoid expanding the heap by more than approximately 50% of its current total size.
2014-04-02 17:57:15 -04:00
unsigned mmap_step;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
} mal;
#define SIZE_ALIGN (4*sizeof(size_t))
#define SIZE_MASK (-SIZE_ALIGN)
#define OVERHEAD (2*sizeof(size_t))
#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN)
#define DONTCARE 16
#define RECLAIM 163840
fix issue with excessive mremap syscalls on realloc CHUNK_SIZE macro was defined incorrectly and shaving off at least one significant bit in the size of mmapped chunks, resulting in the test for oldlen==newlen always failing and incurring a syscall. fortunately i don't think this issue caused any other observable behavior; the definition worked correctly for all non-mmapped chunks where its correctness matters more, since their lengths are always multiples of the alignment.
2011-11-16 23:59:28 -05:00
#define CHUNK_SIZE(c) ((c)->csize & -2)
#define CHUNK_PSIZE(c) ((c)->psize & -2)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
#define PREV_CHUNK(c) ((struct chunk *)((char *)(c) - CHUNK_PSIZE(c)))
#define NEXT_CHUNK(c) ((struct chunk *)((char *)(c) + CHUNK_SIZE(c)))
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
#define MEM_TO_CHUNK(p) (struct chunk *)((char *)(p) - OVERHEAD)
#define CHUNK_TO_MEM(c) (void *)((char *)(c) + OVERHEAD)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head))
#define C_INUSE ((size_t)1)
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
#define IS_MMAPPED(c) !((c)->csize & (C_INUSE))
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
/* Synchronization tools */
workaround gcc got-register-reload performance problems in malloc with this patch, the malloc in libc.so built with -Os is nearly the same speed as the one built with -O3. thus it solves the performance regression that resulted from removing the forced -O3 when building libc.so; now libc.so can be both small and fast.
2012-09-14 23:52:51 -04:00
static inline void lock(volatile int *lk)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
{
fix potential deadlock bug in libc-internal locking logic if a multithreaded program became non-multithreaded (i.e. all other threads exited) while one thread held an internal lock, the remaining thread would fail to release the lock. the the program then became multithreaded again at a later time, any further attempts to obtain the lock would deadlock permanently. the underlying cause is that the value of libc.threads_minus_1 at unlock time might not match the value at lock time. one solution would be returning a flag to the caller indicating whether the lock was taken and needs to be unlocked, but there is a simpler solution: using the lock itself as such a flag. note that this flag is not needed anyway for correctness; if the lock is not held, the unlock code is harmless. however, the memory synchronization properties associated with a_store are costly on some archs, so it's best to avoid executing the unlock code when it is unnecessary.
2013-09-20 02:00:27 -04:00
if (libc.threads_minus_1)
while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
workaround gcc got-register-reload performance problems in malloc with this patch, the malloc in libc.so built with -Os is nearly the same speed as the one built with -O3. thus it solves the performance regression that resulted from removing the forced -O3 when building libc.so; now libc.so can be both small and fast.
2012-09-14 23:52:51 -04:00
static inline void unlock(volatile int *lk)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
{
fix potential deadlock bug in libc-internal locking logic if a multithreaded program became non-multithreaded (i.e. all other threads exited) while one thread held an internal lock, the remaining thread would fail to release the lock. the the program then became multithreaded again at a later time, any further attempts to obtain the lock would deadlock permanently. the underlying cause is that the value of libc.threads_minus_1 at unlock time might not match the value at lock time. one solution would be returning a flag to the caller indicating whether the lock was taken and needs to be unlocked, but there is a simpler solution: using the lock itself as such a flag. note that this flag is not needed anyway for correctness; if the lock is not held, the unlock code is harmless. however, the memory synchronization properties associated with a_store are costly on some archs, so it's best to avoid executing the unlock code when it is unnecessary.
2013-09-20 02:00:27 -04:00
if (lk[0]) {
a_store(lk, 0);
if (lk[1]) __wake(lk, 1, 1);
}
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
workaround gcc got-register-reload performance problems in malloc with this patch, the malloc in libc.so built with -Os is nearly the same speed as the one built with -O3. thus it solves the performance regression that resulted from removing the forced -O3 when building libc.so; now libc.so can be both small and fast.
2012-09-14 23:52:51 -04:00
static inline void lock_bin(int i)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
{
fix potential deadlock bug in libc-internal locking logic if a multithreaded program became non-multithreaded (i.e. all other threads exited) while one thread held an internal lock, the remaining thread would fail to release the lock. the the program then became multithreaded again at a later time, any further attempts to obtain the lock would deadlock permanently. the underlying cause is that the value of libc.threads_minus_1 at unlock time might not match the value at lock time. one solution would be returning a flag to the caller indicating whether the lock was taken and needs to be unlocked, but there is a simpler solution: using the lock itself as such a flag. note that this flag is not needed anyway for correctness; if the lock is not held, the unlock code is harmless. however, the memory synchronization properties associated with a_store are costly on some archs, so it's best to avoid executing the unlock code when it is unnecessary.
2013-09-20 02:00:27 -04:00
lock(mal.bins[i].lock);
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
if (!mal.bins[i].head)
mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i);
}
workaround gcc got-register-reload performance problems in malloc with this patch, the malloc in libc.so built with -Os is nearly the same speed as the one built with -O3. thus it solves the performance regression that resulted from removing the forced -O3 when building libc.so; now libc.so can be both small and fast.
2012-09-14 23:52:51 -04:00
static inline void unlock_bin(int i)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
{
unlock(mal.bins[i].lock);
}
static int first_set(uint64_t x)
{
#if 1
return a_ctz_64(x);
#else
static const char debruijn64[64] = {
0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
};
static const char debruijn32[32] = {
0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
};
if (sizeof(long) < 8) {
uint32_t y = x;
if (!y) {
y = x>>32;
return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
}
return debruijn32[(y&-y)*0x076be629 >> 27];
}
return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
#endif
}
static int bin_index(size_t x)
{
x = x / SIZE_ALIGN - 1;
if (x <= 32) return x;
if (x > 0x1c00) return 63;
malloc: cast size down to int in bin_index functions even if size_t was 32-bit already, the fact that the value was unsigned and that gcc is too stupid to figure out it would be positive as a signed quantity (due to the immediately-prior arithmetic and conditionals) results in gcc compiling the integer-to-float conversion as zero extension to 64 bits followed by an "fildll" (64 bit) instruction rather than a simple "fildl" (32 bit) instruction on x86. reportedly fildll is very slow on certain p4-class machines; even if not, the new code is slightly smaller.
2011-06-12 10:53:42 -04:00
return ((union { float v; uint32_t r; }){(int)x}.r>>21) - 496;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
static int bin_index_up(size_t x)
{
x = x / SIZE_ALIGN - 1;
if (x <= 32) return x;
malloc: cast size down to int in bin_index functions even if size_t was 32-bit already, the fact that the value was unsigned and that gcc is too stupid to figure out it would be positive as a signed quantity (due to the immediately-prior arithmetic and conditionals) results in gcc compiling the integer-to-float conversion as zero extension to 64 bits followed by an "fildll" (64 bit) instruction rather than a simple "fildl" (32 bit) instruction on x86. reportedly fildll is very slow on certain p4-class machines; even if not, the new code is slightly smaller.
2011-06-12 10:53:42 -04:00
return ((union { float v; uint32_t r; }){(int)x}.r+0x1fffff>>21) - 496;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
#if 0
void __dump_heap(int x)
{
struct chunk *c;
int i;
for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c))
fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n",
c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)),
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
c->csize & 15,
NEXT_CHUNK(c)->psize & 15);
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
for (i=0; i<64; i++) {
if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) {
fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head);
if (!(mal.binmap & 1ULL<<i))
fprintf(stderr, "missing from binmap!\n");
} else if (mal.binmap & 1ULL<<i)
fprintf(stderr, "binmap wrongly contains %d!\n", i);
}
}
#endif
static struct chunk *expand_heap(size_t n)
{
fix init race that could lead to deadlock in malloc init code the malloc init code provided its own version of pthread_once type logic, including the exact same bug that was fixed in pthread_once in commit 0d0c2f40344640a2a6942dda156509593f51db5d. since this code is called adjacent to expand_heap, which takes a lock, there is no reason to have pthread_once-type initialization. simply moving the init code into the interval where expand_heap already holds its lock on the brk achieves the same result with much less synchronization logic, and allows the buggy code to be eliminated rather than just fixed.
2015-03-04 09:29:39 -05:00
static int init;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
struct chunk *w;
uintptr_t new;
lock(mal.brk_lock);
fix init race that could lead to deadlock in malloc init code the malloc init code provided its own version of pthread_once type logic, including the exact same bug that was fixed in pthread_once in commit 0d0c2f40344640a2a6942dda156509593f51db5d. since this code is called adjacent to expand_heap, which takes a lock, there is no reason to have pthread_once-type initialization. simply moving the init code into the interval where expand_heap already holds its lock on the brk achieves the same result with much less synchronization logic, and allows the buggy code to be eliminated rather than just fixed.
2015-03-04 09:29:39 -05:00
if (!init) {
mal.brk = __brk(0);
#ifdef SHARED
mal.brk = mal.brk + PAGE_SIZE-1 & -PAGE_SIZE;
#endif
mal.brk = mal.brk + 2*SIZE_ALIGN-1 & -SIZE_ALIGN;
mal.heap = (void *)mal.brk;
init = 1;
}
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
if (n > SIZE_MAX - mal.brk - 2*PAGE_SIZE) goto fail;
new = mal.brk + n + SIZE_ALIGN + PAGE_SIZE - 1 & -PAGE_SIZE;
n = new - mal.brk;
avoid malloc failure for small requests when brk can't be extended this issue mainly affects PIE binaries and execution of programs via direct invocation of the dynamic linker binary: depending on kernel behavior, in these cases the initial brk may be placed at at location where it cannot be extended, due to conflicting adjacent maps. when brk fails, mmap is used instead to expand the heap. in order to avoid expensive bookkeeping for managing fragmentation by merging these new heap regions, the minimum size for new heap regions increases exponentially in the number of regions. this limits the number of regions, and thereby the number of fixed fragmentation points, to a quantity which is logarithmic with respect to the size of virtual address space and thus negligible. the exponential growth is tuned so as to avoid expanding the heap by more than approximately 50% of its current total size.
2014-04-02 17:57:15 -04:00
if (__brk(new) != new) {
size_t min = (size_t)PAGE_SIZE << mal.mmap_step/2;
n += -n & PAGE_SIZE-1;
if (n < min) n = min;
void *area = __mmap(0, n, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (area == MAP_FAILED) goto fail;
mal.mmap_step++;
area = (char *)area + SIZE_ALIGN - OVERHEAD;
w = area;
n -= SIZE_ALIGN;
w->psize = 0 | C_INUSE;
w->csize = n | C_INUSE;
w = NEXT_CHUNK(w);
w->psize = n | C_INUSE;
w->csize = 0 | C_INUSE;
unlock(mal.brk_lock);
return area;
}
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
fix init race that could lead to deadlock in malloc init code the malloc init code provided its own version of pthread_once type logic, including the exact same bug that was fixed in pthread_once in commit 0d0c2f40344640a2a6942dda156509593f51db5d. since this code is called adjacent to expand_heap, which takes a lock, there is no reason to have pthread_once-type initialization. simply moving the init code into the interval where expand_heap already holds its lock on the brk achieves the same result with much less synchronization logic, and allows the buggy code to be eliminated rather than just fixed.
2015-03-04 09:29:39 -05:00
w = MEM_TO_CHUNK(mal.heap);
w->psize = 0 | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
w = MEM_TO_CHUNK(new);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
w->psize = n | C_INUSE;
w->csize = 0 | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
w = MEM_TO_CHUNK(mal.brk);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
w->csize = n | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
mal.brk = new;
unlock(mal.brk_lock);
return w;
fail:
unlock(mal.brk_lock);
fix failure of malloc to set errno on heap (brk) exhaustion I wrongly assumed the brk syscall would set errno, but on failure it returns the old value of the brk rather than an error code.
2013-10-05 11:59:21 -04:00
errno = ENOMEM;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
return 0;
}
static int adjust_size(size_t *n)
{
/* Result of pointer difference must fit in ptrdiff_t. */
make malloc(0) return unique pointers rather than NULL this change is made with some reluctance, but i think it's for the best. correct programs must handle either behavior, so there is little advantage to having malloc(0) return NULL. and i managed to actually make the malloc code slightly smaller with this change.
2011-02-20 16:16:33 -05:00
if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) {
if (*n) {
errno = ENOMEM;
return -1;
} else {
*n = SIZE_ALIGN;
return 0;
}
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
*n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK;
return 0;
}
static void unbin(struct chunk *c, int i)
{
if (c->prev == c->next)
a_and_64(&mal.binmap, ~(1ULL<<i));
c->prev->next = c->next;
c->next->prev = c->prev;
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
c->csize |= C_INUSE;
NEXT_CHUNK(c)->psize |= C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
static int alloc_fwd(struct chunk *c)
{
int i;
size_t k;
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
while (!((k=c->csize) & C_INUSE)) {
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
i = bin_index(k);
lock_bin(i);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
if (c->csize == k) {
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
unbin(c, i);
unlock_bin(i);
return 1;
}
unlock_bin(i);
}
return 0;
}
static int alloc_rev(struct chunk *c)
{
int i;
size_t k;
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
while (!((k=c->psize) & C_INUSE)) {
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
i = bin_index(k);
lock_bin(i);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
if (c->psize == k) {
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
unbin(PREV_CHUNK(c), i);
unlock_bin(i);
return 1;
}
unlock_bin(i);
}
return 0;
}
/* pretrim - trims a chunk _prior_ to removing it from its bin.
* Must be called with i as the ideal bin for size n, j the bin
* for the _free_ chunk self, and bin j locked. */
static int pretrim(struct chunk *self, size_t n, int i, int j)
{
size_t n1;
struct chunk *next, *split;
/* We cannot pretrim if it would require re-binning. */
if (j < 40) return 0;
if (j < i+3) {
if (j != 63) return 0;
n1 = CHUNK_SIZE(self);
if (n1-n <= MMAP_THRESHOLD) return 0;
} else {
n1 = CHUNK_SIZE(self);
}
if (bin_index(n1-n) != j) return 0;
next = NEXT_CHUNK(self);
split = (void *)((char *)self + n);
split->prev = self->prev;
split->next = self->next;
split->prev->next = split;
split->next->prev = split;
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
split->psize = n | C_INUSE;
split->csize = n1-n;
next->psize = n1-n;
self->csize = n | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
return 1;
}
static void trim(struct chunk *self, size_t n)
{
size_t n1 = CHUNK_SIZE(self);
struct chunk *next, *split;
if (n >= n1 - DONTCARE) return;
next = NEXT_CHUNK(self);
split = (void *)((char *)self + n);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
split->psize = n | C_INUSE;
split->csize = n1-n | C_INUSE;
next->psize = n1-n | C_INUSE;
self->csize = n | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
free(CHUNK_TO_MEM(split));
}
void *malloc(size_t n)
{
struct chunk *c;
int i, j;
make malloc(0) return unique pointers rather than NULL this change is made with some reluctance, but i think it's for the best. correct programs must handle either behavior, so there is little advantage to having malloc(0) return NULL. and i managed to actually make the malloc code slightly smaller with this change.
2011-02-20 16:16:33 -05:00
if (adjust_size(&n) < 0) return 0;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
if (n > MMAP_THRESHOLD) {
fix rare but nasty under-allocation bug in malloc with large requests the bug appeared only with requests roughly 2*sizeof(size_t) to 4*sizeof(size_t) bytes smaller than a multiple of the page size, and only for requests large enough to be serviced by mmap instead of the normal heap. it was only ever observed on 64-bit machines but presumably could also affect 32-bit (albeit with a smaller window of opportunity).
2011-04-04 17:26:41 -04:00
size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
char *base = __mmap(0, len, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (base == (void *)-1) return 0;
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
c = (void *)(base + SIZE_ALIGN - OVERHEAD);
c->csize = len - (SIZE_ALIGN - OVERHEAD);
c->psize = SIZE_ALIGN - OVERHEAD;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
return CHUNK_TO_MEM(c);
}
i = bin_index_up(n);
for (;;) {
uint64_t mask = mal.binmap & -(1ULL<<i);
if (!mask) {
c = expand_heap(n);
if (!c) return 0;
if (alloc_rev(c)) {
struct chunk *x = c;
c = PREV_CHUNK(c);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
NEXT_CHUNK(x)->psize = c->csize =
x->csize + CHUNK_SIZE(c);
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
}
break;
}
j = first_set(mask);
lock_bin(j);
c = mal.bins[j].head;
remove useless check of bin match in malloc this re-check idiom seems to have been copied from the alloc_fwd and alloc_rev functions, which guess a bin based on non-synchronized memory access to adjacent chunk headers then need to confirm, after locking the bin, that the chunk is actually in the bin they locked. the check being removed, however, was being performed on a chunk obtained from the already-locked bin. there is no race to account for here; the check could only fail in the event of corrupt free lists, and even then it would not catch them but simply continue running. since the bin_index function is mildly expensive, it seems preferable to remove the check rather than trying to convert it into a useful consistency check. casual testing shows a 1-5% reduction in run time.
2015-03-04 10:48:00 -05:00
if (c != BIN_TO_CHUNK(j)) {
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
if (!pretrim(c, n, i, j)) unbin(c, j);
unlock_bin(j);
break;
}
unlock_bin(j);
}
/* Now patch up in case we over-allocated */
trim(c, n);
return CHUNK_TO_MEM(c);
}
void *realloc(void *p, size_t n)
{
struct chunk *self, *next;
size_t n0, n1;
void *new;
if (!p) return malloc(n);
if (adjust_size(&n) < 0) return 0;
self = MEM_TO_CHUNK(p);
n1 = n0 = CHUNK_SIZE(self);
if (IS_MMAPPED(self)) {
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
size_t extra = self->psize;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
char *base = (char *)self - extra;
size_t oldlen = n0 + extra;
size_t newlen = n + extra;
very cheap double-free checks in malloc
2011-03-23 13:24:00 -04:00
/* Crash on realloc of freed chunk */
use new a_crash() asm to optimize double-free handler. gcc generates extremely bad code (7 byte immediate mov) for the old null pointer write approach. it should be generating something like "xor %eax,%eax ; mov %al,(%eax)". in any case, using a dedicated crashing opcode accomplishes the same thing in one byte.
2011-08-23 09:43:45 -04:00
if (extra & 1) a_crash();
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
if (newlen < PAGE_SIZE && (new = malloc(n))) {
memcpy(new, p, n-OVERHEAD);
free(p);
return new;
}
newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE;
if (oldlen == newlen) return p;
base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE);
if (base == (void *)-1)
return newlen < oldlen ? p : 0;
self = (void *)(base + extra);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
self->csize = newlen - extra;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
return CHUNK_TO_MEM(self);
}
next = NEXT_CHUNK(self);
harden realloc/free to detect simple overflows the sizes in the header and footer for a chunk should always match. if they don't, the program has definitely invoked undefined behavior, and the most likely cause is a simple overflow, either of a buffer in the block being freed or the one just below it. crashing here should not only improve security of buggy programs, but also aid in debugging, since the crash happens in a context where you have a pointer to the likely-overflowed buffer.
2013-07-19 20:00:11 -04:00
/* Crash on corrupted footer (likely from buffer overflow) */
if (next->psize != self->csize) a_crash();
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
/* Merge adjacent chunks if we need more space. This is not
* a waste of time even if we fail to get enough space, because our
* subsequent call to free would otherwise have to do the merge. */
if (n > n1 && alloc_fwd(next)) {
n1 += CHUNK_SIZE(next);
next = NEXT_CHUNK(next);
}
/* FIXME: find what's wrong here and reenable it..? */
if (0 && n > n1 && alloc_rev(self)) {
self = PREV_CHUNK(self);
n1 += CHUNK_SIZE(self);
}
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
self->csize = n1 | C_INUSE;
next->psize = n1 | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
/* If we got enough space, split off the excess and return */
if (n <= n1) {
//memmove(CHUNK_TO_MEM(self), p, n0-OVERHEAD);
trim(self, n);
return CHUNK_TO_MEM(self);
}
/* As a last resort, allocate a new chunk and copy to it. */
new = malloc(n-OVERHEAD);
if (!new) return 0;
memcpy(new, p, n0-OVERHEAD);
free(CHUNK_TO_MEM(self));
return new;
}
void free(void *p)
{
struct chunk *self = MEM_TO_CHUNK(p);
struct chunk *next;
size_t final_size, new_size, size;
int reclaim=0;
int i;
if (!p) return;
if (IS_MMAPPED(self)) {
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
size_t extra = self->psize;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
char *base = (char *)self - extra;
size_t len = CHUNK_SIZE(self) + extra;
very cheap double-free checks in malloc
2011-03-23 13:24:00 -04:00
/* Crash on double free */
use new a_crash() asm to optimize double-free handler. gcc generates extremely bad code (7 byte immediate mov) for the old null pointer write approach. it should be generating something like "xor %eax,%eax ; mov %al,(%eax)". in any case, using a dedicated crashing opcode accomplishes the same thing in one byte.
2011-08-23 09:43:45 -04:00
if (extra & 1) a_crash();
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
__munmap(base, len);
return;
}
final_size = new_size = CHUNK_SIZE(self);
next = NEXT_CHUNK(self);
harden realloc/free to detect simple overflows the sizes in the header and footer for a chunk should always match. if they don't, the program has definitely invoked undefined behavior, and the most likely cause is a simple overflow, either of a buffer in the block being freed or the one just below it. crashing here should not only improve security of buggy programs, but also aid in debugging, since the crash happens in a context where you have a pointer to the likely-overflowed buffer.
2013-07-19 20:00:11 -04:00
/* Crash on corrupted footer (likely from buffer overflow) */
if (next->psize != self->csize) a_crash();
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
for (;;) {
/* Replace middle of large chunks with fresh zero pages */
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
if (reclaim && (self->psize & next->csize & C_INUSE)) {
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE;
uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE;
#if 1
__madvise((void *)a, b-a, MADV_DONTNEED);
#else
__mmap((void *)a, b-a, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
#endif
}
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
if (self->psize & next->csize & C_INUSE) {
self->csize = final_size | C_INUSE;
next->psize = final_size | C_INUSE;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
i = bin_index(final_size);
lock_bin(i);
lock(mal.free_lock);
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
if (self->psize & next->csize & C_INUSE)
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
break;
unlock(mal.free_lock);
unlock_bin(i);
}
if (alloc_rev(self)) {
self = PREV_CHUNK(self);
size = CHUNK_SIZE(self);
final_size += size;
if (new_size+size > RECLAIM && (new_size+size^size) > size)
reclaim = 1;
}
if (alloc_fwd(next)) {
size = CHUNK_SIZE(next);
final_size += size;
if (new_size+size > RECLAIM && (new_size+size^size) > size)
reclaim = 1;
next = NEXT_CHUNK(next);
}
}
eliminate OOB array hacks in malloc
2011-06-26 16:12:43 -04:00
self->csize = final_size;
next->psize = final_size;
initial check-in, version 0.5.0
2011-02-12 00:22:29 -05:00
unlock(mal.free_lock);
self->next = BIN_TO_CHUNK(i);
self->prev = mal.bins[i].tail;
self->next->prev = self;
self->prev->next = self;
if (!(mal.binmap & 1ULL<<i))
a_or_64(&mal.binmap, 1ULL<<i);
unlock_bin(i);
}
Reference in New Issue Copy Permalink