////////////// TLSF (Two-Level Segregate Fit) Memory Allocator ////////////////
// ╒══════════════ Block size interpretation (32-bit) ═════════════╕
// 3 2 1
// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits
// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─┴─╫─┴─┴─┤
// │ | FL │ SB = SL + AL │ ◄─ usize
// └───────────────────────────────────────────────┴─────────╨─────┘
// FL: first level, SL: second level, AL: alignment, SB: small block
const AL_BITS: u32 = sizeof<usize>() == sizeof<u32>() ? 2 : 3;
const AL_SIZE: usize = 1 << <usize>AL_BITS;
const AL_MASK: usize = AL_SIZE - 1;
const SL_BITS: u32 = 5;
const SL_SIZE: usize = 1 << <usize>SL_BITS;
const SB_BITS: usize = <usize>(SL_BITS + AL_BITS);
const SB_SIZE: usize = 1 << <usize>SB_BITS;
const SB_MASK: usize = SB_SIZE - 1;
const FL_BITS: u32 = (sizeof<usize>() == sizeof<u32>()
? 30 // ^= up to 1GB per block
: 32 // ^= up to 4GB per block
) - SB_BITS;
// ╒════════════════ Block structure layout (32-bit) ══════════════╕
// 3 2 1
// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits
// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┼─┤
// │ size │L│F│ ◄─┐
// ╞═══════════════════════════════════════════════════════════╧═╧═╡ │ ┐
// │ if free: ◄ prev │ ◄─┤ usize
// ├───────────────────────────────────────────────────────────────┤ │
// │ if free: next ► │ ◄─┤
// ├───────────────────────────────────────────────────────────────┤ │
// │ ... unused free space >= 0 ... │ │ = 0
// ├ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┤ │
// │ if free: jump ▲ │ ◄─┘
// └───────────────────────────────────────────────────────────────┘ MIN SIZE ┘
// F: FREE, L: LEFT_FREE
/** Tag indicating that this block is free. */
const FREE: usize = 1 << 0;
/** Tag indicating that this block's left block is free. */
const LEFT_FREE: usize = 1 << 1;
/** Mask to obtain all tags. */
const TAGS: usize = FREE | LEFT_FREE;
assert(AL_BITS >= 2); // alignment must be large enough to store all tags
/** Block structure. */
@unmanaged
class Block {
/** Info field holding this block's size and tags. */
info: usize;
/** End offset of the {@link Block#info} field. User data starts here. */
static readonly INFO: usize = sizeof<usize>();
/** Previous free block, if any. Only valid if free. */
prev: Block | null;
/** Next free block, if any. Only valid if free. */
next: Block | null;
/** Minimum size of a block, excluding {@link Block#info}. */
static readonly MIN_SIZE: usize = 3 * sizeof<usize>(); // prev + next + jump
/** Maximum size of a used block, excluding {@link Block#info}. */
static readonly MAX_SIZE: usize = 1 << (FL_BITS + SB_BITS);
/** Gets this block's left (free) block in memory. */
get left(): Block {
assert(this.info & LEFT_FREE); // must be free to contain a jump
return assert(
load<Block>(changetype<usize>(this) - sizeof<usize>())
); // can't be null
}
/** Gets this block's right block in memory. */
get right(): Block {
assert(this.info & ~TAGS); // can't skip beyond the tail block
return assert(
changetype<Block>(
changetype<usize>(this) + Block.INFO + (this.info & ~TAGS)
)
); // can't be null
}
}
// ╒════════════════ Root structure layout (32-bit) ═══════════════╕
// 3 2 1
// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits
// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐
// │ 0 | flMap S│ ◄────┐
// ╞═══════════════════════════════════════════════════════════════╡ │
// │ slMap[0] S │ ◄─┐ │
// ├───────────────────────────────────────────────────────────────┤ │ │
// │ slMap[1] │ ◄─┤ │
// ├───────────────────────────────────────────────────────────────┤ u32 │
// │ ... │ ◄─┤ │
// ├───────────────────────────────────────────────────────────────┤ │ │
// │ slMap[22] P │ ◄─┘ │
// ╞═══════════════════════════════════════════════════════════════╡ usize
// │ head[0] │ ◄────┤
// ├───────────────────────────────────────────────────────────────┤ │
// │ ... │ ◄────┤
// ├───────────────────────────────────────────────────────────────┤ │
// │ head[736] │ ◄────┤
// ╞═══════════════════════════════════════════════════════════════╡ │
// │ tailRef │ ◄────┘
// └───────────────────────────────────────────────────────────────┘ SIZE ┘
// S: Small blocks map, P: Possibly padded if 64-bit
assert((1 << SL_BITS) <= 32); // second level must fit into 32 bits
/** Root structure. */
@unmanaged
class Root {
/** First level bitmap. */
flMap: usize = 0;
/** Start offset of second level maps. */
private static readonly SL_START: usize = sizeof<usize>();
// Using *one* SL map per *FL bit*
/** Gets the second level map for the specified first level. */
getSLMap(fl: usize): u32 {
assert(fl < FL_BITS); // fl out of range
return load<u32>(changetype<usize>(this) + fl * 4, Root.SL_START);
}
/** Sets the second level map for the specified first level. */
setSLMap(fl: usize, value: u32): void {
assert(fl < FL_BITS); // fl out of range
store<u32>(changetype<usize>(this) + fl * 4, value, Root.SL_START);
}
/** End offset of second level maps. */
private static readonly SL_END: usize = Root.SL_START + FL_BITS * 4;
// Using *number bits per SL* heads per *FL bit*
/** Start offset of FL/SL heads. */
private static readonly HL_START: usize = (Root.SL_END + AL_MASK) & ~AL_MASK;
/** Gets the head of the specified first and second level index. */
getHead(fl: usize, sl: u32): Block | null {
assert(fl < FL_BITS); // fl out of range
assert(sl < SL_SIZE); // sl out of range
return changetype<Block>(load<usize>(
changetype<usize>(this) + (fl * SL_SIZE + <usize>sl) * sizeof<usize>()
, Root.HL_START));
}
/** Sets the head of the specified first and second level index. */
setHead(fl: usize, sl: u32, value: Block | null): void {
assert(fl < FL_BITS); // fl out of range
assert(sl < SL_SIZE); // sl out of range
store<usize>(
changetype<usize>(this) + (fl * SL_SIZE + <usize>sl) * sizeof<usize>()
, changetype<usize>(value)
, Root.HL_START);
}
/** End offset of FL/SL heads. */
private static readonly HL_END: usize = (
Root.HL_START + FL_BITS * SL_SIZE * sizeof<usize>()
);
get tailRef(): usize { return load<usize>(0, Root.HL_END); }
set tailRef(value: usize) { store<usize>(0, value, Root.HL_END); }
/** Total size of the {@link Root} structure. */
static readonly SIZE: usize = Root.HL_END + sizeof<usize>();
/** Inserts a previously used block back into the free list. */
insert(block: Block): void {
// check as much as possible here to prevent invalid free blocks
assert(block); // cannot be null
var blockInfo = block.info;
assert(blockInfo & FREE); // must be free
var size: usize;
assert(
(size = block.info & ~TAGS) >= Block.MIN_SIZE && size < Block.MAX_SIZE
); // must be valid, not necessary to compute yet if noAssert=true
var right: Block = assert(block.right); // can't be null
var rightInfo = right.info;
// merge with right block if also free
if (rightInfo & FREE) {
this.remove(right);
block.info = (blockInfo += Block.INFO + (rightInfo & ~TAGS));
right = block.right;
rightInfo = right.info;
// jump is set below
}
// merge with left block if also free
if (blockInfo & LEFT_FREE) {
var left: Block = assert(block.left); // can't be null
var leftInfo = left.info;
assert(leftInfo & FREE); // must be free according to tags
this.remove(left);
left.info = (leftInfo += Block.INFO + (blockInfo & ~TAGS));
block = left;
blockInfo = leftInfo;
// jump is set below
}
right.info = rightInfo | LEFT_FREE;
this.setJump(block, right);
// right is no longer used now, hence rightInfo is not synced
size = blockInfo & ~TAGS;
assert(size >= Block.MIN_SIZE && size < Block.MAX_SIZE); // must be valid
// mapping_insert
var fl: usize, sl: u32;
if (size < SB_SIZE) {
fl = 0;
sl = <u32>(size / AL_SIZE);
} else {
fl = fls<usize>(size);
sl = <u32>((size >> (fl - SL_BITS)) ^ (1 << SL_BITS));
fl -= SB_BITS - 1;
}
// perform insertion
var head = this.getHead(fl, sl);
block.prev = null;
block.next = head;
if (head) head.prev = block;
this.setHead(fl, sl, block);
// update first and second level maps
this.flMap |= (1 << fl);
this.setSLMap(fl, this.getSLMap(fl) | (1 << sl));
}
/**
* Removes a free block from FL/SL maps. Does not alter left/jump because it
* is likely that splitting is performed afterwards, invalidating any changes
* again.
*/
private remove(block: Block): void {
var blockInfo = block.info;
assert(blockInfo & FREE); // must be free
var size = blockInfo & ~TAGS;
assert(size >= Block.MIN_SIZE && size < Block.MAX_SIZE); // must be valid
// mapping_insert
var fl: usize, sl: u32;
if (size < SB_SIZE) {
fl = 0;
sl = <u32>(size / AL_SIZE);
} else {
fl = fls<usize>(size);
sl = <u32>((size >> (fl - SL_BITS)) ^ (1 << SL_BITS));
fl -= SB_BITS - 1;
}
// link previous and next free block
var prev = block.prev;
var next = block.next;
if (prev)
prev.next = next;
if (next)
next.prev = prev;
// update head if we are removing it
if (block == this.getHead(fl, sl)) {
this.setHead(fl, sl, next);
// clear second level map if head is empty now
if (!next) {
var slMap = this.getSLMap(fl);
this.setSLMap(fl, slMap &= ~(1 << sl));
// clear first level map if second level is empty now
if (!slMap)
this.flMap &= ~(1 << fl);
}
}
}
/** Searches for a free block of at least the specified size. */
search(size: usize): Block | null {
assert(size >= Block.MIN_SIZE && size < Block.MAX_SIZE);
// mapping_search
var fl: usize, sl: u32;
if (size < SB_SIZE) {
fl = 0;
sl = <u32>(size / AL_SIZE);
} else {
// (*) size += (1 << (fls<usize>(size) - SL_BITS)) - 1;
fl = fls<usize>(size);
sl = <u32>((size >> (fl - SL_BITS)) ^ (1 << SL_BITS));
fl -= SB_BITS - 1;
// (*) instead of rounding up, use next second level list for better fit
if (sl < SL_SIZE - 1) ++sl;
else ++fl, sl = 0;
}
// search second level
var slMap = this.getSLMap(fl) & (~0 << sl);
var head: Block | null;
if (!slMap) {
// search next larger first level
var flMap = this.flMap & (~0 << (fl + 1));
if (!flMap) {
head = null;
} else {
fl = ffs<usize>(flMap);
slMap = assert(this.getSLMap(fl)); // can't be zero if fl points here
head = this.getHead(fl, ffs<u32>(slMap));
}
} else
head = this.getHead(fl, ffs<u32>(slMap));
return head;
}
/** Links a free left with its right block in memory. */
private setJump(left: Block, right: Block): void {
assert(left.info & FREE); // must be free
assert(left.right == right); // right block must match
assert(right.info & LEFT_FREE); // right block must be tagged as LEFT_FREE
store<Block>(
changetype<usize>(right) - sizeof<usize>()
, left); // last word in left block's (free) data region
}
/**
* Uses the specified free block, removing it from internal maps and
* splitting it if possible, and returns its data pointer.
*/
use(block: Block, size: usize): usize {
var blockInfo = block.info;
assert(blockInfo & FREE); // must be free so we can use it
assert(size >= Block.MIN_SIZE && size < Block.MAX_SIZE); // must be valid
assert(!(size & AL_MASK)); // size must be aligned so the new block is
this.remove(block);
// split if the block can hold another MIN_SIZE block
var remaining = (blockInfo & ~TAGS) - size;
if (remaining >= Block.INFO + Block.MIN_SIZE) {
block.info = size | (blockInfo & LEFT_FREE); // also discards FREE
var spare = changetype<Block>(
changetype<usize>(block) + Block.INFO + size
);
spare.info = (remaining - Block.INFO) | FREE; // not LEFT_FREE
this.insert(spare); // also sets jump
// otherwise tag block as no longer FREE and right as no longer LEFT_FREE
} else {
block.info = blockInfo & ~FREE;
var right: Block = assert(block.right); // can't be null (tail)
right.info &= ~LEFT_FREE;
}
return changetype<usize>(block) + Block.INFO;
}
/** Adds more memory to the pool. */
addMemory(start: usize, end: usize): bool {
assert(start <= end);
assert(!(start & AL_MASK)); // must be aligned
assert(!(end & AL_MASK)); // must be aligned
var tailRef = this.tailRef;
var tailInfo: usize = 0;
if (tailRef) {
assert(start >= tailRef + sizeof<usize>()); // starts after tail
// merge with current tail if adjacent
if (start - Block.INFO == tailRef) {
start -= Block.INFO;
tailInfo = changetype<Block>(tailRef).info;
}
} else
assert(start >= changetype<usize>(this) + Root.SIZE); // starts after root
// check if size is large enough for a free block and the tail block
var size = end - start;
if (size < Block.INFO + Block.MIN_SIZE + Block.INFO)
return false;
// left size is total minus its own and the zero-length tail's header
var leftSize = size - 2 * Block.INFO;
var left = changetype<Block>(start);
left.info = leftSize | FREE | (tailInfo & LEFT_FREE);
left.prev = null;
left.next = null;
// tail is a zero-length used block
var tail = changetype<Block>(start + size - Block.INFO);
tail.info = 0 | LEFT_FREE;
this.tailRef = changetype<usize>(tail);
this.insert(left); // also merges with free left before tail / sets jump
return true;
}
}
/** Determines the first (LSB to MSB) set bit's index of a word. */
function ffs<T>(word: T): T {
assert(word != 0); // word cannot be 0
return ctz<T>(word); // differs from ffs only for 0
}
/** Determines the last (LSB to MSB) set bit's index of a word. */
function fls<T>(word: T): T {
assert(word != 0); // word cannot be 0
const inv: T = (sizeof<T>() << 3) - 1;
return inv - clz<T>(word);
}
/** Reference to the initialized {@link Root} structure, once initialized. */
var ROOT: Root = changetype<Root>(0);
// External interface
/** Allocates a chunk of memory. */
export function allocate_memory(size: usize): usize {
// initialize if necessary
var root = ROOT;
if (!root) {
var rootOffset = (HEAP_BASE + AL_MASK) & ~AL_MASK;
ROOT = root = changetype<Root>(rootOffset);
root.tailRef = 0;
root.flMap = 0;
for (var fl: usize = 0; fl < FL_BITS; ++fl) {
root.setSLMap(fl, 0);
for (var sl: u32 = 0; sl < SL_SIZE; ++sl)
root.setHead(fl, sl, null);
}
root.addMemory(rootOffset + Root.SIZE, current_memory() << 16);
}
// search for a suitable block
var data: usize = 0;
if (size && size < Block.MAX_SIZE) {
size = max<usize>((size + AL_MASK) & ~AL_MASK, Block.MIN_SIZE);
var block = root.search(size);
if (!block) {
// request more memory
var pagesBefore = current_memory();
var pagesWanted = max(pagesBefore, ((size + 0xffff) & ~0xffff) >>> 16);
if (grow_memory(pagesWanted) < 0)
unreachable(); // out of memory
var pagesAfter = current_memory();
root.addMemory(<usize>pagesBefore << 16, <usize>pagesAfter << 16);
block = assert(root.search(size)); // must be found now
}
assert((block.info & ~TAGS) >= size);
data = root.use(block, size);
}
return data;
}
/** Frees the chunk of memory at the specified address. */
export function free_memory(data: usize): void {
if (data) {
var root = ROOT;
if (root) {
var block = changetype<Block>(data - Block.INFO);
var blockInfo = block.info;
assert(!(blockInfo & FREE)); // must be used
block.info = blockInfo | FREE;
root.insert(changetype<Block>(data - Block.INFO));
}
}
}