123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669 |
-
- #include <stddef.h>
- #include <stdalign.h>
-
- #include "allocator_internal.h"
-
- #define IS_BLACK_NODE(n) (n == NULL || n->color == COLOR_BLACK)
- #define IS_RED_NODE(n) (n != NULL && n->color == COLOR_RED)
-
- #ifdef DEBUG
- #include <stdio.h>
-
- int debug_tree_black_height(TreeAlloc *node) {
- if (node == NULL) {
- return 1;
- }
- return (IS_BLACK_NODE(node) ? 1 : 0) + debug_tree_black_height(node->left);
- }
-
- void debug_print_node(int indent, TreeAlloc *node, int bad) {
- for (int ii = 0; ii < indent; ii++)
- printf(" ");
-
- if (IS_RED_NODE(node))
- printf("\e[31m");
- else if (bad)
- printf("\e[30m]");
-
- if (bad)
- printf("\e[43m");
-
- if (node)
- printf("%p %lu\n", node, node->size);
- else
- printf("(nil) 0\n");
- printf("\e[37m");
-
- if (bad)
- printf("\e[40m");
-
- }
-
- void debug_print_tree(int indent, void *p, int safe) {
- TreeAlloc *node = (TreeAlloc*) p;
- if (node != NULL) {
- int bad = debug_tree_black_height(node->left) != debug_tree_black_height(node->right);
- bad |= IS_RED_NODE(node) && (
- IS_RED_NODE(node->left) ||
- IS_RED_NODE(node->right) ||
- IS_RED_NODE(node->parent) );
- bad &= !safe;
- debug_print_tree(indent + 1, node->left, safe);
- debug_print_node(indent, node, bad);
- debug_print_tree(indent + 1, node->right, safe);
- }
- }
-
- #endif
-
- TreeAlloc *insert_node_at(void *address, uintptr_t padding, uintptr_t align, uintptr_t size) {
- return NULL;
- }
-
- /*
- * Search for the node whose allocated region contains an address.
- */
- TreeAlloc *search_by_address(TreeAlloc *root, void *address) {
- TreeAlloc *head = root;
- while (1) {
- void *region_start = head;
- void *region_end = head + head->size;
- if (address < region_start) {
- // The requested address is before this region's start.
- if (head->left) {
- // There is another region that comes before this one
- // in memory.
- head = head->left;
- } else {
- // This address is before any of the allocated regions.
- return NULL;
- }
- } else if (address <= region_end) {
- // The requested address is within the range of this region.
- return head;
- } else {
- // The requested address is after this region's end.
- if (head->right) {
- // There is another region that comes after this one
- // in memory.
- head = head->right;
- } else {
- // This address is after any of the allocated regions.
- return NULL;
- }
- }
- }
- }
-
- static uintptr_t effective_size(TreeAlloc *head, uintptr_t padding, uintptr_t align) {
- return head->size - (align_after(head + padding, align) - (void*) head);
- }
-
- /*
- * This is the most optimistic estimate of size that we can use which also preserves the ordering over
- * the tree. I had planned to use effective_size before I realized that it would break the tree
- * ordering.
- */
- static uintptr_t pessimistic_size(TreeAlloc *head, uintptr_t padding, uintptr_t align) {
- return head->size - padding - align + 1;
- }
-
- TreeAlloc *search_by_size(TreeAlloc *root, uintptr_t padding, uintptr_t align, uintptr_t size) {
- TreeAlloc *head = root;
- while (1) {
- uintptr_t esize = pessimistic_size(head, padding, align);
- if (esize <= size) {
- if (head->right == NULL) {
- return NULL;
- } else {
- head = head->right;
- }
- } else {
- if (head->left == NULL || pessimistic_size(head->left, padding, align) < size) {
- return head;
- } else {
- head = head->left;
- }
- }
- }
- }
-
- TreeAlloc *succ(TreeAlloc *el) {
- if (el->right != NULL) {
- el = el->right;
- while (el->left != NULL) {
- el = el->left;
- }
- return el;
- }
- while (el->parent != NULL && el == el->parent->right) {
- el = el->parent;
- }
- return el->parent;
- }
-
- TreeAlloc *pred(TreeAlloc *el) {
- if (el->left != NULL) {
- el = el->left;
- while (el->right != NULL) {
- el = el->right;
- }
- return el;
- }
- while (el->parent != NULL && el == el->parent->left) {
- el = el->parent;
- }
- return el->parent;
- }
-
- TreeAlloc *get_sibling(TreeAlloc *p, TreeAlloc *ta) {
- if (!p)
- return NULL;
- else if (p->left == ta)
- return p->right;
- else
- return p->left;
- }
-
- void rotate_left(TreeAlloc **root_ptr, TreeAlloc *ta) {
- TreeAlloc *parent, *tmp;
-
- tmp = ta->right;
- parent = ta->parent;
-
- if (tmp) {
- ta->right = tmp->left;
- tmp->left = ta;
- tmp->parent = parent;
- }
-
- if (ta->right) ta->right->parent = ta;
- ta->parent = tmp;
-
- if (!parent) {
- *root_ptr = tmp;
- } else if (ta == parent->left) {
- parent->left = tmp;
- } else {
- parent->right = tmp;
- }
- }
-
- void rotate_right(TreeAlloc **root_ptr, TreeAlloc *ta) {
- TreeAlloc *parent, *tmp;
-
- tmp = ta->left;
- parent = ta->parent;
-
- if (tmp) {
- ta->left = tmp->right;
- tmp->right = ta;
- tmp->parent = parent;
- }
-
- if (ta->left) ta->left->parent = ta;
- ta->parent = tmp;
-
- if (!parent) {
- *root_ptr = tmp;
- } else if (ta == parent->left) {
- parent->left = tmp;
- } else {
- parent->right = tmp;
- }
- }
-
- void repair_tree_after_insert(TreeAlloc **root_ptr, TreeAlloc *ta) {
- TreeAlloc *parent = ta->parent;
- if (ta == *root_ptr) {
- ta->color = COLOR_BLACK;
- } else if (IS_BLACK_NODE(parent)) {
- return;
- } else {
- TreeAlloc *uncle = get_sibling(parent->parent, parent);
- TreeAlloc *grandparent = parent->parent;
-
- if (IS_RED_NODE(uncle)) {
- parent->color = COLOR_BLACK;
- uncle->color = COLOR_BLACK;
- grandparent->color = COLOR_RED;
- repair_tree_after_insert(root_ptr, grandparent);
- } else {
- if (parent->left == ta) {
- if (grandparent->left == parent) {
- rotate_right(root_ptr, grandparent);
- grandparent->color = COLOR_RED;
- parent->color = COLOR_BLACK;
- } else {
- rotate_right(root_ptr, parent);
- rotate_left(root_ptr, grandparent);
- grandparent->color = COLOR_RED;
- ta->color = COLOR_BLACK;
- }
- } else {
- if (grandparent->left == parent) {
- rotate_left(root_ptr, parent);
- rotate_right(root_ptr, grandparent);
- grandparent->color = COLOR_RED;
- ta->color = COLOR_BLACK;
- } else {
- rotate_left(root_ptr, grandparent);
- grandparent->color = COLOR_RED;
- parent->color = COLOR_BLACK;
- }
- }
- }
- }
-
- }
- // Inserts a node into an empty tree.
- void insert_singleton(TreeAlloc **root_ptr, TreeAlloc *to_insert) {
- #ifdef DEBUG
- printf("= PRE-INSERT-SINGLETON =\n");
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- *root_ptr = to_insert;
- to_insert->parent = NULL;
- to_insert->color = COLOR_BLACK;
- #ifdef DEBUG
- printf("= POST-INSERT-SINGLETON =\n");
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- }
-
- void insert_by_size(TreeAlloc** root_ptr, TreeAlloc* to_insert) {
- #ifdef DEBUG
- printf("=== PRE-INSERT-BY-SIZE ===\n");
- printf("===== INSERTING =====\n");
- debug_print_node(0, to_insert, 0);
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- TreeAlloc *tree_ptr = *root_ptr;
- while (1) {
- if (to_insert->size < tree_ptr->size) {
- if (tree_ptr->left) {
- tree_ptr = tree_ptr->left;
- } else {
- tree_ptr->left = to_insert;
- to_insert->parent = tree_ptr;
- break;
- }
- } else {
- if (tree_ptr->right) {
- tree_ptr = tree_ptr->right;
- } else {
- tree_ptr->right = to_insert;
- to_insert->parent = tree_ptr;
- break;
- }
- }
- }
- to_insert->color = COLOR_RED;
- repair_tree_after_insert(root_ptr, to_insert);
- #ifdef DEBUG
- printf("== POST-INSERT-FIXUP ===\n");
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- }
-
- void insert_by_addr(TreeAlloc** root_ptr, TreeAlloc* to_insert) {
- #ifdef DEBUG
- printf("=== PRE-INSERT-BY-ADDR ===\n");
- printf("===== INSERTING =====\n");
- debug_print_tree(0, to_insert, 0);
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- TreeAlloc *tree_ptr = *root_ptr;
- while (1) {
- if (to_insert < tree_ptr) {
- if (tree_ptr->left) {
- tree_ptr = tree_ptr->left;
- } else {
- tree_ptr->left = to_insert;
- to_insert->parent = tree_ptr;
- break;
- }
- } else {
- if (tree_ptr->right) {
- tree_ptr = tree_ptr->right;
- } else {
- tree_ptr->right = to_insert;
- to_insert->parent = tree_ptr;
- break;
- }
- }
- }
- to_insert->color = COLOR_RED;
- repair_tree_after_insert(root_ptr, to_insert);
- #ifdef DEBUG
- printf("== POST-INSERT-FIXUP ===\n");
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- }
-
- void replace_node(TreeAlloc **root_ptr, TreeAlloc *node, TreeAlloc *replace) {
- if (!node->parent) {
- *root_ptr = replace;
- } else {
- if (node == node->parent->left)
- node->parent->left = replace;
- else
- node->parent->right = replace;
- }
- if (replace) replace->parent = node->parent;
- }
-
- void repair_after_remove(TreeAlloc **root_ptr, TreeAlloc *parent, TreeAlloc *node) {
- #ifdef DEBUG
- printf("delete fixup at %p -> %p\n", parent, node);
- #endif
- // In theory, the last two conditions should be the same ...
- if (IS_RED_NODE(node) || (node != NULL && node == *root_ptr)) {
- node->color = COLOR_BLACK;
- } else {
- TreeAlloc *sibling = get_sibling(parent, node);
- if (IS_RED_NODE(sibling)) {
- if (parent->left == node) {
- rotate_left(root_ptr, parent);
- } else {
- rotate_right(root_ptr, parent);
- }
- // The rotate shouldn't touch the parent relationship of `node`
- parent->parent->color = COLOR_BLACK;
- parent->color = COLOR_RED;
- sibling = get_sibling(parent, node);
- }
-
- if (IS_BLACK_NODE(sibling->left) && IS_BLACK_NODE(sibling->right)) {
- if (node != NULL)
- node->color = COLOR_BLACK;
- sibling->color = COLOR_RED;
- repair_after_remove(root_ptr, parent->parent, parent);
- } else {
- if (parent->left == node && IS_BLACK_NODE(sibling->right)) {
- rotate_right(root_ptr, sibling);
- sibling->color = COLOR_RED;
- sibling = parent->right;
- sibling->color = COLOR_BLACK;
- }
- if (parent->right == node && IS_BLACK_NODE(sibling->left)) {
- rotate_left(root_ptr, sibling);
- sibling->color = COLOR_RED;
- sibling = parent->left;
- sibling->color = COLOR_BLACK;
- }
- if (parent->left == node) {
- rotate_left(root_ptr, parent);
- } else {
- rotate_right(root_ptr, parent);
- }
- if (node != NULL)
- node->color = COLOR_BLACK;
- TreeAlloc *uncle = get_sibling(parent->parent, parent);
- if (uncle != NULL)
- uncle->color = COLOR_BLACK;
- char swap = parent->color;
- parent->color = parent->parent->color;
- parent->parent->color = swap;
- }
- }
- }
-
- void remove_node(TreeAlloc **root_ptr, TreeAlloc *to_remove) {
- char do_repair = 0;
- char old_color;
- #ifdef DEBUG
- printf("====== PRE-REMOVE ======\n");
- printf("======= REMOVING =======\n");
- debug_print_node(0, to_remove, 0);
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- TreeAlloc *replace, *parent_of_replace;
- TreeAlloc *parent = to_remove->parent;
- if (!to_remove->left) {
- #ifdef DEBUG
- printf("code path 1l\n");
- #endif
- replace = to_remove->right;
- parent_of_replace = to_remove->parent;
- do_repair = to_remove->color == COLOR_BLACK;
- replace_node(root_ptr, to_remove, replace);
- } else if (!to_remove->right) {
- #ifdef DEBUG
- printf("code path 1r\n");
- #endif
- replace = to_remove->left;
- parent_of_replace = to_remove->parent;
- do_repair = to_remove->color == COLOR_BLACK;
- replace_node(root_ptr, to_remove, replace);
- } else {
- #ifdef DEBUG
- printf("code path 2\n");
- #endif
- TreeAlloc *tmp = succ(to_remove);
- replace = tmp->right;
- do_repair = tmp->color == COLOR_BLACK;
- if (tmp != to_remove->right) {
- replace_node(root_ptr, tmp, replace);
- tmp->right = to_remove->right;
- to_remove->right->parent = tmp;
- parent_of_replace = tmp->parent;
- } else {
- parent_of_replace = tmp;
- }
- replace_node(root_ptr, to_remove, tmp);
- tmp->color = to_remove->color;
- tmp->left = to_remove->left;
- to_remove->left->parent = tmp;
- }
-
- // Make sure that it doesn't have any tree pointers it shouldn't have.
- to_remove->parent = to_remove->left = to_remove->right = NULL;
-
- #ifdef DEBUG
- printf("==== PRE-REMOVE-FIXUP ===\n");
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 1);
- printf("===== END OF TREES =====\n");
- printf("considering fixing up %p -> %p\n", parent_of_replace, replace);
- #endif
- if (replace && parent_of_replace == NULL) {
- replace->color = COLOR_BLACK;
- } else if (parent_of_replace != NULL && do_repair) {
- repair_after_remove(root_ptr, parent_of_replace, replace);
- }
- #ifdef DEBUG
- printf("=== POST-REMOVE ===\n");
- printf("===== CURRENT TREE =====\n");
- debug_print_tree(0, *root_ptr, 0);
- printf("===== END OF TREES =====\n");
- #endif
- }
-
- TreeAlloc *get_new_region(Arena *arena, uintptr_t size, uintptr_t padding, uintptr_t align) {
- uintptr_t realsize = size + align + alignof(WatermarkAlloc) + padding - 1;
- #ifdef DEBUG
- printf("Attemping request of size %ld\n", realsize);
- #endif
- if (realsize < MIN_NEW_MEM_SIZE) {
- realsize = MIN_NEW_MEM_SIZE;
- }
- TreeAlloc *reg = (TreeAlloc *) arena->get_new_region(realsize);
- if (reg == NULL) {
- arena->error("can't allocate a new memory region!");
- } else {
- reg->parent = NULL;
- reg->left = NULL;
- reg->right = NULL;
- reg->before = NULL;
- reg->after = NULL;
- reg->size = realsize;
- }
- return reg;
- }
-
- void unalloc(Arena *arena, void *addr) {
- #ifdef DEBUG
- printf("==== UNALLOCATING ====\n");
- printf("=== FREESPACE TREE ===\n");
- debug_print_tree(0, arena->root_freespace, 0);
- printf("=== TREEALLOC TREE ===\n");
- debug_print_tree(0, arena->root_treealloc, 0);
- printf("==== END OF TREES ====\n");
- #endif
- if (arena->root_treealloc == NULL) {
- arena->error("attempt to unallocate when there are no allocations!");
- return;
- }
- // Find the node this address belongs to
- TreeAlloc *node = search_by_address(arena->root_treealloc, addr);
- if (node == NULL) {
- arena->error("attempt to free memory outside any allocations!");
- return;
- }
- // Handle the watermark allocator in this region
- if (node->type == RT_WATERMARK) {
- // TODO: handle watermark deallocation
- return;
- }
- // Get rid of it
- remove_node(&arena->root_treealloc, node);
- // If there's free space on either side of it, merge it with the free space into a bigger chunk of
- // free space.
- uintptr_t size = node->size;
- FreeSpace *start = (FreeSpace*) node;
- if (node->before != NULL && node->before->type == RT_FREESPACE) {
- start = (FreeSpace*) node->before;
- size += node->before->size;
- remove_node((TreeAlloc**) &arena->root_freespace, node->before);
- }
- if (node->after != NULL && node->after->type == RT_FREESPACE) {
- size += node->after->size;
- remove_node((TreeAlloc**) &arena->root_freespace, node->after);
- }
- start->type = RT_FREESPACE;
- start->size = size;
- // And finally, insert the resulting free space.
- if (arena->root_freespace == NULL) {
- insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start);
- } else {
- TreeAlloc *insert_point = search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, size);
- if (insert_point == NULL) {
- insert_by_size((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start);
- } else {
- insert_by_size((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start);
- }
- }
- }
-
- void *alloc(Arena *arena, uintptr_t size, uintptr_t align) {
- uintptr_t actual_align = lcm(alignof(struct WatermarkAlloc), align);
-
- #ifdef DEBUG
- printf("==== ALLOCATING =====\n");
- printf("=== FREESPACE TREE ===\n");
- debug_print_tree(0, arena->root_freespace, 0);
- printf("=== TREEALLOC TREE ===\n");
- debug_print_tree(0, arena->root_treealloc, 0);
- printf("==== END OF TREES ====\n");
- #endif
-
- TreeAlloc *region;
- if (arena->root_freespace == NULL) {
- // Handle being out of freespace.
- #ifdef DEBUG
- printf("Out of freespace nodes; getting more\n");
- #endif
- region = get_new_region(arena, size, sizeof(TreeAlloc), actual_align);
- } else {
- region = search_by_size((TreeAlloc*) arena->root_freespace, sizeof(TreeAlloc), actual_align, size);
- if (region == NULL) {
- // Handle insufficient freespace or fragmentation.
- #ifdef DEBUG
- printf("Out of sufficiently large freespace nodes; getting more\n");
- #endif
- region = get_new_region(arena, size, sizeof(TreeAlloc), actual_align);
- } else {
- remove_node((TreeAlloc**) &arena->root_freespace, region);
- }
- }
-
- void *true_end = align_after(align_after(((void*) region) + sizeof(TreeAlloc), actual_align) + size, alignof(WatermarkAlloc));
- // The size of the new allocation (adjusted for region header and alignment
- uintptr_t new_size = true_end - (void*) region;
- // The size of the free space region following the new allocation
- uintptr_t new_free_size = region->size - new_size;
- region->right = NULL;
- region->left = NULL;
- region->type = RT_TREE_NODE;
- region->size = size;
-
- #ifdef DEBUG
- printf("start: %p, end: %p, adjusted end: %p\n", region, ((void*) region) + size, true_end);
- printf("size: %lu -> %lu\n", size, new_size);
- printf("new_free_size: %lu\n", new_free_size);
- #endif
-
- if (arena->root_treealloc == NULL) {
- insert_singleton((TreeAlloc**) &arena->root_treealloc, region);
- } else {
- insert_by_addr(&arena->root_treealloc, region);
- }
- if (region->size >= new_size + sizeof(FreeSpace)) {
- // If there's enough free space after the allocation, use it!
- region->size = new_size; // Safe because the allocated region tree is not sorted by size.
- FreeSpace *new_free = (FreeSpace*) ((void*) region + new_size);
- new_free->left = NULL;
- new_free->right = NULL;
- new_free->parent = NULL;
- new_free->type = RT_FREESPACE;
- new_free->size = new_free_size;
- if (arena->root_freespace == NULL) {
- insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free);
- } else {
- insert_by_size((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free);
- }
- // Set the region following this one to be the new free space
- region->after = (TreeAlloc*) new_free;
- } else {
- // There isn't a free space after this one, so put the `next` pointer at the next allocated
- // region if there is one.
- region->after = search_by_address((TreeAlloc *) &arena->root_treealloc, region + region->size + 1);
- }
-
- // Are there any allocations before this one?
- region->before = search_by_address((TreeAlloc *) &arena->root_treealloc, region - 1);
-
- #ifdef DEBUG
- printf("region is still at %p\n", region);
- printf("=== POST-ALLOCATION ===\n");
- printf("=== FREESPACE TREE ===\n");
- debug_print_tree(0, arena->root_freespace, 0);
- printf("=== TREEALLOC TREE ===\n");
- debug_print_tree(0, arena->root_treealloc, 0);
- printf("==== END OF TREES ====\n");
- #endif
- return align_after((void*) region + sizeof(TreeAlloc), actual_align);
- }
-
- void *alloc_growable(Arena *arena, uintptr_t size, uintptr_t align) {
- // TODO: Basically the same as above, but put the allocated region in the center of the largest free
- // space. Due to alignment and whatnot, the code will be gory.
- return NULL;
- }
|