|
|
#include <stddef.h>
#include <stdalign.h>
#include "allocator_internal.h"
#ifdef DEBUG
#include <stdio.h>
int debug_tree_black_height(TreeAlloc *node) { if (node == NULL) { return 1; } return ((node->color == COLOR_BLACK) ? 1 : 0) + debug_tree_black_height(node->left); }
void debug_print_tree(int indent, void *p, int safe) { TreeAlloc *node = (TreeAlloc*) p; if (node != NULL) { int bad = debug_tree_black_height(node->left) != debug_tree_black_height(node->right); bad |= node->color == COLOR_RED && ((node->left != NULL && node->left->color == COLOR_RED) || (node->right != NULL && node->right->color == COLOR_RED) || (node->parent != NULL && node->parent->color == COLOR_RED)); bad &= !safe; debug_print_tree(indent + 1, node->left, safe); for (int ii = 0; ii < indent; ii++) { printf(" "); } if (node->color == COLOR_RED) { printf("\e[31m"); } else if (bad) { printf("\e[30m]"); } if (bad) { printf("\e[43m"); } printf("%p %lu\n", node, node->size); printf("\e[37m"); if (bad) { printf("\e[40m"); } debug_print_tree(indent + 1, node->right, safe); } }
#endif
TreeAlloc *insert_node_at(void *address, uintptr_t padding, uintptr_t align, uintptr_t size) { return NULL; }
/*
* Search for the node whose allocated region contains an address. */ TreeAlloc *search_by_address(TreeAlloc *root, void *address) { TreeAlloc *head = root; while (1) { if (head > (TreeAlloc*) address) { if (head->left == NULL) { return NULL; } else { head = head->left; } } else { if (head->right == NULL || head->right > (TreeAlloc*) address) { return head; } else { head = head->right; } } } }
static uintptr_t effective_size(TreeAlloc *head, uintptr_t padding, uintptr_t align) { return head->size - (align_after(head + padding, align) - (void*) head); }
/*
* This is the most optimistic estimate of size that we can use which also preserves the ordering over * the tree. I had planned to use effective_size before I realized that it would break the tree * ordering. */ static uintptr_t pessimistic_size(TreeAlloc *head, uintptr_t padding, uintptr_t align) { return head->size - padding - align + 1; }
TreeAlloc *search_by_size(TreeAlloc *root, uintptr_t padding, uintptr_t align, uintptr_t size) { TreeAlloc *head = root; while (1) { uintptr_t esize = pessimistic_size(head, padding, align); if (esize < size) { if (head->right == NULL) { return NULL; } else { head = head->right; } } else { if (head->left == NULL || pessimistic_size(head->left, padding, align) < size) { return head; } else { head = head->left; } } } }
TreeAlloc *succ(TreeAlloc *el) { if (el->right != NULL) { el = el->right; while (el->left != NULL) { el = el->left; } return el; } while (el->parent != NULL && el == el->parent->right) { el = el->parent; } return el->parent; }
TreeAlloc *pred(TreeAlloc *el) { if (el->left != NULL) { el = el->left; while (el->right != NULL) { el = el->right; } return el; } while (el->parent != NULL && el == el->parent->left) { el = el->parent; } return el->parent; }
TreeAlloc *get_sibling(TreeAlloc *p, TreeAlloc *ta) { if (!p) return NULL; else if (p->left == ta) return p->right; else return p->left; }
void rotate_left(TreeAlloc **root_ptr, TreeAlloc *ta) { TreeAlloc *parent, *tmp;
tmp = ta->right; parent = ta->parent;
if (tmp) { ta->right = tmp->left; tmp->left = ta; tmp->parent = parent; }
if (ta->right) ta->right->parent = ta; ta->parent = tmp;
if (!parent) { *root_ptr = tmp; } else if (ta == parent->left) { parent->left = tmp; } else { parent->right = tmp; } }
void rotate_right(TreeAlloc **root_ptr, TreeAlloc *ta) { TreeAlloc *parent, *tmp;
tmp = ta->left; parent = ta->parent;
if (tmp) { ta->left = tmp->right; tmp->right = ta; tmp->parent = parent; }
if (ta->left) ta->left->parent = ta; ta->parent = tmp;
if (!parent) { *root_ptr = tmp; } else if (ta == parent->left) { parent->left = tmp; } else { parent->right = tmp; } }
#define IS_BLACK_NODE(n) (n == NULL || n->color == COLOR_BLACK)
#define IS_RED_NODE(n) (n != NULL && n->color == COLOR_RED)
void repair_tree_after_insert(TreeAlloc **root_ptr, TreeAlloc *ta) { TreeAlloc *parent = ta->parent; if (ta == *root_ptr) { ta->color = COLOR_BLACK; } else if (IS_BLACK_NODE(parent)) { return; } else { TreeAlloc *uncle = get_sibling(parent->parent, parent); TreeAlloc *grandparent = parent->parent;
if (IS_RED_NODE(uncle)) { parent->color = COLOR_BLACK; uncle->color = COLOR_BLACK; grandparent->color = COLOR_RED; repair_tree_after_insert(root_ptr, grandparent); } else { if (parent->left == ta) { if (grandparent->left == parent) { rotate_right(root_ptr, grandparent); grandparent->color = COLOR_RED; parent->color = COLOR_BLACK; } else { rotate_right(root_ptr, parent); rotate_left(root_ptr, grandparent); grandparent->color = COLOR_RED; ta->color = COLOR_BLACK; } } else { if (grandparent->left == parent) { rotate_left(root_ptr, parent); rotate_right(root_ptr, grandparent); grandparent->color = COLOR_RED; ta->color = COLOR_BLACK; } else { rotate_left(root_ptr, grandparent); grandparent->color = COLOR_RED; parent->color = COLOR_BLACK; } } } }
} // Inserts a node into an empty tree.
void insert_singleton(TreeAlloc **root_ptr, TreeAlloc *to_insert) { #ifdef DEBUG
printf("= PRE-INSERT-SINGLETON =\n"); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
*root_ptr = to_insert; to_insert->parent = NULL; to_insert->color = COLOR_BLACK; #ifdef DEBUG
printf("= POST-INSERT-SINGLETON =\n"); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
}
void insert_right(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* after) { #ifdef DEBUG
printf("=== PRE-INSERT-RIGHT ===\n"); printf("===== INSERTING =====\n"); debug_print_tree(0, to_insert, 0); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
if (after->right != NULL) { after = after->right; while (after->left != NULL) { after = after->left; } after->left = to_insert; to_insert->parent = after; } else { after->right = to_insert; to_insert->parent = after; } to_insert->color = COLOR_RED; repair_tree_after_insert(root_ptr, to_insert); #ifdef DEBUG
printf("== POST-INSERT-FIXUP ===\n"); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
}
void insert_left(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* before) { #ifdef DEBUG
printf("=== PRE-INSERT-LEFT ====\n"); printf("===== INSERTING =====\n"); debug_print_tree(0, to_insert, 0); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
if (before->left != NULL) { before = before->left; while (before->right != NULL) { before = before->right; } before->right = to_insert; to_insert->parent = before; } else { before->left = to_insert; to_insert->parent = before; } to_insert->color = COLOR_RED; repair_tree_after_insert(root_ptr, to_insert); #ifdef DEBUG
printf("== POST-INSERT-FIXUP ===\n"); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
}
void replace_node(TreeAlloc **root_ptr, TreeAlloc *node, TreeAlloc *replace) { if (!node->parent) { *root_ptr = replace; } else { if (node == node->parent->left) node->parent->left = replace; else node->parent->right = replace; } if (replace) replace->parent = node->parent; }
void repair_after_remove(TreeAlloc **root_ptr, TreeAlloc *parent, TreeAlloc *node) { #ifdef DEBUG
printf("delete fixup at %p -> %p\n", parent, node); #endif
// In theory, the last two conditions should be the same ...
if (IS_RED_NODE(node) || (node != NULL && node == *root_ptr)) { node->color = COLOR_BLACK; } else { TreeAlloc *sibling = get_sibling(parent, node); if (IS_RED_NODE(sibling)) { if (parent->left == node) { rotate_left(root_ptr, parent); } else { rotate_right(root_ptr, parent); } // The rotate shouldn't touch the parent relationship of `node`
parent->parent->color = COLOR_BLACK; parent->color = COLOR_RED; sibling = get_sibling(parent, node); }
if (IS_BLACK_NODE(sibling->left) && IS_BLACK_NODE(sibling->right)) { if (node != NULL) node->color = COLOR_BLACK; sibling->color = COLOR_RED; repair_after_remove(root_ptr, parent->parent, parent); } else { if (parent->left == node && IS_BLACK_NODE(sibling->right)) { rotate_right(root_ptr, sibling); sibling->color = COLOR_RED; sibling = parent->right; sibling->color = COLOR_BLACK; } if (parent->right == node && IS_BLACK_NODE(sibling->left)) { rotate_left(root_ptr, sibling); sibling->color = COLOR_RED; sibling = parent->left; sibling->color = COLOR_BLACK; } if (parent->left == node) { rotate_left(root_ptr, parent); } else { rotate_right(root_ptr, parent); } if (node != NULL) node->color = COLOR_BLACK; TreeAlloc *uncle = get_sibling(parent->parent, parent); if (uncle != NULL) uncle->color = COLOR_BLACK; char swap = parent->color; parent->color = parent->parent->color; parent->parent->color = swap; } } }
void remove_node(TreeAlloc **root_ptr, TreeAlloc *to_remove) { char do_repair = 0; char old_color; #ifdef DEBUG
printf("====== PRE-REMOVE ======\n"); printf("======= REMOVING =======\n"); debug_print_tree(0, to_remove, 0); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
TreeAlloc *replace, *parent_of_replace; TreeAlloc *parent = to_remove->parent; if (!to_remove->left) { #ifdef DEBUG
printf("code path 1l\n"); #endif
replace = to_remove->right; parent_of_replace = to_remove->parent; do_repair = to_remove->color == COLOR_BLACK; replace_node(root_ptr, to_remove, replace); } else if (!to_remove->right) { #ifdef DEBUG
printf("code path 1r\n"); #endif
replace = to_remove->left; parent_of_replace = to_remove->parent; do_repair = to_remove->color == COLOR_BLACK; replace_node(root_ptr, to_remove, replace); } else { #ifdef DEBUG
printf("code path 2\n"); #endif
TreeAlloc *tmp = succ(to_remove); replace = tmp->right; do_repair = tmp->color == COLOR_BLACK; if (tmp != to_remove->right) { replace_node(root_ptr, tmp, replace); tmp->right = to_remove->right; to_remove->right->parent = tmp; parent_of_replace = tmp->parent; } else { parent_of_replace = tmp; } replace_node(root_ptr, to_remove, tmp); tmp->color = to_remove->color; tmp->left = to_remove->left; to_remove->left->parent = tmp; }
// Make sure that it doesn't have any tree pointers it shouldn't have.
to_remove->parent = to_remove->left = to_remove->right = NULL;
#ifdef DEBUG
printf("==== PRE-REMOVE-FIXUP ===\n"); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 1); printf("===== END OF TREES =====\n"); printf("considering fixing up %p -> %p\n", parent_of_replace, replace); #endif
if (replace && parent_of_replace == NULL) { replace->color = COLOR_BLACK; } else if (parent_of_replace != NULL && do_repair) { repair_after_remove(root_ptr, parent_of_replace, replace); } #ifdef DEBUG
printf("=== POST-REMOVE ===\n"); printf("===== CURRENT TREE =====\n"); debug_print_tree(0, *root_ptr, 0); printf("===== END OF TREES =====\n"); #endif
}
int add_new_region(Arena *arena, uintptr_t size, uintptr_t padding, uintptr_t align) { uintptr_t realsize = size + align + alignof(WatermarkAlloc) + padding - 1; if (realsize < MIN_NEW_MEM_SIZE) { realsize = MIN_NEW_MEM_SIZE; } FreeSpace *reg = (FreeSpace*) arena->get_new_region(realsize); if (reg == NULL) { arena->error("can't allocate a new memory region!"); return 0; } FreeSpace *newreg = align_after(reg, alignof(WatermarkAlloc)); newreg->left = NULL; newreg->right = NULL; realsize -= (void*) newreg - (void*) reg; realsize -= realsize % alignof(WatermarkAlloc); newreg->size = realsize; if (arena->root_freespace == NULL) { insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) newreg); } else { FreeSpace *head = arena->root_freespace; while (head->right != NULL) { head = head->right; } insert_right((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) newreg, (TreeAlloc*) head); } #ifdef DEBUG
printf("= POST-REGION-CREATION =\n"); printf("==== FREESPACE TREE ====\n"); debug_print_tree(0, arena->root_freespace, 0); printf("==== TREEALLOC TREE ====\n"); debug_print_tree(0, arena->root_treealloc, 0); printf("===== END OF TREES =====\n"); #endif
return 1; }
void unalloc(Arena *arena, void *addr) { #ifdef DEBUG
printf("==== UNALLOCATING ====\n"); printf("=== FREESPACE TREE ===\n"); debug_print_tree(0, arena->root_freespace, 0); printf("=== TREEALLOC TREE ===\n"); debug_print_tree(0, arena->root_treealloc, 0); printf("==== END OF TREES ====\n"); #endif
if (arena->root_treealloc == NULL) { arena->error("attempt to unallocate when there are no allocations!"); return; } // Find the node this address belongs to
TreeAlloc *node = search_by_address(arena->root_treealloc, addr); if (node == NULL) { arena->error("attempt to free memory outside any allocations!"); return; } // Handle the watermark allocator in this region
if (node->type == RT_WATERMARK) { // TODO: handle watermark deallocation
return; } // Get rid of it
remove_node(&arena->root_treealloc, node); // If there's free space on either side of it, merge it with the free space into a bigger chunk of
// free space.
uintptr_t size = node->size; FreeSpace *start = (FreeSpace*) node; if (node->before != NULL && node->before->type == RT_FREESPACE) { start = (FreeSpace*) node->before; size += node->before->size; remove_node((TreeAlloc**) &arena->root_freespace, node->before); } if (node->after != NULL && node->after->type == RT_FREESPACE) { size += node->after->size; remove_node((TreeAlloc**) &arena->root_freespace, node->after); } start->type = RT_FREESPACE; start->size = size; // And finally, insert the resulting free space.
if (arena->root_freespace == NULL) { insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start); } else { TreeAlloc *insert_point = search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, size); if (insert_point == NULL) { TreeAlloc *head = (TreeAlloc*) arena->root_freespace; while (head->right != NULL) { head = head->right; } insert_right((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start, head); } else { insert_left((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start, insert_point); } } }
void *alloc(Arena *arena, uintptr_t size, uintptr_t align) { uintptr_t actual_align = lcm(alignof(struct WatermarkAlloc), align); #ifdef DEBUG
printf("==== ALLOCATING =====\n"); printf("=== FREESPACE TREE ===\n"); debug_print_tree(0, arena->root_freespace, 0); printf("=== TREEALLOC TREE ===\n"); debug_print_tree(0, arena->root_treealloc, 0); printf("==== END OF TREES ====\n"); #endif
if (arena->root_freespace == NULL) { // Handle being out of freespace.
#ifdef DEBUG
printf("Out of freespace nodes; getting more\n"); #endif
if (!add_new_region(arena, size, sizeof(TreeAlloc), actual_align)) { return NULL; } return alloc(arena, size, align); } else { TreeAlloc *region = search_by_size((TreeAlloc*) arena->root_freespace, sizeof(TreeAlloc), actual_align, size); if (region == NULL) { // Handle insufficient freespace or fragmentation.
#ifdef DEBUG
printf("Out of sufficiently large freespace nodes; getting more\n"); #endif
if (!add_new_region(arena, size, sizeof(TreeAlloc), actual_align)) { return NULL; } return alloc(arena, size, align); } remove_node((TreeAlloc**) &arena->root_freespace, region); void *true_end = align_after(align_after(((void*) region) + sizeof(TreeAlloc), actual_align) + size, alignof(WatermarkAlloc)); // The size of the new allocation (adjusted for region header and alignment
uintptr_t new_size = true_end - (void*) region; // The size of the free space region following the new allocation
uintptr_t new_free_size = region->size - new_size; region->right = NULL; region->left = NULL; region->type = RT_TREE_NODE; #ifdef DEBUG
printf("start: %p, end: %p, adjusted end: %p\n", region, ((void*) region) + size, true_end); printf("size: %lu -> %lu\n", size, new_size); #endif
if (arena->root_treealloc == NULL) { insert_singleton((TreeAlloc**) &arena->root_treealloc, region); } else { #ifdef DEBUG
printf("searching for an insert point\n"); #endif
TreeAlloc *insert_point = search_by_address((TreeAlloc*) arena->root_treealloc, region); if (insert_point == NULL) { TreeAlloc *head = arena->root_treealloc; while (head->left != NULL) { head = head->left; } #ifdef DEBUG
printf("none found; inserting before %p\n", head); #endif
insert_left(&arena->root_treealloc, region, head); } else { #ifdef DEBUG
printf("found one: %p\n", insert_point); #endif
insert_right(&arena->root_treealloc, region, insert_point); } } if (new_free_size >= sizeof(FreeSpace)) { // If there's enough free space after the allocation, use it!
region->size = new_size; // Safe because the allocated region tree is not sorted by size.
FreeSpace *new_free = (FreeSpace*) ((void*) region + new_size); new_free->left = NULL; new_free->right = NULL; new_free->type = RT_FREESPACE; new_free->size = new_free_size; if (arena->root_freespace == NULL) { insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free); } else { FreeSpace *insert_point = (FreeSpace*) search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, new_free_size); insert_left((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free, (TreeAlloc*) insert_point); } // Set the region following this one to be the new free space
region->after = (TreeAlloc*) new_free; } else { // There isn't a free space after this one, so put the `next` pointer at the next allocated
// region.
region->after = succ(region); } // I seem to have forgotten about the fact that memory may not be contiguous
if (region->after != NULL && region->after != (void*) region + region->size) { region->after = NULL; } // Also make sure the `before` pointer is correct.
TreeAlloc *before_alloc = pred(region); if (before_alloc == NULL || ((void*) before_alloc) + before_alloc->size < (void*) region) { region->before = search_by_address((TreeAlloc*) arena->root_freespace, region); } else { region->before = before_alloc; } // I seem to have forgotten about the fact that memory may not be contiguous
if (region->before != NULL && region->before != (void*) region->before + region->before->size) { region->before = NULL; } #ifdef DEBUG
printf("region is still at %p\n", region); #endif
return align_after((void*) region + sizeof(TreeAlloc), actual_align); } }
void *alloc_growable(Arena *arena, uintptr_t size, uintptr_t align) { // TODO: Basically the same as above, but put the allocated region in the center of the largest free
// space. Due to alignment and whatnot, the code will be gory.
return NULL; }
|