|
@@ -4,12 +4,12 @@
|
4
|
4
|
|
5
|
5
|
#include "allocator_internal.h"
|
6
|
6
|
|
7
|
|
-#ifdef DEBUG
|
8
|
|
-#include <stdio.h>
|
9
|
|
-
|
10
|
7
|
#define IS_BLACK_NODE(n) (n == NULL || n->color == COLOR_BLACK)
|
11
|
8
|
#define IS_RED_NODE(n) (n != NULL && n->color == COLOR_RED)
|
12
|
9
|
|
|
10
|
+#ifdef DEBUG
|
|
11
|
+#include <stdio.h>
|
|
12
|
+
|
13
|
13
|
int debug_tree_black_height(TreeAlloc *node) {
|
14
|
14
|
if (node == NULL) {
|
15
|
15
|
return 1;
|
|
@@ -45,9 +45,9 @@ void debug_print_tree(int indent, void *p, int safe) {
|
45
|
45
|
if (node != NULL) {
|
46
|
46
|
int bad = debug_tree_black_height(node->left) != debug_tree_black_height(node->right);
|
47
|
47
|
bad |= IS_RED_NODE(node) && (
|
48
|
|
- IS_RED_NODE(node->left) ||
|
49
|
|
- IS_RED_NODE(node->right) ||
|
50
|
|
- IS_RED_NODE(node->parent) );
|
|
48
|
+ IS_RED_NODE(node->left) ||
|
|
49
|
+ IS_RED_NODE(node->right) ||
|
|
50
|
+ IS_RED_NODE(node->parent) );
|
51
|
51
|
bad &= !safe;
|
52
|
52
|
debug_print_tree(indent + 1, node->left, safe);
|
53
|
53
|
debug_print_node(indent, node, bad);
|
|
@@ -67,17 +67,30 @@ TreeAlloc *insert_node_at(void *address, uintptr_t padding, uintptr_t align, uin
|
67
|
67
|
TreeAlloc *search_by_address(TreeAlloc *root, void *address) {
|
68
|
68
|
TreeAlloc *head = root;
|
69
|
69
|
while (1) {
|
70
|
|
- if (head >= (TreeAlloc*) address) {
|
71
|
|
- if (head->left == NULL) {
|
72
|
|
- return NULL;
|
73
|
|
- } else {
|
|
70
|
+ void *region_start = head;
|
|
71
|
+ void *region_end = head + head->size;
|
|
72
|
+ if (address < region_start) {
|
|
73
|
+ // The requested address is before this region's start.
|
|
74
|
+ if (head->left) {
|
|
75
|
+ // There is another region that comes before this one
|
|
76
|
+ // in memory.
|
74
|
77
|
head = head->left;
|
|
78
|
+ } else {
|
|
79
|
+ // This address is before any of the allocated regions.
|
|
80
|
+ return NULL;
|
75
|
81
|
}
|
|
82
|
+ } else if (address <= region_end) {
|
|
83
|
+ // The requested address is within the range of this region.
|
|
84
|
+ return head;
|
76
|
85
|
} else {
|
77
|
|
- if (head->right == NULL || head->right > (TreeAlloc*) address) {
|
78
|
|
- return head;
|
79
|
|
- } else {
|
|
86
|
+ // The requested address is after this region's end.
|
|
87
|
+ if (head->right) {
|
|
88
|
+ // There is another region that comes after this one
|
|
89
|
+ // in memory.
|
80
|
90
|
head = head->right;
|
|
91
|
+ } else {
|
|
92
|
+ // This address is after any of the allocated regions.
|
|
93
|
+ return NULL;
|
81
|
94
|
}
|
82
|
95
|
}
|
83
|
96
|
}
|
|
@@ -263,27 +276,34 @@ void insert_singleton(TreeAlloc **root_ptr, TreeAlloc *to_insert) {
|
263
|
276
|
#endif
|
264
|
277
|
}
|
265
|
278
|
|
266
|
|
-void insert_right(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* after) {
|
|
279
|
+void insert_by_size(TreeAlloc** root_ptr, TreeAlloc* to_insert) {
|
267
|
280
|
#ifdef DEBUG
|
268
|
|
- printf("=== PRE-INSERT-RIGHT ===\n");
|
|
281
|
+ printf("=== PRE-INSERT-BY-SIZE ===\n");
|
269
|
282
|
printf("===== INSERTING =====\n");
|
270
|
283
|
debug_print_node(0, to_insert, 0);
|
271
|
|
- printf("\tafter\n");
|
272
|
|
- debug_print_node(0, after, 0);
|
273
|
284
|
printf("===== CURRENT TREE =====\n");
|
274
|
285
|
debug_print_tree(0, *root_ptr, 0);
|
275
|
286
|
printf("===== END OF TREES =====\n");
|
276
|
287
|
#endif
|
277
|
|
- if (after->right != NULL) {
|
278
|
|
- after = after->right;
|
279
|
|
- while (after->left != NULL) {
|
280
|
|
- after = after->left;
|
|
288
|
+ TreeAlloc *tree_ptr = *root_ptr;
|
|
289
|
+ while (1) {
|
|
290
|
+ if (to_insert->size < tree_ptr->size) {
|
|
291
|
+ if (tree_ptr->left) {
|
|
292
|
+ tree_ptr = tree_ptr->left;
|
|
293
|
+ } else {
|
|
294
|
+ tree_ptr->left = to_insert;
|
|
295
|
+ to_insert->parent = tree_ptr;
|
|
296
|
+ break;
|
|
297
|
+ }
|
|
298
|
+ } else {
|
|
299
|
+ if (tree_ptr->right) {
|
|
300
|
+ tree_ptr = tree_ptr->right;
|
|
301
|
+ } else {
|
|
302
|
+ tree_ptr->right = to_insert;
|
|
303
|
+ to_insert->parent = tree_ptr;
|
|
304
|
+ break;
|
|
305
|
+ }
|
281
|
306
|
}
|
282
|
|
- after->left = to_insert;
|
283
|
|
- to_insert->parent = after;
|
284
|
|
- } else {
|
285
|
|
- after->right = to_insert;
|
286
|
|
- to_insert->parent = after;
|
287
|
307
|
}
|
288
|
308
|
to_insert->color = COLOR_RED;
|
289
|
309
|
repair_tree_after_insert(root_ptr, to_insert);
|
|
@@ -295,30 +315,34 @@ void insert_right(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* after)
|
295
|
315
|
#endif
|
296
|
316
|
}
|
297
|
317
|
|
298
|
|
-void insert_left(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* before) {
|
|
318
|
+void insert_by_addr(TreeAlloc** root_ptr, TreeAlloc* to_insert) {
|
299
|
319
|
#ifdef DEBUG
|
300
|
|
- printf("=== PRE-INSERT-LEFT ====\n");
|
|
320
|
+ printf("=== PRE-INSERT-BY-ADDR ===\n");
|
301
|
321
|
printf("===== INSERTING =====\n");
|
302
|
322
|
debug_print_tree(0, to_insert, 0);
|
303
|
|
- printf("\tbefore\n");
|
304
|
|
- debug_print_node(0, before, 0);
|
305
|
323
|
printf("===== CURRENT TREE =====\n");
|
306
|
324
|
debug_print_tree(0, *root_ptr, 0);
|
307
|
325
|
printf("===== END OF TREES =====\n");
|
308
|
326
|
#endif
|
309
|
|
- if (!before)
|
310
|
|
- before = *root_ptr;
|
311
|
|
-
|
312
|
|
- if (before->left != NULL) {
|
313
|
|
- before = before->left;
|
314
|
|
- while (before->right != NULL) {
|
315
|
|
- before = before->right;
|
|
327
|
+ TreeAlloc *tree_ptr = *root_ptr;
|
|
328
|
+ while (1) {
|
|
329
|
+ if (to_insert < tree_ptr) {
|
|
330
|
+ if (tree_ptr->left) {
|
|
331
|
+ tree_ptr = tree_ptr->left;
|
|
332
|
+ } else {
|
|
333
|
+ tree_ptr->left = to_insert;
|
|
334
|
+ to_insert->parent = tree_ptr;
|
|
335
|
+ break;
|
|
336
|
+ }
|
|
337
|
+ } else {
|
|
338
|
+ if (tree_ptr->right) {
|
|
339
|
+ tree_ptr = tree_ptr->right;
|
|
340
|
+ } else {
|
|
341
|
+ tree_ptr->right = to_insert;
|
|
342
|
+ to_insert->parent = tree_ptr;
|
|
343
|
+ break;
|
|
344
|
+ }
|
316
|
345
|
}
|
317
|
|
- before->right = to_insert;
|
318
|
|
- to_insert->parent = before;
|
319
|
|
- } else {
|
320
|
|
- before->left = to_insert;
|
321
|
|
- to_insert->parent = before;
|
322
|
346
|
}
|
323
|
347
|
to_insert->color = COLOR_RED;
|
324
|
348
|
repair_tree_after_insert(root_ptr, to_insert);
|
|
@@ -471,40 +495,26 @@ void remove_node(TreeAlloc **root_ptr, TreeAlloc *to_remove) {
|
471
|
495
|
#endif
|
472
|
496
|
}
|
473
|
497
|
|
474
|
|
-int add_new_region(Arena *arena, uintptr_t size, uintptr_t padding, uintptr_t align) {
|
|
498
|
+TreeAlloc *get_new_region(Arena *arena, uintptr_t size, uintptr_t padding, uintptr_t align) {
|
475
|
499
|
uintptr_t realsize = size + align + alignof(WatermarkAlloc) + padding - 1;
|
|
500
|
+#ifdef DEBUG
|
|
501
|
+ printf("Attemping request of size %ld\n", realsize);
|
|
502
|
+#endif
|
476
|
503
|
if (realsize < MIN_NEW_MEM_SIZE) {
|
477
|
504
|
realsize = MIN_NEW_MEM_SIZE;
|
478
|
505
|
}
|
479
|
|
- FreeSpace *reg = (FreeSpace*) arena->get_new_region(realsize);
|
|
506
|
+ TreeAlloc *reg = (TreeAlloc *) arena->get_new_region(realsize);
|
480
|
507
|
if (reg == NULL) {
|
481
|
508
|
arena->error("can't allocate a new memory region!");
|
482
|
|
- return 0;
|
483
|
|
- }
|
484
|
|
- FreeSpace *newreg = align_after(reg, alignof(WatermarkAlloc));
|
485
|
|
- newreg->left = NULL;
|
486
|
|
- newreg->right = NULL;
|
487
|
|
- realsize -= (void*) newreg - (void*) reg;
|
488
|
|
- realsize -= realsize % alignof(WatermarkAlloc);
|
489
|
|
- newreg->size = realsize;
|
490
|
|
- if (arena->root_freespace == NULL) {
|
491
|
|
- insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) newreg);
|
492
|
509
|
} else {
|
493
|
|
- FreeSpace *head = arena->root_freespace;
|
494
|
|
- while (head->right != NULL) {
|
495
|
|
- head = head->right;
|
496
|
|
- }
|
497
|
|
- insert_right((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) newreg, (TreeAlloc*) head);
|
|
510
|
+ reg->parent = NULL;
|
|
511
|
+ reg->left = NULL;
|
|
512
|
+ reg->right = NULL;
|
|
513
|
+ reg->before = NULL;
|
|
514
|
+ reg->after = NULL;
|
|
515
|
+ reg->size = realsize;
|
498
|
516
|
}
|
499
|
|
-#ifdef DEBUG
|
500
|
|
- printf("= POST-REGION-CREATION =\n");
|
501
|
|
- printf("==== FREESPACE TREE ====\n");
|
502
|
|
- debug_print_tree(0, arena->root_freespace, 0);
|
503
|
|
- printf("==== TREEALLOC TREE ====\n");
|
504
|
|
- debug_print_tree(0, arena->root_treealloc, 0);
|
505
|
|
- printf("===== END OF TREES =====\n");
|
506
|
|
-#endif
|
507
|
|
- return 1;
|
|
517
|
+ return reg;
|
508
|
518
|
}
|
509
|
519
|
|
510
|
520
|
void unalloc(Arena *arena, void *addr) {
|
|
@@ -554,19 +564,16 @@ void unalloc(Arena *arena, void *addr) {
|
554
|
564
|
} else {
|
555
|
565
|
TreeAlloc *insert_point = search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, size);
|
556
|
566
|
if (insert_point == NULL) {
|
557
|
|
- TreeAlloc *head = (TreeAlloc*) arena->root_freespace;
|
558
|
|
- while (head->right != NULL) {
|
559
|
|
- head = head->right;
|
560
|
|
- }
|
561
|
|
- insert_right((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start, head);
|
|
567
|
+ insert_by_size((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start);
|
562
|
568
|
} else {
|
563
|
|
- insert_left((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start, insert_point);
|
|
569
|
+ insert_by_size((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start);
|
564
|
570
|
}
|
565
|
571
|
}
|
566
|
572
|
}
|
567
|
573
|
|
568
|
574
|
void *alloc(Arena *arena, uintptr_t size, uintptr_t align) {
|
569
|
575
|
uintptr_t actual_align = lcm(alignof(struct WatermarkAlloc), align);
|
|
576
|
+
|
570
|
577
|
#ifdef DEBUG
|
571
|
578
|
printf("==== ALLOCATING =====\n");
|
572
|
579
|
printf("=== FREESPACE TREE ===\n");
|
|
@@ -575,106 +582,83 @@ void *alloc(Arena *arena, uintptr_t size, uintptr_t align) {
|
575
|
582
|
debug_print_tree(0, arena->root_treealloc, 0);
|
576
|
583
|
printf("==== END OF TREES ====\n");
|
577
|
584
|
#endif
|
|
585
|
+
|
|
586
|
+ TreeAlloc *region;
|
578
|
587
|
if (arena->root_freespace == NULL) {
|
579
|
588
|
// Handle being out of freespace.
|
580
|
589
|
#ifdef DEBUG
|
581
|
590
|
printf("Out of freespace nodes; getting more\n");
|
582
|
591
|
#endif
|
583
|
|
- if (!add_new_region(arena, size, sizeof(TreeAlloc), actual_align)) {
|
584
|
|
- return NULL;
|
585
|
|
- }
|
586
|
|
- return alloc(arena, size, align);
|
|
592
|
+ region = get_new_region(arena, size, sizeof(TreeAlloc), actual_align);
|
587
|
593
|
} else {
|
588
|
|
- TreeAlloc *region = search_by_size((TreeAlloc*) arena->root_freespace, sizeof(TreeAlloc), actual_align, size);
|
|
594
|
+ region = search_by_size((TreeAlloc*) arena->root_freespace, sizeof(TreeAlloc), actual_align, size);
|
589
|
595
|
if (region == NULL) {
|
590
|
596
|
// Handle insufficient freespace or fragmentation.
|
591
|
597
|
#ifdef DEBUG
|
592
|
598
|
printf("Out of sufficiently large freespace nodes; getting more\n");
|
593
|
599
|
#endif
|
594
|
|
- if (!add_new_region(arena, size, sizeof(TreeAlloc), actual_align)) {
|
595
|
|
- return NULL;
|
596
|
|
- }
|
597
|
|
- return alloc(arena, size, align);
|
598
|
|
- }
|
599
|
|
- remove_node((TreeAlloc**) &arena->root_freespace, region);
|
600
|
|
- void *true_end = align_after(align_after(((void*) region) + sizeof(TreeAlloc), actual_align) + size, alignof(WatermarkAlloc));
|
601
|
|
- // The size of the new allocation (adjusted for region header and alignment
|
602
|
|
- uintptr_t new_size = true_end - (void*) region;
|
603
|
|
- // The size of the free space region following the new allocation
|
604
|
|
- uintptr_t new_free_size = region->size - new_size;
|
605
|
|
- region->right = NULL;
|
606
|
|
- region->left = NULL;
|
607
|
|
- region->type = RT_TREE_NODE;
|
608
|
|
-#ifdef DEBUG
|
609
|
|
- printf("start: %p, end: %p, adjusted end: %p\n", region, ((void*) region) + size, true_end);
|
610
|
|
- printf("size: %lu -> %lu\n", size, new_size);
|
611
|
|
- printf("new_free_size: %lu\n", new_free_size);
|
612
|
|
-#endif
|
613
|
|
- if (arena->root_treealloc == NULL) {
|
614
|
|
- insert_singleton((TreeAlloc**) &arena->root_treealloc, region);
|
|
600
|
+ region = get_new_region(arena, size, sizeof(TreeAlloc), actual_align);
|
615
|
601
|
} else {
|
|
602
|
+ remove_node((TreeAlloc**) &arena->root_freespace, region);
|
|
603
|
+ }
|
|
604
|
+ }
|
|
605
|
+
|
|
606
|
+ void *true_end = align_after(align_after(((void*) region) + sizeof(TreeAlloc), actual_align) + size, alignof(WatermarkAlloc));
|
|
607
|
+ // The size of the new allocation (adjusted for region header and alignment
|
|
608
|
+ uintptr_t new_size = true_end - (void*) region;
|
|
609
|
+ // The size of the free space region following the new allocation
|
|
610
|
+ uintptr_t new_free_size = region->size - new_size;
|
|
611
|
+ region->right = NULL;
|
|
612
|
+ region->left = NULL;
|
|
613
|
+ region->type = RT_TREE_NODE;
|
|
614
|
+ region->size = size;
|
|
615
|
+
|
616
|
616
|
#ifdef DEBUG
|
617
|
|
- printf("searching for an insert point\n");
|
618
|
|
-#endif
|
619
|
|
- TreeAlloc *insert_point = search_by_address((TreeAlloc*) arena->root_treealloc, region);
|
620
|
|
- if (insert_point == NULL) {
|
621
|
|
- TreeAlloc *head = arena->root_treealloc;
|
622
|
|
- while (head->left != NULL) {
|
623
|
|
- head = head->left;
|
624
|
|
- }
|
625
|
|
-#ifdef DEBUG
|
626
|
|
- printf("none found; inserting before %p\n", head);
|
627
|
|
-#endif
|
628
|
|
- insert_left(&arena->root_treealloc, region, head);
|
629
|
|
- } else {
|
630
|
|
-#ifdef DEBUG
|
631
|
|
- printf("found one: %p\n", insert_point);
|
|
617
|
+ printf("start: %p, end: %p, adjusted end: %p\n", region, ((void*) region) + size, true_end);
|
|
618
|
+ printf("size: %lu -> %lu\n", size, new_size);
|
|
619
|
+ printf("new_free_size: %lu\n", new_free_size);
|
632
|
620
|
#endif
|
633
|
|
- insert_right(&arena->root_treealloc, region, insert_point);
|
634
|
|
- }
|
635
|
|
- }
|
636
|
|
- if (region->size >= new_size + sizeof(FreeSpace)) {
|
637
|
|
- // If there's enough free space after the allocation, use it!
|
638
|
|
- region->size = new_size; // Safe because the allocated region tree is not sorted by size.
|
639
|
|
- FreeSpace *new_free = (FreeSpace*) ((void*) region + new_size);
|
640
|
|
- new_free->left = NULL;
|
641
|
|
- new_free->right = NULL;
|
642
|
|
- new_free->parent = NULL;
|
643
|
|
- new_free->type = RT_FREESPACE;
|
644
|
|
- new_free->size = new_free_size;
|
645
|
|
- if (arena->root_freespace == NULL) {
|
646
|
|
- insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free);
|
647
|
|
- } else {
|
648
|
|
- FreeSpace *insert_point = (FreeSpace*) search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, new_free_size);
|
649
|
|
- insert_left((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free, (TreeAlloc*) insert_point);
|
650
|
|
- }
|
651
|
|
- // Set the region following this one to be the new free space
|
652
|
|
- region->after = (TreeAlloc*) new_free;
|
653
|
|
- } else {
|
654
|
|
- // There isn't a free space after this one, so put the `next` pointer at the next allocated
|
655
|
|
- // region.
|
656
|
|
- region->after = succ(region);
|
657
|
|
- }
|
658
|
|
- // I seem to have forgotten about the fact that memory may not be contiguous
|
659
|
|
- if (region->after != NULL && region->after != (void*) region + region->size) {
|
660
|
|
- region->after = NULL;
|
661
|
|
- }
|
662
|
|
- // Also make sure the `before` pointer is correct.
|
663
|
|
- TreeAlloc *before_alloc = pred(region);
|
664
|
|
- if (before_alloc == NULL || ((void*) before_alloc) + before_alloc->size < (void*) region) {
|
665
|
|
- region->before = search_by_address((TreeAlloc*) arena->root_freespace, region);
|
|
621
|
+
|
|
622
|
+ if (arena->root_treealloc == NULL) {
|
|
623
|
+ insert_singleton((TreeAlloc**) &arena->root_treealloc, region);
|
|
624
|
+ } else {
|
|
625
|
+ insert_by_addr(&arena->root_treealloc, region);
|
|
626
|
+ }
|
|
627
|
+ if (region->size >= new_size + sizeof(FreeSpace)) {
|
|
628
|
+ // If there's enough free space after the allocation, use it!
|
|
629
|
+ region->size = new_size; // Safe because the allocated region tree is not sorted by size.
|
|
630
|
+ FreeSpace *new_free = (FreeSpace*) ((void*) region + new_size);
|
|
631
|
+ new_free->left = NULL;
|
|
632
|
+ new_free->right = NULL;
|
|
633
|
+ new_free->parent = NULL;
|
|
634
|
+ new_free->type = RT_FREESPACE;
|
|
635
|
+ new_free->size = new_free_size;
|
|
636
|
+ if (arena->root_freespace == NULL) {
|
|
637
|
+ insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free);
|
666
|
638
|
} else {
|
667
|
|
- region->before = before_alloc;
|
668
|
|
- }
|
669
|
|
- // I seem to have forgotten about the fact that memory may not be contiguous
|
670
|
|
- if (region->before != NULL && region->before != (void*) region->before + region->before->size) {
|
671
|
|
- region->before = NULL;
|
|
639
|
+ insert_by_size((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free);
|
672
|
640
|
}
|
|
641
|
+ // Set the region following this one to be the new free space
|
|
642
|
+ region->after = (TreeAlloc*) new_free;
|
|
643
|
+ } else {
|
|
644
|
+ // There isn't a free space after this one, so put the `next` pointer at the next allocated
|
|
645
|
+ // region if there is one.
|
|
646
|
+ region->after = search_by_address((TreeAlloc *) &arena->root_treealloc, region + region->size + 1);
|
|
647
|
+ }
|
|
648
|
+
|
|
649
|
+ // Are there any allocations before this one?
|
|
650
|
+ region->before = search_by_address((TreeAlloc *) &arena->root_treealloc, region - 1);
|
|
651
|
+
|
673
|
652
|
#ifdef DEBUG
|
674
|
|
- printf("region is still at %p\n", region);
|
|
653
|
+ printf("region is still at %p\n", region);
|
|
654
|
+ printf("=== POST-ALLOCATION ===\n");
|
|
655
|
+ printf("=== FREESPACE TREE ===\n");
|
|
656
|
+ debug_print_tree(0, arena->root_freespace, 0);
|
|
657
|
+ printf("=== TREEALLOC TREE ===\n");
|
|
658
|
+ debug_print_tree(0, arena->root_treealloc, 0);
|
|
659
|
+ printf("==== END OF TREES ====\n");
|
675
|
660
|
#endif
|
676
|
|
- return align_after((void*) region + sizeof(TreeAlloc), actual_align);
|
677
|
|
- }
|
|
661
|
+ return align_after((void*) region + sizeof(TreeAlloc), actual_align);
|
678
|
662
|
}
|
679
|
663
|
|
680
|
664
|
void *alloc_growable(Arena *arena, uintptr_t size, uintptr_t align) {
|