No Description
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

tree_alloc.c 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593
  1. #include <stddef.h>
  2. #include <stdalign.h>
  3. #include "allocator_internal.h"
  4. #ifdef DEBUG
  5. #include <stdio.h>
  6. int debug_tree_black_height(TreeAlloc *node) {
  7. if (node == NULL) {
  8. return 1;
  9. }
  10. return ((node->color == COLOR_BLACK) ? 1 : 0) + debug_tree_black_height(node->left);
  11. }
  12. void debug_print_tree(int indent, void *p) {
  13. TreeAlloc *node = (TreeAlloc*) p;
  14. if (node != NULL) {
  15. int bad = debug_tree_black_height(node->left) != debug_tree_black_height(node->right);
  16. bad |= node->color == COLOR_RED && ((node->left != NULL && node->left->color == COLOR_RED) ||
  17. (node->right != NULL && node->right->color == COLOR_RED) || (node->parent != NULL &&
  18. node->parent->color == COLOR_RED));
  19. debug_print_tree(indent + 1, node->left);
  20. for (int ii = 0; ii < indent; ii++) { printf(" "); }
  21. if (node->color == COLOR_RED) { printf(bad ? "\e[30m]" : "\e[31m"); }
  22. if (bad) { printf("\e[43m"); }
  23. printf("%p %lu\n", node, node->size);
  24. if (node->color == COLOR_RED) { printf("\e[37m"); }
  25. if (bad) { printf("\e[40m"); }
  26. debug_print_tree(indent + 1, node->right);
  27. }
  28. }
  29. #endif
  30. TreeAlloc *insert_node_at(void *address, uintptr_t padding, uintptr_t align, uintptr_t size) {
  31. return NULL;
  32. }
  33. /*
  34. * Search for the node whose allocated region contains an address.
  35. */
  36. TreeAlloc *search_by_address(TreeAlloc *root, void *address) {
  37. TreeAlloc *head = root;
  38. while (1) {
  39. if (head > (TreeAlloc*) address) {
  40. if (head->left == NULL) {
  41. return NULL;
  42. } else {
  43. head = head->left;
  44. }
  45. } else {
  46. if (head->right == NULL || head->right > (TreeAlloc*) address) {
  47. return head;
  48. } else {
  49. head = head->right;
  50. }
  51. }
  52. }
  53. }
  54. static uintptr_t effective_size(TreeAlloc *head, uintptr_t padding, uintptr_t align) {
  55. return head->size - (align_after(head + padding, align) - (void*) head);
  56. }
  57. /*
  58. * This is the most optimistic estimate of size that we can use which also preserves the ordering over
  59. * the tree. I had planned to use effective_size before I realized that it would break the tree
  60. * ordering.
  61. */
  62. static uintptr_t pessimistic_size(TreeAlloc *head, uintptr_t padding, uintptr_t align) {
  63. return head->size - padding - align + 1;
  64. }
  65. TreeAlloc *search_by_size(TreeAlloc *root, uintptr_t padding, uintptr_t align, uintptr_t size) {
  66. TreeAlloc *head = root;
  67. while (1) {
  68. uintptr_t esize = pessimistic_size(head, padding, align);
  69. if (esize < size) {
  70. if (head->right == NULL) {
  71. return NULL;
  72. } else {
  73. head = head->right;
  74. }
  75. } else {
  76. if (head->left == NULL || pessimistic_size(head->left, padding, align) < size) {
  77. return head;
  78. } else {
  79. head = head->left;
  80. }
  81. }
  82. }
  83. }
  84. TreeAlloc *succ(TreeAlloc *el) {
  85. if (el->right != NULL) {
  86. el = el->right;
  87. while (el->left != NULL) {
  88. el = el->left;
  89. }
  90. return el;
  91. }
  92. while (el->parent != NULL && el == el->parent->right) {
  93. el = el->parent;
  94. }
  95. return el->parent;
  96. }
  97. TreeAlloc *pred(TreeAlloc *el) {
  98. if (el->left != NULL) {
  99. el = el->left;
  100. while (el->right != NULL) {
  101. el = el->right;
  102. }
  103. return el;
  104. }
  105. while (el->parent != NULL && el == el->parent->left) {
  106. el = el->parent;
  107. }
  108. return el->parent;
  109. }
  110. TreeAlloc *get_sibling(TreeAlloc *ta) {
  111. TreeAlloc *p = ta->parent;
  112. if (!p)
  113. return NULL;
  114. else if (p->left == ta)
  115. return p->right;
  116. else
  117. return p->left;
  118. }
  119. void rotate_left(TreeAlloc **root_ptr, TreeAlloc *ta) {
  120. TreeAlloc *parent, *tmp;
  121. parent = ta->parent;
  122. tmp = ta->right;
  123. if (!tmp) return;
  124. ta->right = tmp->left;
  125. tmp->left = ta;
  126. ta->parent = tmp;
  127. if (ta->right) ta->right->parent = ta;
  128. if (parent == NULL) {
  129. *root_ptr = tmp;
  130. } else {
  131. if (ta == parent->left)
  132. parent->left = tmp;
  133. else
  134. parent->right = tmp;
  135. }
  136. tmp->parent = parent;
  137. }
  138. void rotate_right(TreeAlloc **root_ptr, TreeAlloc *ta) {
  139. TreeAlloc *parent, *tmp;
  140. parent = ta->parent;
  141. tmp = ta->left;
  142. if (!tmp) return;
  143. ta->left = tmp->right;
  144. tmp->right = ta;
  145. ta->parent = tmp;
  146. if (ta->left) ta->left->parent = ta;
  147. if (parent == NULL) {
  148. *root_ptr = tmp;
  149. } else {
  150. if (ta == parent->left)
  151. parent->left = tmp;
  152. else
  153. parent->right = tmp;
  154. }
  155. tmp->parent = parent;
  156. }
  157. #define IS_BLACK_NODE(n) (n == NULL || n->color == COLOR_BLACK)
  158. #define IS_RED_NODE(n) (n != NULL && n->color == COLOR_RED)
  159. void repair_tree_after_insert(TreeAlloc **root_ptr, TreeAlloc *ta) {
  160. TreeAlloc *parent = ta->parent;
  161. #ifdef DEBUG
  162. printf("=== PRE-INSERT-FIXUP ===\n");
  163. printf("===== CURRENT TREE =====\n");
  164. debug_print_tree(0, *root_ptr);
  165. printf("===== END OF TREES =====\n");
  166. #endif
  167. if (parent == NULL) {
  168. ta->color = COLOR_BLACK;
  169. return;
  170. }
  171. TreeAlloc *grandparent = parent->parent;
  172. TreeAlloc *uncle = get_sibling(parent);
  173. if (IS_RED_NODE(parent)) {
  174. if (IS_RED_NODE(uncle)) {
  175. parent->color = COLOR_BLACK;
  176. uncle->color = COLOR_BLACK;
  177. grandparent->color = COLOR_RED;
  178. repair_tree_after_insert(root_ptr, grandparent);
  179. } else {
  180. if (ta == parent->left && parent == grandparent->left) {
  181. rotate_left(root_ptr, parent);
  182. ta = ta->left;
  183. parent = parent->left;
  184. } else {
  185. rotate_right(root_ptr, parent);
  186. ta = ta->right;
  187. parent = parent->right;
  188. }
  189. grandparent = parent->parent;
  190. if (ta == parent->left) {
  191. rotate_right(root_ptr, grandparent);
  192. } else {
  193. rotate_left(root_ptr, grandparent);
  194. }
  195. parent->color = COLOR_BLACK;
  196. grandparent->color = COLOR_RED;
  197. }
  198. }
  199. #ifdef DEBUG
  200. printf("== POST-INSERT-FIXUP ===\n");
  201. printf("===== CURRENT TREE =====\n");
  202. debug_print_tree(0, *root_ptr);
  203. printf("===== END OF TREES =====\n");
  204. #endif
  205. }
  206. void replace_node(TreeAlloc **root_ptr, TreeAlloc *node, TreeAlloc *replace) {
  207. if (!node->parent) {
  208. *root_ptr = replace;
  209. } else {
  210. if (node == node->parent->left)
  211. node->parent->left = replace;
  212. else
  213. node->parent->right = replace;
  214. }
  215. if (replace) replace->parent = node->parent;
  216. }
  217. void repair_after_remove(TreeAlloc **root_ptr, TreeAlloc *node) {
  218. if (IS_RED_NODE(node)) {
  219. node->color = COLOR_BLACK;
  220. } else {
  221. TreeAlloc *sibling = get_sibling(node);
  222. if (IS_RED_NODE(sibling)) {
  223. if (node->parent->left == node) {
  224. rotate_left(root_ptr, node->parent);
  225. } else {
  226. rotate_right(root_ptr, node->parent);
  227. }
  228. node->parent->parent->color = node->parent->color = COLOR_BLACK;
  229. }
  230. if (IS_BLACK_NODE(sibling->left) && IS_BLACK_NODE(sibling->right)) {
  231. node->color = COLOR_BLACK;
  232. sibling->color = COLOR_RED;
  233. repair_after_remove(root_ptr, node->parent);
  234. } else {
  235. if (node->parent->left == node && IS_BLACK_NODE(sibling)) {
  236. rotate_right(root_ptr, sibling);
  237. sibling = get_sibling(node);
  238. sibling->color = COLOR_RED;
  239. sibling->right->color = COLOR_RED;
  240. rotate_left(root_ptr, node->parent);
  241. node->color = get_sibling(node->parent)->color = COLOR_BLACK;
  242. } else if (node->parent->right == node && IS_BLACK_NODE(sibling)) {
  243. rotate_left(root_ptr, sibling);
  244. sibling = get_sibling(node);
  245. sibling->color = COLOR_RED;
  246. sibling->left->color = COLOR_RED;
  247. rotate_right(root_ptr, sibling);
  248. node->color = get_sibling(node->parent)->color = COLOR_BLACK;
  249. }
  250. node->parent->color ^= node->parent->parent->color;
  251. node->parent->parent->color ^= node->parent->color;
  252. node->parent->color ^= node->parent->parent->color;
  253. }
  254. }
  255. }
  256. void remove_node(TreeAlloc **root_ptr, TreeAlloc *node) {
  257. char do_repair = 0;
  258. #ifdef DEBUG
  259. printf("====== PRE-REMOVE ======\n");
  260. printf("===== CURRENT TREE =====\n");
  261. debug_print_tree(0, *root_ptr);
  262. printf("===== END OF TREES =====\n");
  263. #endif
  264. TreeAlloc *replace;
  265. TreeAlloc *parent = node->parent;
  266. if (!node->left) {
  267. replace = node->right;
  268. do_repair = node->color == COLOR_BLACK;
  269. replace_node(root_ptr, node, replace);
  270. } else if (!node->right) {
  271. replace = node->left;
  272. do_repair = node->color == COLOR_BLACK;
  273. replace_node(root_ptr, node, replace);
  274. } else {
  275. TreeAlloc *tmp = node->right;
  276. while (tmp->left) tmp = tmp->left;
  277. replace = tmp->right;
  278. do_repair = tmp->color == COLOR_BLACK;
  279. if (tmp != node->right) {
  280. replace_node(root_ptr, tmp, replace);
  281. tmp->right = node->right;
  282. node->right->parent = tmp;
  283. }
  284. replace_node(root_ptr, node, tmp);
  285. tmp->color = node->color;
  286. tmp->left = node->left;
  287. node->left->parent = tmp;
  288. }
  289. if (do_repair && replace && replace->parent != NULL) {
  290. #ifdef DEBUG
  291. printf("=== PRE-REMOVE-FIXUP ===\n");
  292. printf("===== CURRENT TREE =====\n");
  293. debug_print_tree(0, *root_ptr);
  294. printf("===== END OF TREES =====\n");
  295. #endif
  296. repair_after_remove(root_ptr, replace);
  297. }
  298. }
  299. // Inserts a node into an empty tree.
  300. void insert_singleton(TreeAlloc **root_ptr, TreeAlloc *to_insert) {
  301. #ifdef DEBUG
  302. printf("= PRE-INSERT-SINGLETON =\n");
  303. printf("===== CURRENT TREE =====\n");
  304. debug_print_tree(0, *root_ptr);
  305. printf("===== END OF TREES =====\n");
  306. #endif
  307. *root_ptr = to_insert;
  308. to_insert->parent = NULL;
  309. to_insert->color = COLOR_BLACK;
  310. }
  311. void insert_right(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* after) {
  312. #ifdef DEBUG
  313. printf("=== PRE-INSERT-RIGHT ===\n");
  314. printf("===== CURRENT TREE =====\n");
  315. debug_print_tree(0, *root_ptr);
  316. printf("===== END OF TREES =====\n");
  317. #endif
  318. if (after->right != NULL) {
  319. after->right->parent = to_insert;
  320. to_insert->right = after->right;
  321. }
  322. after->right = to_insert;
  323. to_insert->parent = after;
  324. to_insert->color = COLOR_RED;
  325. repair_tree_after_insert(root_ptr, to_insert);
  326. }
  327. void insert_left(TreeAlloc** root_ptr, TreeAlloc* to_insert, TreeAlloc* before) {
  328. #ifdef DEBUG
  329. printf("=== PRE-INSERT-LEFT ====\n");
  330. printf("===== CURRENT TREE =====\n");
  331. debug_print_tree(0, *root_ptr);
  332. printf("===== END OF TREES =====\n");
  333. #endif
  334. if (before->left != NULL) {
  335. before->left->parent = to_insert;
  336. to_insert->left = before->left;
  337. }
  338. before->left = to_insert;
  339. to_insert->parent = before;
  340. to_insert->color = COLOR_RED;
  341. repair_tree_after_insert(root_ptr, to_insert);
  342. }
  343. int add_new_region(Arena *arena, uintptr_t size, uintptr_t padding, uintptr_t align) {
  344. uintptr_t realsize = size + align + alignof(WatermarkAlloc) + padding - 1;
  345. if (realsize < MIN_NEW_MEM_SIZE) {
  346. realsize = MIN_NEW_MEM_SIZE;
  347. }
  348. FreeSpace *reg = (FreeSpace*) arena->get_new_region(realsize);
  349. if (reg == NULL) {
  350. arena->error("can't allocate a new memory region!");
  351. return 0;
  352. }
  353. FreeSpace *newreg = align_after(reg, alignof(WatermarkAlloc));
  354. newreg->left = NULL;
  355. newreg->right = NULL;
  356. realsize -= (void*) newreg - (void*) reg;
  357. realsize -= realsize % alignof(WatermarkAlloc);
  358. newreg->size = realsize;
  359. if (arena->root_freespace == NULL) {
  360. insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) newreg);
  361. } else {
  362. FreeSpace *head = arena->root_freespace;
  363. while (head->right != NULL) {
  364. head = head->right;
  365. }
  366. insert_right((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) newreg, (TreeAlloc*) head);
  367. }
  368. #ifdef DEBUG
  369. printf("= POST-REGION-CREATION =\n");
  370. printf("==== FREESPACE TREE ====\n");
  371. debug_print_tree(0, arena->root_freespace);
  372. printf("==== TREEALLOC TREE ====\n");
  373. debug_print_tree(0, arena->root_treealloc);
  374. printf("===== END OF TREES =====\n");
  375. #endif
  376. return 1;
  377. }
  378. void unalloc(Arena *arena, void *addr) {
  379. #ifdef DEBUG
  380. printf("==== UNALLOCATING ====\n");
  381. printf("=== FREESPACE TREE ===\n");
  382. debug_print_tree(0, arena->root_freespace);
  383. printf("=== TREEALLOC TREE ===\n");
  384. debug_print_tree(0, arena->root_treealloc);
  385. printf("==== END OF TREES ====\n");
  386. #endif
  387. if (arena->root_treealloc == NULL) {
  388. arena->error("attempt to unallocate when there are no allocations!");
  389. return;
  390. }
  391. // Find the node this address belongs to
  392. TreeAlloc *node = search_by_address(arena->root_treealloc, addr);
  393. if (node == NULL) {
  394. arena->error("attempt to free memory outside any allocations!");
  395. return;
  396. }
  397. // Handle the watermark allocator in this region
  398. if (node->type == RT_WATERMARK) {
  399. // TODO: handle watermark deallocation
  400. return;
  401. }
  402. // Get rid of it
  403. remove_node(&arena->root_treealloc, node);
  404. // If there's free space on either side of it, merge it with the free space into a bigger chunk of
  405. // free space.
  406. uintptr_t size = node->size;
  407. FreeSpace *start = (FreeSpace*) node;
  408. if (node->before != NULL && node->before->type == RT_FREESPACE) {
  409. start = (FreeSpace*) node->before;
  410. size += node->before->size;
  411. remove_node((TreeAlloc**) &arena->root_freespace, node->before);
  412. }
  413. if (node->after != NULL && node->after->type == RT_FREESPACE) {
  414. size += node->after->size;
  415. remove_node((TreeAlloc**) &arena->root_freespace, node->after);
  416. }
  417. start->type = RT_FREESPACE;
  418. start->size = size;
  419. // And finally, insert the resulting free space.
  420. if (arena->root_freespace == NULL) {
  421. insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start);
  422. } else {
  423. TreeAlloc *insert_point = search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, size);
  424. if (insert_point == NULL) {
  425. TreeAlloc *head = (TreeAlloc*) arena->root_freespace;
  426. while (head->right != NULL) {
  427. head = head->right;
  428. }
  429. insert_right((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start, head);
  430. } else {
  431. insert_left((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) start, insert_point);
  432. }
  433. }
  434. }
  435. void *alloc(Arena *arena, uintptr_t size, uintptr_t align) {
  436. uintptr_t actual_align = lcm(alignof(struct WatermarkAlloc), align);
  437. #ifdef DEBUG
  438. printf("==== ALLOCATING =====\n");
  439. printf("=== FREESPACE TREE ===\n");
  440. debug_print_tree(0, arena->root_freespace);
  441. printf("=== TREEALLOC TREE ===\n");
  442. debug_print_tree(0, arena->root_treealloc);
  443. printf("==== END OF TREES ====\n");
  444. #endif
  445. if (arena->root_freespace == NULL) {
  446. // Handle being out of freespace.
  447. #ifdef DEBUG
  448. printf("Out of freespace nodes; getting more\n");
  449. #endif
  450. if (!add_new_region(arena, size, sizeof(TreeAlloc), actual_align)) {
  451. return NULL;
  452. }
  453. return alloc(arena, size, align);
  454. } else {
  455. TreeAlloc *region = search_by_size((TreeAlloc*) arena->root_freespace, sizeof(TreeAlloc), actual_align, size);
  456. if (region == NULL) {
  457. // Handle insufficient freespace or fragmentation.
  458. #ifdef DEBUG
  459. printf("Out of sufficiently large freespace nodes; getting more\n");
  460. #endif
  461. if (!add_new_region(arena, size, sizeof(TreeAlloc), actual_align)) {
  462. return NULL;
  463. }
  464. return alloc(arena, size, align);
  465. }
  466. remove_node((TreeAlloc**) &arena->root_freespace, region);
  467. void *true_end = align_after(align_after(((void*) region) + sizeof(TreeAlloc), actual_align) + size, alignof(WatermarkAlloc));
  468. // The size of the new allocation (adjusted for region header and alignment
  469. uintptr_t new_size = true_end - (void*) region;
  470. // The size of the free space region following the new allocation
  471. uintptr_t new_free_size = region->size - new_size;
  472. region->right = NULL;
  473. region->left = NULL;
  474. region->type = RT_TREE_NODE;
  475. #ifdef DEBUG
  476. printf("start: %p, end: %p, adjusted end: %p\n", region, ((void*) region) + size, true_end);
  477. printf("size: %lu -> %lu\n", size, new_size);
  478. #endif
  479. if (arena->root_treealloc == NULL) {
  480. insert_singleton((TreeAlloc**) &arena->root_treealloc, region);
  481. } else {
  482. #ifdef DEBUG
  483. printf("searching for an insert point\n");
  484. #endif
  485. TreeAlloc *insert_point = search_by_address((TreeAlloc*) arena->root_treealloc, region);
  486. if (insert_point == NULL) {
  487. TreeAlloc *head = arena->root_treealloc;
  488. while (head->left != NULL) {
  489. head = head->left;
  490. }
  491. #ifdef DEBUG
  492. printf("none found; inserting before %p\n", head);
  493. #endif
  494. insert_left(&arena->root_treealloc, region, head);
  495. } else {
  496. #ifdef DEBUG
  497. printf("found one: %p\n", insert_point);
  498. #endif
  499. insert_right(&arena->root_treealloc, region, insert_point);
  500. }
  501. }
  502. if (new_free_size >= sizeof(FreeSpace)) {
  503. // If there's enough free space after the allocation, use it!
  504. region->size = new_size; // Safe because the allocated region tree is not sorted by size.
  505. FreeSpace *new_free = (FreeSpace*) ((void*) region + new_size);
  506. new_free->left = NULL;
  507. new_free->right = NULL;
  508. new_free->type = RT_FREESPACE;
  509. new_free->size = new_free_size;
  510. if (arena->root_freespace == NULL) {
  511. insert_singleton((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free);
  512. } else {
  513. FreeSpace *insert_point = (FreeSpace*) search_by_size((TreeAlloc*) arena->root_freespace, 0, 1, new_free_size);
  514. insert_left((TreeAlloc**) &arena->root_freespace, (TreeAlloc*) new_free, (TreeAlloc*) insert_point);
  515. }
  516. // Set the region following this one to be the new free space
  517. region->after = (TreeAlloc*) new_free;
  518. } else {
  519. // There isn't a free space after this one, so put the `next` pointer at the next allocated
  520. // region.
  521. region->after = succ(region);
  522. }
  523. // I seem to have forgotten about the fact that memory may not be contiguous
  524. if (region->after != NULL && region->after != (void*) region + region->size) {
  525. region->after = NULL;
  526. }
  527. // Also make sure the `before` pointer is correct.
  528. TreeAlloc *before_alloc = pred(region);
  529. if (before_alloc == NULL || ((void*) before_alloc) + before_alloc->size < (void*) region) {
  530. region->before = search_by_address((TreeAlloc*) arena->root_freespace, region);
  531. } else {
  532. region->before = before_alloc;
  533. }
  534. // I seem to have forgotten about the fact that memory may not be contiguous
  535. if (region->before != NULL && region->before != (void*) region->before + region->before->size) {
  536. region->before = NULL;
  537. }
  538. #ifdef DEBUG
  539. printf("region is still at %p\n", region);
  540. #endif
  541. return align_after((void*) region + sizeof(TreeAlloc), actual_align);
  542. }
  543. }
  544. void *alloc_growable(Arena *arena, uintptr_t size, uintptr_t align) {
  545. // TODO: Basically the same as above, but put the allocated region in the center of the largest free
  546. // space. Due to alignment and whatnot, the code will be gory.
  547. return NULL;
  548. }