Ruby 2.7.7p221 (2022-11-24 revision 168ec2b1e5ad0e4688e963d9de019557c78feed9)
transient_heap.c
Go to the documentation of this file.
1/**********************************************************************
2
3 transient_heap.c - implement transient_heap.
4
5 Copyright (C) 2018 Koichi Sasada
6
7**********************************************************************/
8
9#include "ruby/ruby.h"
10#include "ruby/debug.h"
11#include "vm_debug.h"
12#include "gc.h"
13#include "internal.h"
14#include "ruby_assert.h"
15#include "transient_heap.h"
16#include "debug_counter.h"
17
18#if USE_TRANSIENT_HEAP /* USE_TRANSIENT_HEAP */
19/*
20 * 1: enable assertions
21 * 2: enable verify all transient heaps
22 */
23#ifndef TRANSIENT_HEAP_CHECK_MODE
24#define TRANSIENT_HEAP_CHECK_MODE 0
25#endif
26#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
27
28/*
29 * 1: show events
30 * 2: show dump at events
31 * 3: show all operations
32 */
33#define TRANSIENT_HEAP_DEBUG 0
34
35/* For Debug: Provide blocks infinitely.
36 * This mode generates blocks unlimitedly
37 * and prohibit access free'ed blocks to check invalid access.
38 */
39#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
40
41#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
42#include <sys/mman.h>
43#include <errno.h>
44#endif
45
46/* For Debug: Prohibit promoting to malloc space.
47 */
48#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
49
50/* size configuration */
51#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
52
53 /* K M */
54#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
55#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
56#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
57#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
58
59#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
60#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
61
62#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
63#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
64
69};
70
73 int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */
80};
81
90
94
96 int arena_index; /* increment only */
97};
98
105};
106
107static struct transient_heap global_transient_heap;
108
109static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
110static const void *transient_heap_ptr(VALUE obj, int error);
111static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
112
113#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
114
115static void
116transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
117{
118 int i=0, n=0;
119
120 while (i<block->info.index) {
121 void *ptr = &block->buff[i];
122 struct transient_alloc_header *header = ptr;
123 fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, ptr, header->size, header->next_marked_index, rb_obj_info(header->obj));
124 i += header->size;
125 n++;
126 }
127}
128
129static void
130transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
131{
132 while (block) {
133 fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
134 type_str, (void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (void *)block->info.next_block);
135
136 transient_heap_block_dump(theap, block);
137 block = block->info.next_block;
138 }
139}
140
141static void
142transient_heap_dump(struct transient_heap* theap)
143{
144 fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
145 transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
146 transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
147 transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
148}
149
150/* Debug: dump all tarnsient_heap blocks */
151void
153{
154 transient_heap_dump(&global_transient_heap);
155}
156
157#if TRANSIENT_HEAP_CHECK_MODE >= 2
158ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void transient_heap_ptr_check(struct transient_heap *theap, VALUE obj));
159static void
160transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
161{
162 if (obj != Qundef) {
163 const void *ptr = transient_heap_ptr(obj, FALSE);
164 TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
165 }
166}
167
168ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static int transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block));
169static int
170transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
171{
172 int i=0, n=0;
173 struct transient_alloc_header *header;
174
175 while (i<block->info.index) {
176 header = (void *)&block->buff[i];
178 transient_heap_ptr_check(theap, header->obj);
179 n ++;
180 i += header->size;
181 }
182 TH_ASSERT(block->info.objects == n);
183
184 return n;
185}
186
187static int
188transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
189{
190 int n = 0;
191 struct transient_heap_block *block = blocks;
192 while (block) {
193 n += transient_heap_block_verify(theap, block);
194 *block_num_ptr += 1;
195 block = block->info.next_block;
196 }
197
198 return n;
199}
200#endif
201
202static void
203transient_heap_verify(struct transient_heap *theap)
204{
205#if TRANSIENT_HEAP_CHECK_MODE >= 2
206 int n=0, block_num=0;
207
208 n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
209 n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
210
211 TH_ASSERT(n == theap->total_objects);
213 TH_ASSERT(block_num == theap->total_blocks);
214#endif
215}
216
217/* Debug: check assertions for all transient_heap blocks */
218void
220{
221 transient_heap_verify(&global_transient_heap);
222}
223
224static struct transient_heap*
225transient_heap_get(void)
226{
227 struct transient_heap* theap = &global_transient_heap;
228 transient_heap_verify(theap);
229 return theap;
230}
231
232static void
233reset_block(struct transient_heap_block *block)
234{
235 __msan_allocated_memory(block, sizeof block);
236 block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header);
237 block->info.index = 0;
238 block->info.objects = 0;
240 block->info.next_block = NULL;
241 __asan_poison_memory_region(&block->buff, sizeof block->buff);
242}
243
244static void
245connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
246{
247 block->info.next_block = theap->free_blocks;
248 theap->free_blocks = block;
249}
250
251static void
252connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
253{
254 block->info.next_block = theap->using_blocks;
255 theap->using_blocks = block;
256}
257
258#if 0
259static void
260connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
261{
262 block->info.next_block = theap->marked_blocks;
263 theap->marked_blocks = block;
264}
265#endif
266
267static void
268append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
269{
270 if (theap->marked_blocks) {
271 struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
272 while (block) {
273 last_block = block;
274 block = block->info.next_block;
275 }
276
277 TH_ASSERT(last_block->info.next_block == NULL);
278 last_block->info.next_block = append_blocks;
279 }
280 else {
281 theap->marked_blocks = append_blocks;
282 }
283}
284
285static struct transient_heap_block *
286transient_heap_block_alloc(struct transient_heap* theap)
287{
288 struct transient_heap_block *block;
289#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
290 block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
291 MAP_PRIVATE | MAP_ANONYMOUS,
292 -1, 0);
293 if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
294#else
295 if (theap->arena == NULL) {
297 if (theap->arena == NULL) {
298 rb_bug("transient_heap_block_alloc: failed\n");
299 }
300 }
301
303 block = &theap->arena[theap->arena_index++];
304 TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
305#endif
306 reset_block(block);
307
309 if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (void *)block);
310 return block;
311}
312
313
314static struct transient_heap_block *
315transient_heap_allocatable_block(struct transient_heap* theap)
316{
317 struct transient_heap_block *block;
318
319#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
320 block = transient_heap_block_alloc(theap);
321 theap->total_blocks++;
322#else
323 /* get one block from free_blocks */
324 block = theap->free_blocks;
325 if (block) {
326 theap->free_blocks = block->info.next_block;
327 block->info.next_block = NULL;
328 theap->total_blocks++;
329 }
330#endif
331
332 return block;
333}
334
335static struct transient_alloc_header *
336transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
337{
338 struct transient_heap_block *block = theap->using_blocks;
339
340 while (block) {
341 TH_ASSERT(block->info.size >= block->info.index);
342
343 if (block->info.size - block->info.index >= (int32_t)size) {
344 struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
345 block->info.index += size;
346 block->info.objects++;
347 return header;
348 }
349 else {
350 block = transient_heap_allocatable_block(theap);
351 if (block) connect_to_using_blocks(theap, block);
352 }
353 }
354
355 return NULL;
356}
357
358void *
360{
361 struct transient_heap* theap = transient_heap_get();
362 size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
363
367 RB_TYPE_P(obj, T_HASH)); /* supported types */
368
370 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
371 return NULL;
372 }
373#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
374 else if (RB_OBJ_PROMOTED_RAW(obj)) {
375 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
376 return NULL;
377 }
378#else
379 else if (RBASIC_CLASS(obj) == 0) {
380 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
381 return NULL;
382 }
383#endif
384 else {
385 struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
386 if (header) {
387 void *ptr;
388
389 /* header is poisoned to prevent buffer overflow, should
390 * unpoison first... */
391 asan_unpoison_memory_region(header, sizeof *header, true);
392
393 header->size = size;
396 header->obj = obj; /* TODO: can we eliminate it? */
397
398 /* header is fixed; shall poison again */
399 asan_poison_memory_region(header, sizeof *header);
400 ptr = header + 1;
401
402 theap->total_objects++; /* statistics */
403
404#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
406 transient_heap_promote_add(theap, obj);
407 }
408#endif
409 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (void *)header, ptr, (int)size, rb_obj_info(obj));
410
411 RB_DEBUG_COUNTER_INC(theap_alloc);
412
413 /* ptr is set up; OK to unpoison. */
414 asan_unpoison_memory_region(ptr, size - sizeof *header, true);
415 return ptr;
416 }
417 else {
418 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
419 RB_DEBUG_COUNTER_INC(theap_alloc_fail);
420 return NULL;
421 }
422 }
423}
424
425void
427{
428 int i, block_num;
429 struct transient_heap* theap = transient_heap_get();
430
431#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
432 block_num = 0;
433#else
435 block_num = TRANSIENT_HEAP_BLOCK_NUM;
436#endif
437 for (i=0; i<block_num; i++) {
438 connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
439 }
440 theap->using_blocks = transient_heap_allocatable_block(theap);
441
443 theap->promoted_objects_index = 0;
444 /* should not use ALLOC_N to be free from GC */
445 theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
447 integer_overflow,
449 if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
450}
451
452static struct transient_heap_block *
453blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
454{
455 struct transient_heap_block *block = blocks;
456
457 while (block) {
458 if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) {
459 return block;
460 }
461 block = block->info.next_block;
462 }
463
464 return NULL;
465}
466
467static struct transient_heap_block *
468alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
469{
470 struct transient_heap_block *block;
471
472 if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
473 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
474 return block;
475 }
476 else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
477 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
478 return block;
479 }
480 else {
481 return NULL;
482 }
483}
484
485static struct transient_alloc_header *
486ptr_to_alloc_header(const void *ptr)
487{
488 struct transient_alloc_header *header = (void *)ptr;
489 header -= 1;
490 return header;
491}
492
493static int
494transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
495{
496 if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
497 return TRUE;
498 }
499 else {
500 return FALSE;
501 }
502}
503
504
505int
507{
508 return transient_header_managed_ptr_p(transient_heap_get(), ptr);
509}
510
511static struct transient_heap_block *
512alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
513{
514 struct transient_heap_block *block;
515#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
516 block = alloc_header_to_block_verbose(theap, header);
517 if (block == NULL) {
518 transient_heap_dump(theap);
519 rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
520 }
521#else
522 block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
523 TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
524#endif
525 return block;
526}
527
528void
530{
531 struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
532 asan_unpoison_memory_region(header, sizeof *header, false);
533 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header, %s (%p)", rb_obj_info(obj), ptr);
534 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
535
536#if TRANSIENT_HEAP_CHECK_MODE > 0
537 {
538 struct transient_heap* theap = transient_heap_get();
540 TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
541
542 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
543 transient_heap_dump(theap);
544 rb_bug("rb_transient_heap_mark: magic is broken");
545 }
546 else if (header->obj != obj) {
547 // transient_heap_dump(theap);
548 rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
549 rb_obj_info(header->obj), rb_obj_info(obj));
550 }
551 }
552#endif
553
555 /* already marked */
556 return;
557 }
558 else {
559 struct transient_heap* theap = transient_heap_get();
560 struct transient_heap_block *block = alloc_header_to_block(theap, header);
561 __asan_unpoison_memory_region(&block->info, sizeof block->info);
563 block->info.last_marked_index = (int)((char *)header - block->buff);
564 theap->total_marked_objects++;
565
566 transient_heap_verify(theap);
567 }
568}
569
570ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error));
571static const void *
572transient_heap_ptr(VALUE obj, int error)
573{
574 const void *ptr = NULL;
575
576 switch (BUILTIN_TYPE(obj)) {
577 case T_ARRAY:
578 if (RARRAY_TRANSIENT_P(obj)) {
580 ptr = RARRAY(obj)->as.heap.ptr;
581 }
582 break;
583 case T_OBJECT:
584 if (ROBJ_TRANSIENT_P(obj)) {
586 }
587 break;
588 case T_STRUCT:
590 ptr = rb_struct_const_heap_ptr(obj);
591 }
592 break;
593 case T_HASH:
594 if (RHASH_TRANSIENT_P(obj)) {
596 ptr = (VALUE *)(RHASH(obj)->as.ar);
597 }
598 else {
599 ptr = NULL;
600 }
601 break;
602 default:
603 if (error) {
604 rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
605 }
606 }
607
608 return ptr;
609}
610
611static void
612transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
613{
614 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
615
617 /* duplicate check */
618 int i;
619 for (i=0; i<theap->promoted_objects_index; i++) {
620 if (theap->promoted_objects[i] == obj) return;
621 }
622 }
623
624 if (theap->promoted_objects_size <= theap->promoted_objects_index) {
625 theap->promoted_objects_size *= 2;
626 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
627 if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) {
628 /* realloc failure due to integer overflow */
629 theap->promoted_objects = NULL;
630 }
631 else {
632 theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
633 }
634 if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
635 }
637}
638
639void
641{
642 if (transient_heap_ptr(obj, FALSE)) {
643 struct transient_heap* theap = transient_heap_get();
644 transient_heap_promote_add(theap, obj);
645 }
646 else {
647 /* ignore */
648 }
649}
650
651static struct transient_alloc_header *
652alloc_header(struct transient_heap_block* block, int index)
653{
654 return (void *)&block->buff[index];
655}
656
657static void
658transient_heap_reset(void)
659{
660 struct transient_heap* theap = transient_heap_get();
661 struct transient_heap_block* block;
662
663 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
664
665 block = theap->marked_blocks;
666 while (block) {
667 struct transient_heap_block *next_block = block->info.next_block;
668 theap->total_objects -= block->info.objects;
669#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
670 if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
671 rb_bug("madvise err:%d", errno);
672 }
673 if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
674 rb_bug("mprotect err:%d", errno);
675 }
676#else
677 reset_block(block);
678 connect_to_free_blocks(theap, block);
679#endif
680 theap->total_blocks--;
681 block = next_block;
682 }
683
684 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
685
686 theap->marked_blocks = NULL;
687 theap->total_marked_objects = 0;
688}
689
690static void
691transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
692{
693 int marked_index = block->info.last_marked_index;
695
696 while (marked_index >= 0) {
697 struct transient_alloc_header *header = alloc_header(block, marked_index);
698 VALUE obj = header->obj;
700 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header %s\n", rb_obj_info(obj));
701
702 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", (void *)header, rb_obj_info(obj));
703
704 if (obj != Qnil) {
705 RB_DEBUG_COUNTER_INC(theap_evacuate);
706
707 switch (BUILTIN_TYPE(obj)) {
708 case T_ARRAY:
710 break;
711 case T_OBJECT:
713 break;
714 case T_STRUCT:
716 break;
717 case T_HASH:
719 break;
720 default:
721 rb_bug("unsupporeted: %s\n", rb_obj_info(obj));
722 }
723 header->obj = Qundef; /* for debug */
724 }
725 marked_index = header->next_marked_index;
726 }
727}
728
729static void
730transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
731{
732 TH_ASSERT(theap->status != status);
733 theap->status = status;
734}
735
736static void
737transient_heap_evacuate(void *dmy)
738{
739 struct transient_heap* theap = transient_heap_get();
740
741 if (theap->status == transient_heap_marking) {
742 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
743 }
744 else {
745 VALUE gc_disabled = rb_gc_disable_no_rest();
746 struct transient_heap_block* block;
747
748 if (TRANSIENT_HEAP_DEBUG >= 1) {
749 int i;
750 fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
751 if (TRANSIENT_HEAP_DEBUG >= 4) {
752 for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
753 }
754 }
755 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
756
758 transient_heap_update_status(theap, transient_heap_escaping);
759
760 /* evacuate from marked blocks */
761 block = theap->marked_blocks;
762 while (block) {
763 transient_heap_block_evacuate(theap, block);
764 block = block->info.next_block;
765 }
766
767 /* evacuate from using blocks
768 only affect incremental marking */
769 block = theap->using_blocks;
770 while (block) {
771 transient_heap_block_evacuate(theap, block);
772 block = block->info.next_block;
773 }
774
775 /* all objects in marked_objects are escaped. */
776 transient_heap_reset();
777
778 if (TRANSIENT_HEAP_DEBUG > 0) {
779 fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
780 }
781
782 transient_heap_verify(theap);
783 transient_heap_update_status(theap, transient_heap_none);
784 if (gc_disabled != Qtrue) rb_gc_enable();
785 }
786}
787
788static void
789clear_marked_index(struct transient_heap_block* block)
790{
791 int marked_index = block->info.last_marked_index;
792
793 while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
794 struct transient_alloc_header *header = alloc_header(block, marked_index);
795 /* header is poisoned to prevent buffer overflow, should
796 * unpoison first... */
797 asan_unpoison_memory_region(header, sizeof *header, false);
799 if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", (void *)block, marked_index);
800
801 marked_index = header->next_marked_index;
803 }
804
806}
807
808static void
809blocks_clear_marked_index(struct transient_heap_block* block)
810{
811 while (block) {
812 clear_marked_index(block);
813 block = block->info.next_block;
814 }
815}
816
817static void
818transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block)
819{
820 int i=0, n=0;
821
822 while (i<block->info.index) {
823 void *ptr = &block->buff[i];
824 struct transient_alloc_header *header = ptr;
825
826 asan_unpoison_memory_region(header, sizeof *header, false);
827
828 void *poisoned = __asan_region_is_poisoned((void *)header->obj, SIZEOF_VALUE);
829 asan_unpoison_object(header->obj, false);
830
831 header->obj = rb_gc_location(header->obj);
832
833 if (poisoned) {
834 asan_poison_object(header->obj);
835 }
836
837 i += header->size;
838 asan_poison_memory_region(header, sizeof *header);
839 n++;
840 }
841}
842
843static void
844transient_heap_blocks_update_refs(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
845{
846 while (block) {
847 transient_heap_block_update_refs(theap, block);
848 block = block->info.next_block;
849 }
850}
851
852void
854{
855 struct transient_heap* theap = transient_heap_get();
856 int i;
857
858 transient_heap_blocks_update_refs(theap, theap->using_blocks, "using_blocks");
859 transient_heap_blocks_update_refs(theap, theap->marked_blocks, "marked_blocks");
860
861 for (i=0; i<theap->promoted_objects_index; i++) {
862 VALUE obj = theap->promoted_objects[i];
864 }
865}
866
867void
869{
870 struct transient_heap* theap = transient_heap_get();
871
872 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promtoed:%d full_marking:%d\n",
873 theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
874 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
875
876 blocks_clear_marked_index(theap->marked_blocks);
877 blocks_clear_marked_index(theap->using_blocks);
878
879 if (theap->using_blocks) {
880 if (theap->using_blocks->info.objects > 0) {
881 append_to_marked_blocks(theap, theap->using_blocks);
882 theap->using_blocks = NULL;
883 }
884 else {
885 append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
887 }
888 }
889
890 if (theap->using_blocks == NULL) {
891 theap->using_blocks = transient_heap_allocatable_block(theap);
892 }
893
895 transient_heap_update_status(theap, transient_heap_marking);
896 theap->total_marked_objects = 0;
897
898 if (full_marking) {
899 theap->promoted_objects_index = 0;
900 }
901 else { /* mark promoted objects */
902 int i;
903 for (i=0; i<theap->promoted_objects_index; i++) {
904 VALUE obj = theap->promoted_objects[i];
905 const void *ptr = transient_heap_ptr(obj, TRUE);
906 if (ptr) {
908 }
909 }
910 }
911
912 transient_heap_verify(theap);
913}
914
915void
917{
918 struct transient_heap* theap = transient_heap_get();
919
920 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n",
921 theap->total_objects,
922 theap->total_marked_objects);
923 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
924
926
928 transient_heap_update_status(theap, transient_heap_none);
929
930 if (theap->total_marked_objects > 0) {
931 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
932 rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
933 }
934 else {
935 transient_heap_reset();
936 }
937
938 transient_heap_verify(theap);
939}
940#endif /* USE_TRANSIENT_HEAP */
int errno
void rb_ary_transient_heap_evacuate(VALUE ary, int promote)
Definition: array.c:402
struct RIMemo * ptr
Definition: debug.c:65
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
Definition: vm_trace.c:1614
void * rb_aligned_malloc(size_t alignment, size_t size)
Definition: gc.c:9643
@ RARRAY_EMBED_FLAG
Definition: ruby.h:1029
void rb_bug(const char *fmt,...)
Definition: error.c:636
void rb_hash_transient_heap_evacuate(VALUE hash, int promote)
Definition: hash.c:1244
#define __asan_unpoison_memory_region(x, y)
Definition: internal.h:109
#define __asan_region_is_poisoned(x, y)
Definition: internal.h:110
#define __asan_poison_memory_region(x, y)
Definition: internal.h:108
#define __msan_allocated_memory(x, y)
Definition: internal.h:120
#define RHASH_TRANSIENT_P(hash)
#define NULL
void * malloc(size_t) __attribute__((__malloc__)) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(1)))
__uint16_t uint16_t
#define Qundef
__int16_t int16_t
const VALUE VALUE obj
#define SIZE_MAX
VALUE rb_gc_enable(void)
Definition: gc.c:9225
const rb_iseq_t const char * error
#define RHASH_AR_TABLE_P(hash)
#define T_STRUCT
__intptr_t intptr_t
int fprintf(FILE *__restrict__, const char *__restrict__,...) __attribute__((__format__(__printf__
const char size_t n
#define stderr
void * realloc(void *, size_t) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(2)))
#define RARRAY(obj)
uint32_t i
__int32_t int32_t
#define FL_TEST_RAW(x, f)
#define STATIC_ASSERT(name, expr)
#define T_HASH
VALUE rb_gc_location(VALUE)
Definition: gc.c:8127
#define RSTRUCT_TRANSIENT_P(st)
#define TRUE
#define FALSE
unsigned int size
#define Qtrue
#define UNLIKELY(x)
const char * rb_obj_info(VALUE obj)
Definition: gc.c:11713
#define Qnil
#define T_ARRAY
#define T_OBJECT
#define RARRAY_TRANSIENT_P(ary)
#define RB_TYPE_P(obj, type)
#define RHASH(obj)
VALUE rb_gc_disable_no_rest(void)
Definition: gc.c:9247
__inline__ int
#define RB_OBJ_PROMOTED_RAW(x)
#define ROBJ_TRANSIENT_P(obj)
#define ROBJECT_IVPTR(o)
#define RBASIC_CLASS(obj)
#define BUILTIN_TYPE(x)
#define SIZEOF_VALUE
#define RB_DEBUG_COUNTER_INC(type)
unsigned long VALUE
Definition: ruby.h:102
void rb_struct_transient_heap_evacuate(VALUE obj, int promote)
Definition: struct.c:681
struct transient_heap_block::transient_heap_block_header info
char buff[TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)]
VALUE * promoted_objects
int promoted_objects_index
struct transient_heap_block * using_blocks
struct transient_heap_block * arena
struct transient_heap_block * free_blocks
struct transient_heap_block * marked_blocks
enum transient_heap_status status
void * rb_transient_heap_alloc(VALUE obj, size_t req_size)
void rb_transient_heap_promote(VALUE obj)
#define TRANSIENT_HEAP_ALLOC_MAX
int rb_transient_heap_managed_ptr_p(const void *ptr)
#define TRANSIENT_HEAP_TOTAL_SIZE
#define TRANSIENT_HEAP_ALLOC_MAGIC
#define ROUND_UP(v, a)
#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE
#define TRANSIENT_HEAP_BLOCK_NUM
void rb_transient_heap_verify(void)
void Init_TransientHeap(void)
#define TH_ASSERT(expr)
#define TRANSIENT_HEAP_ALLOC_ALIGN
#define TRANSIENT_HEAP_BLOCK_SIZE
void rb_transient_heap_mark(VALUE obj, const void *ptr)
transient_heap_status
@ transient_heap_none
@ transient_heap_escaping
@ transient_heap_marking
#define TRANSIENT_HEAP_DEBUG
void rb_transient_heap_finish_marking(void)
void rb_transient_heap_dump(void)
void rb_transient_heap_start_marking(int full_marking)
#define TRANSIENT_HEAP_ALLOC_MARKING_FREE
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error))
void rb_transient_heap_update_references(void)
#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
#define TRANSIENT_HEAP_ALLOC_MARKING_LAST
void rb_obj_transient_heap_evacuate(VALUE obj, int promote)
Definition: variable.c:1215