23#ifndef TRANSIENT_HEAP_CHECK_MODE
24#define TRANSIENT_HEAP_CHECK_MODE 0
26#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
33#define TRANSIENT_HEAP_DEBUG 0
39#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
41#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
48#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
51#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
54#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 )
55#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32)
56#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 )
57#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
59#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
60#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
62#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
63#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
111static int transient_header_managed_ptr_p(
struct transient_heap* theap,
const void *
ptr);
113#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
120 while (i<block->info.index) {
133 fprintf(
stderr,
"- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
134 type_str, (
void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (
void *)block->info.next_block);
136 transient_heap_block_dump(theap, block);
145 transient_heap_blocks_dump(theap, theap->
using_blocks,
"using_blocks");
146 transient_heap_blocks_dump(theap, theap->
marked_blocks,
"marked_blocks");
147 transient_heap_blocks_dump(theap, theap->
free_blocks,
"free_blocks");
154 transient_heap_dump(&global_transient_heap);
157#if TRANSIENT_HEAP_CHECK_MODE >= 2
175 while (i<block->info.index) {
176 header = (
void *)&block->
buff[
i];
178 transient_heap_ptr_check(theap, header->
obj);
193 n += transient_heap_block_verify(theap, block);
205#if TRANSIENT_HEAP_CHECK_MODE >= 2
206 int n=0, block_num=0;
208 n += transient_heap_blocks_verify(theap, theap->
using_blocks, &block_num);
209 n += transient_heap_blocks_verify(theap, theap->
marked_blocks, &block_num);
221 transient_heap_verify(&global_transient_heap);
225transient_heap_get(
void)
228 transient_heap_verify(theap);
278 last_block->info.next_block = append_blocks;
289#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
291 MAP_PRIVATE | MAP_ANONYMOUS,
293 if (block == MAP_FAILED)
rb_bug(
"transient_heap_block_alloc: err:%d\n",
errno);
298 rb_bug(
"transient_heap_block_alloc: failed\n");
319#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
320 block = transient_heap_block_alloc(theap);
350 block = transient_heap_allocatable_block(theap);
351 if (block) connect_to_using_blocks(theap, block);
373#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
391 asan_unpoison_memory_region(header,
sizeof *header,
true);
399 asan_poison_memory_region(header,
sizeof *header);
404#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
406 transient_heap_promote_add(theap,
obj);
414 asan_unpoison_memory_region(
ptr,
size -
sizeof *header,
true);
431#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
437 for (
i=0;
i<block_num;
i++) {
438 connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
440 theap->
using_blocks = transient_heap_allocatable_block(theap);
458 if (block->
buff <= (
char *)header && (
char *)header < block->
buff + block->
info.
size) {
472 if ((block = blocks_alloc_header_to_block(theap, theap->
marked_blocks, header)) !=
NULL) {
476 else if ((block = blocks_alloc_header_to_block(theap, theap->
using_blocks, header)) !=
NULL) {
486ptr_to_alloc_header(
const void *
ptr)
496 if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(
ptr))) {
508 return transient_header_managed_ptr_p(transient_heap_get(),
ptr);
515#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
516 block = alloc_header_to_block_verbose(theap, header);
518 transient_heap_dump(theap);
519 rb_bug(
"alloc_header_to_block: not found in mark_blocks (%p)\n", header);
523 TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
532 asan_unpoison_memory_region(header,
sizeof *header,
false);
536#if TRANSIENT_HEAP_CHECK_MODE > 0
543 transient_heap_dump(theap);
544 rb_bug(
"rb_transient_heap_mark: magic is broken");
546 else if (header->
obj !=
obj) {
548 rb_bug(
"rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
566 transient_heap_verify(theap);
590 ptr = rb_struct_const_heap_ptr(
obj);
642 if (transient_heap_ptr(
obj,
FALSE)) {
644 transient_heap_promote_add(theap,
obj);
658transient_heap_reset(
void)
669#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
678 connect_to_free_blocks(theap, block);
696 while (marked_index >= 0) {
725 marked_index = header->next_marked_index;
737transient_heap_evacuate(
void *dmy)
763 transient_heap_block_evacuate(theap, block);
771 transient_heap_block_evacuate(theap, block);
776 transient_heap_reset();
782 transient_heap_verify(theap);
797 asan_unpoison_memory_region(header,
sizeof *header,
false);
799 if (0)
fprintf(
stderr,
"clear_marked_index - block:%p mark_index:%d\n", (
void *)block, marked_index);
812 clear_marked_index(block);
822 while (i<block->info.index) {
826 asan_unpoison_memory_region(header,
sizeof *header,
false);
829 asan_unpoison_object(header->
obj,
false);
834 asan_poison_object(header->
obj);
838 asan_poison_memory_region(header,
sizeof *header);
847 transient_heap_block_update_refs(theap, block);
858 transient_heap_blocks_update_refs(theap, theap->
using_blocks,
"using_blocks");
859 transient_heap_blocks_update_refs(theap, theap->
marked_blocks,
"marked_blocks");
891 theap->
using_blocks = transient_heap_allocatable_block(theap);
905 const void *
ptr = transient_heap_ptr(
obj,
TRUE);
912 transient_heap_verify(theap);
935 transient_heap_reset();
938 transient_heap_verify(theap);
void rb_ary_transient_heap_evacuate(VALUE ary, int promote)
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
void * rb_aligned_malloc(size_t alignment, size_t size)
void rb_bug(const char *fmt,...)
void rb_hash_transient_heap_evacuate(VALUE hash, int promote)
#define __asan_unpoison_memory_region(x, y)
#define __asan_region_is_poisoned(x, y)
#define __asan_poison_memory_region(x, y)
#define __msan_allocated_memory(x, y)
void rb_struct_transient_heap_evacuate(VALUE obj, int promote)
struct transient_heap_block::transient_heap_block_header info
char buff[TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)]
int promoted_objects_size
int promoted_objects_index
struct transient_heap_block * using_blocks
struct transient_heap_block * arena
struct transient_heap_block * free_blocks
struct transient_heap_block * marked_blocks
enum transient_heap_status status
void * rb_transient_heap_alloc(VALUE obj, size_t req_size)
void rb_transient_heap_promote(VALUE obj)
#define TRANSIENT_HEAP_ALLOC_MAX
int rb_transient_heap_managed_ptr_p(const void *ptr)
#define TRANSIENT_HEAP_TOTAL_SIZE
#define TRANSIENT_HEAP_ALLOC_MAGIC
#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE
#define TRANSIENT_HEAP_BLOCK_NUM
void rb_transient_heap_verify(void)
void Init_TransientHeap(void)
#define TRANSIENT_HEAP_ALLOC_ALIGN
#define TRANSIENT_HEAP_BLOCK_SIZE
void rb_transient_heap_mark(VALUE obj, const void *ptr)
@ transient_heap_escaping
#define TRANSIENT_HEAP_DEBUG
void rb_transient_heap_finish_marking(void)
void rb_transient_heap_dump(void)
void rb_transient_heap_start_marking(int full_marking)
#define TRANSIENT_HEAP_ALLOC_MARKING_FREE
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error))
void rb_transient_heap_update_references(void)
#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
#define TRANSIENT_HEAP_ALLOC_MARKING_LAST
void rb_obj_transient_heap_evacuate(VALUE obj, int promote)