25static const int DEBUG = 0;
27#define RB_PAGE_SIZE (pagesize)
28#define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1))
32static VALUE rb_cContinuation;
33static VALUE rb_cFiber;
34static VALUE rb_eFiberError;
35#ifdef RB_EXPERIMENTAL_FIBER_POOL
36static VALUE rb_cFiberPool;
39#define CAPTURE_JUST_VALID_VM_STACK 1
42#ifdef COROUTINE_LIMITED_ADDRESS_SPACE
43#define FIBER_POOL_ALLOCATION_FREE
44#define FIBER_POOL_INITIAL_SIZE 8
45#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 32
47#define FIBER_POOL_INITIAL_SIZE 32
48#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 1024
58#ifdef CAPTURE_JUST_VALID_VM_STACK
95#ifdef FIBER_POOL_ALLOCATION_FREE
138#ifdef FIBER_POOL_ALLOCATION_FREE
146#ifdef FIBER_POOL_ALLOCATION_FREE
220#define FIBER_CREATED_P(fiber) ((fiber)->status == FIBER_CREATED)
221#define FIBER_RESUMED_P(fiber) ((fiber)->status == FIBER_RESUMED)
222#define FIBER_SUSPENDED_P(fiber) ((fiber)->status == FIBER_SUSPENDED)
223#define FIBER_TERMINATED_P(fiber) ((fiber)->status == FIBER_TERMINATED)
224#define FIBER_RUNNABLE_P(fiber) (FIBER_CREATED_P(fiber) || FIBER_SUSPENDED_P(fiber))
248#if defined(MAP_STACK) && !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__)
249#define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON | MAP_STACK)
251#define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON)
254#define ERRNOMSG strerror(errno)
259fiber_pool_vacancy_pointer(
void * base,
size_t size)
313 fiber_pool_stack_reset(&vacancy->
stack);
322 vacancy->
next = head;
324#ifdef FIBER_POOL_ALLOCATION_FREE
326 head->previous = vacancy;
327 vacancy->previous =
NULL;
334#ifdef FIBER_POOL_ALLOCATION_FREE
339 vacancy->
next->previous = vacancy->previous;
342 if (vacancy->previous) {
343 vacancy->previous->
next = vacancy->
next;
352fiber_pool_vacancy_pop(
struct fiber_pool * pool)
357 fiber_pool_vacancy_remove(vacancy);
364fiber_pool_vacancy_pop(
struct fiber_pool * pool)
387 fiber_pool_vacancy_reset(vacancy);
391 return fiber_pool_vacancy_push(vacancy, vacancies);
399fiber_pool_allocate_memory(
size_t *
count,
size_t stride)
409 void * base = VirtualAlloc(0, (*
count)*stride, MEM_COMMIT, PAGE_READWRITE);
412 *
count = (*count) >> 1;
421 if (base == MAP_FAILED) {
423 *
count = (*count) >> 1;
461#ifdef FIBER_POOL_ALLOCATION_FREE
462 allocation->used = 0;
472 for (
size_t i = 0;
i <
count;
i += 1) {
479 if (!VirtualProtect(page,
RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) {
480 VirtualFree(allocation->
base, 0, MEM_RELEASE);
490 vacancies = fiber_pool_vacancy_initialize(
496#ifdef FIBER_POOL_ALLOCATION_FREE
504#ifdef FIBER_POOL_ALLOCATION_FREE
505 if (allocation->
next) {
506 allocation->
next->previous = allocation;
509 allocation->previous =
NULL;
539#ifdef FIBER_POOL_ALLOCATION_FREE
551 for (
i = 0;
i < allocation->
count;
i += 1) {
557 fiber_pool_vacancy_remove(vacancy);
561 VirtualFree(allocation->
base, 0, MEM_RELEASE);
566 if (allocation->previous) {
567 allocation->previous->
next = allocation->
next;
574 if (allocation->
next) {
575 allocation->
next->previous = allocation->previous;
613#ifdef FIBER_POOL_ALLOCATION_FREE
617 fiber_pool_stack_reset(&vacancy->
stack);
619 return vacancy->
stack;
627 void * base = fiber_pool_stack_base(
stack);
635#if VM_CHECK_MODE > 0 && defined(MADV_DONTNEED)
637 madvise(base,
size, MADV_DONTNEED);
638#elif defined(MADV_FREE_REUSABLE)
639 madvise(base,
size, MADV_FREE_REUSABLE);
640#elif defined(MADV_FREE)
641 madvise(base,
size, MADV_FREE);
642#elif defined(MADV_DONTNEED)
643 madvise(base,
size, MADV_DONTNEED);
645 VirtualAlloc(base,
size, MEM_RESET, PAGE_READWRITE);
665 fiber_pool_vacancy_reset(vacancy);
671#ifdef FIBER_POOL_ALLOCATION_FREE
674 allocation->used -= 1;
677 if (allocation->used == 0) {
678 fiber_pool_allocation_free(allocation);
681 fiber_pool_stack_free(&vacancy->
stack);
686 fiber_pool_stack_free(&vacancy->
stack);
699fiber_initialize_coroutine(
rb_fiber_t *fiber,
size_t * vm_stack_size)
703 void * vm_stack =
NULL;
711#ifdef COROUTINE_PRIVATE_STACK
737 if (DEBUG)
fprintf(
stderr,
"fiber_stack_release: %p, stack.base=%p\n", (
void*)fiber, fiber->stack.base);
741 fiber_pool_stack_release(&fiber->
stack);
768 switch (fiber->status) {
829 if (!fiber)
rb_raise(rb_eFiberError,
"uninitialized fiber");
836#define THREAD_MUST_BE_RUNNING(th) do { \
837 if (!(th)->ec->tag) rb_raise(rb_eThreadError, "not running thread"); \
847cont_compact(
void *
ptr)
873#ifdef CAPTURE_JUST_VALID_VM_STACK
922 coroutine_destroy(&fiber->
context);
923 fiber_stack_release(fiber);
937cont_memsize(
const void *
ptr)
942 size =
sizeof(*cont);
944#ifdef CAPTURE_JUST_VALID_VM_STACK
982fiber_compact(
void *
ptr)
989 cont_compact(&fiber->
cont);
1001 cont_mark(&fiber->
cont);
1006fiber_free(
void *
ptr)
1017 cont_free(&fiber->
cont);
1022fiber_memsize(
const void *
ptr)
1025 size_t size =
sizeof(*fiber);
1027 const rb_thread_t *th = rb_ec_thread_ptr(saved_ec);
1035 size += cont_memsize(&fiber->
cont);
1079 {cont_mark, cont_free, cont_memsize, cont_compact},
1111 cont_save_thread(cont, th);
1116 cont_init_mjit_cont(cont);
1123 volatile VALUE contval;
1128 cont->
self = contval;
1129 cont_init(cont, th);
1138 cont_init_mjit_cont(&fiber->
cont);
1146 while (p < ec->
cfp->
sp) {
1158 while (
cfp != end_of_cfp) {
1173cont_capture(
volatile int *
volatile stat)
1177 volatile VALUE contval;
1182 cont = cont_new(rb_cContinuation);
1183 contval = cont->
self;
1185#ifdef CAPTURE_JUST_VALID_VM_STACK
1203 cont_save_machine_stack(th, cont);
1216 *entry++ = p->
entry;
1225 value = cont->
value;
1241 ec_switch(th, fiber);
1264 ec_switch(th, fiber);
1272#ifdef CAPTURE_JUST_VALID_VM_STACK
1329 fiber_restore_thread(th, new_fiber);
1345 cont_restore_thread(cont);
1353 _JUMP_BUFFER *
bp = (
void*)&cont->
jmpbuf;
1354 bp->Frame = ((_JUMP_BUFFER*)((
void*)&
buf))->Frame;
1373#define STACK_PAD_SIZE 1
1375#define STACK_PAD_SIZE 1024
1379#if !STACK_GROW_DIRECTION
1380 if (addr_in_prev_frame > &space[0]) {
1383#if STACK_GROW_DIRECTION <= 0
1385 if (&space[0] > end) {
1390 cont_restore_0(cont, &space[0]);
1394#if !STACK_GROW_DIRECTION
1399#if STACK_GROW_DIRECTION >= 0
1410#if !STACK_GROW_DIRECTION
1414 cont_restore_1(cont);
1501rb_callcc(
VALUE self)
1503 volatile int called;
1504 volatile VALUE val = cont_capture(&called);
1544lookup_rollback_func(
e_proc *ensure_func)
1566 for (p=current; p; p=p->
next)
1569 for (entry=target; entry->
marker; entry++)
1574 base_point = cur_size;
1575 while (base_point) {
1576 if (target_size >= base_point &&
1577 p->
entry.
marker == target[target_size - base_point].marker)
1584 for (
i=0;
i < target_size - base_point;
i++) {
1585 if (!lookup_rollback_func(target[
i].
e_proc)) {
1590 while (cur_size > base_point) {
1593 current = current->
next;
1597 for (j = 0; j <
i; j++) {
1598 func = lookup_rollback_func(target[
i - j - 1].
e_proc);
1600 (*func)(target[
i - j - 1].
data2);
1626 if (cont_thread_value(cont) != th->
self) {
1642 cont_restore_0(cont, &contval);
1715 {fiber_mark, fiber_free, fiber_memsize, fiber_compact,},
1726fiber_t_alloc(
VALUE fiber_value)
1739 cont_init(&fiber->
cont, th);
1790 return fiber_initialize(
self,
rb_block_proc(), &shared_fiber_pool);
1796 return fiber_initialize(fiber_alloc(rb_cFiber),
rb_proc_new(func,
obj), &shared_fiber_pool);
1801#define PASS_KW_SPLAT (rb_empty_keyword_given_p() ? RB_PASS_EMPTY_KEYWORDS : rb_keyword_given_p())
1810 int need_interrupt =
TRUE;
1847 need_interrupt =
TRUE;
1850 rb_fiber_terminate(fiber, need_interrupt);
1857 VALUE fiber_value = fiber_alloc(rb_cFiber);
1868#ifdef COROUTINE_PRIVATE_STACK
1869 fiber->
stack = fiber_pool_stack_acquire(&shared_fiber_pool);
1872 coroutine_initialize_main(&fiber->
context);
1927 root_fiber_alloc(rb_ec_thread_ptr(ec));
1944 if (root_fiber == fiber) {
1945 rb_raise(rb_eFiberError,
"can't yield from root fiber");
1972 fiber = root_fiber_alloc(th);
1976 fiber_prepare_stack(next_fiber);
1985 fiber_setcontext(next_fiber, fiber);
2009 return make_passing_arg(
argc,
argv);
2012 if (cont_thread_value(cont) != th->
self) {
2013 rb_raise(rb_eFiberError,
"fiber called across threads");
2016 rb_raise(rb_eFiberError,
"fiber called across stack rewinding barrier");
2019 value =
rb_exc_new2(rb_eFiberError,
"dead fiber called");
2033 cont->
value = value;
2042 fiber->
prev = fiber_current();
2051 value = fiber_store(fiber, th);
2054 fiber_stack_release(fiber);
2077rb_fiber_terminate(
rb_fiber_t *fiber,
int need_interrupt)
2085 coroutine_destroy(&fiber->
context);
2090 next_fiber = return_fiber();
2102 rb_raise(rb_eFiberError,
"cannot raise exception on unborn fiber");
2105 if (fiber->
prev != 0 || fiber_is_root_p(fiber)) {
2106 rb_raise(rb_eFiberError,
"double resume");
2110 rb_raise(rb_eFiberError,
"cannot resume transferred Fiber");
2113 return fiber_switch(fiber,
argc,
argv, 1, kw_splat);
2125 return fiber_switch(return_fiber(),
argc,
argv, 0, kw_splat);
2303fiber_to_s(
VALUE fiber_value)
2305 const rb_fiber_t *fiber = fiber_ptr(fiber_value);
2307 char status_info[0x20];
2310 snprintf(status_info, 0x20,
" (%s, transferred)", fiber_status_name(fiber->status));
2313 snprintf(status_info, 0x20,
" (%s)", fiber_status_name(fiber->status));
2318 strlcat(status_info,
">",
sizeof(status_info));
2327#ifdef HAVE_WORKING_FORK
2340#ifdef RB_EXPERIMENTAL_FIBER_POOL
2342fiber_pool_free(
void *
ptr)
2354fiber_pool_memsize(
const void *
ptr)
2357 size_t size =
sizeof(*fiber_pool);
2366 {
NULL, fiber_pool_free, fiber_pool_memsize,},
2431 GetSystemInfo(&info);
2432 pagesize = info.dwPageSize;
2440 char * fiber_shared_fiber_pool_free_stacks =
getenv(
"RUBY_SHARED_FIBER_POOL_FREE_STACKS");
2441 if (fiber_shared_fiber_pool_free_stacks) {
2442 shared_fiber_pool.
free_stacks =
atoi(fiber_shared_fiber_pool_free_stacks);
2455#ifdef RB_EXPERIMENTAL_FIBER_POOL
2458 rb_define_method(rb_cFiberPool,
"initialize", rb_fiber_pool_initialize, -1);
struct coroutine_context * coroutine_transfer(struct coroutine_context *current, struct coroutine_context *target)
NOINLINE(static VALUE cont_capture(volatile int *volatile stat))
NORETURN(NOINLINE(static void cont_restore_0(rb_context_t *, VALUE *)))
void rb_threadptr_root_fiber_setup(rb_thread_t *th)
void rb_fiber_update_self(rb_fiber_t *fiber)
#define THREAD_MUST_BE_RUNNING(th)
void rb_fiber_reset_root_local_storage(rb_thread_t *th)
void ruby_Init_Fiber_as_Coroutine(void)
void ruby_register_rollback_func_for_ensure(e_proc *ensure_func, e_proc *rollback_func)
#define FIBER_RUNNABLE_P(fiber)
VALUE rb_fiber_current(void)
VALUE rb_fiber_resume_kw(VALUE fiber_value, int argc, const VALUE *argv, int kw_splat)
VALUE rb_fiber_transfer(VALUE fiber_value, int argc, const VALUE *argv)
VALUE rb_fiber_yield_kw(int argc, const VALUE *argv, int kw_splat)
#define FIBER_POOL_INITIAL_SIZE
VALUE rb_fiber_resume(VALUE fiber_value, int argc, const VALUE *argv)
#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE
VALUE rb_fiber_alive_p(VALUE fiber_value)
void rb_fiber_init_mjit_cont(struct rb_fiber_struct *fiber)
RUBY_SYMBOL_EXPORT_BEGIN void ruby_Init_Continuation_body(void)
VALUE rb_obj_is_fiber(VALUE obj)
void rb_fiber_close(rb_fiber_t *fiber)
#define FIBER_RESUMED_P(fiber)
#define FIBER_STACK_FLAGS
VALUE rb_fiber_yield(int argc, const VALUE *argv)
void rb_fiber_mark_self(const rb_fiber_t *fiber)
VALUE rb_fiber_new(rb_block_call_func_t func, VALUE obj)
#define FIBER_CREATED_P(fiber)
struct rb_context_struct rb_context_t
void rb_threadptr_root_fiber_release(rb_thread_t *th)
#define FIBER_SUSPENDED_P(fiber)
void rb_threadptr_root_fiber_terminate(rb_thread_t *th)
#define FIBER_TERMINATED_P(fiber)
void rb_fiber_start(void)
char str[HTML_ESCAPE_MAX_LEN+1]
void ruby_stop(int ex)
Calls ruby_cleanup() and exits the process.
VALUE rb_define_class(const char *, VALUE)
Defines a top-level class.
void rb_undef_method(VALUE, const char *)
void rb_define_alias(VALUE, const char *, const char *)
Defines an alias of a method.
VALUE rb_cObject
Object class.
void rb_raise(VALUE exc, const char *fmt,...)
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
void rb_bug(const char *fmt,...)
VALUE rb_make_exception(int, const VALUE *)
Make an Exception object from the list of arguments in a manner similar to Kernel#raise.
VALUE rb_any_to_s(VALUE)
Default implementation of #to_s.
unsigned char buf[MIME_BUF_SIZE]
void st_free_table(st_table *tab)
size_t st_memsize(const st_table *tab)
st_table * st_init_numtable(void)
int st_insert(st_table *tab, st_data_t key, st_data_t value)
int st_lookup(st_table *tab, st_data_t key, st_data_t *value)
struct fiber_pool_allocation * next
struct fiber_pool_allocation * allocation
struct fiber_pool_vacancy * next
struct fiber_pool_stack stack
struct fiber_pool_allocation * allocations
struct fiber_pool_vacancy * vacancies
rb_execution_context_t saved_ec
struct mjit_cont * mjit_cont
rb_ensure_entry_t * ensure_array
struct cont_saved_vm_stack saved_vm_stack
struct rb_context_struct::@65 machine
struct rb_ensure_entry entry
struct rb_ensure_list * next
VALUE local_storage_recursive_hash_for_trace
struct rb_thread_struct * thread_ptr
struct rb_execution_context_struct::@55 machine
rb_ensure_list_t * ensure_list
struct rb_vm_protect_tag * protect_tag
struct rb_trace_arg_struct * trace_arg
VALUE local_storage_recursive_hash
struct coroutine_context context
BITFIELD(enum fiber_status, status, 2)
struct fiber_pool_stack stack
struct rb_fiber_struct * prev
struct rb_iseq_constant_body * body
const struct rb_block block
rb_execution_context_t * ec
enum rb_thread_status status
struct rb_thread_struct * main_thread
size_t fiber_vm_stack_size
size_t fiber_machine_stack_size
struct rb_vm_struct::@52 default_params