Ruby 4.0.5p0 (2026-05-20 revision 64336ffd0ee9e1f4c05891695a3d7b49cb709721)
vm_insnhelper.c
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/random.h"
28#include "internal/variable.h"
29#include "internal/set_table.h"
30#include "internal/struct.h"
31#include "variable.h"
32
33/* finish iseq array */
34#include "insns.inc"
35#include "insns_info.inc"
36
37extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
38extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
39extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
40extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
41 int argc, const VALUE *argv, int priv);
42
43static const struct rb_callcache vm_empty_cc;
44static const struct rb_callcache vm_empty_cc_for_super;
45
46/* control stack frame */
47
48static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
49
50VALUE
51ruby_vm_special_exception_copy(VALUE exc)
52{
53 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
54 rb_obj_copy_ivar(e, exc);
55 return e;
56}
57
58NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
59static void
60ec_stack_overflow(rb_execution_context_t *ec, int setup)
61{
62 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
63 ec->raised_flag = RAISED_STACKOVERFLOW;
64 if (setup) {
65 VALUE at = rb_ec_backtrace_object(ec);
66 mesg = ruby_vm_special_exception_copy(mesg);
67 rb_ivar_set(mesg, idBt, at);
68 rb_ivar_set(mesg, idBt_locations, at);
69 }
70 ec->errinfo = mesg;
71 EC_JUMP_TAG(ec, TAG_RAISE);
72}
73
74NORETURN(static void vm_stackoverflow(void));
75
76static void
77vm_stackoverflow(void)
78{
79 ec_stack_overflow(GET_EC(), TRUE);
80}
81
82void
83rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
84{
85 if (rb_during_gc()) {
86 rb_bug("system stack overflow during GC. Faulty native extension?");
87 }
88 if (crit >= rb_stack_overflow_fatal) {
89 ec->raised_flag = RAISED_STACKOVERFLOW;
90 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
91 EC_JUMP_TAG(ec, TAG_RAISE);
92 }
93 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
94}
95
96static inline void stack_check(rb_execution_context_t *ec);
97
98#if VM_CHECK_MODE > 0
99static int
100callable_class_p(VALUE klass)
101{
102#if VM_CHECK_MODE >= 2
103 if (!klass) return FALSE;
104 switch (RB_BUILTIN_TYPE(klass)) {
105 default:
106 break;
107 case T_ICLASS:
108 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
109 case T_MODULE:
110 return TRUE;
111 }
112 while (klass) {
113 if (klass == rb_cBasicObject) {
114 return TRUE;
115 }
116 klass = RCLASS_SUPER(klass);
117 }
118 return FALSE;
119#else
120 return klass != 0;
121#endif
122}
123
124static int
125callable_method_entry_p(const rb_callable_method_entry_t *cme)
126{
127 if (cme == NULL) {
128 return TRUE;
129 }
130 else {
131 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
132
133 if (callable_class_p(cme->defined_class)) {
134 return TRUE;
135 }
136 else {
137 return FALSE;
138 }
139 }
140}
141
142static void
143vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
144{
145 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
146 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
147
148 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
149 cref_or_me_type = imemo_type(cref_or_me);
150 }
151 if (type & VM_FRAME_FLAG_BMETHOD) {
152 req_me = TRUE;
153 }
154
155 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
156 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
157 }
158 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
159 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
160 }
161
162 if (req_me) {
163 if (cref_or_me_type != imemo_ment) {
164 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
165 }
166 }
167 else {
168 if (req_cref && cref_or_me_type != imemo_cref) {
169 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
170 }
171 else { /* cref or Qfalse */
172 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
173 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
174 /* ignore */
175 }
176 else {
177 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
178 }
179 }
180 }
181 }
182
183 if (cref_or_me_type == imemo_ment) {
184 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
185
186 if (!callable_method_entry_p(me)) {
187 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
188 }
189 }
190
191 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
192 VM_ASSERT(iseq == NULL ||
193 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
194 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
195 );
196 }
197 else {
198 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
199 }
200}
201
202static void
203vm_check_frame(VALUE type,
204 VALUE specval,
205 VALUE cref_or_me,
206 const rb_iseq_t *iseq)
207{
208 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
209 VM_ASSERT(FIXNUM_P(type));
210
211#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
212 case magic: \
213 vm_check_frame_detail(type, req_block, req_me, req_cref, \
214 specval, cref_or_me, is_cframe, iseq); \
215 break
216 switch (given_magic) {
217 /* BLK ME CREF CFRAME */
218 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
219 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
220 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
221 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
222 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
223 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
224 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
225 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
226 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
227 default:
228 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
229 }
230#undef CHECK
231}
232
233static VALUE vm_stack_canary; /* Initialized later */
234static bool vm_stack_canary_was_born = false;
235
236// Return the index of the instruction right before the given PC.
237// This is needed because insn_entry advances PC before the insn body.
238static unsigned int
239previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
240{
241 unsigned int pos = 0;
242 while (pos < ISEQ_BODY(iseq)->iseq_size) {
243 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
244 unsigned int next_pos = pos + insn_len(opcode);
245 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
246 return pos;
247 }
248 pos = next_pos;
249 }
250 rb_bug("failed to find the previous insn");
251}
252
253void
254rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
255{
256 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
257 const struct rb_iseq_struct *iseq;
258
259 if (! LIKELY(vm_stack_canary_was_born)) {
260 return; /* :FIXME: isn't it rather fatal to enter this branch? */
261 }
262 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
263 /* This is at the very beginning of a thread. cfp does not exist. */
264 return;
265 }
266 else if (! (iseq = GET_ISEQ())) {
267 return;
268 }
269 else if (LIKELY(sp[0] != vm_stack_canary)) {
270 return;
271 }
272 else {
273 /* we are going to call methods below; squash the canary to
274 * prevent infinite loop. */
275 sp[0] = Qundef;
276 }
277
278 const VALUE *orig = rb_iseq_original_iseq(iseq);
279 const VALUE iseqw = rb_iseqw_new(iseq);
280 const VALUE inspection = rb_inspect(iseqw);
281 const char *stri = rb_str_to_cstr(inspection);
282 const VALUE disasm = rb_iseq_disasm(iseq);
283 const char *strd = rb_str_to_cstr(disasm);
284 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
285 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
286 const char *name = insn_name(insn);
287
288 /* rb_bug() is not capable of outputting this large contents. It
289 is designed to run form a SIGSEGV handler, which tends to be
290 very restricted. */
291 ruby_debug_printf(
292 "We are killing the stack canary set by %s, "
293 "at %s@pc=%"PRIdPTR"\n"
294 "watch out the C stack trace.\n"
295 "%s",
296 name, stri, pos, strd);
297 rb_bug("see above.");
298}
299#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
300
301#else
302#define vm_check_canary(ec, sp)
303#define vm_check_frame(a, b, c, d)
304#endif /* VM_CHECK_MODE > 0 */
305
306#if USE_DEBUG_COUNTER
307static void
308vm_push_frame_debug_counter_inc(
309 const struct rb_execution_context_struct *ec,
310 const struct rb_control_frame_struct *reg_cfp,
311 VALUE type)
312{
313 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
314
315 RB_DEBUG_COUNTER_INC(frame_push);
316
317 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
318 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
319 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
320 if (prev) {
321 if (curr) {
322 RB_DEBUG_COUNTER_INC(frame_R2R);
323 }
324 else {
325 RB_DEBUG_COUNTER_INC(frame_R2C);
326 }
327 }
328 else {
329 if (curr) {
330 RB_DEBUG_COUNTER_INC(frame_C2R);
331 }
332 else {
333 RB_DEBUG_COUNTER_INC(frame_C2C);
334 }
335 }
336 }
337
338 switch (type & VM_FRAME_MAGIC_MASK) {
339 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
340 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
341 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
342 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
343 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
344 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
345 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
346 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
347 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
348 }
349
350 rb_bug("unreachable");
351}
352#else
353#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
354#endif
355
356// Return a poison value to be set above the stack top to verify leafness.
357VALUE
358rb_vm_stack_canary(void)
359{
360#if VM_CHECK_MODE > 0
361 return vm_stack_canary;
362#else
363 return 0;
364#endif
365}
366
367STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
368STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
369STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
370
371static void
372vm_push_frame(rb_execution_context_t *ec,
373 const rb_iseq_t *iseq,
374 VALUE type,
375 VALUE self,
376 VALUE specval,
377 VALUE cref_or_me,
378 const VALUE *pc,
379 VALUE *sp,
380 int local_size,
381 int stack_max)
382{
383 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
384
385 vm_check_frame(type, specval, cref_or_me, iseq);
386 VM_ASSERT(local_size >= 0);
387
388 /* check stack overflow */
389 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
390 vm_check_canary(ec, sp);
391
392 /* setup vm value stack */
393
394 /* initialize local variables */
395 for (int i=0; i < local_size; i++) {
396 *sp++ = Qnil;
397 }
398
399 /* setup ep with managing data */
400 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
401 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
402 *sp++ = type; /* ep[-0] / ENV_FLAGS */
403
404 /* setup new frame */
405 *cfp = (const struct rb_control_frame_struct) {
406 .pc = pc,
407 .sp = sp,
408 .iseq = iseq,
409 .self = self,
410 .ep = sp - 1,
411 .block_code = NULL,
412#if VM_DEBUG_BP_CHECK
413 .bp_check = sp,
414#endif
415 .jit_return = NULL,
416 };
417
418 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
419 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
420 future/untested compilers/platforms. */
421
422 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
423 atomic_signal_fence(memory_order_seq_cst);
424 #endif
425
426 ec->cfp = cfp;
427
428 if (VMDEBUG == 2) {
429 SDR();
430 }
431 vm_push_frame_debug_counter_inc(ec, cfp, type);
432}
433
434void
435rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
436{
437 rb_control_frame_t *cfp = ec->cfp;
438
439 if (VMDEBUG == 2) SDR();
440
441 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
442}
443
444/* return TRUE if the frame is finished */
445static inline int
446vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
447{
448 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
449
450 if (VMDEBUG == 2) SDR();
451
452 RUBY_VM_CHECK_INTS(ec);
453 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
454
455 return flags & VM_FRAME_FLAG_FINISH;
456}
457
458void
459rb_vm_pop_frame(rb_execution_context_t *ec)
460{
461 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
462}
463
464// it pushes pseudo-frame with fname filename.
465VALUE
466rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
467{
468 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
469 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
470
471 vm_push_frame(ec,
472 dmy_iseq, //const rb_iseq_t *iseq,
473 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
474 ec->cfp->self, // VALUE self,
475 VM_BLOCK_HANDLER_NONE, // VALUE specval,
476 Qfalse, // VALUE cref_or_me,
477 NULL, // const VALUE *pc,
478 ec->cfp->sp, // VALUE *sp,
479 0, // int local_size,
480 0); // int stack_max
481
482 return (VALUE)dmy_iseq;
483}
484
485/* method dispatch */
486static inline VALUE
487rb_arity_error_new(int argc, int min, int max)
488{
489 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
490 if (min == max) {
491 /* max is not needed */
492 }
493 else if (max == UNLIMITED_ARGUMENTS) {
494 rb_str_cat_cstr(err_mess, "+");
495 }
496 else {
497 rb_str_catf(err_mess, "..%d", max);
498 }
499 rb_str_cat_cstr(err_mess, ")");
500 return rb_exc_new3(rb_eArgError, err_mess);
501}
502
503void
504rb_error_arity(int argc, int min, int max)
505{
506 rb_exc_raise(rb_arity_error_new(argc, min, max));
507}
508
509/* lvar */
510
511NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
512
513static void
514vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
515{
516 /* remember env value forcely */
517 rb_gc_writebarrier_remember(VM_ENV_ENVVAL(ep));
518 VM_FORCE_WRITE(&ep[index], v);
519 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
520 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
521}
522
523// YJIT assumes this function never runs GC
524static inline void
525vm_env_write(const VALUE *ep, int index, VALUE v)
526{
527 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
528 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
529 VM_STACK_ENV_WRITE(ep, index, v);
530 }
531 else {
532 vm_env_write_slowpath(ep, index, v);
533 }
534}
535
536void
537rb_vm_env_write(const VALUE *ep, int index, VALUE v)
538{
539 vm_env_write(ep, index, v);
540}
541
542VALUE
543rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
544{
545 if (block_handler == VM_BLOCK_HANDLER_NONE) {
546 return Qnil;
547 }
548 else {
549 switch (vm_block_handler_type(block_handler)) {
550 case block_handler_type_iseq:
551 case block_handler_type_ifunc:
552 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
553 case block_handler_type_symbol:
554 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
555 case block_handler_type_proc:
556 return VM_BH_TO_PROC(block_handler);
557 default:
558 VM_UNREACHABLE(rb_vm_bh_to_procval);
559 }
560 }
561}
562
563/* svar */
564
565#if VM_CHECK_MODE > 0
566static int
567vm_svar_valid_p(VALUE svar)
568{
569 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
570 switch (imemo_type(svar)) {
571 case imemo_svar:
572 case imemo_cref:
573 case imemo_ment:
574 return TRUE;
575 default:
576 break;
577 }
578 }
579 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
580 return FALSE;
581}
582#endif
583
584static inline struct vm_svar *
585lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
586{
587 VALUE svar;
588
589 if (lep && (ec == NULL || ec->root_lep != lep)) {
590 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
591 }
592 else {
593 svar = ec->root_svar;
594 }
595
596 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
597
598 return (struct vm_svar *)svar;
599}
600
601static inline void
602lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
603{
604 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
605
606 if (lep && (ec == NULL || ec->root_lep != lep)) {
607 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
608 }
609 else {
610 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
611 }
612}
613
614static VALUE
615lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
616{
617 const struct vm_svar *svar = lep_svar(ec, lep);
618
619 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
620
621 switch (key) {
622 case VM_SVAR_LASTLINE:
623 return svar->lastline;
624 case VM_SVAR_BACKREF:
625 return svar->backref;
626 default: {
627 const VALUE ary = svar->others;
628
629 if (NIL_P(ary)) {
630 return Qnil;
631 }
632 else {
633 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
634 }
635 }
636 }
637}
638
639static struct vm_svar *
640svar_new(VALUE obj)
641{
642 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
643 *((VALUE *)&svar->lastline) = Qnil;
644 *((VALUE *)&svar->backref) = Qnil;
645 *((VALUE *)&svar->others) = Qnil;
646
647 return svar;
648}
649
650static void
651lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
652{
653 struct vm_svar *svar = lep_svar(ec, lep);
654
655 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
656 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
657 }
658
659 switch (key) {
660 case VM_SVAR_LASTLINE:
661 RB_OBJ_WRITE(svar, &svar->lastline, val);
662 return;
663 case VM_SVAR_BACKREF:
664 RB_OBJ_WRITE(svar, &svar->backref, val);
665 return;
666 default: {
667 VALUE ary = svar->others;
668
669 if (NIL_P(ary)) {
670 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
671 }
672 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
673 }
674 }
675}
676
677static inline VALUE
678vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
679{
680 VALUE val;
681
682 if (type == 0) {
683 val = lep_svar_get(ec, lep, key);
684 }
685 else {
686 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
687
688 if (type & 0x01) {
689 switch (type >> 1) {
690 case '&':
691 val = rb_reg_last_match(backref);
692 break;
693 case '`':
694 val = rb_reg_match_pre(backref);
695 break;
696 case '\'':
697 val = rb_reg_match_post(backref);
698 break;
699 case '+':
700 val = rb_reg_match_last(backref);
701 break;
702 default:
703 rb_bug("unexpected back-ref");
704 }
705 }
706 else {
707 val = rb_reg_nth_match((int)(type >> 1), backref);
708 }
709 }
710 return val;
711}
712
713static inline VALUE
714vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
715{
716 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
717 int nth = 0;
718
719 if (type & 0x01) {
720 switch (type >> 1) {
721 case '&':
722 case '`':
723 case '\'':
724 break;
725 case '+':
726 return rb_reg_last_defined(backref);
727 default:
728 rb_bug("unexpected back-ref");
729 }
730 }
731 else {
732 nth = (int)(type >> 1);
733 }
734 return rb_reg_nth_defined(nth, backref);
735}
736
737PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
738static rb_callable_method_entry_t *
739check_method_entry(VALUE obj, int can_be_svar)
740{
741 if (obj == Qfalse) return NULL;
742
743#if VM_CHECK_MODE > 0
744 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
745#endif
746
747 switch (imemo_type(obj)) {
748 case imemo_ment:
749 return (rb_callable_method_entry_t *)obj;
750 case imemo_cref:
751 return NULL;
752 case imemo_svar:
753 if (can_be_svar) {
754 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
755 }
756 default:
757#if VM_CHECK_MODE > 0
758 rb_bug("check_method_entry: svar should not be there:");
759#endif
760 return NULL;
761 }
762}
763
764static rb_callable_method_entry_t *
765env_method_entry_unchecked(VALUE obj, int can_be_svar)
766{
767 if (obj == Qfalse) return NULL;
768
769 switch (imemo_type(obj)) {
770 case imemo_ment:
771 return (rb_callable_method_entry_t *)obj;
772 case imemo_cref:
773 return NULL;
774 case imemo_svar:
775 if (can_be_svar) {
776 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
777 }
778 default:
779 return NULL;
780 }
781}
782
783const rb_callable_method_entry_t *
784rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
785{
786 const VALUE *ep = cfp->ep;
787 rb_callable_method_entry_t *me;
788
789 while (!VM_ENV_LOCAL_P(ep)) {
790 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
791 ep = VM_ENV_PREV_EP(ep);
792 }
793
794 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
795}
796
797const rb_callable_method_entry_t *
798rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
799{
800 const VALUE *ep = cfp->ep;
801 rb_callable_method_entry_t *me;
802
803 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
804 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
805 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
806 }
807
808 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
809}
810
811static const rb_iseq_t *
812method_entry_iseqptr(const rb_callable_method_entry_t *me)
813{
814 switch (me->def->type) {
815 case VM_METHOD_TYPE_ISEQ:
816 return me->def->body.iseq.iseqptr;
817 default:
818 return NULL;
819 }
820}
821
822static rb_cref_t *
823method_entry_cref(const rb_callable_method_entry_t *me)
824{
825 switch (me->def->type) {
826 case VM_METHOD_TYPE_ISEQ:
827 return me->def->body.iseq.cref;
828 default:
829 return NULL;
830 }
831}
832
833#if VM_CHECK_MODE == 0
834PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
835#endif
836static rb_cref_t *
837check_cref(VALUE obj, int can_be_svar)
838{
839 if (obj == Qfalse) return NULL;
840
841#if VM_CHECK_MODE > 0
842 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
843#endif
844
845 switch (imemo_type(obj)) {
846 case imemo_ment:
847 return method_entry_cref((rb_callable_method_entry_t *)obj);
848 case imemo_cref:
849 return (rb_cref_t *)obj;
850 case imemo_svar:
851 if (can_be_svar) {
852 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
853 }
854 default:
855#if VM_CHECK_MODE > 0
856 rb_bug("check_method_entry: svar should not be there:");
857#endif
858 return NULL;
859 }
860}
861
862static inline rb_cref_t *
863vm_env_cref(const VALUE *ep)
864{
865 rb_cref_t *cref;
866
867 while (!VM_ENV_LOCAL_P(ep)) {
868 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
869 ep = VM_ENV_PREV_EP(ep);
870 }
871
872 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
873}
874
875static int
876is_cref(const VALUE v, int can_be_svar)
877{
878 if (RB_TYPE_P(v, T_IMEMO)) {
879 switch (imemo_type(v)) {
880 case imemo_cref:
881 return TRUE;
882 case imemo_svar:
883 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
884 default:
885 break;
886 }
887 }
888 return FALSE;
889}
890
891static int
892vm_env_cref_by_cref(const VALUE *ep)
893{
894 while (!VM_ENV_LOCAL_P(ep)) {
895 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
896 ep = VM_ENV_PREV_EP(ep);
897 }
898 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
899}
900
901static rb_cref_t *
902cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
903{
904 const VALUE v = *vptr;
905 rb_cref_t *cref, *new_cref;
906
907 if (RB_TYPE_P(v, T_IMEMO)) {
908 switch (imemo_type(v)) {
909 case imemo_cref:
910 cref = (rb_cref_t *)v;
911 new_cref = vm_cref_dup(cref);
912 if (parent) {
913 RB_OBJ_WRITE(parent, vptr, new_cref);
914 }
915 else {
916 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
917 }
918 return (rb_cref_t *)new_cref;
919 case imemo_svar:
920 if (can_be_svar) {
921 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
922 }
923 /* fall through */
924 case imemo_ment:
925 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
926 default:
927 break;
928 }
929 }
930 return NULL;
931}
932
933static rb_cref_t *
934vm_cref_replace_with_duplicated_cref(const VALUE *ep)
935{
936 if (vm_env_cref_by_cref(ep)) {
937 rb_cref_t *cref;
938 VALUE envval;
939
940 while (!VM_ENV_LOCAL_P(ep)) {
941 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
942 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
943 return cref;
944 }
945 ep = VM_ENV_PREV_EP(ep);
946 }
947 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
948 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
949 }
950 else {
951 rb_bug("vm_cref_dup: unreachable");
952 }
953}
954
955static rb_cref_t *
956vm_get_cref(const VALUE *ep)
957{
958 rb_cref_t *cref = vm_env_cref(ep);
959
960 if (cref != NULL) {
961 return cref;
962 }
963 else {
964 rb_bug("vm_get_cref: unreachable");
965 }
966}
967
968rb_cref_t *
969rb_vm_get_cref(const VALUE *ep)
970{
971 return vm_get_cref(ep);
972}
973
974static rb_cref_t *
975vm_ec_cref(const rb_execution_context_t *ec)
976{
977 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
978
979 if (cfp == NULL) {
980 return NULL;
981 }
982 return vm_get_cref(cfp->ep);
983}
984
985static const rb_cref_t *
986vm_get_const_key_cref(const VALUE *ep)
987{
988 const rb_cref_t *cref = vm_get_cref(ep);
989 const rb_cref_t *key_cref = cref;
990
991 while (cref) {
992 if (RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
993 RCLASS_CLONED_P(CREF_CLASS(cref)) ) {
994 return key_cref;
995 }
996 cref = CREF_NEXT(cref);
997 }
998
999 /* does not include singleton class */
1000 return NULL;
1001}
1002
1003rb_cref_t *
1004rb_vm_rewrite_cref(rb_cref_t *cref, VALUE old_klass, VALUE new_klass)
1005{
1006 rb_cref_t *new_cref_head = NULL;
1007 rb_cref_t *new_cref_tail = NULL;
1008
1009 #define ADD_NEW_CREF(new_cref) \
1010 if (new_cref_tail) { \
1011 RB_OBJ_WRITE(new_cref_tail, &new_cref_tail->next, new_cref); \
1012 } \
1013 else { \
1014 new_cref_head = new_cref; \
1015 } \
1016 new_cref_tail = new_cref;
1017
1018 while (cref) {
1019 rb_cref_t *new_cref;
1020 if (CREF_CLASS(cref) == old_klass) {
1021 new_cref = vm_cref_new_use_prev(new_klass, METHOD_VISI_UNDEF, FALSE, cref, FALSE);
1022 ADD_NEW_CREF(new_cref);
1023 return new_cref_head;
1024 }
1025 new_cref = vm_cref_new_use_prev(CREF_CLASS(cref), METHOD_VISI_UNDEF, FALSE, cref, FALSE);
1026 cref = CREF_NEXT(cref);
1027 ADD_NEW_CREF(new_cref);
1028 }
1029
1030 #undef ADD_NEW_CREF
1031
1032 // Could we just reuse the original cref?
1033 return new_cref_head;
1034}
1035
1036static rb_cref_t *
1037vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1038{
1039 rb_cref_t *prev_cref = NULL;
1040
1041 if (ep) {
1042 prev_cref = vm_env_cref(ep);
1043 }
1044 else {
1045 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1046
1047 if (cfp) {
1048 prev_cref = vm_env_cref(cfp->ep);
1049 }
1050 }
1051
1052 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1053}
1054
1055static inline VALUE
1056vm_get_cbase(const VALUE *ep)
1057{
1058 const rb_cref_t *cref = vm_get_cref(ep);
1059
1060 return CREF_CLASS_FOR_DEFINITION(cref);
1061}
1062
1063static inline VALUE
1064vm_get_const_base(const VALUE *ep)
1065{
1066 const rb_cref_t *cref = vm_get_cref(ep);
1067
1068 while (cref) {
1069 if (!CREF_PUSHED_BY_EVAL(cref)) {
1070 return CREF_CLASS_FOR_DEFINITION(cref);
1071 }
1072 cref = CREF_NEXT(cref);
1073 }
1074
1075 return Qundef;
1076}
1077
1078static inline void
1079vm_check_if_namespace(VALUE klass)
1080{
1081 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1082 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1083 }
1084}
1085
1086static inline void
1087vm_ensure_not_refinement_module(VALUE self)
1088{
1089 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1090 rb_warn("not defined at the refinement, but at the outer class/module");
1091 }
1092}
1093
1094static inline VALUE
1095vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1096{
1097 return klass;
1098}
1099
1100static inline VALUE
1101vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1102{
1103 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1104 VALUE val;
1105
1106 if (NIL_P(orig_klass) && allow_nil) {
1107 /* in current lexical scope */
1108 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1109 const rb_cref_t *cref;
1110 VALUE klass = Qnil;
1111
1112 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1113 root_cref = CREF_NEXT(root_cref);
1114 }
1115 cref = root_cref;
1116 while (cref && CREF_NEXT(cref)) {
1117 if (CREF_PUSHED_BY_EVAL(cref)) {
1118 klass = Qnil;
1119 }
1120 else {
1121 klass = CREF_CLASS(cref);
1122 }
1123 cref = CREF_NEXT(cref);
1124
1125 if (!NIL_P(klass)) {
1126 VALUE av, am = 0;
1127 rb_const_entry_t *ce;
1128 search_continue:
1129 if ((ce = rb_const_lookup(klass, id))) {
1130 rb_const_warn_if_deprecated(ce, klass, id);
1131 val = ce->value;
1132 if (UNDEF_P(val)) {
1133 if (am == klass) break;
1134 am = klass;
1135 if (is_defined) return 1;
1136 if (rb_autoloading_value(klass, id, &av, NULL)) return av;
1137 rb_autoload_load(klass, id);
1138 goto search_continue;
1139 }
1140 else {
1141 if (is_defined) {
1142 return 1;
1143 }
1144 else {
1145 if (UNLIKELY(!rb_ractor_main_p())) {
1146 if (!rb_ractor_shareable_p(val)) {
1147 rb_raise(rb_eRactorIsolationError,
1148 "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" by non-main ractor.", rb_class_path(klass), rb_id2str(id));
1149 }
1150 }
1151 return val;
1152 }
1153 }
1154 }
1155 }
1156 }
1157
1158 /* search self */
1159 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1160 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1161 }
1162 else {
1163 klass = CLASS_OF(ec->cfp->self);
1164 }
1165
1166 if (is_defined) {
1167 return rb_const_defined(klass, id);
1168 }
1169 else {
1170 return rb_const_get(klass, id);
1171 }
1172 }
1173 else {
1174 vm_check_if_namespace(orig_klass);
1175 if (is_defined) {
1176 return rb_public_const_defined_from(orig_klass, id);
1177 }
1178 else {
1179 return rb_public_const_get_from(orig_klass, id);
1180 }
1181 }
1182}
1183
1184VALUE
1185rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1186{
1187 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1188}
1189
1190static inline VALUE
1191vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1192{
1193 VALUE val = Qnil;
1194 int idx = 0;
1195 int allow_nil = TRUE;
1196 if (segments[0] == idNULL) {
1197 val = rb_cObject;
1198 idx++;
1199 allow_nil = FALSE;
1200 }
1201 while (segments[idx]) {
1202 ID id = segments[idx++];
1203 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1204 allow_nil = FALSE;
1205 }
1206 return val;
1207}
1208
1209
1210static inline VALUE
1211vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1212{
1213 VALUE klass;
1214
1215 if (!cref) {
1216 rb_bug("vm_get_cvar_base: no cref");
1217 }
1218
1219 while (CREF_NEXT(cref) &&
1220 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1221 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1222 cref = CREF_NEXT(cref);
1223 }
1224 if (top_level_raise && !CREF_NEXT(cref)) {
1225 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1226 }
1227
1228 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1229
1230 if (NIL_P(klass)) {
1231 rb_raise(rb_eTypeError, "no class variables available");
1232 }
1233 return klass;
1234}
1235
1236ALWAYS_INLINE(static void fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, attr_index_t index, shape_id_t shape_id));
1237static inline void
1238fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, attr_index_t index, shape_id_t shape_id)
1239{
1240 if (is_attr) {
1241 vm_cc_attr_index_set(cc, index, shape_id);
1242 }
1243 else {
1244 vm_ic_attr_index_set(iseq, ic, index, shape_id);
1245 }
1246}
1247
1248#define ractor_incidental_shareable_p(cond, val) \
1249 (!(cond) || rb_ractor_shareable_p(val))
1250#define ractor_object_incidental_shareable_p(obj, val) \
1251 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1252
1253ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1254static inline VALUE
1255vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1256{
1257 VALUE fields_obj;
1258#if OPT_IC_FOR_IVAR
1259 if (SPECIAL_CONST_P(obj)) {
1260 return default_value;
1261 }
1262
1263 switch (BUILTIN_TYPE(obj)) {
1264 case T_OBJECT:
1265 fields_obj = obj;
1266 break;
1267 case T_CLASS:
1268 case T_MODULE:
1269 {
1270 if (UNLIKELY(!rb_ractor_main_p())) {
1271 // For two reasons we can only use the fast path on the main
1272 // ractor.
1273 // First, only the main ractor is allowed to set ivars on classes
1274 // and modules. So we can skip locking.
1275 // Second, other ractors need to check the shareability of the
1276 // values returned from the class ivars.
1277
1278 if (default_value == Qundef) { // defined?
1279 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1280 }
1281 else {
1282 goto general_path;
1283 }
1284 }
1285
1286 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1287 break;
1288 }
1289 default:
1290 fields_obj = rb_obj_fields(obj, id);
1291 }
1292
1293 if (!fields_obj) {
1294 return default_value;
1295 }
1296
1297 VALUE val = Qundef;
1298
1299 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1300 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1301
1302 shape_id_t cached_id;
1303 attr_index_t index;
1304
1305 if (is_attr) {
1306 vm_cc_atomic_shape_and_index(cc, &cached_id, &index);
1307 }
1308 else {
1309 vm_ic_atomic_shape_and_index(ic, &cached_id, &index);
1310 }
1311
1312 if (LIKELY(cached_id == shape_id)) {
1313 RUBY_ASSERT(!rb_shape_too_complex_p(cached_id));
1314
1315 if (index == ATTR_INDEX_NOT_SET) {
1316 return default_value;
1317 }
1318
1319 val = ivar_list[index];
1320#if USE_DEBUG_COUNTER
1321 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1322
1323 if (RB_TYPE_P(obj, T_OBJECT)) {
1324 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1325 }
1326#endif
1327 RUBY_ASSERT(!UNDEF_P(val));
1328 }
1329 else { // cache miss case
1330#if USE_DEBUG_COUNTER
1331 if (is_attr) {
1332 if (cached_id != INVALID_SHAPE_ID) {
1333 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1334 }
1335 else {
1336 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1337 }
1338 }
1339 else {
1340 if (cached_id != INVALID_SHAPE_ID) {
1341 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1342 }
1343 else {
1344 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1345 }
1346 }
1347 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1348
1349 if (RB_TYPE_P(obj, T_OBJECT)) {
1350 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1351 }
1352#endif
1353
1354 if (UNLIKELY(rb_shape_too_complex_p(shape_id))) {
1355 st_table *table = (st_table *)ivar_list;
1356
1357 RUBY_ASSERT(table);
1358 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1359
1360 if (!st_lookup(table, id, &val)) {
1361 val = default_value;
1362 }
1363 }
1364 else {
1365 shape_id_t previous_cached_id = cached_id;
1366 if (rb_shape_get_iv_index_with_hint(shape_id, id, &index, &cached_id)) {
1367 // This fills in the cache with the shared cache object.
1368 // "ent" is the shared cache object
1369 if (cached_id != previous_cached_id) {
1370 fill_ivar_cache(iseq, ic, cc, is_attr, index, cached_id);
1371 }
1372
1373 if (index == ATTR_INDEX_NOT_SET) {
1374 val = default_value;
1375 }
1376 else {
1377 // We fetched the ivar list above
1378 val = ivar_list[index];
1379 RUBY_ASSERT(!UNDEF_P(val));
1380 }
1381 }
1382 else {
1383 if (is_attr) {
1384 vm_cc_attr_index_initialize(cc, shape_id);
1385 }
1386 else {
1387 vm_ic_attr_index_initialize(ic, shape_id);
1388 }
1389
1390 val = default_value;
1391 }
1392 }
1393 }
1394
1395 if (!UNDEF_P(default_value)) {
1396 RUBY_ASSERT(!UNDEF_P(val));
1397 }
1398
1399 return val;
1400
1401general_path:
1402#endif /* OPT_IC_FOR_IVAR */
1403 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1404
1405 if (is_attr) {
1406 return rb_attr_get(obj, id);
1407 }
1408 else {
1409 return rb_ivar_get(obj, id);
1410 }
1411}
1412
1413static void
1414populate_cache(attr_index_t index, shape_id_t next_shape_id, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, bool is_attr)
1415{
1416 RUBY_ASSERT(!rb_shape_too_complex_p(next_shape_id));
1417
1418 // Cache population code
1419 if (is_attr) {
1420 vm_cc_attr_index_set(cc, index, next_shape_id);
1421 }
1422 else {
1423 vm_ic_attr_index_set(iseq, ic, index, next_shape_id);
1424 }
1425}
1426
1427ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1428NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1429NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1430
1431static VALUE
1432vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1433{
1434#if OPT_IC_FOR_IVAR
1435 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1436
1437 rb_check_frozen(obj);
1438
1439 attr_index_t index = rb_ivar_set_index(obj, id, val);
1440 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1441
1442 if (!rb_shape_too_complex_p(next_shape_id)) {
1443 populate_cache(index, next_shape_id, id, iseq, ic, cc, is_attr);
1444 }
1445
1446 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1447 return val;
1448#else
1449 return rb_ivar_set(obj, id, val);
1450#endif
1451}
1452
1453static VALUE
1454vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1455{
1456 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1457}
1458
1459static VALUE
1460vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1461{
1462 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1463}
1464
1465NOINLINE(static VALUE vm_setivar_class(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index));
1466static VALUE
1467vm_setivar_class(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index)
1468{
1469 if (UNLIKELY(!rb_ractor_main_p())) {
1470 return Qundef;
1471 }
1472
1473 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1474 if (UNLIKELY(!fields_obj)) {
1475 return Qundef;
1476 }
1477
1478 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1479
1480 // Cache hit case
1481 if (shape_id == dest_shape_id) {
1482 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1483 }
1484 else if (dest_shape_id != INVALID_SHAPE_ID) {
1485 if (RSHAPE_DIRECT_CHILD_P(shape_id, dest_shape_id) && RSHAPE_EDGE_NAME(dest_shape_id) == id && RSHAPE_CAPACITY(shape_id) == RSHAPE_CAPACITY(dest_shape_id)) {
1486 RUBY_ASSERT(index < RSHAPE_CAPACITY(dest_shape_id));
1487 }
1488 else {
1489 return Qundef;
1490 }
1491 }
1492 else {
1493 return Qundef;
1494 }
1495
1496 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[index], val);
1497
1498 if (shape_id != dest_shape_id) {
1499 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1500 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1501 }
1502
1503 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1504
1505 return val;
1506}
1507
1508NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index));
1509static VALUE
1510vm_setivar_default(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index)
1511{
1512 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1513
1514 // Cache hit case
1515 if (shape_id == dest_shape_id) {
1516 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1517 }
1518 else if (dest_shape_id != INVALID_SHAPE_ID) {
1519 if (RSHAPE_DIRECT_CHILD_P(shape_id, dest_shape_id) && RSHAPE_EDGE_NAME(dest_shape_id) == id && RSHAPE_CAPACITY(shape_id) == RSHAPE_CAPACITY(dest_shape_id)) {
1520 RUBY_ASSERT(index < RSHAPE_CAPACITY(dest_shape_id));
1521 }
1522 else {
1523 return Qundef;
1524 }
1525 }
1526 else {
1527 return Qundef;
1528 }
1529
1530 VALUE fields_obj = rb_obj_fields(obj, id);
1531 RUBY_ASSERT(fields_obj);
1532 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[index], val);
1533
1534 if (shape_id != dest_shape_id) {
1535 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1536 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1537 }
1538
1539 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1540
1541 return val;
1542}
1543
1544static inline VALUE
1545vm_setivar(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index)
1546{
1547#if OPT_IC_FOR_IVAR
1548 switch (BUILTIN_TYPE(obj)) {
1549 case T_OBJECT:
1550 {
1551 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1552
1553 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1554 RUBY_ASSERT(dest_shape_id == INVALID_SHAPE_ID || !rb_shape_too_complex_p(dest_shape_id));
1555
1556 if (LIKELY(shape_id == dest_shape_id)) {
1557 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1558 VM_ASSERT(!rb_ractor_shareable_p(obj));
1559 }
1560 else if (dest_shape_id != INVALID_SHAPE_ID) {
1561 if (RSHAPE_DIRECT_CHILD_P(shape_id, dest_shape_id) && RSHAPE_EDGE_NAME(dest_shape_id) == id && RSHAPE_CAPACITY(shape_id) == RSHAPE_CAPACITY(dest_shape_id)) {
1562 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1563
1564 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1565
1566 RUBY_ASSERT(index < RSHAPE_CAPACITY(dest_shape_id));
1567 }
1568 else {
1569 break;
1570 }
1571 }
1572 else {
1573 break;
1574 }
1575
1576 VALUE *ptr = ROBJECT_FIELDS(obj);
1577
1578 RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
1579 RB_OBJ_WRITE(obj, &ptr[index], val);
1580
1581 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1582 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1583 return val;
1584 }
1585 break;
1586 case T_CLASS:
1587 case T_MODULE:
1588 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1589 default:
1590 break;
1591 }
1592
1593 return Qundef;
1594#endif /* OPT_IC_FOR_IVAR */
1595}
1596
1597static VALUE
1598update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1599{
1600 VALUE defined_class = 0;
1601 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1602
1603 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1604 defined_class = RBASIC(defined_class)->klass;
1605 }
1606
1607 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1608 if (!rb_cvc_tbl) {
1609 rb_bug("the cvc table should be set");
1610 }
1611
1612 VALUE ent_data;
1613 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1614 rb_bug("should have cvar cache entry");
1615 }
1616
1617 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1618
1619 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1620 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1621 ic->entry = ent;
1622
1623 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1624 RB_OBJ_WRITTEN(iseq, Qundef, ent->class_value);
1625 RB_OBJ_WRITTEN(ent->class_value, Qundef, ent->cref);
1626
1627 return cvar_value;
1628}
1629
1630static inline VALUE
1631vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1632{
1633 const rb_cref_t *cref;
1634 cref = vm_get_cref(GET_EP());
1635
1636 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1637 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1638
1639 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1640 RUBY_ASSERT(!UNDEF_P(v));
1641
1642 return v;
1643 }
1644
1645 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1646
1647 return update_classvariable_cache(iseq, klass, id, cref, ic);
1648}
1649
1650VALUE
1651rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1652{
1653 return vm_getclassvariable(iseq, cfp, id, ic);
1654}
1655
1656static inline void
1657vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1658{
1659 const rb_cref_t *cref;
1660 cref = vm_get_cref(GET_EP());
1661
1662 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1663 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1664
1665 rb_class_ivar_set(ic->entry->class_value, id, val);
1666 return;
1667 }
1668
1669 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1670
1671 rb_cvar_set(klass, id, val);
1672
1673 update_classvariable_cache(iseq, klass, id, cref, ic);
1674}
1675
1676void
1677rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1678{
1679 vm_setclassvariable(iseq, cfp, id, val, ic);
1680}
1681
1682ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1683static inline VALUE
1684vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1685{
1686 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1687}
1688
1689static inline void
1690vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1691{
1692 if (RB_SPECIAL_CONST_P(obj)) {
1693 rb_error_frozen_object(obj);
1694 return;
1695 }
1696
1697 shape_id_t dest_shape_id;
1698 attr_index_t index;
1699 vm_ic_atomic_shape_and_index(ic, &dest_shape_id, &index);
1700
1701 if (UNLIKELY(UNDEF_P(vm_setivar(obj, id, val, dest_shape_id, index)))) {
1702 switch (BUILTIN_TYPE(obj)) {
1703 case T_OBJECT:
1704 break;
1705 case T_CLASS:
1706 case T_MODULE:
1707 if (!UNDEF_P(vm_setivar_class(obj, id, val, dest_shape_id, index))) {
1708 return;
1709 }
1710 break;
1711 default:
1712 if (!UNDEF_P(vm_setivar_default(obj, id, val, dest_shape_id, index))) {
1713 return;
1714 }
1715 }
1716 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1717 }
1718}
1719
1720void
1721rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1722{
1723 vm_setinstancevariable(iseq, obj, id, val, ic);
1724}
1725
1726VALUE
1727rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1728{
1729 return vm_getinstancevariable(iseq, obj, id, ic);
1730}
1731
1732static VALUE
1733vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1734{
1735 /* continue throw */
1736
1737 if (FIXNUM_P(err)) {
1738 ec->tag->state = RUBY_TAG_FATAL;
1739 }
1740 else if (SYMBOL_P(err)) {
1741 ec->tag->state = TAG_THROW;
1742 }
1743 else if (THROW_DATA_P(err)) {
1744 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1745 }
1746 else {
1747 ec->tag->state = TAG_RAISE;
1748 }
1749 return err;
1750}
1751
1752static VALUE
1753vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1754 const int flag, const VALUE throwobj)
1755{
1756 const rb_control_frame_t *escape_cfp = NULL;
1757 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1758
1759 if (flag != 0) {
1760 /* do nothing */
1761 }
1762 else if (state == TAG_BREAK) {
1763 int is_orphan = 1;
1764 const VALUE *ep = GET_EP();
1765 const rb_iseq_t *base_iseq = GET_ISEQ();
1766 escape_cfp = reg_cfp;
1767
1768 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1769 if (ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) {
1770 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1771 ep = escape_cfp->ep;
1772 base_iseq = escape_cfp->iseq;
1773 }
1774 else {
1775 ep = VM_ENV_PREV_EP(ep);
1776 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1777 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1778 VM_ASSERT(escape_cfp->iseq == base_iseq);
1779 }
1780 }
1781
1782 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1783 /* lambda{... break ...} */
1784 is_orphan = 0;
1785 state = TAG_RETURN;
1786 }
1787 else {
1788 ep = VM_ENV_PREV_EP(ep);
1789
1790 while (escape_cfp < eocfp) {
1791 if (escape_cfp->ep == ep) {
1792 const rb_iseq_t *const iseq = escape_cfp->iseq;
1793 const VALUE epc = escape_cfp->pc - ISEQ_BODY(iseq)->iseq_encoded;
1794 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1795 unsigned int i;
1796
1797 if (!ct) break;
1798 for (i=0; i < ct->size; i++) {
1799 const struct iseq_catch_table_entry *const entry =
1800 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1801
1802 if (entry->type == CATCH_TYPE_BREAK &&
1803 entry->iseq == base_iseq &&
1804 entry->start < epc && entry->end >= epc) {
1805 if (entry->cont == epc) { /* found! */
1806 is_orphan = 0;
1807 }
1808 break;
1809 }
1810 }
1811 break;
1812 }
1813
1814 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1815 }
1816 }
1817
1818 if (is_orphan) {
1819 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1820 }
1821 }
1822 else if (state == TAG_RETRY) {
1823 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1824
1825 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1826 }
1827 else if (state == TAG_RETURN) {
1828 const VALUE *current_ep = GET_EP();
1829 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1830 int in_class_frame = 0;
1831 int toplevel = 1;
1832 escape_cfp = reg_cfp;
1833
1834 // find target_lep, target_ep
1835 while (!VM_ENV_LOCAL_P(ep)) {
1836 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1837 target_ep = ep;
1838 }
1839 ep = VM_ENV_PREV_EP(ep);
1840 }
1841 target_lep = ep;
1842
1843 while (escape_cfp < eocfp) {
1844 const VALUE *lep = VM_CF_LEP(escape_cfp);
1845
1846 if (!target_lep) {
1847 target_lep = lep;
1848 }
1849
1850 if (lep == target_lep &&
1851 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1852 ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) {
1853 in_class_frame = 1;
1854 target_lep = 0;
1855 }
1856
1857 if (lep == target_lep) {
1858 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1859 toplevel = 0;
1860 if (in_class_frame) {
1861 /* lambda {class A; ... return ...; end} */
1862 goto valid_return;
1863 }
1864 else {
1865 const VALUE *tep = current_ep;
1866
1867 while (target_lep != tep) {
1868 if (escape_cfp->ep == tep) {
1869 /* in lambda */
1870 if (tep == target_ep) {
1871 goto valid_return;
1872 }
1873 else {
1874 goto unexpected_return;
1875 }
1876 }
1877 tep = VM_ENV_PREV_EP(tep);
1878 }
1879 }
1880 }
1881 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1882 switch (ISEQ_BODY(escape_cfp->iseq)->type) {
1883 case ISEQ_TYPE_TOP:
1884 case ISEQ_TYPE_MAIN:
1885 if (toplevel) {
1886 if (in_class_frame) goto unexpected_return;
1887 if (target_ep == NULL) {
1888 goto valid_return;
1889 }
1890 else {
1891 goto unexpected_return;
1892 }
1893 }
1894 break;
1895 case ISEQ_TYPE_EVAL: {
1896 const rb_iseq_t *is = escape_cfp->iseq;
1897 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1898 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1899 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1900 t = ISEQ_BODY(is)->type;
1901 }
1902 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1903 break;
1904 }
1905 case ISEQ_TYPE_CLASS:
1906 toplevel = 0;
1907 break;
1908 default:
1909 break;
1910 }
1911 }
1912 }
1913
1914 if (escape_cfp->ep == target_lep && ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_METHOD) {
1915 if (target_ep == NULL) {
1916 goto valid_return;
1917 }
1918 else {
1919 goto unexpected_return;
1920 }
1921 }
1922
1923 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1924 }
1925 unexpected_return:;
1926 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1927
1928 valid_return:;
1929 /* do nothing */
1930 }
1931 else {
1932 rb_bug("isns(throw): unsupported throw type");
1933 }
1934
1935 ec->tag->state = state;
1936 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1937}
1938
1939static VALUE
1940vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1941 rb_num_t throw_state, VALUE throwobj)
1942{
1943 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1944 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1945
1946 if (state != 0) {
1947 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1948 }
1949 else {
1950 return vm_throw_continue(ec, throwobj);
1951 }
1952}
1953
1954VALUE
1955rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1956{
1957 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1958}
1959
1960static inline void
1961vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1962{
1963 int is_splat = flag & 0x01;
1964 const VALUE *ptr;
1965 rb_num_t len;
1966 const VALUE obj = ary;
1967
1968 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1969 ary = obj;
1970 ptr = &ary;
1971 len = 1;
1972 }
1973 else {
1974 ptr = RARRAY_CONST_PTR(ary);
1975 len = (rb_num_t)RARRAY_LEN(ary);
1976 }
1977
1978 if (num + is_splat == 0) {
1979 /* no space left on stack */
1980 }
1981 else if (flag & 0x02) {
1982 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1983 rb_num_t i = 0, j;
1984
1985 if (len < num) {
1986 for (i = 0; i < num - len; i++) {
1987 *cfp->sp++ = Qnil;
1988 }
1989 }
1990
1991 for (j = 0; i < num; i++, j++) {
1992 VALUE v = ptr[len - j - 1];
1993 *cfp->sp++ = v;
1994 }
1995
1996 if (is_splat) {
1997 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1998 }
1999 }
2000 else {
2001 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
2002 if (is_splat) {
2003 if (num > len) {
2004 *cfp->sp++ = rb_ary_new();
2005 }
2006 else {
2007 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
2008 }
2009 }
2010
2011 if (num > len) {
2012 rb_num_t i = 0;
2013 for (; i < num - len; i++) {
2014 *cfp->sp++ = Qnil;
2015 }
2016
2017 for (rb_num_t j = 0; i < num; i++, j++) {
2018 *cfp->sp++ = ptr[len - j - 1];
2019 }
2020 }
2021 else {
2022 for (rb_num_t j = 0; j < num; j++) {
2023 *cfp->sp++ = ptr[num - j - 1];
2024 }
2025 }
2026 }
2027
2028 RB_GC_GUARD(ary);
2029}
2030
2031static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2032
2033static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
2034
2035static struct rb_class_cc_entries *
2036vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
2037{
2038 int initial_capa = 2;
2039 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
2040#if VM_CHECK_MODE > 0
2041 ccs->debug_sig = ~(VALUE)ccs;
2042#endif
2043 ccs->capa = initial_capa;
2044 ccs->len = 0;
2045 ccs->cme = cme;
2046 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
2047
2048 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2049 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
2050 return ccs;
2051}
2052
2053static void
2054vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
2055{
2056 if (! vm_cc_markable(cc)) {
2057 return;
2058 }
2059
2060 if (UNLIKELY(ccs->len == ccs->capa)) {
2061 RUBY_ASSERT(ccs->capa > 0);
2062 ccs->capa *= 2;
2063 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
2064#if VM_CHECK_MODE > 0
2065 ccs->debug_sig = ~(VALUE)ccs;
2066#endif
2067 // GC?
2068 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2069 }
2070 VM_ASSERT(ccs->len < ccs->capa);
2071
2072 const int pos = ccs->len++;
2073 ccs->entries[pos].argc = vm_ci_argc(ci);
2074 ccs->entries[pos].flag = vm_ci_flag(ci);
2075 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
2076
2077 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
2078 // for tuning
2079 // vm_mtbl_dump(klass, 0);
2080 }
2081}
2082
2083#if VM_CHECK_MODE > 0
2084void
2085rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
2086{
2087 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
2088 for (int i=0; i<ccs->len; i++) {
2089 ruby_debug_printf("CCS CI ID:flag:%x argc:%u\n",
2090 ccs->entries[i].flag,
2091 ccs->entries[i].argc);
2092 rp(ccs->entries[i].cc);
2093 }
2094}
2095
2096static int
2097vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
2098{
2099 VM_ASSERT(vm_ccs_p(ccs));
2100 VM_ASSERT(ccs->len <= ccs->capa);
2101
2102 for (int i=0; i<ccs->len; i++) {
2103 const struct rb_callcache *cc = ccs->entries[i].cc;
2104
2105 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2106 VM_ASSERT(vm_cc_class_check(cc, klass));
2107 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
2108 VM_ASSERT(!vm_cc_super_p(cc));
2109 VM_ASSERT(!vm_cc_refinement_p(cc));
2110 }
2111 return TRUE;
2112}
2113#endif
2114
2115const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
2116
2117static void
2118vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2119{
2120 ASSERT_vm_locking();
2121
2122 if (rb_multi_ractor_p()) {
2123 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2124 // Another ractor updated the CC table while we were waiting on the VM lock.
2125 // We have to retry.
2126 return;
2127 }
2128
2129 VALUE ccs_obj = 0;
2130 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2131 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2132
2133 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2134 // Another ractor replaced that entry while we were waiting on the VM lock.
2135 return;
2136 }
2137
2138 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2139 rb_vm_cc_table_delete(new_table, mid);
2140 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2141 }
2142 else {
2143 rb_vm_cc_table_delete(cc_tbl, mid);
2144 }
2145}
2146
2147static const struct rb_callcache *
2148vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2149{
2150 ASSERT_vm_locking();
2151
2152 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2153
2154 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2155
2156 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2157
2158 if (cme == NULL) {
2159 // undef or not found: can't cache the information
2160 VM_ASSERT(vm_cc_cme(&vm_empty_cc) == NULL);
2161 return &vm_empty_cc;
2162 }
2163
2164 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2165 const VALUE original_cc_table = cc_tbl;
2166 if (!cc_tbl) {
2167 // Is this possible after rb_callable_method_entry ?
2168 cc_tbl = rb_vm_cc_table_create(1);
2169 }
2170 else if (rb_multi_ractor_p()) {
2171 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2172 }
2173
2174 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2175
2176 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2177
2178 VM_ASSERT(cc_tbl);
2179
2180 struct rb_class_cc_entries *ccs = NULL;
2181 {
2182 VALUE ccs_obj;
2183 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2184 ccs = (struct rb_class_cc_entries *)ccs_obj;
2185 }
2186 else {
2187 // TODO: required?
2188 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2189 }
2190 }
2191
2192 cme = rb_check_overloaded_cme(cme, ci);
2193
2194 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2195 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2196
2197 VM_ASSERT(vm_cc_cme(cc) != NULL);
2198 VM_ASSERT(cme->called_id == mid);
2199 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2200
2201 if (original_cc_table != cc_tbl) {
2202 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2203 }
2204
2205 return cc;
2206}
2207
2208static const struct rb_callcache *
2209vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2210{
2211 VALUE cc_tbl;
2212 struct rb_class_cc_entries *ccs;
2213retry:
2214 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2215 ccs = NULL;
2216
2217 if (cc_tbl) {
2218 // CCS data is keyed on method id, so we don't need the method id
2219 // for doing comparisons in the `for` loop below.
2220
2221 VALUE ccs_obj;
2222 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2223 ccs = (struct rb_class_cc_entries *)ccs_obj;
2224 const int ccs_len = ccs->len;
2225
2226 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2227 RB_VM_LOCKING() {
2228 vm_evict_cc(klass, cc_tbl, mid);
2229 }
2230 goto retry;
2231 }
2232 else {
2233 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2234
2235 // We already know the method id is correct because we had
2236 // to look up the ccs_data by method id. All we need to
2237 // compare is argc and flag
2238 unsigned int argc = vm_ci_argc(ci);
2239 unsigned int flag = vm_ci_flag(ci);
2240
2241 for (int i=0; i<ccs_len; i++) {
2242 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2243 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2244 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2245
2246 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2247
2248 if (ccs_ci_argc == argc && ccs_ci_flag == flag) {
2249 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2250
2251 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2252 VM_ASSERT(ccs_cc->klass == klass);
2253 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2254
2255 return ccs_cc;
2256 }
2257 }
2258 }
2259 }
2260 }
2261
2262 RB_GC_GUARD(cc_tbl);
2263 return NULL;
2264}
2265
2266static const struct rb_callcache *
2267vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2268{
2269 const ID mid = vm_ci_mid(ci);
2270
2271 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2272 if (cc) {
2273 return cc;
2274 }
2275
2276 RB_VM_LOCKING() {
2277 if (rb_multi_ractor_p()) {
2278 // The CC may have been populated by another ractor while we were waiting on the lock,
2279 // so we must lookup a second time.
2280 cc = vm_lookup_cc(klass, ci, mid);
2281 }
2282
2283 if (!cc) {
2284 cc = vm_populate_cc(klass, ci, mid);
2285 }
2286 }
2287
2288 return cc;
2289}
2290
2291const struct rb_callcache *
2292rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2293{
2294 const struct rb_callcache *cc;
2295
2296 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2297
2298 cc = vm_search_cc(klass, ci);
2299
2300 VM_ASSERT(cc);
2301 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2302 VM_ASSERT(cc == vm_cc_empty() || cc->klass == klass);
2303 VM_ASSERT(cc == vm_cc_empty() || callable_method_entry_p(vm_cc_cme(cc)));
2304 VM_ASSERT(cc == vm_cc_empty() || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2305 VM_ASSERT(cc == vm_cc_empty() || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2306
2307 return cc;
2308}
2309
2310static const struct rb_callcache *
2311vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2312{
2313#if USE_DEBUG_COUNTER
2314 const struct rb_callcache *old_cc = cd->cc;
2315#endif
2316
2317 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2318
2319#if OPT_INLINE_METHOD_CACHE
2320 cd->cc = cc;
2321
2322 const struct rb_callcache *empty_cc = &vm_empty_cc;
2323 if (cd_owner && cc != empty_cc) {
2324 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2325 }
2326
2327#if USE_DEBUG_COUNTER
2328 if (!old_cc || old_cc == empty_cc) {
2329 // empty
2330 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2331 }
2332 else if (old_cc == cc) {
2333 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2334 }
2335 else if (vm_cc_cme(old_cc) == vm_cc_cme(cc)) {
2336 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2337 }
2338 else if (vm_cc_cme(old_cc) && vm_cc_cme(cc) &&
2339 vm_cc_cme(old_cc)->def == vm_cc_cme(cc)->def) {
2340 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2341 }
2342 else {
2343 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2344 }
2345#endif
2346#endif // OPT_INLINE_METHOD_CACHE
2347
2348 VM_ASSERT(vm_cc_cme(cc) == NULL ||
2349 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci));
2350
2351 return cc;
2352}
2353
2354ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass));
2355static const struct rb_callcache *
2356vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2357{
2358 const struct rb_callcache *cc = cd->cc;
2359
2360#if OPT_INLINE_METHOD_CACHE
2361 if (LIKELY(vm_cc_class_check(cc, klass))) {
2362 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2363 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2364 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2365 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2366 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2367 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2368
2369 return cc;
2370 }
2371 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2372 }
2373 else {
2374 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2375 }
2376#endif
2377
2378 return vm_search_method_slowpath0(cd_owner, cd, klass);
2379}
2380
2381static const struct rb_callable_method_entry_struct *
2382vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2383{
2384 VALUE klass = CLASS_OF(recv);
2385 VM_ASSERT(klass != Qfalse);
2386 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2387
2388 const struct rb_callcache *cc = vm_search_method_fastpath(cd_owner, cd, klass);
2389 return vm_cc_cme(cc);
2390}
2391
2392const struct rb_callable_method_entry_struct *
2393rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2394{
2395 return vm_search_method(cd_owner, cd, recv);
2396}
2397
2398#if __has_attribute(transparent_union)
2399typedef union {
2400 VALUE (*anyargs)(ANYARGS);
2401 VALUE (*f00)(VALUE);
2402 VALUE (*f01)(VALUE, VALUE);
2403 VALUE (*f02)(VALUE, VALUE, VALUE);
2404 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2405 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2406 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2407 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2408 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2409 VALUE (*f08)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2410 VALUE (*f09)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2411 VALUE (*f10)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2412 VALUE (*f11)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2413 VALUE (*f12)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2414 VALUE (*f13)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2415 VALUE (*f14)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2416 VALUE (*f15)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2417 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2418} __attribute__((__transparent_union__)) cfunc_type;
2419# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2420#else
2421typedef VALUE (*cfunc_type)(ANYARGS);
2422# define make_cfunc_type(f) (cfunc_type)(f)
2423#endif
2424
2425static inline int
2426check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2427{
2428 if (! me) {
2429 return false;
2430 }
2431 else {
2432 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2433 VM_ASSERT(callable_method_entry_p(me));
2434 VM_ASSERT(me->def);
2435 if (me->def->type != VM_METHOD_TYPE_CFUNC) {
2436 return false;
2437 }
2438 else {
2439#if __has_attribute(transparent_union)
2440 return me->def->body.cfunc.func == func.anyargs;
2441#else
2442 return me->def->body.cfunc.func == func;
2443#endif
2444 }
2445 }
2446}
2447
2448static inline int
2449check_method_basic_definition(const rb_callable_method_entry_t *me)
2450{
2451 return me && METHOD_ENTRY_BASIC(me);
2452}
2453
2454static inline int
2455vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2456{
2457 VM_ASSERT(iseq != NULL);
2458 const struct rb_callable_method_entry_struct *cme = vm_search_method((VALUE)iseq, cd, recv);
2459 return check_cfunc(cme, func);
2460}
2461
2462bool
2463rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2464{
2465 return check_cfunc(me, func);
2466}
2467
2468int
2469rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2470{
2471 return vm_method_cfunc_is(iseq, cd, recv, func);
2472}
2473
2474#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2475#define vm_method_cfunc_is(iseq, cd, recv, func) vm_method_cfunc_is(iseq, cd, recv, make_cfunc_type(func))
2476
2477#define EQ_UNREDEFINED_P(t) BASIC_OP_UNREDEFINED_P(BOP_EQ, t##_REDEFINED_OP_FLAG)
2478
2479static inline bool
2480FIXNUM_2_P(VALUE a, VALUE b)
2481{
2482 /* FIXNUM_P(a) && FIXNUM_P(b)
2483 * == ((a & 1) && (b & 1))
2484 * == a & b & 1 */
2485 SIGNED_VALUE x = a;
2486 SIGNED_VALUE y = b;
2487 SIGNED_VALUE z = x & y & 1;
2488 return z == 1;
2489}
2490
2491static inline bool
2492FLONUM_2_P(VALUE a, VALUE b)
2493{
2494#if USE_FLONUM
2495 /* FLONUM_P(a) && FLONUM_P(b)
2496 * == ((a & 3) == 2) && ((b & 3) == 2)
2497 * == ! ((a ^ 2) | (b ^ 2) & 3)
2498 */
2499 SIGNED_VALUE x = a;
2500 SIGNED_VALUE y = b;
2501 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2502 return !z;
2503#else
2504 return false;
2505#endif
2506}
2507
2508static VALUE
2509opt_equality_specialized(VALUE recv, VALUE obj)
2510{
2511 if (FIXNUM_2_P(recv, obj) && EQ_UNREDEFINED_P(INTEGER)) {
2512 goto compare_by_identity;
2513 }
2514 else if (FLONUM_2_P(recv, obj) && EQ_UNREDEFINED_P(FLOAT)) {
2515 goto compare_by_identity;
2516 }
2517 else if (STATIC_SYM_P(recv) && STATIC_SYM_P(obj) && EQ_UNREDEFINED_P(SYMBOL)) {
2518 goto compare_by_identity;
2519 }
2520 else if (SPECIAL_CONST_P(recv)) {
2521 //
2522 }
2523 else if (RBASIC_CLASS(recv) == rb_cFloat && RB_FLOAT_TYPE_P(obj) && EQ_UNREDEFINED_P(FLOAT)) {
2524 double a = RFLOAT_VALUE(recv);
2525 double b = RFLOAT_VALUE(obj);
2526
2527 return RBOOL(a == b);
2528 }
2529 else if (RBASIC_CLASS(recv) == rb_cString && EQ_UNREDEFINED_P(STRING)) {
2530 if (recv == obj) {
2531 return Qtrue;
2532 }
2533 else if (RB_TYPE_P(obj, T_STRING)) {
2534 return rb_str_eql_internal(obj, recv);
2535 }
2536 }
2537 return Qundef;
2538
2539 compare_by_identity:
2540 return RBOOL(recv == obj);
2541}
2542
2543static VALUE
2544opt_equality(const rb_iseq_t *cd_owner, VALUE recv, VALUE obj, CALL_DATA cd)
2545{
2546 VM_ASSERT(cd_owner != NULL);
2547
2548 VALUE val = opt_equality_specialized(recv, obj);
2549 if (!UNDEF_P(val)) return val;
2550
2551 if (!vm_method_cfunc_is(cd_owner, cd, recv, rb_obj_equal)) {
2552 return Qundef;
2553 }
2554 else {
2555 return RBOOL(recv == obj);
2556 }
2557}
2558
2559#undef EQ_UNREDEFINED_P
2560
2561static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2562NOINLINE(static VALUE opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid));
2563
2564static VALUE
2565opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid)
2566{
2567 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2568
2569 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2570 return RBOOL(recv == obj);
2571 }
2572 else {
2573 return Qundef;
2574 }
2575}
2576
2577static VALUE
2578opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2579{
2580 VALUE val = opt_equality_specialized(recv, obj);
2581 if (!UNDEF_P(val)) {
2582 return val;
2583 }
2584 else {
2585 return opt_equality_by_mid_slowpath(recv, obj, mid);
2586 }
2587}
2588
2589VALUE
2590rb_equal_opt(VALUE obj1, VALUE obj2)
2591{
2592 return opt_equality_by_mid(obj1, obj2, idEq);
2593}
2594
2595VALUE
2596rb_eql_opt(VALUE obj1, VALUE obj2)
2597{
2598 return opt_equality_by_mid(obj1, obj2, idEqlP);
2599}
2600
2601extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2602extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2603
2604static VALUE
2605check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2606{
2607 switch (type) {
2608 case VM_CHECKMATCH_TYPE_WHEN:
2609 return pattern;
2610 case VM_CHECKMATCH_TYPE_RESCUE:
2611 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2612 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2613 }
2614 /* fall through */
2615 case VM_CHECKMATCH_TYPE_CASE: {
2616 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2617 }
2618 default:
2619 rb_bug("check_match: unreachable");
2620 }
2621}
2622
2623
2624static inline VALUE
2625double_cmp_lt(double a, double b)
2626{
2627 return RBOOL(a < b);
2628}
2629
2630static inline VALUE
2631double_cmp_le(double a, double b)
2632{
2633 return RBOOL(a <= b);
2634}
2635
2636static inline VALUE
2637double_cmp_gt(double a, double b)
2638{
2639 return RBOOL(a > b);
2640}
2641
2642static inline VALUE
2643double_cmp_ge(double a, double b)
2644{
2645 return RBOOL(a >= b);
2646}
2647
2648// Copied by vm_dump.c
2649static inline VALUE *
2650vm_base_ptr(const rb_control_frame_t *cfp)
2651{
2652 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2653
2654 if (cfp->iseq && VM_FRAME_RUBYFRAME_P(cfp)) {
2655 VALUE *bp = prev_cfp->sp + ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE;
2656
2657 if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2658 int lts = ISEQ_BODY(cfp->iseq)->local_table_size;
2659 int params = ISEQ_BODY(cfp->iseq)->param.size;
2660
2661 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2662 bp += vm_ci_argc(ci);
2663 }
2664
2665 if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2666 /* adjust `self' */
2667 bp += 1;
2668 }
2669#if VM_DEBUG_BP_CHECK
2670 if (bp != cfp->bp_check) {
2671 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2672 (long)(cfp->bp_check - GET_EC()->vm_stack),
2673 (long)(bp - GET_EC()->vm_stack));
2674 rb_bug("vm_base_ptr: unreachable");
2675 }
2676#endif
2677 return bp;
2678 }
2679 else {
2680 return NULL;
2681 }
2682}
2683
2684VALUE *
2685rb_vm_base_ptr(const rb_control_frame_t *cfp)
2686{
2687 return vm_base_ptr(cfp);
2688}
2689
2690/* method call processes with call_info */
2691
2692#include "vm_args.c"
2693
2694static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size);
2695ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2696static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc);
2697static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2698static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2699static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2700static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2701
2702static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2703
2704static VALUE
2705vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2706{
2707 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2708
2709 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0);
2710}
2711
2712static VALUE
2713vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2714{
2715 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2716
2717 const struct rb_callcache *cc = calling->cc;
2718 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2719 int param = ISEQ_BODY(iseq)->param.size;
2720 int local = ISEQ_BODY(iseq)->local_table_size;
2721 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2722}
2723
2724bool
2725rb_simple_iseq_p(const rb_iseq_t *iseq)
2726{
2727 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2728 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2729 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2730 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2731 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2732 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2733 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2734 ISEQ_BODY(iseq)->param.flags.has_block == FALSE;
2735}
2736
2737bool
2738rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2739{
2740 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2741 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2742 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2743 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2744 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2745 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2746 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2747 ISEQ_BODY(iseq)->param.flags.has_block == FALSE;
2748}
2749
2750bool
2751rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2752{
2753 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2754 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2755 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2756 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2757 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2758 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2759 ISEQ_BODY(iseq)->param.flags.has_block == FALSE;
2760}
2761
2762#define ALLOW_HEAP_ARGV (-2)
2763#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2764
2765static inline bool
2766vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2767{
2768 vm_check_canary(GET_EC(), cfp->sp);
2769 bool ret = false;
2770
2771 if (!NIL_P(ary)) {
2772 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2773 long len = RARRAY_LEN(ary);
2774 int argc = calling->argc;
2775
2776 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2777 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2778 * a temporary array, instead of trying to keeping arguments on the VM stack.
2779 */
2780 VALUE *argv = cfp->sp - argc;
2781 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2782 rb_ary_cat(argv_ary, argv, argc);
2783 rb_ary_cat(argv_ary, ptr, len);
2784 cfp->sp -= argc - 1;
2785 cfp->sp[-1] = argv_ary;
2786 calling->argc = 1;
2787 calling->heap_argv = argv_ary;
2788 RB_GC_GUARD(ary);
2789 }
2790 else {
2791 long i;
2792
2793 if (max_args >= 0 && len + argc > max_args) {
2794 /* If only a given max_args is allowed, copy up to max args.
2795 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2796 * where additional arguments are ignored.
2797 *
2798 * Also, copy up to one more argument than the maximum,
2799 * in case it is an empty keyword hash that will be removed.
2800 */
2801 calling->argc += len - (max_args - argc + 1);
2802 len = max_args - argc + 1;
2803 ret = true;
2804 }
2805 else {
2806 /* Unset heap_argv if set originally. Can happen when
2807 * forwarding modified arguments, where heap_argv was used
2808 * originally, but heap_argv not supported by the forwarded
2809 * method in all cases.
2810 */
2811 calling->heap_argv = 0;
2812 }
2813 CHECK_VM_STACK_OVERFLOW(cfp, len);
2814
2815 for (i = 0; i < len; i++) {
2816 *cfp->sp++ = ptr[i];
2817 }
2818 calling->argc += i;
2819 }
2820 }
2821
2822 return ret;
2823}
2824
2825static inline void
2826vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2827{
2828 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2829 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2830 const VALUE h = rb_hash_new_with_size(kw_len);
2831 VALUE *sp = cfp->sp;
2832 int i;
2833
2834 for (i=0; i<kw_len; i++) {
2835 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2836 }
2837 (sp-kw_len)[0] = h;
2838
2839 cfp->sp -= kw_len - 1;
2840 calling->argc -= kw_len - 1;
2841 calling->kw_splat = 1;
2842}
2843
2844static inline VALUE
2845vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2846{
2847 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2848 if (keyword_hash != Qnil) {
2849 /* Convert a non-hash keyword splat to a new hash */
2850 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2851 }
2852 }
2853 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2854 /* Convert a hash keyword splat to a new hash unless
2855 * a mutable keyword splat was passed.
2856 * Skip allocating new hash for empty keyword splat, as empty
2857 * keyword splat will be ignored by both callers.
2858 */
2859 keyword_hash = rb_hash_dup(keyword_hash);
2860 }
2861 return keyword_hash;
2862}
2863
2864static inline void
2865CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2866 struct rb_calling_info *restrict calling,
2867 const struct rb_callinfo *restrict ci, int max_args)
2868{
2869 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2870 if (IS_ARGS_KW_SPLAT(ci)) {
2871 // f(*a, **kw)
2872 VM_ASSERT(calling->kw_splat == 1);
2873
2874 cfp->sp -= 2;
2875 calling->argc -= 2;
2876 VALUE ary = cfp->sp[0];
2877 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2878
2879 // splat a
2880 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2881
2882 // put kw
2883 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2884 if (UNLIKELY(calling->heap_argv)) {
2885 rb_ary_push(calling->heap_argv, kwh);
2886 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2887 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2888 calling->kw_splat = 0;
2889 }
2890 }
2891 else {
2892 cfp->sp[0] = kwh;
2893 cfp->sp++;
2894 calling->argc++;
2895
2896 VM_ASSERT(calling->kw_splat == 1);
2897 }
2898 }
2899 else {
2900 calling->kw_splat = 0;
2901 }
2902 }
2903 else {
2904 // f(*a)
2905 VM_ASSERT(calling->kw_splat == 0);
2906
2907 cfp->sp -= 1;
2908 calling->argc -= 1;
2909 VALUE ary = cfp->sp[0];
2910
2911 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2912 goto check_keyword;
2913 }
2914
2915 // check the last argument
2916 VALUE last_hash, argv_ary;
2917 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2918 if (!IS_ARGS_KEYWORD(ci) &&
2919 RARRAY_LEN(argv_ary) > 0 &&
2920 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2921 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2922
2923 rb_ary_pop(argv_ary);
2924 if (!RHASH_EMPTY_P(last_hash)) {
2925 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2926 calling->kw_splat = 1;
2927 }
2928 }
2929 }
2930 else {
2931check_keyword:
2932 if (!IS_ARGS_KEYWORD(ci) &&
2933 calling->argc > 0 &&
2934 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2935 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2936
2937 if (RHASH_EMPTY_P(last_hash)) {
2938 calling->argc--;
2939 cfp->sp -= 1;
2940 }
2941 else {
2942 cfp->sp[-1] = rb_hash_dup(last_hash);
2943 calling->kw_splat = 1;
2944 }
2945 }
2946 }
2947 }
2948 }
2949 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2950 // f(**kw)
2951 VM_ASSERT(calling->kw_splat == 1);
2952 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2953
2954 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2955 cfp->sp--;
2956 calling->argc--;
2957 calling->kw_splat = 0;
2958 }
2959 else {
2960 cfp->sp[-1] = kwh;
2961 }
2962 }
2963 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
2964 // f(k1:1, k2:2)
2965 VM_ASSERT(calling->kw_splat == 0);
2966
2967 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
2968 * by creating a keyword hash.
2969 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
2970 */
2971 vm_caller_setup_arg_kw(cfp, calling, ci);
2972 }
2973}
2974
2975#define USE_OPT_HIST 0
2976
2977#if USE_OPT_HIST
2978#define OPT_HIST_MAX 64
2979static int opt_hist[OPT_HIST_MAX+1];
2980
2981__attribute__((destructor))
2982static void
2983opt_hist_show_results_at_exit(void)
2984{
2985 for (int i=0; i<OPT_HIST_MAX; i++) {
2986 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
2987 }
2988}
2989#endif
2990
2991static VALUE
2992vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2993 struct rb_calling_info *calling)
2994{
2995 const struct rb_callcache *cc = calling->cc;
2996 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2997 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
2998 const int opt = calling->argc - lead_num;
2999 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3000 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3001 const int param = ISEQ_BODY(iseq)->param.size;
3002 const int local = ISEQ_BODY(iseq)->local_table_size;
3003 const int delta = opt_num - opt;
3004
3005 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3006
3007#if USE_OPT_HIST
3008 if (opt_pc < OPT_HIST_MAX) {
3009 opt_hist[opt]++;
3010 }
3011 else {
3012 opt_hist[OPT_HIST_MAX]++;
3013 }
3014#endif
3015
3016 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
3017}
3018
3019static VALUE
3020vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3021 struct rb_calling_info *calling)
3022{
3023 const struct rb_callcache *cc = calling->cc;
3024 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3025 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3026 const int opt = calling->argc - lead_num;
3027 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3028
3029 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3030
3031#if USE_OPT_HIST
3032 if (opt_pc < OPT_HIST_MAX) {
3033 opt_hist[opt]++;
3034 }
3035 else {
3036 opt_hist[OPT_HIST_MAX]++;
3037 }
3038#endif
3039
3040 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3041}
3042
3043static void
3044args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
3045 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
3046 VALUE *const locals);
3047
3048static VALUE
3049vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3050 struct rb_calling_info *calling)
3051{
3052 const struct rb_callcache *cc = calling->cc;
3053 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3054 int param_size = ISEQ_BODY(iseq)->param.size;
3055 int local_size = ISEQ_BODY(iseq)->local_table_size;
3056
3057 // Setting up local size and param size
3058 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3059
3060 local_size = local_size + vm_ci_argc(calling->cd->ci);
3061 param_size = param_size + vm_ci_argc(calling->cd->ci);
3062
3063 cfp->sp[0] = (VALUE)calling->cd->ci;
3064
3065 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
3066}
3067
3068static VALUE
3069vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3070 struct rb_calling_info *calling)
3071{
3072 const struct rb_callinfo *ci = calling->cd->ci;
3073 const struct rb_callcache *cc = calling->cc;
3074
3075 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
3076 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
3077
3078 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3079 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3080 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3081 const int ci_kw_len = kw_arg->keyword_len;
3082 const VALUE * const ci_keywords = kw_arg->keywords;
3083 VALUE *argv = cfp->sp - calling->argc;
3084 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3085 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3086 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3087 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3088 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3089
3090 int param = ISEQ_BODY(iseq)->param.size;
3091 int local = ISEQ_BODY(iseq)->local_table_size;
3092 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3093}
3094
3095static VALUE
3096vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3097 struct rb_calling_info *calling)
3098{
3099 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
3100 const struct rb_callcache *cc = calling->cc;
3101
3102 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
3103 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3104
3105 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3106 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3107 VALUE * const argv = cfp->sp - calling->argc;
3108 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3109
3110 int i;
3111 for (i=0; i<kw_param->num; i++) {
3112 klocals[i] = kw_param->default_values[i];
3113 }
3114 klocals[i] = INT2FIX(0); // kw specify flag
3115 // NOTE:
3116 // nobody check this value, but it should be cleared because it can
3117 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3118
3119 int param = ISEQ_BODY(iseq)->param.size;
3120 int local = ISEQ_BODY(iseq)->local_table_size;
3121 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3122}
3123
3124static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3125
3126static VALUE
3127vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3128 struct rb_calling_info *calling)
3129{
3130 const struct rb_builtin_function *bf = calling->cc->aux_.bf;
3131 cfp->sp -= (calling->argc + 1);
3132 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3133 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3134}
3135
3136VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3137
3138static void
3139warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3140{
3141 rb_vm_t *vm = GET_VM();
3142 set_table *dup_check_table = vm->unused_block_warning_table;
3143 st_data_t key;
3144 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3145
3146 union {
3147 VALUE v;
3148 unsigned char b[SIZEOF_VALUE];
3149 } k1 = {
3150 .v = (VALUE)pc,
3151 }, k2 = {
3152 .v = (VALUE)cme->def,
3153 };
3154
3155 // relax check
3156 if (!strict_unused_block) {
3157 key = (st_data_t)cme->def->original_id;
3158
3159 if (set_table_lookup(dup_check_table, key)) {
3160 return;
3161 }
3162 }
3163
3164 // strict check
3165 // make unique key from pc and me->def pointer
3166 key = 0;
3167 for (int i=0; i<SIZEOF_VALUE; i++) {
3168 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3169 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3170 }
3171
3172 if (0) {
3173 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3174 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3175 fprintf(stderr, "key:%p\n", (void *)key);
3176 }
3177
3178 // duplication check
3179 if (set_insert(dup_check_table, key)) {
3180 // already shown
3181 }
3182 else if (RTEST(ruby_verbose) || strict_unused_block) {
3183 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3184 VALUE name = rb_gen_method_name(cme->defined_class, ISEQ_BODY(iseq)->location.base_label);
3185
3186 if (!NIL_P(m_loc)) {
3187 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3188 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3189 }
3190 else {
3191 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3192 }
3193 }
3194}
3195
3196static inline int
3197vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3198 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3199{
3200 const struct rb_callinfo *ci = calling->cd->ci;
3201 const struct rb_callcache *cc = calling->cc;
3202
3203 VM_ASSERT((vm_ci_argc(ci), 1));
3204 VM_ASSERT(vm_cc_cme(cc) != NULL);
3205
3206 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3207 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3208 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3209 warn_unused_block(vm_cc_cme(cc), iseq, (void *)ec->cfp->pc);
3210 }
3211
3212 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3213 if (LIKELY(rb_simple_iseq_p(iseq))) {
3214 rb_control_frame_t *cfp = ec->cfp;
3215 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3216 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3217
3218 if (calling->argc != lead_num) {
3219 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3220 }
3221
3222 //VM_ASSERT(ci == calling->cd->ci);
3223 VM_ASSERT(cc == calling->cc);
3224
3225 if (vm_call_iseq_optimizable_p(ci, cc)) {
3226 if ((iseq->body->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3227 VM_ASSERT(iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF);
3228 vm_cc_bf_set(cc, (void *)iseq->body->iseq_encoded[1]);
3229 CC_SET_FASTPATH(cc, vm_call_single_noarg_leaf_builtin, true);
3230 }
3231 else {
3232 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3233 }
3234 }
3235 return 0;
3236 }
3237 else if (rb_iseq_only_optparam_p(iseq)) {
3238 rb_control_frame_t *cfp = ec->cfp;
3239
3240 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3241 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3242
3243 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3244 const int argc = calling->argc;
3245 const int opt = argc - lead_num;
3246
3247 if (opt < 0 || opt > opt_num) {
3248 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3249 }
3250
3251 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3252 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3253 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3254 vm_call_cacheable(ci, cc));
3255 }
3256 else {
3257 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3258 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3259 vm_call_cacheable(ci, cc));
3260 }
3261
3262 /* initialize opt vars for self-references */
3263 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3264 for (int i=argc; i<lead_num + opt_num; i++) {
3265 argv[i] = Qnil;
3266 }
3267 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3268 }
3269 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3270 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3271 const int argc = calling->argc;
3272 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3273
3274 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3275 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3276
3277 if (argc - kw_arg->keyword_len == lead_num) {
3278 const int ci_kw_len = kw_arg->keyword_len;
3279 const VALUE * const ci_keywords = kw_arg->keywords;
3280 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3281 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3282
3283 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3284 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3285
3286 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3287 vm_call_cacheable(ci, cc));
3288
3289 return 0;
3290 }
3291 }
3292 else if (argc == lead_num) {
3293 /* no kwarg */
3294 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3295 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3296
3297 if (klocals[kw_param->num] == INT2FIX(0)) {
3298 /* copy from default_values */
3299 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3300 vm_call_cacheable(ci, cc));
3301 }
3302
3303 return 0;
3304 }
3305 }
3306 }
3307
3308 // Called iseq is using ... param
3309 // def foo(...) # <- iseq for foo will have "forwardable"
3310 //
3311 // We want to set the `...` local to the caller's CI
3312 // foo(1, 2) # <- the ci for this should end up as `...`
3313 //
3314 // So hopefully the stack looks like:
3315 //
3316 // => 1
3317 // => 2
3318 // => *
3319 // => **
3320 // => &
3321 // => ... # <- points at `foo`s CI
3322 // => cref_or_me
3323 // => specval
3324 // => type
3325 //
3326 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3327 bool can_fastpath = true;
3328
3329 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3330 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3331 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3332 ci = vm_ci_new_runtime(
3333 vm_ci_mid(ci),
3334 vm_ci_flag(ci),
3335 vm_ci_argc(ci),
3336 vm_ci_kwarg(ci));
3337 }
3338 else {
3339 ci = forward_cd->caller_ci;
3340 }
3341 can_fastpath = false;
3342 }
3343 // C functions calling iseqs will stack allocate a CI,
3344 // so we need to convert it to heap allocated
3345 if (!vm_ci_markable(ci)) {
3346 ci = vm_ci_new_runtime(
3347 vm_ci_mid(ci),
3348 vm_ci_flag(ci),
3349 vm_ci_argc(ci),
3350 vm_ci_kwarg(ci));
3351 can_fastpath = false;
3352 }
3353 argv[param_size - 1] = (VALUE)ci;
3354 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3355 return 0;
3356 }
3357
3358 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3359}
3360
3361static void
3362vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3363{
3364 // This case is when the caller is using a ... parameter.
3365 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3366 // In this case the caller's caller's CI will be on the stack.
3367 //
3368 // For example:
3369 //
3370 // def bar(a, b); a + b; end
3371 // def foo(...); bar(...); end
3372 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3373 //
3374 // Stack layout will be:
3375 //
3376 // > 1
3377 // > 2
3378 // > CI for foo(1, 2)
3379 // > cref_or_me
3380 // > specval
3381 // > type
3382 // > receiver
3383 // > CI for foo(1, 2), via `getlocal ...`
3384 // > ( SP points here )
3385 const VALUE * lep = VM_CF_LEP(cfp);
3386
3387 const rb_iseq_t *iseq;
3388
3389 // If we're in an escaped environment (lambda for example), get the iseq
3390 // from the captured env.
3391 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3392 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3393 iseq = env->iseq;
3394 }
3395 else { // Otherwise use the lep to find the caller
3396 iseq = rb_vm_search_cf_from_ep(ec, cfp, lep)->iseq;
3397 }
3398
3399 // Our local storage is below the args we need to copy
3400 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3401
3402 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3403 VALUE * to = cfp->sp - 1; // clobber the CI
3404
3405 if (RTEST(splat)) {
3406 to -= 1; // clobber the splat array
3407 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3408 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3409 to += RARRAY_LEN(splat);
3410 }
3411
3412 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3413 MEMCPY(to, from, VALUE, argc);
3414 cfp->sp = to + argc;
3415
3416 // Stack layout should now be:
3417 //
3418 // > 1
3419 // > 2
3420 // > CI for foo(1, 2)
3421 // > cref_or_me
3422 // > specval
3423 // > type
3424 // > receiver
3425 // > 1
3426 // > 2
3427 // > ( SP points here )
3428}
3429
3430static VALUE
3431vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3432{
3433 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3434
3435 const struct rb_callcache *cc = calling->cc;
3436 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3437 int param_size = ISEQ_BODY(iseq)->param.size;
3438 int local_size = ISEQ_BODY(iseq)->local_table_size;
3439
3440 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3441
3442 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3443 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3444}
3445
3446static VALUE
3447vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3448{
3449 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3450
3451 const struct rb_callcache *cc = calling->cc;
3452 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3453 int param_size = ISEQ_BODY(iseq)->param.size;
3454 int local_size = ISEQ_BODY(iseq)->local_table_size;
3455
3456 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3457
3458 // Setting up local size and param size
3459 local_size = local_size + vm_ci_argc(calling->cd->ci);
3460 param_size = param_size + vm_ci_argc(calling->cd->ci);
3461
3462 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3463 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3464}
3465
3466static inline VALUE
3467vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3468 int opt_pc, int param_size, int local_size)
3469{
3470 const struct rb_callinfo *ci = calling->cd->ci;
3471 const struct rb_callcache *cc = calling->cc;
3472
3473 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3474 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3475 }
3476 else {
3477 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3478 }
3479}
3480
3481static inline VALUE
3482vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3483 int opt_pc, int param_size, int local_size)
3484{
3485 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3486 VALUE *argv = cfp->sp - calling->argc;
3487 VALUE *sp = argv + param_size;
3488 cfp->sp = argv - 1 /* recv */;
3489
3490 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3491 calling->block_handler, (VALUE)me,
3492 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3493 local_size - param_size,
3494 ISEQ_BODY(iseq)->stack_max);
3495 return Qundef;
3496}
3497
3498static inline VALUE
3499vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc)
3500{
3501 const struct rb_callcache *cc = calling->cc;
3502 unsigned int i;
3503 VALUE *argv = cfp->sp - calling->argc;
3504 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3505 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3506 VALUE *src_argv = argv;
3507 VALUE *sp_orig, *sp;
3508 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3509
3510 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3511 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3512 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3513 dst_captured->code.val = src_captured->code.val;
3514 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3515 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3516 }
3517 else {
3518 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3519 }
3520 }
3521
3522 vm_pop_frame(ec, cfp, cfp->ep);
3523 cfp = ec->cfp;
3524
3525 sp_orig = sp = cfp->sp;
3526
3527 /* push self */
3528 sp[0] = calling->recv;
3529 sp++;
3530
3531 /* copy arguments */
3532 for (i=0; i < ISEQ_BODY(iseq)->param.size; i++) {
3533 *sp++ = src_argv[i];
3534 }
3535
3536 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3537 calling->recv, calling->block_handler, (VALUE)me,
3538 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3539 ISEQ_BODY(iseq)->local_table_size - ISEQ_BODY(iseq)->param.size,
3540 ISEQ_BODY(iseq)->stack_max);
3541
3542 cfp->sp = sp_orig;
3543
3544 return Qundef;
3545}
3546
3547static void
3548ractor_unsafe_check(void)
3549{
3550 if (!rb_ractor_main_p()) {
3551 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3552 }
3553}
3554
3555static VALUE
3556call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3557{
3558 ractor_unsafe_check();
3559 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3560 return (*f)(recv, rb_ary_new4(argc, argv));
3561}
3562
3563static VALUE
3564call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3565{
3566 ractor_unsafe_check();
3567 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3568 return (*f)(argc, argv, recv);
3569}
3570
3571static VALUE
3572call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3573{
3574 ractor_unsafe_check();
3575 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3576 return (*f)(recv);
3577}
3578
3579static VALUE
3580call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3581{
3582 ractor_unsafe_check();
3583 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3584 return (*f)(recv, argv[0]);
3585}
3586
3587static VALUE
3588call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3589{
3590 ractor_unsafe_check();
3591 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3592 return (*f)(recv, argv[0], argv[1]);
3593}
3594
3595static VALUE
3596call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3597{
3598 ractor_unsafe_check();
3599 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3600 return (*f)(recv, argv[0], argv[1], argv[2]);
3601}
3602
3603static VALUE
3604call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3605{
3606 ractor_unsafe_check();
3607 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3608 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3609}
3610
3611static VALUE
3612call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3613{
3614 ractor_unsafe_check();
3615 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3616 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3617}
3618
3619static VALUE
3620call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3621{
3622 ractor_unsafe_check();
3623 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3624 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3625}
3626
3627static VALUE
3628call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3629{
3630 ractor_unsafe_check();
3631 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3632 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3633}
3634
3635static VALUE
3636call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3637{
3638 ractor_unsafe_check();
3639 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3640 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3641}
3642
3643static VALUE
3644call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3645{
3646 ractor_unsafe_check();
3647 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3648 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3649}
3650
3651static VALUE
3652call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3653{
3654 ractor_unsafe_check();
3655 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3656 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3657}
3658
3659static VALUE
3660call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3661{
3662 ractor_unsafe_check();
3663 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3664 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3665}
3666
3667static VALUE
3668call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3669{
3670 ractor_unsafe_check();
3671 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3672 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3673}
3674
3675static VALUE
3676call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3677{
3678 ractor_unsafe_check();
3679 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3680 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3681}
3682
3683static VALUE
3684call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3685{
3686 ractor_unsafe_check();
3687 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3688 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3689}
3690
3691static VALUE
3692call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3693{
3694 ractor_unsafe_check();
3695 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3696 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3697}
3698
3699static VALUE
3700ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3701{
3702 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3703 return (*f)(recv, rb_ary_new4(argc, argv));
3704}
3705
3706static VALUE
3707ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3708{
3709 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3710 return (*f)(argc, argv, recv);
3711}
3712
3713static VALUE
3714ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3715{
3716 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3717 return (*f)(recv);
3718}
3719
3720static VALUE
3721ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3722{
3723 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3724 return (*f)(recv, argv[0]);
3725}
3726
3727static VALUE
3728ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3729{
3730 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3731 return (*f)(recv, argv[0], argv[1]);
3732}
3733
3734static VALUE
3735ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3736{
3737 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3738 return (*f)(recv, argv[0], argv[1], argv[2]);
3739}
3740
3741static VALUE
3742ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3743{
3744 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3745 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3746}
3747
3748static VALUE
3749ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3750{
3751 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3752 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3753}
3754
3755static VALUE
3756ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3757{
3758 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3759 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3760}
3761
3762static VALUE
3763ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3764{
3765 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3766 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3767}
3768
3769static VALUE
3770ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3771{
3772 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3773 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3774}
3775
3776static VALUE
3777ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3778{
3779 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3780 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3781}
3782
3783static VALUE
3784ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3785{
3786 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3787 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3788}
3789
3790static VALUE
3791ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3792{
3793 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3794 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3795}
3796
3797static VALUE
3798ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3799{
3800 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3801 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3802}
3803
3804static VALUE
3805ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3806{
3807 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3808 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3809}
3810
3811static VALUE
3812ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3813{
3814 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3815 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3816}
3817
3818static VALUE
3819ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3820{
3821 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3822 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3823}
3824
3825static inline int
3826vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3827{
3828 const int ov_flags = RAISED_STACKOVERFLOW;
3829 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3830 if (rb_ec_raised_p(ec, ov_flags)) {
3831 rb_ec_raised_reset(ec, ov_flags);
3832 return TRUE;
3833 }
3834 return FALSE;
3835}
3836
3837#define CHECK_CFP_CONSISTENCY(func) \
3838 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3839 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3840
3841static inline
3842const rb_method_cfunc_t *
3843vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3844{
3845#if VM_DEBUG_VERIFY_METHOD_CACHE
3846 switch (me->def->type) {
3847 case VM_METHOD_TYPE_CFUNC:
3848 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3849 break;
3850# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3851 METHOD_BUG(ISEQ);
3852 METHOD_BUG(ATTRSET);
3853 METHOD_BUG(IVAR);
3854 METHOD_BUG(BMETHOD);
3855 METHOD_BUG(ZSUPER);
3856 METHOD_BUG(UNDEF);
3857 METHOD_BUG(OPTIMIZED);
3858 METHOD_BUG(MISSING);
3859 METHOD_BUG(REFINED);
3860 METHOD_BUG(ALIAS);
3861# undef METHOD_BUG
3862 default:
3863 rb_bug("wrong method type: %d", me->def->type);
3864 }
3865#endif
3866 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3867}
3868
3869static VALUE
3870vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3871 int argc, VALUE *argv, VALUE *stack_bottom)
3872{
3873 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3874 const struct rb_callinfo *ci = calling->cd->ci;
3875 const struct rb_callcache *cc = calling->cc;
3876 VALUE val;
3877 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3878 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3879
3880 VALUE recv = calling->recv;
3881 VALUE block_handler = calling->block_handler;
3882 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3883
3884 if (UNLIKELY(calling->kw_splat)) {
3885 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3886 }
3887
3888 VM_ASSERT(reg_cfp == ec->cfp);
3889
3890 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3891 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3892
3893 vm_push_frame(ec, NULL, frame_type, recv,
3894 block_handler, (VALUE)me,
3895 0, ec->cfp->sp, 0, 0);
3896
3897 int len = cfunc->argc;
3898 if (len >= 0) rb_check_arity(argc, len, len);
3899
3900 reg_cfp->sp = stack_bottom;
3901 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3902
3903 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3904
3905 rb_vm_pop_frame(ec);
3906
3907 VM_ASSERT(ec->cfp->sp == stack_bottom);
3908
3909 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3910 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3911
3912 return val;
3913}
3914
3915// Push a C method frame for a given cme. This is called when JIT code skipped
3916// pushing a frame but the C method reached a point where a frame is needed.
3917void
3918rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3919{
3920 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3921 rb_execution_context_t *ec = GET_EC();
3922 VALUE *sp = ec->cfp->sp;
3923 VALUE recv = *(sp - recv_idx - 1);
3924 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3925 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3926#if VM_CHECK_MODE > 0
3927 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3928 *(GET_EC()->cfp->sp) = Qfalse;
3929#endif
3930 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3931}
3932
3933// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3934bool
3935rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3936{
3937 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3938}
3939
3940static VALUE
3941vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3942{
3943 int argc = calling->argc;
3944 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
3945 VALUE *argv = &stack_bottom[1];
3946
3947 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3948}
3949
3950static VALUE
3951vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3952{
3953 const struct rb_callinfo *ci = calling->cd->ci;
3954 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
3955
3956 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
3957 VALUE argv_ary;
3958 if (UNLIKELY(argv_ary = calling->heap_argv)) {
3959 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
3960 int argc = RARRAY_LENINT(argv_ary);
3961 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3962 VALUE *stack_bottom = reg_cfp->sp - 2;
3963
3964 VM_ASSERT(calling->argc == 1);
3965 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
3966 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
3967
3968 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3969 }
3970 else {
3971 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
3972
3973 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
3974 }
3975}
3976
3977static inline VALUE
3978vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
3979{
3980 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
3981 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
3982
3983 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
3984 return vm_call_cfunc_other(ec, reg_cfp, calling);
3985 }
3986
3987 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3988 calling->kw_splat = 0;
3989 int i;
3990 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
3991 VALUE *sp = stack_bottom;
3992 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
3993 for(i = 0; i < argc; i++) {
3994 *++sp = argv[i];
3995 }
3996 reg_cfp->sp = sp+1;
3997
3998 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
3999}
4000
4001static inline VALUE
4002vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4003{
4004 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
4005 VALUE argv_ary = reg_cfp->sp[-1];
4006 int argc = RARRAY_LENINT(argv_ary);
4007 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4008 VALUE last_hash;
4009 int argc_offset = 0;
4010
4011 if (UNLIKELY(argc > 0 &&
4012 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
4013 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
4014 if (!RHASH_EMPTY_P(last_hash)) {
4015 return vm_call_cfunc_other(ec, reg_cfp, calling);
4016 }
4017 argc_offset++;
4018 }
4019 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
4020}
4021
4022static inline VALUE
4023vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4024{
4025 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
4026 VALUE keyword_hash = reg_cfp->sp[-1];
4027
4028 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
4029 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
4030 }
4031
4032 return vm_call_cfunc_other(ec, reg_cfp, calling);
4033}
4034
4035static VALUE
4036vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4037{
4038 const struct rb_callinfo *ci = calling->cd->ci;
4039 RB_DEBUG_COUNTER_INC(ccf_cfunc);
4040
4041 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4042 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
4043 // f(*a)
4044 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
4045 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
4046 }
4047 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
4048 // f(*a, **kw)
4049 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
4050 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
4051 }
4052 }
4053
4054 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
4055 return vm_call_cfunc_other(ec, reg_cfp, calling);
4056}
4057
4058static VALUE
4059vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4060{
4061 const struct rb_callcache *cc = calling->cc;
4062 RB_DEBUG_COUNTER_INC(ccf_ivar);
4063 cfp->sp -= 1;
4064 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
4065 return ivar;
4066}
4067
4068static VALUE
4069vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
4070{
4071 RB_DEBUG_COUNTER_INC(ccf_attrset);
4072 VALUE val = *(cfp->sp - 1);
4073 cfp->sp -= 2;
4074 attr_index_t index;
4075 shape_id_t dest_shape_id;
4076 vm_cc_atomic_shape_and_index(cc, &dest_shape_id, &index);
4077 ID id = vm_cc_cme(cc)->def->body.attr.id;
4078 rb_check_frozen(obj);
4079 VALUE res = vm_setivar(obj, id, val, dest_shape_id, index);
4080 if (UNDEF_P(res)) {
4081 switch (BUILTIN_TYPE(obj)) {
4082 case T_OBJECT:
4083 break;
4084 case T_CLASS:
4085 case T_MODULE:
4086 {
4087 res = vm_setivar_class(obj, id, val, dest_shape_id, index);
4088 if (!UNDEF_P(res)) {
4089 return res;
4090 }
4091 }
4092 break;
4093 default:
4094 {
4095 res = vm_setivar_default(obj, id, val, dest_shape_id, index);
4096 if (!UNDEF_P(res)) {
4097 return res;
4098 }
4099 }
4100 }
4101 res = vm_setivar_slowpath_attr(obj, id, val, cc);
4102 }
4103 return res;
4104}
4105
4106static VALUE
4107vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4108{
4109 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4110}
4111
4112static inline VALUE
4113vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4114{
4115 rb_proc_t *proc;
4116 VALUE val;
4117 const struct rb_callcache *cc = calling->cc;
4118 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4119 VALUE procv = cme->def->body.bmethod.proc;
4120
4121 if (!RB_OBJ_SHAREABLE_P(procv) &&
4122 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4123 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4124 }
4125
4126 /* control block frame */
4127 GetProcPtr(procv, proc);
4128 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4129
4130 return val;
4131}
4132
4133static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4134
4135static VALUE
4136vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4137{
4138 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4139
4140 const struct rb_callcache *cc = calling->cc;
4141 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4142 VALUE procv = cme->def->body.bmethod.proc;
4143
4144 if (!RB_OBJ_SHAREABLE_P(procv) &&
4145 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4146 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4147 }
4148
4149 rb_proc_t *proc;
4150 GetProcPtr(procv, proc);
4151 const struct rb_block *block = &proc->block;
4152
4153 while (vm_block_type(block) == block_type_proc) {
4154 block = vm_proc_block(block->as.proc);
4155 }
4156 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4157
4158 const struct rb_captured_block *captured = &block->as.captured;
4159 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4160 VALUE * const argv = cfp->sp - calling->argc;
4161 const int arg_size = ISEQ_BODY(iseq)->param.size;
4162
4163 int opt_pc;
4164 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4165 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4166 }
4167 else {
4168 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4169 }
4170
4171 cfp->sp = argv - 1; // -1 for the receiver
4172
4173 vm_push_frame(ec, iseq,
4174 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4175 calling->recv,
4176 VM_GUARDED_PREV_EP(captured->ep),
4177 (VALUE)cme,
4178 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4179 argv + arg_size,
4180 ISEQ_BODY(iseq)->local_table_size - arg_size,
4181 ISEQ_BODY(iseq)->stack_max);
4182
4183 return Qundef;
4184}
4185
4186static VALUE
4187vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4188{
4189 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4190
4191 VALUE *argv;
4192 int argc;
4193 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4194 if (UNLIKELY(calling->heap_argv)) {
4195 argv = RARRAY_PTR(calling->heap_argv);
4196 cfp->sp -= 2;
4197 }
4198 else {
4199 argc = calling->argc;
4200 argv = ALLOCA_N(VALUE, argc);
4201 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4202 cfp->sp += - argc - 1;
4203 }
4204
4205 return vm_call_bmethod_body(ec, calling, argv);
4206}
4207
4208static VALUE
4209vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4210{
4211 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4212
4213 const struct rb_callcache *cc = calling->cc;
4214 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4215 VALUE procv = cme->def->body.bmethod.proc;
4216 rb_proc_t *proc;
4217 GetProcPtr(procv, proc);
4218 const struct rb_block *block = &proc->block;
4219
4220 while (vm_block_type(block) == block_type_proc) {
4221 block = vm_proc_block(block->as.proc);
4222 }
4223 if (vm_block_type(block) == block_type_iseq) {
4224 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4225 return vm_call_iseq_bmethod(ec, cfp, calling);
4226 }
4227
4228 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4229 return vm_call_noniseq_bmethod(ec, cfp, calling);
4230}
4231
4232VALUE
4233rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4234{
4235 VALUE klass = current_class;
4236
4237 /* for prepended Module, then start from cover class */
4238 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4239 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4240 klass = RBASIC_CLASS(klass);
4241 }
4242
4243 while (RTEST(klass)) {
4244 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4245 if (owner == target_owner) {
4246 return klass;
4247 }
4248 klass = RCLASS_SUPER(klass);
4249 }
4250
4251 return current_class; /* maybe module function */
4252}
4253
4254static const rb_callable_method_entry_t *
4255aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4256{
4257 const rb_method_entry_t *orig_me = me->def->body.alias.original_me;
4258 const rb_callable_method_entry_t *cme;
4259
4260 if (orig_me->defined_class == 0) {
4261 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4262 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4263 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4264
4265 if (me->def->reference_count == 1) {
4266 RB_OBJ_WRITE(me, &me->def->body.alias.original_me, cme);
4267 }
4268 else {
4269 rb_method_definition_t *def =
4270 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4271 rb_method_definition_set((rb_method_entry_t *)me, def, (void *)cme);
4272 }
4273 }
4274 else {
4275 cme = (const rb_callable_method_entry_t *)orig_me;
4276 }
4277
4278 VM_ASSERT(callable_method_entry_p(cme));
4279 return cme;
4280}
4281
4282const rb_callable_method_entry_t *
4283rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4284{
4285 return aliased_callable_method_entry(me);
4286}
4287
4288static VALUE
4289vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4290{
4291 calling->cc = &VM_CC_ON_STACK(Qundef,
4292 vm_call_general,
4293 {{0}},
4294 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4295
4296 return vm_call_method_each_type(ec, cfp, calling);
4297}
4298
4299static enum method_missing_reason
4300ci_missing_reason(const struct rb_callinfo *ci)
4301{
4302 enum method_missing_reason stat = MISSING_NOENTRY;
4303 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4304 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4305 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4306 return stat;
4307}
4308
4309static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4310
4311static VALUE
4312vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4313 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4314{
4315 ASSUME(calling->argc >= 0);
4316
4317 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4318 int argc = calling->argc;
4319 VALUE recv = calling->recv;
4320 VALUE klass = CLASS_OF(recv);
4321 ID mid = rb_check_id(&symbol);
4322 flags |= VM_CALL_OPT_SEND;
4323
4324 if (UNLIKELY(! mid)) {
4325 mid = idMethodMissing;
4326 missing_reason = ci_missing_reason(ci);
4327 ec->method_missing_reason = missing_reason;
4328
4329 VALUE argv_ary;
4330 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4331 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4332 rb_ary_unshift(argv_ary, symbol);
4333
4334 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4335 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4336 VALUE exc = rb_make_no_method_exception(
4337 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4338
4339 rb_exc_raise(exc);
4340 }
4341 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4342 }
4343 else {
4344 /* E.g. when argc == 2
4345 *
4346 * | | | | TOPN
4347 * | | +------+
4348 * | | +---> | arg1 | 0
4349 * +------+ | +------+
4350 * | arg1 | -+ +-> | arg0 | 1
4351 * +------+ | +------+
4352 * | arg0 | ---+ | sym | 2
4353 * +------+ +------+
4354 * | recv | | recv | 3
4355 * --+------+--------+------+------
4356 */
4357 int i = argc;
4358 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4359 INC_SP(1);
4360 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4361 argc = ++calling->argc;
4362
4363 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4364 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4365 TOPN(i) = symbol;
4366 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4367 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4368 VALUE exc = rb_make_no_method_exception(
4369 rb_eNoMethodError, 0, recv, argc, argv, priv);
4370
4371 rb_exc_raise(exc);
4372 }
4373 else {
4374 TOPN(i) = rb_str_intern(symbol);
4375 }
4376 }
4377 }
4378
4379 struct rb_forwarding_call_data new_fcd = {
4380 .cd = {
4381 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4382 .cc = NULL,
4383 },
4384 .caller_ci = NULL,
4385 };
4386
4387 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4388 calling->cd = &new_fcd.cd;
4389 }
4390 else {
4391 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4392 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4393 new_fcd.caller_ci = caller_ci;
4394 calling->cd = (struct rb_call_data *)&new_fcd;
4395 }
4396 calling->cc = &VM_CC_ON_STACK(klass,
4397 vm_call_general,
4398 { .method_missing_reason = missing_reason },
4399 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4400
4401 if (flags & VM_CALL_FCALL) {
4402 return vm_call_method(ec, reg_cfp, calling);
4403 }
4404
4405 const struct rb_callcache *cc = calling->cc;
4406 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4407
4408 if (vm_cc_cme(cc) != NULL) {
4409 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4410 case METHOD_VISI_PUBLIC: /* likely */
4411 return vm_call_method_each_type(ec, reg_cfp, calling);
4412 case METHOD_VISI_PRIVATE:
4413 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4414 break;
4415 case METHOD_VISI_PROTECTED:
4416 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4417 break;
4418 default:
4419 VM_UNREACHABLE(vm_call_method);
4420 }
4421 return vm_call_method_missing(ec, reg_cfp, calling);
4422 }
4423
4424 return vm_call_method_nome(ec, reg_cfp, calling);
4425}
4426
4427static VALUE
4428vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4429{
4430 const struct rb_callinfo *ci = calling->cd->ci;
4431 int i;
4432 VALUE sym;
4433
4434 i = calling->argc - 1;
4435
4436 if (calling->argc == 0) {
4437 rb_raise(rb_eArgError, "no method name given");
4438 }
4439
4440 sym = TOPN(i);
4441 /* E.g. when i == 2
4442 *
4443 * | | | | TOPN
4444 * +------+ | |
4445 * | arg1 | ---+ | | 0
4446 * +------+ | +------+
4447 * | arg0 | -+ +-> | arg1 | 1
4448 * +------+ | +------+
4449 * | sym | +---> | arg0 | 2
4450 * +------+ +------+
4451 * | recv | | recv | 3
4452 * --+------+--------+------+------
4453 */
4454 /* shift arguments */
4455 if (i > 0) {
4456 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4457 }
4458 calling->argc -= 1;
4459 DEC_SP(1);
4460
4461 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4462}
4463
4464static VALUE
4465vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4466{
4467 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4468 const struct rb_callinfo *ci = calling->cd->ci;
4469 int flags = VM_CALL_FCALL;
4470 VALUE sym;
4471
4472 VALUE argv_ary;
4473 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4474 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4475 sym = rb_ary_shift(argv_ary);
4476 flags |= VM_CALL_ARGS_SPLAT;
4477 if (calling->kw_splat) {
4478 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4479 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4480 calling->kw_splat = 0;
4481 }
4482 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4483 }
4484
4485 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4486 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4487}
4488
4489static VALUE
4490vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4491{
4492 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4493 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4494}
4495
4496static VALUE
4497vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4498{
4499 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4500
4501 const struct rb_callinfo *ci = calling->cd->ci;
4502 int flags = vm_ci_flag(ci);
4503
4504 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4505 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4506 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4507 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4508 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4509 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4510 }
4511
4512 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4513 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4514}
4515
4516static VALUE
4517vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4518 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4519{
4520 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4521
4522 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4523 unsigned int argc, flag;
4524
4525 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4526 argc = ++calling->argc;
4527
4528 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4529 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4530 vm_check_canary(ec, reg_cfp->sp);
4531 if (argc > 1) {
4532 MEMMOVE(argv+1, argv, VALUE, argc-1);
4533 }
4534 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4535 INC_SP(1);
4536
4537 ec->method_missing_reason = reason;
4538
4539 struct rb_forwarding_call_data new_fcd = {
4540 .cd = {
4541 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4542 .cc = NULL,
4543 },
4544 .caller_ci = NULL,
4545 };
4546
4547 if (!(flag & VM_CALL_FORWARDING)) {
4548 calling->cd = &new_fcd.cd;
4549 }
4550 else {
4551 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4552 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4553 new_fcd.caller_ci = caller_ci;
4554 calling->cd = (struct rb_call_data *)&new_fcd;
4555 }
4556
4557 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4558 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4559 return vm_call_method(ec, reg_cfp, calling);
4560}
4561
4562static VALUE
4563vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4564{
4565 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4566}
4567
4568static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4569static VALUE
4570vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4571{
4572 klass = RCLASS_SUPER(klass);
4573
4574 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4575 if (cme == NULL) {
4576 return vm_call_method_nome(ec, cfp, calling);
4577 }
4578 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4579 cme->def->body.refined.orig_me) {
4580 cme = refined_method_callable_without_refinement(cme);
4581 }
4582
4583 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4584
4585 return vm_call_method_each_type(ec, cfp, calling);
4586}
4587
4588static inline VALUE
4589find_refinement(VALUE refinements, VALUE klass)
4590{
4591 if (NIL_P(refinements)) {
4592 return Qnil;
4593 }
4594 return rb_hash_lookup(refinements, klass);
4595}
4596
4597PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4598static rb_control_frame_t *
4599current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4600{
4601 rb_control_frame_t *top_cfp = cfp;
4602
4603 if (cfp->iseq && ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_BLOCK) {
4604 const rb_iseq_t *local_iseq = ISEQ_BODY(cfp->iseq)->local_iseq;
4605
4606 do {
4607 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4608 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4609 /* TODO: orphan block */
4610 return top_cfp;
4611 }
4612 } while (cfp->iseq != local_iseq);
4613 }
4614 return cfp;
4615}
4616
4617static const rb_callable_method_entry_t *
4618refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4619{
4620 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4621 const rb_callable_method_entry_t *cme;
4622
4623 if (orig_me->defined_class == 0) {
4624 cme = NULL;
4625 rb_notimplement();
4626 }
4627 else {
4628 cme = (const rb_callable_method_entry_t *)orig_me;
4629 }
4630
4631 VM_ASSERT(callable_method_entry_p(cme));
4632
4633 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4634 cme = NULL;
4635 }
4636
4637 return cme;
4638}
4639
4640static const rb_callable_method_entry_t *
4641search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4642{
4643 ID mid = vm_ci_mid(calling->cd->ci);
4644 const rb_cref_t *cref = vm_get_cref(cfp->ep);
4645 const struct rb_callcache * const cc = calling->cc;
4646 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4647
4648 for (; cref; cref = CREF_NEXT(cref)) {
4649 const VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4650 if (NIL_P(refinement)) continue;
4651
4652 const rb_callable_method_entry_t *const ref_me =
4653 rb_callable_method_entry(refinement, mid);
4654
4655 if (ref_me) {
4656 if (vm_cc_call(cc) == vm_call_super_method) {
4657 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4658 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4659 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4660 continue;
4661 }
4662 }
4663
4664 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4665 cme->def != ref_me->def) {
4666 cme = ref_me;
4667 }
4668 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4669 return cme;
4670 }
4671 }
4672 else {
4673 return NULL;
4674 }
4675 }
4676
4677 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4678 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4679 }
4680 else {
4681 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4682 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4683 return cme;
4684 }
4685}
4686
4687static VALUE
4688vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4689{
4690 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4691
4692 if (ref_cme) {
4693 if (calling->cd->cc) {
4694 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4695 RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc);
4696 return vm_call_method(ec, cfp, calling);
4697 }
4698 else {
4699 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4700 calling->cc= ref_cc;
4701 return vm_call_method(ec, cfp, calling);
4702 }
4703 }
4704 else {
4705 return vm_call_method_nome(ec, cfp, calling);
4706 }
4707}
4708
4709static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4710
4711NOINLINE(static VALUE
4712 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4713 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4714
4715static VALUE
4716vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4717 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4718{
4719 int argc = calling->argc;
4720
4721 /* remove self */
4722 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4723 DEC_SP(1);
4724
4725 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4726}
4727
4728static VALUE
4729vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4730{
4731 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4732
4733 const struct rb_callinfo *ci = calling->cd->ci;
4734 VALUE procval = calling->recv;
4735 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4736}
4737
4738static VALUE
4739vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4740{
4741 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4742
4743 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4744 const struct rb_callinfo *ci = calling->cd->ci;
4745
4746 if (BASIC_OP_UNREDEFINED_P(BOP_CALL, PROC_REDEFINED_OP_FLAG)) {
4747 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4748 }
4749 else {
4750 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4751 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4752 return vm_call_general(ec, reg_cfp, calling);
4753 }
4754}
4755
4756static VALUE
4757vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4758{
4759 VALUE recv = calling->recv;
4760
4761 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4762 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4763 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4764
4765 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4766 return internal_RSTRUCT_GET(recv, off);
4767}
4768
4769static VALUE
4770vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4771{
4772 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4773
4774 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4775 reg_cfp->sp -= 1;
4776 return ret;
4777}
4778
4779static VALUE
4780vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4781{
4782 VALUE recv = calling->recv;
4783
4784 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4785 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4786 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4787
4788 rb_check_frozen(recv);
4789
4790 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4791 internal_RSTRUCT_SET(recv, off, val);
4792
4793 return val;
4794}
4795
4796static VALUE
4797vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4798{
4799 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4800
4801 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4802 reg_cfp->sp -= 2;
4803 return ret;
4804}
4805
4806NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4807 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4808
4809#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4810 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4811 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4812 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4813 var = func; \
4814 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4815 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4816 } \
4817 else { \
4818 nohook; \
4819 var = func; \
4820 }
4821
4822static VALUE
4823vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4824 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4825{
4826 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4827 case OPTIMIZED_METHOD_TYPE_SEND:
4828 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4829 return vm_call_opt_send(ec, cfp, calling);
4830 case OPTIMIZED_METHOD_TYPE_CALL:
4831 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
4832 return vm_call_opt_call(ec, cfp, calling);
4833 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
4834 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
4835 return vm_call_opt_block_call(ec, cfp, calling);
4836 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
4837 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4838 rb_check_arity(calling->argc, 0, 0);
4839
4840 VALUE v;
4841 VM_CALL_METHOD_ATTR(v,
4842 vm_call_opt_struct_aref(ec, cfp, calling),
4843 set_vm_cc_ivar(cc); \
4844 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4845 return v;
4846 }
4847 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
4848 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4849 rb_check_arity(calling->argc, 1, 1);
4850
4851 VALUE v;
4852 VM_CALL_METHOD_ATTR(v,
4853 vm_call_opt_struct_aset(ec, cfp, calling),
4854 set_vm_cc_ivar(cc); \
4855 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4856 return v;
4857 }
4858 default:
4859 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
4860 }
4861}
4862
4863static VALUE
4864vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4865{
4866 const struct rb_callinfo *ci = calling->cd->ci;
4867 const struct rb_callcache *cc = calling->cc;
4868 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4869 VALUE v;
4870
4871 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
4872
4873 switch (cme->def->type) {
4874 case VM_METHOD_TYPE_ISEQ:
4875 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
4876 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
4877 return vm_call_iseq_fwd_setup(ec, cfp, calling);
4878 }
4879 else {
4880 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
4881 return vm_call_iseq_setup(ec, cfp, calling);
4882 }
4883
4884 case VM_METHOD_TYPE_NOTIMPLEMENTED:
4885 case VM_METHOD_TYPE_CFUNC:
4886 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
4887 return vm_call_cfunc(ec, cfp, calling);
4888
4889 case VM_METHOD_TYPE_ATTRSET:
4890 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4891
4892 rb_check_arity(calling->argc, 1, 1);
4893
4894 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
4895
4896 if (vm_cc_markable(cc)) {
4897 vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID);
4898 VM_CALL_METHOD_ATTR(v,
4899 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4900 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4901 }
4902 else {
4903 cc = &((struct rb_callcache) {
4904 .flags = T_IMEMO |
4905 (imemo_callcache << FL_USHIFT) |
4906 VM_CALLCACHE_UNMARKABLE |
4907 VM_CALLCACHE_ON_STACK,
4908 .klass = cc->klass,
4909 .cme_ = cc->cme_,
4910 .call_ = cc->call_,
4911 .aux_ = {
4912 .attr = {
4913 .value = vm_pack_shape_and_index(INVALID_SHAPE_ID, ATTR_INDEX_NOT_SET),
4914 }
4915 },
4916 });
4917
4918 VM_CALL_METHOD_ATTR(v,
4919 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4920 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4921 }
4922 return v;
4923
4924 case VM_METHOD_TYPE_IVAR:
4925 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4926 rb_check_arity(calling->argc, 0, 0);
4927 vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID);
4928 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
4929 VM_CALL_METHOD_ATTR(v,
4930 vm_call_ivar(ec, cfp, calling),
4931 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
4932 return v;
4933
4934 case VM_METHOD_TYPE_MISSING:
4935 vm_cc_method_missing_reason_set(cc, 0);
4936 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
4937 return vm_call_method_missing(ec, cfp, calling);
4938
4939 case VM_METHOD_TYPE_BMETHOD:
4940 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
4941 return vm_call_bmethod(ec, cfp, calling);
4942
4943 case VM_METHOD_TYPE_ALIAS:
4944 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
4945 return vm_call_alias(ec, cfp, calling);
4946
4947 case VM_METHOD_TYPE_OPTIMIZED:
4948 return vm_call_optimized(ec, cfp, calling, ci, cc);
4949
4950 case VM_METHOD_TYPE_UNDEF:
4951 break;
4952
4953 case VM_METHOD_TYPE_ZSUPER:
4954 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
4955
4956 case VM_METHOD_TYPE_REFINED:
4957 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
4958 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
4959 return vm_call_refined(ec, cfp, calling);
4960 }
4961
4962 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
4963}
4964
4965NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
4966
4967static VALUE
4968vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4969{
4970 /* method missing */
4971 const struct rb_callinfo *ci = calling->cd->ci;
4972 const int stat = ci_missing_reason(ci);
4973
4974 if (vm_ci_mid(ci) == idMethodMissing) {
4975 if (UNLIKELY(calling->heap_argv)) {
4976 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
4977 }
4978 else {
4979 rb_control_frame_t *reg_cfp = cfp;
4980 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4981 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
4982 }
4983 }
4984 else {
4985 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
4986 }
4987}
4988
4989/* Protected method calls and super invocations need to check that the receiver
4990 * (self for super) inherits the module on which the method is defined.
4991 * In the case of refinements, it should consider the original class not the
4992 * refinement.
4993 */
4994static VALUE
4995vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
4996{
4997 VALUE defined_class = me->defined_class;
4998 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
4999 return NIL_P(refined_class) ? defined_class : refined_class;
5000}
5001
5002static inline VALUE
5003vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5004{
5005 const struct rb_callinfo *ci = calling->cd->ci;
5006 const struct rb_callcache *cc = calling->cc;
5007
5008 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
5009
5010 if (vm_cc_cme(cc) != NULL) {
5011 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
5012 case METHOD_VISI_PUBLIC: /* likely */
5013 return vm_call_method_each_type(ec, cfp, calling);
5014
5015 case METHOD_VISI_PRIVATE:
5016 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
5017 enum method_missing_reason stat = MISSING_PRIVATE;
5018 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
5019
5020 vm_cc_method_missing_reason_set(cc, stat);
5021 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5022 return vm_call_method_missing(ec, cfp, calling);
5023 }
5024 return vm_call_method_each_type(ec, cfp, calling);
5025
5026 case METHOD_VISI_PROTECTED:
5027 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
5028 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
5029 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
5030 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
5031 return vm_call_method_missing(ec, cfp, calling);
5032 }
5033 else {
5034 /* caching method info to dummy cc */
5035 VM_ASSERT(vm_cc_cme(cc) != NULL);
5036 struct rb_callcache cc_on_stack = *cc;
5037 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
5038 calling->cc = &cc_on_stack;
5039 return vm_call_method_each_type(ec, cfp, calling);
5040 }
5041 }
5042 return vm_call_method_each_type(ec, cfp, calling);
5043
5044 default:
5045 rb_bug("unreachable");
5046 }
5047 }
5048 else {
5049 return vm_call_method_nome(ec, cfp, calling);
5050 }
5051}
5052
5053static VALUE
5054vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5055{
5056 RB_DEBUG_COUNTER_INC(ccf_general);
5057 return vm_call_method(ec, reg_cfp, calling);
5058}
5059
5060void
5061rb_vm_cc_general(const struct rb_callcache *cc)
5062{
5063 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
5064 VM_ASSERT(cc != vm_cc_empty());
5065
5066 *(vm_call_handler *)&cc->call_ = vm_call_general;
5067}
5068
5069static VALUE
5070vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5071{
5072 RB_DEBUG_COUNTER_INC(ccf_super_method);
5073
5074 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
5075 // can merge the function and the address of the function becomes same.
5076 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
5077 if (ec == NULL) rb_bug("unreachable");
5078
5079 /* this check is required to distinguish with other functions. */
5080 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
5081 return vm_call_method(ec, reg_cfp, calling);
5082}
5083
5084/* super */
5085
5086static inline VALUE
5087vm_search_normal_superclass(VALUE klass)
5088{
5089 if (BUILTIN_TYPE(klass) == T_ICLASS &&
5090 RB_TYPE_P(RBASIC(klass)->klass, T_MODULE) &&
5091 FL_TEST_RAW(RBASIC(klass)->klass, RMODULE_IS_REFINEMENT)) {
5092 klass = RBASIC(klass)->klass;
5093 }
5094 klass = RCLASS_ORIGIN(klass);
5095 return RCLASS_SUPER(klass);
5096}
5097
5098NORETURN(static void vm_super_outside(void));
5099
5100static void
5101vm_super_outside(void)
5102{
5103 rb_raise(rb_eNoMethodError, "super called outside of method");
5104}
5105
5106static const struct rb_callcache *
5107empty_cc_for_super(void)
5108{
5109 return &vm_empty_cc_for_super;
5110}
5111
5112static const struct rb_callcache *
5113vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5114{
5115 VALUE current_defined_class;
5116 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5117
5118 if (!me) {
5119 vm_super_outside();
5120 }
5121
5122 current_defined_class = vm_defined_class_for_protected_call(me);
5123
5124 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5125 reg_cfp->iseq != method_entry_iseqptr(me) &&
5126 !rb_obj_is_kind_of(recv, current_defined_class)) {
5127 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5128 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5129
5130 if (m) { /* not bound UnboundMethod */
5131 rb_raise(rb_eTypeError,
5132 "self has wrong type to call super in this context: "
5133 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5134 rb_obj_class(recv), m);
5135 }
5136 }
5137
5138 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5139 rb_raise(rb_eRuntimeError,
5140 "implicit argument passing of super from method defined"
5141 " by define_method() is not supported."
5142 " Specify all arguments explicitly.");
5143 }
5144
5145 ID mid = me->def->original_id;
5146
5147 if (!vm_ci_markable(cd->ci)) {
5148 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5149 }
5150 else {
5151 // update iseq. really? (TODO)
5152 cd->ci = vm_ci_new_runtime(mid,
5153 vm_ci_flag(cd->ci),
5154 vm_ci_argc(cd->ci),
5155 vm_ci_kwarg(cd->ci));
5156
5157 RB_OBJ_WRITTEN(reg_cfp->iseq, Qundef, cd->ci);
5158 }
5159
5160 const struct rb_callcache *cc;
5161
5162 VALUE klass = vm_search_normal_superclass(me->defined_class);
5163
5164 if (!klass) {
5165 /* bound instance method of module */
5166 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5167 RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc);
5168 }
5169 else {
5170 cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, klass);
5171 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5172
5173 // define_method can cache for different method id
5174 if (cached_cme == NULL) {
5175 // empty_cc_for_super is not markable object
5176 cd->cc = empty_cc_for_super();
5177 }
5178 else if (cached_cme->called_id != mid) {
5179 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5180 if (cme) {
5181 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5182 RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc);
5183 }
5184 else {
5185 cd->cc = cc = empty_cc_for_super();
5186 }
5187 }
5188 else {
5189 switch (cached_cme->def->type) {
5190 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5191 case VM_METHOD_TYPE_REFINED:
5192 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5193 case VM_METHOD_TYPE_ATTRSET:
5194 case VM_METHOD_TYPE_IVAR:
5195 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5196 break;
5197 default:
5198 break; // use fastpath
5199 }
5200 }
5201 }
5202
5203 VM_ASSERT((vm_cc_cme(cc), true));
5204
5205 return cc;
5206}
5207
5208/* yield */
5209
5210static inline int
5211block_proc_is_lambda(const VALUE procval)
5212{
5213 rb_proc_t *proc;
5214
5215 if (procval) {
5216 GetProcPtr(procval, proc);
5217 return proc->is_lambda;
5218 }
5219 else {
5220 return 0;
5221 }
5222}
5223
5224static VALUE
5225vm_yield_with_cfunc(rb_execution_context_t *ec,
5226 const struct rb_captured_block *captured,
5227 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5228 const rb_callable_method_entry_t *me)
5229{
5230 int is_lambda = FALSE; /* TODO */
5231 VALUE val, arg, blockarg;
5232 int frame_flag;
5233 const struct vm_ifunc *ifunc = captured->code.ifunc;
5234
5235 if (is_lambda) {
5236 arg = rb_ary_new4(argc, argv);
5237 }
5238 else if (argc == 0) {
5239 arg = Qnil;
5240 }
5241 else {
5242 arg = argv[0];
5243 }
5244
5245 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5246
5247 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5248 if (kw_splat) {
5249 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5250 }
5251
5252 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5253 frame_flag,
5254 self,
5255 VM_GUARDED_PREV_EP(captured->ep),
5256 (VALUE)me,
5257 0, ec->cfp->sp, 0, 0);
5258 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5259 rb_vm_pop_frame(ec);
5260
5261 return val;
5262}
5263
5264VALUE
5265rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5266{
5267 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5268}
5269
5270static VALUE
5271vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5272{
5273 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, rb_vm_bh_to_procval(ec, block_handler));
5274}
5275
5276static inline int
5277vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5278{
5279 int i;
5280 long len = RARRAY_LEN(ary);
5281
5282 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5283
5284 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5285 argv[i] = RARRAY_AREF(ary, i);
5286 }
5287
5288 return i;
5289}
5290
5291static inline VALUE
5292vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5293{
5294 VALUE ary, arg0 = argv[0];
5295 ary = rb_check_array_type(arg0);
5296#if 0
5297 argv[0] = arg0;
5298#else
5299 VM_ASSERT(argv[0] == arg0);
5300#endif
5301 return ary;
5302}
5303
5304static int
5305vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5306{
5307 if (rb_simple_iseq_p(iseq)) {
5308 rb_control_frame_t *cfp = ec->cfp;
5309 VALUE arg0;
5310
5311 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5312
5313 if (arg_setup_type == arg_setup_block &&
5314 calling->argc == 1 &&
5315 ISEQ_BODY(iseq)->param.flags.has_lead &&
5316 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5317 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5318 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5319 }
5320
5321 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5322 if (arg_setup_type == arg_setup_block) {
5323 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5324 int i;
5325 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5326 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5327 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5328 }
5329 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5330 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5331 }
5332 }
5333 else {
5334 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5335 }
5336 }
5337
5338 return 0;
5339 }
5340 else {
5341 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5342 }
5343}
5344
5345static int
5346vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5347{
5348 struct rb_calling_info calling_entry, *calling;
5349
5350 calling = &calling_entry;
5351 calling->argc = argc;
5352 calling->block_handler = block_handler;
5353 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5354 calling->recv = Qundef;
5355 calling->heap_argv = 0;
5356 calling->cc = NULL;
5357 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5358
5359 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5360}
5361
5362/* ruby iseq -> ruby block */
5363
5364static VALUE
5365vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5366 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5367 bool is_lambda, VALUE block_handler)
5368{
5369 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5370 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5371 const int arg_size = ISEQ_BODY(iseq)->param.size;
5372 VALUE * const rsp = GET_SP() - calling->argc;
5373 VALUE * const argv = rsp;
5374 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5375 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5376
5377 SET_SP(rsp);
5378
5379 vm_push_frame(ec, iseq,
5380 frame_flag,
5381 captured->self,
5382 VM_GUARDED_PREV_EP(captured->ep), 0,
5383 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5384 rsp + arg_size,
5385 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5386
5387 return Qundef;
5388}
5389
5390static VALUE
5391vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5392 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5393 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5394{
5395 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5396 int flags = vm_ci_flag(ci);
5397
5398 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5399 ((calling->argc == 0) ||
5400 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5401 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5402 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5403 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5404 flags = 0;
5405 if (UNLIKELY(calling->heap_argv)) {
5406#if VM_ARGC_STACK_MAX < 0
5407 if (RARRAY_LEN(calling->heap_argv) < 1) {
5408 rb_raise(rb_eArgError, "no receiver given");
5409 }
5410#endif
5411 calling->recv = rb_ary_shift(calling->heap_argv);
5412 // Modify stack to avoid cfp consistency error
5413 reg_cfp->sp++;
5414 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5415 reg_cfp->sp[-2] = calling->recv;
5416 flags |= VM_CALL_ARGS_SPLAT;
5417 }
5418 else {
5419 if (calling->argc < 1) {
5420 rb_raise(rb_eArgError, "no receiver given");
5421 }
5422 calling->recv = TOPN(--calling->argc);
5423 }
5424 if (calling->kw_splat) {
5425 flags |= VM_CALL_KW_SPLAT;
5426 }
5427 }
5428 else {
5429 if (calling->argc < 1) {
5430 rb_raise(rb_eArgError, "no receiver given");
5431 }
5432 calling->recv = TOPN(--calling->argc);
5433 }
5434
5435 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5436}
5437
5438static VALUE
5439vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5440 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5441 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5442{
5443 VALUE val;
5444 int argc;
5445 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5446 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5447 argc = calling->argc;
5448 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5449 POPN(argc); /* TODO: should put before C/yield? */
5450 return val;
5451}
5452
5453static VALUE
5454vm_proc_to_block_handler(VALUE procval)
5455{
5456 const struct rb_block *block = vm_proc_block(procval);
5457
5458 switch (vm_block_type(block)) {
5459 case block_type_iseq:
5460 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5461 case block_type_ifunc:
5462 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5463 case block_type_symbol:
5464 return VM_BH_FROM_SYMBOL(block->as.symbol);
5465 case block_type_proc:
5466 return VM_BH_FROM_PROC(block->as.proc);
5467 }
5468 VM_UNREACHABLE(vm_yield_with_proc);
5469 return Qundef;
5470}
5471
5472static VALUE
5473vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5474 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5475 bool is_lambda, VALUE block_handler)
5476{
5477 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5478 VALUE proc = VM_BH_TO_PROC(block_handler);
5479 is_lambda = block_proc_is_lambda(proc);
5480 block_handler = vm_proc_to_block_handler(proc);
5481 }
5482
5483 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5484}
5485
5486static inline VALUE
5487vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5488 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5489 bool is_lambda, VALUE block_handler)
5490{
5491 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5492 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5493 bool is_lambda, VALUE block_handler);
5494
5495 switch (vm_block_handler_type(block_handler)) {
5496 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5497 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5498 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5499 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5500 default: rb_bug("vm_invoke_block: unreachable");
5501 }
5502
5503 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5504}
5505
5506static VALUE
5507vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5508{
5509 const rb_execution_context_t *ec = GET_EC();
5510 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5511 struct rb_captured_block *captured;
5512
5513 if (cfp == 0) {
5514 rb_bug("vm_make_proc_with_iseq: unreachable");
5515 }
5516
5517 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5518 captured->code.iseq = blockiseq;
5519
5520 return rb_vm_make_proc(ec, captured, rb_cProc);
5521}
5522
5523static VALUE
5524vm_once_exec(VALUE iseq)
5525{
5526 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5527 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5528}
5529
5530static VALUE
5531vm_once_clear(VALUE data)
5532{
5533 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5534 is->once.running_thread = NULL;
5535 return Qnil;
5536}
5537
5538/* defined insn */
5539
5540static bool
5541check_respond_to_missing(VALUE obj, VALUE v)
5542{
5543 VALUE args[2];
5544 VALUE r;
5545
5546 args[0] = obj; args[1] = Qfalse;
5547 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5548 if (!UNDEF_P(r) && RTEST(r)) {
5549 return true;
5550 }
5551 else {
5552 return false;
5553 }
5554}
5555
5556static bool
5557vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5558{
5559 VALUE klass;
5560 enum defined_type type = (enum defined_type)op_type;
5561
5562 switch (type) {
5563 case DEFINED_IVAR:
5564 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5565 break;
5566 case DEFINED_GVAR:
5567 return rb_gvar_defined(SYM2ID(obj));
5568 break;
5569 case DEFINED_CVAR: {
5570 const rb_cref_t *cref = vm_get_cref(GET_EP());
5571 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5572 return rb_cvar_defined(klass, SYM2ID(obj));
5573 break;
5574 }
5575 case DEFINED_CONST:
5576 case DEFINED_CONST_FROM: {
5577 bool allow_nil = type == DEFINED_CONST;
5578 klass = v;
5579 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5580 break;
5581 }
5582 case DEFINED_FUNC:
5583 klass = CLASS_OF(v);
5584 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5585 break;
5586 case DEFINED_METHOD:{
5587 VALUE klass = CLASS_OF(v);
5588 const rb_method_entry_t *me = rb_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5589
5590 if (me) {
5591 switch (METHOD_ENTRY_VISI(me)) {
5592 case METHOD_VISI_PRIVATE:
5593 break;
5594 case METHOD_VISI_PROTECTED:
5595 if (!rb_obj_is_kind_of(GET_SELF(), rb_class_real(me->defined_class))) {
5596 break;
5597 }
5598 case METHOD_VISI_PUBLIC:
5599 return true;
5600 break;
5601 default:
5602 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(me));
5603 }
5604 }
5605 else {
5606 return check_respond_to_missing(obj, v);
5607 }
5608 break;
5609 }
5610 case DEFINED_YIELD:
5611 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5612 return true;
5613 }
5614 break;
5615 case DEFINED_ZSUPER:
5616 {
5617 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5618
5619 if (me) {
5620 VALUE klass = vm_search_normal_superclass(me->defined_class);
5621 if (!klass) return false;
5622
5623 ID id = me->def->original_id;
5624
5625 return rb_method_boundp(klass, id, 0);
5626 }
5627 }
5628 break;
5629 case DEFINED_REF:
5630 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5631 default:
5632 rb_bug("unimplemented defined? type (VM)");
5633 break;
5634 }
5635
5636 return false;
5637}
5638
5639bool
5640rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5641{
5642 return vm_defined(ec, reg_cfp, op_type, obj, v);
5643}
5644
5645static const VALUE *
5646vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5647{
5648 rb_num_t i;
5649 const VALUE *ep = reg_ep;
5650 for (i = 0; i < lv; i++) {
5651 ep = GET_PREV_EP(ep);
5652 }
5653 return ep;
5654}
5655
5656static VALUE
5657vm_get_special_object(const VALUE *const reg_ep,
5658 enum vm_special_object_type type)
5659{
5660 switch (type) {
5661 case VM_SPECIAL_OBJECT_VMCORE:
5662 return rb_mRubyVMFrozenCore;
5663 case VM_SPECIAL_OBJECT_CBASE:
5664 return vm_get_cbase(reg_ep);
5665 case VM_SPECIAL_OBJECT_CONST_BASE:
5666 return vm_get_const_base(reg_ep);
5667 default:
5668 rb_bug("putspecialobject insn: unknown value_type %d", type);
5669 }
5670}
5671
5672// ZJIT implementation is using the C function
5673// and needs to call a non-static function
5674VALUE
5675rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5676{
5677 return vm_get_special_object(reg_ep, type);
5678}
5679
5680static VALUE
5681vm_concat_array(VALUE ary1, VALUE ary2st)
5682{
5683 const VALUE ary2 = ary2st;
5684 VALUE tmp1 = rb_check_to_array(ary1);
5685 VALUE tmp2 = rb_check_to_array(ary2);
5686
5687 if (NIL_P(tmp1)) {
5688 tmp1 = rb_ary_new3(1, ary1);
5689 }
5690 if (tmp1 == ary1) {
5691 tmp1 = rb_ary_dup(ary1);
5692 }
5693
5694 if (NIL_P(tmp2)) {
5695 return rb_ary_push(tmp1, ary2);
5696 }
5697 else {
5698 return rb_ary_concat(tmp1, tmp2);
5699 }
5700}
5701
5702static VALUE
5703vm_concat_to_array(VALUE ary1, VALUE ary2st)
5704{
5705 /* ary1 must be a newly created array */
5706 const VALUE ary2 = ary2st;
5707
5708 if (NIL_P(ary2)) return ary1;
5709
5710 VALUE tmp2 = rb_check_to_array(ary2);
5711
5712 if (NIL_P(tmp2)) {
5713 return rb_ary_push(ary1, ary2);
5714 }
5715 else {
5716 return rb_ary_concat(ary1, tmp2);
5717 }
5718}
5719
5720// YJIT implementation is using the C function
5721// and needs to call a non-static function
5722VALUE
5723rb_vm_concat_array(VALUE ary1, VALUE ary2st)
5724{
5725 return vm_concat_array(ary1, ary2st);
5726}
5727
5728VALUE
5729rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
5730{
5731 return vm_concat_to_array(ary1, ary2st);
5732}
5733
5734static VALUE
5735vm_splat_array(VALUE flag, VALUE ary)
5736{
5737 if (NIL_P(ary)) {
5738 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
5739 }
5740 VALUE tmp = rb_check_to_array(ary);
5741 if (NIL_P(tmp)) {
5742 return rb_ary_new3(1, ary);
5743 }
5744 else if (RTEST(flag)) {
5745 return rb_ary_dup(tmp);
5746 }
5747 else {
5748 return tmp;
5749 }
5750}
5751
5752// YJIT implementation is using the C function
5753// and needs to call a non-static function
5754VALUE
5755rb_vm_splat_array(VALUE flag, VALUE ary)
5756{
5757 return vm_splat_array(flag, ary);
5758}
5759
5760static VALUE
5761vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5762{
5763 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
5764
5765 if (flag & VM_CHECKMATCH_ARRAY) {
5766 long i;
5767 const long n = RARRAY_LEN(pattern);
5768
5769 for (i = 0; i < n; i++) {
5770 VALUE v = RARRAY_AREF(pattern, i);
5771 VALUE c = check_match(ec, v, target, type);
5772
5773 if (RTEST(c)) {
5774 return c;
5775 }
5776 }
5777 return Qfalse;
5778 }
5779 else {
5780 return check_match(ec, pattern, target, type);
5781 }
5782}
5783
5784VALUE
5785rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5786{
5787 return vm_check_match(ec, target, pattern, flag);
5788}
5789
5790static VALUE
5791vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
5792{
5793 const VALUE kw_bits = *(ep - bits);
5794
5795 if (FIXNUM_P(kw_bits)) {
5796 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
5797 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
5798 return Qfalse;
5799 }
5800 else {
5801 VM_ASSERT(RB_TYPE_P(kw_bits, T_HASH));
5802 if (rb_hash_has_key(kw_bits, INT2FIX(idx))) return Qfalse;
5803 }
5804 return Qtrue;
5805}
5806
5807static void
5808vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
5809{
5810 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
5811 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
5812 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
5813 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
5814
5815 switch (flag) {
5816 case RUBY_EVENT_CALL:
5817 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
5818 return;
5819 case RUBY_EVENT_C_CALL:
5820 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
5821 return;
5822 case RUBY_EVENT_RETURN:
5823 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
5824 return;
5825 case RUBY_EVENT_C_RETURN:
5826 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
5827 return;
5828 }
5829 }
5830}
5831
5832static VALUE
5833vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
5834{
5835 if (!rb_const_defined_at(cbase, id)) {
5836 return 0;
5837 }
5838 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
5839 return rb_public_const_get_at(cbase, id);
5840 }
5841 else {
5842 return rb_const_get_at(cbase, id);
5843 }
5844}
5845
5846static VALUE
5847vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
5848{
5849 if (!RB_TYPE_P(klass, T_CLASS)) {
5850 return 0;
5851 }
5852 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
5853 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
5854
5855 if (tmp != super) {
5856 rb_raise(rb_eTypeError,
5857 "superclass mismatch for class %"PRIsVALUE"",
5858 rb_id2str(id));
5859 }
5860 else {
5861 return klass;
5862 }
5863 }
5864 else {
5865 return klass;
5866 }
5867}
5868
5869static VALUE
5870vm_check_if_module(ID id, VALUE mod)
5871{
5872 if (!RB_TYPE_P(mod, T_MODULE)) {
5873 return 0;
5874 }
5875 else {
5876 return mod;
5877 }
5878}
5879
5880static VALUE
5881declare_under(ID id, VALUE cbase, VALUE c)
5882{
5883 rb_set_class_path_string(c, cbase, rb_id2str(id));
5884 rb_const_set(cbase, id, c);
5885 return c;
5886}
5887
5888static VALUE
5889vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
5890{
5891 /* new class declaration */
5892 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
5893 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
5894 rb_define_alloc_func(c, rb_get_alloc_func(c));
5895 rb_class_inherited(s, c);
5896 return c;
5897}
5898
5899static VALUE
5900vm_declare_module(ID id, VALUE cbase)
5901{
5902 /* new module declaration */
5903 return declare_under(id, cbase, rb_module_new());
5904}
5905
5906NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
5907static void
5908unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
5909{
5910 VALUE name = rb_id2str(id);
5911 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
5912 name, type);
5913 VALUE location = rb_const_source_location_at(cbase, id);
5914 if (!NIL_P(location)) {
5915 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
5916 " previous definition of %"PRIsVALUE" was here",
5917 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
5918 }
5919 rb_exc_raise(rb_exc_new_str(rb_eTypeError, message));
5920}
5921
5922static VALUE
5923vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
5924{
5925 VALUE klass;
5926
5927 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
5928 rb_raise(rb_eTypeError,
5929 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
5930 rb_obj_class(super));
5931 }
5932
5933 vm_check_if_namespace(cbase);
5934
5935 /* find klass */
5936 rb_autoload_load(cbase, id);
5937
5938 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
5939 if (!vm_check_if_class(id, flags, super, klass))
5940 unmatched_redefinition("class", cbase, id, klass);
5941 return klass;
5942 }
5943 else {
5944 return vm_declare_class(id, flags, cbase, super);
5945 }
5946}
5947
5948static VALUE
5949vm_define_module(ID id, rb_num_t flags, VALUE cbase)
5950{
5951 VALUE mod;
5952
5953 vm_check_if_namespace(cbase);
5954 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
5955 if (!vm_check_if_module(id, mod))
5956 unmatched_redefinition("module", cbase, id, mod);
5957 return mod;
5958 }
5959 else {
5960 return vm_declare_module(id, cbase);
5961 }
5962}
5963
5964static VALUE
5965vm_find_or_create_class_by_id(ID id,
5966 rb_num_t flags,
5967 VALUE cbase,
5968 VALUE super)
5969{
5970 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
5971
5972 switch (type) {
5973 case VM_DEFINECLASS_TYPE_CLASS:
5974 /* classdef returns class scope value */
5975 return vm_define_class(id, flags, cbase, super);
5976
5977 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
5978 /* classdef returns class scope value */
5979 return rb_singleton_class(cbase);
5980
5981 case VM_DEFINECLASS_TYPE_MODULE:
5982 /* classdef returns class scope value */
5983 return vm_define_module(id, flags, cbase);
5984
5985 default:
5986 rb_bug("unknown defineclass type: %d", (int)type);
5987 }
5988}
5989
5990static rb_method_visibility_t
5991vm_scope_visibility_get(const rb_execution_context_t *ec)
5992{
5993 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5994
5995 if (!vm_env_cref_by_cref(cfp->ep)) {
5996 return METHOD_VISI_PUBLIC;
5997 }
5998 else {
5999 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
6000 }
6001}
6002
6003static int
6004vm_scope_module_func_check(const rb_execution_context_t *ec)
6005{
6006 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6007
6008 if (!vm_env_cref_by_cref(cfp->ep)) {
6009 return FALSE;
6010 }
6011 else {
6012 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
6013 }
6014}
6015
6016static void
6017vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
6018{
6019 VALUE klass;
6020 rb_method_visibility_t visi;
6021 rb_cref_t *cref = vm_ec_cref(ec);
6022
6023 if (is_singleton) {
6024 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
6025 visi = METHOD_VISI_PUBLIC;
6026 }
6027 else {
6028 klass = CREF_CLASS_FOR_DEFINITION(cref);
6029 visi = vm_scope_visibility_get(ec);
6030 }
6031
6032 if (NIL_P(klass)) {
6033 rb_raise(rb_eTypeError, "no class/module to add method");
6034 }
6035
6036 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
6037 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
6038 if (id == idInitialize && klass != rb_cObject && RB_TYPE_P(klass, T_CLASS) && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
6039 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
6040 }
6041
6042 if (!is_singleton && vm_scope_module_func_check(ec)) {
6043 klass = rb_singleton_class(klass);
6044 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
6045 }
6046}
6047
6048// Return the untagged block handler:
6049// * If it's VM_BLOCK_HANDLER_NONE, return nil
6050// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
6051// * If it's a PROC or SYMBOL, return it as is
6052static VALUE
6053rb_vm_untag_block_handler(VALUE block_handler)
6054{
6055 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
6056
6057 switch (vm_block_handler_type(block_handler)) {
6058 case block_handler_type_iseq:
6059 case block_handler_type_ifunc: {
6060 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
6061 return captured->code.val;
6062 }
6063 case block_handler_type_proc:
6064 case block_handler_type_symbol:
6065 return block_handler;
6066 default:
6067 rb_bug("rb_vm_untag_block_handler: unreachable");
6068 }
6069}
6070
6071VALUE
6072rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
6073{
6074 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
6075}
6076
6077static VALUE
6078vm_invokeblock_i(struct rb_execution_context_struct *ec,
6079 struct rb_control_frame_struct *reg_cfp,
6080 struct rb_calling_info *calling)
6081{
6082 const struct rb_callinfo *ci = calling->cd->ci;
6083 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
6084
6085 if (block_handler == VM_BLOCK_HANDLER_NONE) {
6086 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
6087 }
6088 else {
6089 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
6090 }
6091}
6092
6093enum method_explorer_type {
6094 mexp_search_method,
6095 mexp_search_invokeblock,
6096 mexp_search_super,
6097};
6098
6099static inline VALUE
6100vm_sendish(
6101 struct rb_execution_context_struct *ec,
6102 struct rb_control_frame_struct *reg_cfp,
6103 struct rb_call_data *cd,
6104 VALUE block_handler,
6105 enum method_explorer_type method_explorer
6106) {
6107 VALUE val = Qundef;
6108 const struct rb_callinfo *ci = cd->ci;
6109 const struct rb_callcache *cc;
6110 int argc = vm_ci_argc(ci);
6111 VALUE recv = TOPN(argc);
6112 struct rb_calling_info calling = {
6113 .block_handler = block_handler,
6114 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6115 .recv = recv,
6116 .argc = argc,
6117 .cd = cd,
6118 };
6119
6120 switch (method_explorer) {
6121 case mexp_search_method:
6122 calling.cc = cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, CLASS_OF(recv));
6123 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6124 break;
6125 case mexp_search_super:
6126 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6127 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6128 break;
6129 case mexp_search_invokeblock:
6130 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6131 break;
6132 }
6133 return val;
6134}
6135
6136VALUE
6137rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6138{
6139 stack_check(ec);
6140 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6141 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6142 VM_EXEC(ec, val);
6143 return val;
6144}
6145
6146VALUE
6147rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6148{
6149 stack_check(ec);
6150
6151 struct rb_forwarding_call_data adjusted_cd;
6152 struct rb_callinfo adjusted_ci;
6153
6154 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6155
6156 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6157
6158 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6159 RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
6160 }
6161
6162 VM_EXEC(ec, val);
6163 return val;
6164}
6165
6166VALUE
6167rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6168{
6169 stack_check(ec);
6170 VALUE bh = VM_BLOCK_HANDLER_NONE;
6171 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6172 VM_EXEC(ec, val);
6173 return val;
6174}
6175
6176VALUE
6177rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6178{
6179 stack_check(ec);
6180
6181 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, true);
6182 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_super);
6183
6184 VM_EXEC(ec, val);
6185 return val;
6186}
6187
6188VALUE
6189rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6190{
6191 stack_check(ec);
6192 struct rb_forwarding_call_data adjusted_cd;
6193 struct rb_callinfo adjusted_ci;
6194
6195 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6196
6197 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6198
6199 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6200 RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
6201 }
6202
6203 VM_EXEC(ec, val);
6204 return val;
6205}
6206
6207VALUE
6208rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6209{
6210 stack_check(ec);
6211 VALUE bh = VM_BLOCK_HANDLER_NONE;
6212 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6213 VM_EXEC(ec, val);
6214 return val;
6215}
6216
6217/* object.c */
6218VALUE rb_nil_to_s(VALUE);
6219VALUE rb_true_to_s(VALUE);
6220VALUE rb_false_to_s(VALUE);
6221/* numeric.c */
6222VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6223VALUE rb_fix_to_s(VALUE);
6224/* variable.c */
6225VALUE rb_mod_to_s(VALUE);
6226VALUE rb_mod_name(VALUE);
6227
6228static VALUE
6229vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd)
6230{
6231 int type = TYPE(recv);
6232 if (type == T_STRING) {
6233 return recv;
6234 }
6235
6236 const struct rb_callable_method_entry_struct *cme = vm_search_method((VALUE)iseq, cd, recv);
6237
6238 switch (type) {
6239 case T_SYMBOL:
6240 if (check_method_basic_definition(cme)) {
6241 // rb_sym_to_s() allocates a mutable string, but since we are only
6242 // going to use this string for interpolation, it's fine to use the
6243 // frozen string.
6244 return rb_sym2str(recv);
6245 }
6246 break;
6247 case T_MODULE:
6248 case T_CLASS:
6249 if (check_cfunc(cme, rb_mod_to_s)) {
6250 // rb_mod_to_s() allocates a mutable string, but since we are only
6251 // going to use this string for interpolation, it's fine to use the
6252 // frozen string.
6253 VALUE val = rb_mod_name(recv);
6254 if (NIL_P(val)) {
6255 val = rb_mod_to_s(recv);
6256 }
6257 return val;
6258 }
6259 break;
6260 case T_NIL:
6261 if (check_cfunc(cme, rb_nil_to_s)) {
6262 return rb_nil_to_s(recv);
6263 }
6264 break;
6265 case T_TRUE:
6266 if (check_cfunc(cme, rb_true_to_s)) {
6267 return rb_true_to_s(recv);
6268 }
6269 break;
6270 case T_FALSE:
6271 if (check_cfunc(cme, rb_false_to_s)) {
6272 return rb_false_to_s(recv);
6273 }
6274 break;
6275 case T_FIXNUM:
6276 if (check_cfunc(cme, rb_int_to_s)) {
6277 return rb_fix_to_s(recv);
6278 }
6279 break;
6280 }
6281 return Qundef;
6282}
6283
6284// ZJIT implementation is using the C function
6285// and needs to call a non-static function
6286VALUE
6287rb_vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd)
6288{
6289 return vm_objtostring(iseq, recv, cd);
6290}
6291
6292static VALUE
6293vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6294{
6295 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6296 return ary;
6297 }
6298 else {
6299 return Qundef;
6300 }
6301}
6302
6303static VALUE
6304vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6305{
6306 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6307 return hash;
6308 }
6309 else {
6310 return Qundef;
6311 }
6312}
6313
6314static VALUE
6315vm_opt_str_freeze(VALUE str, int bop, ID id)
6316{
6317 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6318 return str;
6319 }
6320 else {
6321 return Qundef;
6322 }
6323}
6324
6325/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6326#define id_cmp idCmp
6327
6328static VALUE
6329vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6330{
6331 if (BASIC_OP_UNREDEFINED_P(BOP_INCLUDE_P, ARRAY_REDEFINED_OP_FLAG)) {
6332 return rb_ary_includes(ary, target);
6333 }
6334 else {
6335 VALUE args[1] = {target};
6336
6337 // duparray
6338 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6339 VALUE dupary = rb_ary_resurrect(ary);
6340
6341 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6342 }
6343}
6344
6345VALUE
6346rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6347{
6348 return vm_opt_duparray_include_p(ec, ary, target);
6349}
6350
6351static VALUE
6352vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6353{
6354 if (BASIC_OP_UNREDEFINED_P(BOP_MAX, ARRAY_REDEFINED_OP_FLAG)) {
6355 if (array_len == 0) {
6356 return Qnil;
6357 }
6358 else {
6359 VALUE result = *ptr;
6360 rb_snum_t i = array_len - 1;
6361 while (i-- > 0) {
6362 const VALUE v = *++ptr;
6363 if (OPTIMIZED_CMP(v, result) > 0) {
6364 result = v;
6365 }
6366 }
6367 return result;
6368 }
6369 }
6370 else {
6371 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6372 }
6373}
6374
6375VALUE
6376rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6377{
6378 return vm_opt_newarray_max(ec, array_len, ptr);
6379}
6380
6381static VALUE
6382vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6383{
6384 if (BASIC_OP_UNREDEFINED_P(BOP_MIN, ARRAY_REDEFINED_OP_FLAG)) {
6385 if (array_len == 0) {
6386 return Qnil;
6387 }
6388 else {
6389 VALUE result = *ptr;
6390 rb_snum_t i = array_len - 1;
6391 while (i-- > 0) {
6392 const VALUE v = *++ptr;
6393 if (OPTIMIZED_CMP(v, result) < 0) {
6394 result = v;
6395 }
6396 }
6397 return result;
6398 }
6399 }
6400 else {
6401 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6402 }
6403}
6404
6405VALUE
6406rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6407{
6408 return vm_opt_newarray_min(ec, array_len, ptr);
6409}
6410
6411static VALUE
6412vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6413{
6414 // If Array#hash is _not_ monkeypatched, use the optimized call
6415 if (BASIC_OP_UNREDEFINED_P(BOP_HASH, ARRAY_REDEFINED_OP_FLAG)) {
6416 return rb_ary_hash_values(array_len, ptr);
6417 }
6418 else {
6419 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6420 }
6421}
6422
6423VALUE
6424rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6425{
6426 return vm_opt_newarray_hash(ec, array_len, ptr);
6427}
6428
6429VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len);
6430VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6431
6432static VALUE
6433vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6434{
6435 if (BASIC_OP_UNREDEFINED_P(BOP_INCLUDE_P, ARRAY_REDEFINED_OP_FLAG)) {
6436 struct RArray fake_ary = {RBASIC_INIT};
6437 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6438 return rb_ary_includes(ary, target);
6439 }
6440 else {
6441 VALUE args[1] = {target};
6442 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6443 }
6444}
6445
6446VALUE
6447rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6448{
6449 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6450}
6451
6452static VALUE
6453vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6454{
6455 if (BASIC_OP_UNREDEFINED_P(BOP_PACK, ARRAY_REDEFINED_OP_FLAG)) {
6456 struct RArray fake_ary = {RBASIC_INIT};
6457 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6458 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6459 }
6460 else {
6461 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6462 // Setup an array with room for keyword hash.
6463 VALUE args[2];
6464 args[0] = fmt;
6465 int kw_splat = RB_NO_KEYWORDS;
6466 int argc = 1;
6467
6468 if (!UNDEF_P(buffer)) {
6469 args[1] = rb_hash_new_with_size(1);
6470 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6471 kw_splat = RB_PASS_KEYWORDS;
6472 argc++;
6473 }
6474
6475 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6476 }
6477}
6478
6479VALUE
6480rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6481{
6482 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6483}
6484
6485VALUE
6486rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt)
6487{
6488 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, Qundef);
6489}
6490
6491#undef id_cmp
6492
6493static void
6494vm_track_constant_cache(ID id, void *ic)
6495{
6496 rb_vm_t *vm = GET_VM();
6497 struct rb_id_table *const_cache = vm->constant_cache;
6498 VALUE lookup_result;
6499 set_table *ics;
6500
6501 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6502 ics = (set_table *)lookup_result;
6503 }
6504 else {
6505 ics = set_init_numtable();
6506 rb_id_table_insert(const_cache, id, (VALUE)ics);
6507 }
6508
6509 /* The call below to st_insert could allocate which could trigger a GC.
6510 * If it triggers a GC, it may free an iseq that also holds a cache to this
6511 * constant. If that iseq is the last iseq with a cache to this constant, then
6512 * it will free this ST table, which would cause an use-after-free during this
6513 * st_insert.
6514 *
6515 * So to fix this issue, we store the ID that is currently being inserted
6516 * and, in remove_from_constant_cache, we don't free the ST table for ID
6517 * equal to this one.
6518 *
6519 * See [Bug #20921].
6520 */
6521 vm->inserting_constant_cache_id = id;
6522
6523 set_insert(ics, (st_data_t)ic);
6524
6525 vm->inserting_constant_cache_id = (ID)0;
6526}
6527
6528static void
6529vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6530{
6531 RB_VM_LOCKING() {
6532 for (int i = 0; segments[i]; i++) {
6533 ID id = segments[i];
6534 if (id == idNULL) continue;
6535 vm_track_constant_cache(id, ic);
6536 }
6537 }
6538}
6539
6540// For JIT inlining
6541static inline bool
6542vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, const VALUE *reg_ep)
6543{
6544 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || rb_ractor_main_p()) {
6545 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6546
6547 return (ic_cref == NULL || // no need to check CREF
6548 ic_cref == vm_get_cref(reg_ep));
6549 }
6550 return false;
6551}
6552
6553static bool
6554vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6555{
6556 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6557 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, reg_ep);
6558}
6559
6560// YJIT needs this function to never allocate and never raise
6561bool
6562rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6563{
6564 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6565}
6566
6567static void
6568vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6569{
6570 if (ruby_vm_const_missing_count > 0) {
6571 ruby_vm_const_missing_count = 0;
6572 ic->entry = NULL;
6573 return;
6574 }
6575
6576 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6577 RB_OBJ_WRITE(ice, &ice->value, val);
6578 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6579
6580 if (rb_ractor_shareable_p(val)) {
6581 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6582 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6583 }
6584 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6585 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6586 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6587 rb_yjit_constant_ic_update(iseq, ic, pos);
6588}
6589
6590VALUE
6591rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6592{
6593 VALUE val;
6594 const ID *segments = ic->segments;
6595 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6596
6597 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6598 val = ice->value;
6599
6600 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments));
6601 }
6602 else {
6603 ruby_vm_constant_cache_misses++;
6604 val = vm_get_ev_const_chain(ec, segments);
6605 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6606 // Undo the PC increment to get the address to this instruction
6607 // INSN_ATTR(width) == 2
6608 vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2);
6609 }
6610 return val;
6611}
6612
6613static VALUE
6614vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6615{
6616 rb_thread_t *th = rb_ec_thread_ptr(ec);
6617 rb_thread_t *const RUNNING_THREAD_ONCE_DONE = (rb_thread_t *)(0x1);
6618
6619 again:
6620 if (is->once.running_thread == RUNNING_THREAD_ONCE_DONE) {
6621 return is->once.value;
6622 }
6623 else if (is->once.running_thread == NULL) {
6624 VALUE val;
6625 is->once.running_thread = th;
6626 val = rb_ensure(vm_once_exec, (VALUE)iseq, vm_once_clear, (VALUE)is);
6627 // TODO: confirm that it is shareable
6628
6629 if (RB_FL_ABLE(val)) {
6630 RB_OBJ_SET_SHAREABLE(val);
6631 }
6632
6633 RB_OBJ_WRITE(ec->cfp->iseq, &is->once.value, val);
6634
6635 /* is->once.running_thread is cleared by vm_once_clear() */
6636 is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */
6637 return val;
6638 }
6639 else if (is->once.running_thread == th) {
6640 /* recursive once */
6641 return vm_once_exec((VALUE)iseq);
6642 }
6643 else {
6644 /* waiting for finish */
6645 RUBY_VM_CHECK_INTS(ec);
6646 rb_thread_schedule();
6647 goto again;
6648 }
6649}
6650
6651static OFFSET
6652vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
6653{
6654 switch (OBJ_BUILTIN_TYPE(key)) {
6655 case -1:
6656 case T_FLOAT:
6657 case T_SYMBOL:
6658 case T_BIGNUM:
6659 case T_STRING:
6660 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
6661 SYMBOL_REDEFINED_OP_FLAG |
6662 INTEGER_REDEFINED_OP_FLAG |
6663 FLOAT_REDEFINED_OP_FLAG |
6664 NIL_REDEFINED_OP_FLAG |
6665 TRUE_REDEFINED_OP_FLAG |
6666 FALSE_REDEFINED_OP_FLAG |
6667 STRING_REDEFINED_OP_FLAG)) {
6668 st_data_t val;
6669 if (RB_FLOAT_TYPE_P(key)) {
6670 double kval = RFLOAT_VALUE(key);
6671 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
6672 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
6673 }
6674 }
6675 if (rb_hash_stlike_lookup(hash, key, &val)) {
6676 return FIX2LONG((VALUE)val);
6677 }
6678 else {
6679 return else_offset;
6680 }
6681 }
6682 }
6683 return 0;
6684}
6685
6686NORETURN(static void
6687 vm_stack_consistency_error(const rb_execution_context_t *ec,
6688 const rb_control_frame_t *,
6689 const VALUE *));
6690static void
6691vm_stack_consistency_error(const rb_execution_context_t *ec,
6692 const rb_control_frame_t *cfp,
6693 const VALUE *bp)
6694{
6695 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
6696 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
6697 static const char stack_consistency_error[] =
6698 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
6699#if defined RUBY_DEVEL
6700 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
6701 rb_str_cat_cstr(mesg, "\n");
6702 rb_str_append(mesg, rb_iseq_disasm(cfp->iseq));
6703 rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg));
6704#else
6705 rb_bug(stack_consistency_error, nsp, nbp);
6706#endif
6707}
6708
6709static VALUE
6710vm_opt_plus(VALUE recv, VALUE obj)
6711{
6712 if (FIXNUM_2_P(recv, obj) &&
6713 BASIC_OP_UNREDEFINED_P(BOP_PLUS, INTEGER_REDEFINED_OP_FLAG)) {
6714 return rb_fix_plus_fix(recv, obj);
6715 }
6716 else if (FLONUM_2_P(recv, obj) &&
6717 BASIC_OP_UNREDEFINED_P(BOP_PLUS, FLOAT_REDEFINED_OP_FLAG)) {
6718 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6719 }
6720 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6721 return Qundef;
6722 }
6723 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6724 RBASIC_CLASS(obj) == rb_cFloat &&
6725 BASIC_OP_UNREDEFINED_P(BOP_PLUS, FLOAT_REDEFINED_OP_FLAG)) {
6726 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6727 }
6728 else if (RBASIC_CLASS(recv) == rb_cString &&
6729 RBASIC_CLASS(obj) == rb_cString &&
6730 BASIC_OP_UNREDEFINED_P(BOP_PLUS, STRING_REDEFINED_OP_FLAG)) {
6731 return rb_str_opt_plus(recv, obj);
6732 }
6733 else if (RBASIC_CLASS(recv) == rb_cArray &&
6734 RBASIC_CLASS(obj) == rb_cArray &&
6735 BASIC_OP_UNREDEFINED_P(BOP_PLUS, ARRAY_REDEFINED_OP_FLAG)) {
6736 return rb_ary_plus(recv, obj);
6737 }
6738 else {
6739 return Qundef;
6740 }
6741}
6742
6743static VALUE
6744vm_opt_minus(VALUE recv, VALUE obj)
6745{
6746 if (FIXNUM_2_P(recv, obj) &&
6747 BASIC_OP_UNREDEFINED_P(BOP_MINUS, INTEGER_REDEFINED_OP_FLAG)) {
6748 return rb_fix_minus_fix(recv, obj);
6749 }
6750 else if (FLONUM_2_P(recv, obj) &&
6751 BASIC_OP_UNREDEFINED_P(BOP_MINUS, FLOAT_REDEFINED_OP_FLAG)) {
6752 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6753 }
6754 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6755 return Qundef;
6756 }
6757 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6758 RBASIC_CLASS(obj) == rb_cFloat &&
6759 BASIC_OP_UNREDEFINED_P(BOP_MINUS, FLOAT_REDEFINED_OP_FLAG)) {
6760 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6761 }
6762 else {
6763 return Qundef;
6764 }
6765}
6766
6767static VALUE
6768vm_opt_mult(VALUE recv, VALUE obj)
6769{
6770 if (FIXNUM_2_P(recv, obj) &&
6771 BASIC_OP_UNREDEFINED_P(BOP_MULT, INTEGER_REDEFINED_OP_FLAG)) {
6772 return rb_fix_mul_fix(recv, obj);
6773 }
6774 else if (FLONUM_2_P(recv, obj) &&
6775 BASIC_OP_UNREDEFINED_P(BOP_MULT, FLOAT_REDEFINED_OP_FLAG)) {
6776 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6777 }
6778 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6779 return Qundef;
6780 }
6781 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6782 RBASIC_CLASS(obj) == rb_cFloat &&
6783 BASIC_OP_UNREDEFINED_P(BOP_MULT, FLOAT_REDEFINED_OP_FLAG)) {
6784 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6785 }
6786 else {
6787 return Qundef;
6788 }
6789}
6790
6791static VALUE
6792vm_opt_div(VALUE recv, VALUE obj)
6793{
6794 if (FIXNUM_2_P(recv, obj) &&
6795 BASIC_OP_UNREDEFINED_P(BOP_DIV, INTEGER_REDEFINED_OP_FLAG)) {
6796 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
6797 }
6798 else if (FLONUM_2_P(recv, obj) &&
6799 BASIC_OP_UNREDEFINED_P(BOP_DIV, FLOAT_REDEFINED_OP_FLAG)) {
6800 return rb_flo_div_flo(recv, obj);
6801 }
6802 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6803 return Qundef;
6804 }
6805 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6806 RBASIC_CLASS(obj) == rb_cFloat &&
6807 BASIC_OP_UNREDEFINED_P(BOP_DIV, FLOAT_REDEFINED_OP_FLAG)) {
6808 return rb_flo_div_flo(recv, obj);
6809 }
6810 else {
6811 return Qundef;
6812 }
6813}
6814
6815static VALUE
6816vm_opt_mod(VALUE recv, VALUE obj)
6817{
6818 if (FIXNUM_2_P(recv, obj) &&
6819 BASIC_OP_UNREDEFINED_P(BOP_MOD, INTEGER_REDEFINED_OP_FLAG)) {
6820 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
6821 }
6822 else if (FLONUM_2_P(recv, obj) &&
6823 BASIC_OP_UNREDEFINED_P(BOP_MOD, FLOAT_REDEFINED_OP_FLAG)) {
6824 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
6825 }
6826 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6827 return Qundef;
6828 }
6829 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6830 RBASIC_CLASS(obj) == rb_cFloat &&
6831 BASIC_OP_UNREDEFINED_P(BOP_MOD, FLOAT_REDEFINED_OP_FLAG)) {
6832 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
6833 }
6834 else {
6835 return Qundef;
6836 }
6837}
6838
6839static VALUE
6840vm_opt_neq(const rb_iseq_t *iseq, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
6841{
6842 if (vm_method_cfunc_is(iseq, cd, recv, rb_obj_not_equal)) {
6843 VALUE val = opt_equality(iseq, recv, obj, cd_eq);
6844
6845 if (!UNDEF_P(val)) {
6846 return RBOOL(!RTEST(val));
6847 }
6848 }
6849
6850 return Qundef;
6851}
6852
6853static VALUE
6854vm_opt_lt(VALUE recv, VALUE obj)
6855{
6856 if (FIXNUM_2_P(recv, obj) &&
6857 BASIC_OP_UNREDEFINED_P(BOP_LT, INTEGER_REDEFINED_OP_FLAG)) {
6858 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
6859 }
6860 else if (FLONUM_2_P(recv, obj) &&
6861 BASIC_OP_UNREDEFINED_P(BOP_LT, FLOAT_REDEFINED_OP_FLAG)) {
6862 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
6863 }
6864 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6865 return Qundef;
6866 }
6867 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6868 RBASIC_CLASS(obj) == rb_cFloat &&
6869 BASIC_OP_UNREDEFINED_P(BOP_LT, FLOAT_REDEFINED_OP_FLAG)) {
6870 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
6871 }
6872 else {
6873 return Qundef;
6874 }
6875}
6876
6877static VALUE
6878vm_opt_le(VALUE recv, VALUE obj)
6879{
6880 if (FIXNUM_2_P(recv, obj) &&
6881 BASIC_OP_UNREDEFINED_P(BOP_LE, INTEGER_REDEFINED_OP_FLAG)) {
6882 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
6883 }
6884 else if (FLONUM_2_P(recv, obj) &&
6885 BASIC_OP_UNREDEFINED_P(BOP_LE, FLOAT_REDEFINED_OP_FLAG)) {
6886 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
6887 }
6888 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6889 return Qundef;
6890 }
6891 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6892 RBASIC_CLASS(obj) == rb_cFloat &&
6893 BASIC_OP_UNREDEFINED_P(BOP_LE, FLOAT_REDEFINED_OP_FLAG)) {
6894 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
6895 }
6896 else {
6897 return Qundef;
6898 }
6899}
6900
6901static VALUE
6902vm_opt_gt(VALUE recv, VALUE obj)
6903{
6904 if (FIXNUM_2_P(recv, obj) &&
6905 BASIC_OP_UNREDEFINED_P(BOP_GT, INTEGER_REDEFINED_OP_FLAG)) {
6906 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
6907 }
6908 else if (FLONUM_2_P(recv, obj) &&
6909 BASIC_OP_UNREDEFINED_P(BOP_GT, FLOAT_REDEFINED_OP_FLAG)) {
6910 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
6911 }
6912 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6913 return Qundef;
6914 }
6915 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6916 RBASIC_CLASS(obj) == rb_cFloat &&
6917 BASIC_OP_UNREDEFINED_P(BOP_GT, FLOAT_REDEFINED_OP_FLAG)) {
6918 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
6919 }
6920 else {
6921 return Qundef;
6922 }
6923}
6924
6925static VALUE
6926vm_opt_ge(VALUE recv, VALUE obj)
6927{
6928 if (FIXNUM_2_P(recv, obj) &&
6929 BASIC_OP_UNREDEFINED_P(BOP_GE, INTEGER_REDEFINED_OP_FLAG)) {
6930 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
6931 }
6932 else if (FLONUM_2_P(recv, obj) &&
6933 BASIC_OP_UNREDEFINED_P(BOP_GE, FLOAT_REDEFINED_OP_FLAG)) {
6934 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
6935 }
6936 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6937 return Qundef;
6938 }
6939 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6940 RBASIC_CLASS(obj) == rb_cFloat &&
6941 BASIC_OP_UNREDEFINED_P(BOP_GE, FLOAT_REDEFINED_OP_FLAG)) {
6942 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
6943 }
6944 else {
6945 return Qundef;
6946 }
6947}
6948
6949
6950static VALUE
6951vm_opt_ltlt(VALUE recv, VALUE obj)
6952{
6953 if (SPECIAL_CONST_P(recv)) {
6954 return Qundef;
6955 }
6956 else if (RBASIC_CLASS(recv) == rb_cString &&
6957 BASIC_OP_UNREDEFINED_P(BOP_LTLT, STRING_REDEFINED_OP_FLAG)) {
6958 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
6959 return rb_str_buf_append(recv, obj);
6960 }
6961 else {
6962 return rb_str_concat(recv, obj);
6963 }
6964 }
6965 else if (RBASIC_CLASS(recv) == rb_cArray &&
6966 BASIC_OP_UNREDEFINED_P(BOP_LTLT, ARRAY_REDEFINED_OP_FLAG)) {
6967 return rb_ary_push(recv, obj);
6968 }
6969 else {
6970 return Qundef;
6971 }
6972}
6973
6974static VALUE
6975vm_opt_and(VALUE recv, VALUE obj)
6976{
6977 // If recv and obj are both fixnums, then the bottom tag bit
6978 // will be 1 on both. 1 & 1 == 1, so the result value will also
6979 // be a fixnum. If either side is *not* a fixnum, then the tag bit
6980 // will be 0, and we return Qundef.
6981 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
6982
6983 if (FIXNUM_P(ret) &&
6984 BASIC_OP_UNREDEFINED_P(BOP_AND, INTEGER_REDEFINED_OP_FLAG)) {
6985 return ret;
6986 }
6987 else {
6988 return Qundef;
6989 }
6990}
6991
6992static VALUE
6993vm_opt_or(VALUE recv, VALUE obj)
6994{
6995 if (FIXNUM_2_P(recv, obj) &&
6996 BASIC_OP_UNREDEFINED_P(BOP_OR, INTEGER_REDEFINED_OP_FLAG)) {
6997 return recv | obj;
6998 }
6999 else {
7000 return Qundef;
7001 }
7002}
7003
7004static VALUE
7005vm_opt_aref(VALUE recv, VALUE obj)
7006{
7007 if (SPECIAL_CONST_P(recv)) {
7008 if (FIXNUM_2_P(recv, obj) &&
7009 BASIC_OP_UNREDEFINED_P(BOP_AREF, INTEGER_REDEFINED_OP_FLAG)) {
7010 return rb_fix_aref(recv, obj);
7011 }
7012 return Qundef;
7013 }
7014 else if (RBASIC_CLASS(recv) == rb_cArray &&
7015 BASIC_OP_UNREDEFINED_P(BOP_AREF, ARRAY_REDEFINED_OP_FLAG)) {
7016 if (FIXNUM_P(obj)) {
7017 return rb_ary_entry_internal(recv, FIX2LONG(obj));
7018 }
7019 else {
7020 return rb_ary_aref1(recv, obj);
7021 }
7022 }
7023 else if (RBASIC_CLASS(recv) == rb_cHash &&
7024 BASIC_OP_UNREDEFINED_P(BOP_AREF, HASH_REDEFINED_OP_FLAG)) {
7025 return rb_hash_aref(recv, obj);
7026 }
7027 else {
7028 return Qundef;
7029 }
7030}
7031
7032static VALUE
7033vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
7034{
7035 if (SPECIAL_CONST_P(recv)) {
7036 return Qundef;
7037 }
7038 else if (RBASIC_CLASS(recv) == rb_cArray &&
7039 BASIC_OP_UNREDEFINED_P(BOP_ASET, ARRAY_REDEFINED_OP_FLAG) &&
7040 FIXNUM_P(obj)) {
7041 rb_ary_store(recv, FIX2LONG(obj), set);
7042 return set;
7043 }
7044 else if (RBASIC_CLASS(recv) == rb_cHash &&
7045 BASIC_OP_UNREDEFINED_P(BOP_ASET, HASH_REDEFINED_OP_FLAG)) {
7046 rb_hash_aset(recv, obj, set);
7047 return set;
7048 }
7049 else {
7050 return Qundef;
7051 }
7052}
7053
7054static VALUE
7055vm_opt_length(VALUE recv, int bop)
7056{
7057 if (SPECIAL_CONST_P(recv)) {
7058 return Qundef;
7059 }
7060 else if (RBASIC_CLASS(recv) == rb_cString &&
7061 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
7062 if (bop == BOP_EMPTY_P) {
7063 return LONG2NUM(RSTRING_LEN(recv));
7064 }
7065 else {
7066 return rb_str_length(recv);
7067 }
7068 }
7069 else if (RBASIC_CLASS(recv) == rb_cArray &&
7070 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
7071 return LONG2NUM(RARRAY_LEN(recv));
7072 }
7073 else if (RBASIC_CLASS(recv) == rb_cHash &&
7074 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
7075 return INT2FIX(RHASH_SIZE(recv));
7076 }
7077 else {
7078 return Qundef;
7079 }
7080}
7081
7082static VALUE
7083vm_opt_empty_p(VALUE recv)
7084{
7085 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
7086 case Qundef: return Qundef;
7087 case INT2FIX(0): return Qtrue;
7088 default: return Qfalse;
7089 }
7090}
7091
7092VALUE rb_false(VALUE obj);
7093
7094static VALUE
7095vm_opt_nil_p(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv)
7096{
7097 if (NIL_P(recv) &&
7098 BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) {
7099 return Qtrue;
7100 }
7101 else if (vm_method_cfunc_is(iseq, cd, recv, rb_false)) {
7102 return Qfalse;
7103 }
7104 else {
7105 return Qundef;
7106 }
7107}
7108
7109static VALUE
7110fix_succ(VALUE x)
7111{
7112 switch (x) {
7113 case ~0UL:
7114 /* 0xFFFF_FFFF == INT2FIX(-1)
7115 * `-1.succ` is of course 0. */
7116 return INT2FIX(0);
7117 case RSHIFT(~0UL, 1):
7118 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7119 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7120 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7121 default:
7122 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7123 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7124 * == lx*2 + ly*2 + 1
7125 * == (lx*2+1) + (ly*2+1) - 1
7126 * == x + y - 1
7127 *
7128 * Here, if we put y := INT2FIX(1):
7129 *
7130 * == x + INT2FIX(1) - 1
7131 * == x + 2 .
7132 */
7133 return x + 2;
7134 }
7135}
7136
7137static VALUE
7138vm_opt_succ(VALUE recv)
7139{
7140 if (FIXNUM_P(recv) &&
7141 BASIC_OP_UNREDEFINED_P(BOP_SUCC, INTEGER_REDEFINED_OP_FLAG)) {
7142 return fix_succ(recv);
7143 }
7144 else if (SPECIAL_CONST_P(recv)) {
7145 return Qundef;
7146 }
7147 else if (RBASIC_CLASS(recv) == rb_cString &&
7148 BASIC_OP_UNREDEFINED_P(BOP_SUCC, STRING_REDEFINED_OP_FLAG)) {
7149 return rb_str_succ(recv);
7150 }
7151 else {
7152 return Qundef;
7153 }
7154}
7155
7156static VALUE
7157vm_opt_not(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv)
7158{
7159 if (vm_method_cfunc_is(iseq, cd, recv, rb_obj_not)) {
7160 return RBOOL(!RTEST(recv));
7161 }
7162 else {
7163 return Qundef;
7164 }
7165}
7166
7167static VALUE
7168vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7169{
7170 if (SPECIAL_CONST_P(recv)) {
7171 return Qundef;
7172 }
7173 else if (RBASIC_CLASS(recv) == rb_cString &&
7174 CLASS_OF(obj) == rb_cRegexp &&
7175 BASIC_OP_UNREDEFINED_P(BOP_MATCH, STRING_REDEFINED_OP_FLAG)) {
7176 return rb_reg_match(obj, recv);
7177 }
7178 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7179 BASIC_OP_UNREDEFINED_P(BOP_MATCH, REGEXP_REDEFINED_OP_FLAG)) {
7180 return rb_reg_match(recv, obj);
7181 }
7182 else {
7183 return Qundef;
7184 }
7185}
7186
7187rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7188
7189NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7190
7191static inline void
7192vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7193 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7194 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7195{
7196 rb_event_flag_t event = pc_events & target_event;
7197 VALUE self = GET_SELF();
7198
7199 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7200
7201 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7202
7203 if (event & global_hooks->events) {
7204 /* increment PC because source line is calculated with PC-1 */
7205 reg_cfp->pc++;
7206 vm_dtrace(event, ec);
7207 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7208 reg_cfp->pc--;
7209 }
7210
7211 if (local_hooks) local_hooks->running--;
7212 if (local_hooks != NULL) {
7213 if (event & local_hooks->events) {
7214 /* increment PC because source line is calculated with PC-1 */
7215 reg_cfp->pc++;
7216 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7217 reg_cfp->pc--;
7218 }
7219 }
7220}
7221
7222#define VM_TRACE_HOOK(target_event, val) do { \
7223 if ((pc_events & (target_event)) & enabled_flags) { \
7224 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7225 } \
7226} while (0)
7227
7228static VALUE
7229rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7230{
7231 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7232 VM_ASSERT(ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE);
7233 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7234}
7235
7236static void
7237vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7238{
7239 const VALUE *pc = reg_cfp->pc;
7240 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7241 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7242 rb_event_flag_t ractor_events = enabled_flags;
7243
7244 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7245 return;
7246 }
7247 else {
7248 const rb_iseq_t *iseq = reg_cfp->iseq;
7249 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7250 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7251 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7252 rb_hook_list_t *local_hooks = NULL;
7253 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7254 st_data_t val;
7255 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7256 local_hooks = (rb_hook_list_t*)val;
7257 }
7258 }
7259 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7260
7261 rb_hook_list_t *bmethod_local_hooks = NULL;
7262 rb_event_flag_t bmethod_local_events = 0;
7263 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7264 enabled_flags |= iseq_local_events;
7265
7266 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7267
7268 if (bmethod_frame) {
7269 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7270 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7271 unsigned int bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7272 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7273 st_data_t val;
7274 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7275 bmethod_local_hooks = (rb_hook_list_t*)val;
7276 }
7277 if (bmethod_local_hooks) {
7278 bmethod_local_events = bmethod_local_hooks->events;
7279 }
7280 }
7281 }
7282
7283 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7284#if 0
7285 /* disable trace */
7286 /* TODO: incomplete */
7287 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7288#else
7289 /* do not disable trace because of performance problem
7290 * (re-enable overhead)
7291 */
7292#endif
7293 return;
7294 }
7295 else if (ec->trace_arg != NULL) {
7296 /* already tracing */
7297 return;
7298 }
7299 else {
7300 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7301 /* Note, not considering iseq local events here since the same
7302 * iseq could be used in multiple bmethods. */
7303 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7304
7305 if (0) {
7306 ruby_debug_printf("vm_trace>>%4d (%4x) - %s:%d %s\n",
7307 (int)pos,
7308 (int)pc_events,
7309 RSTRING_PTR(rb_iseq_path(iseq)),
7310 (int)rb_iseq_line_no(iseq, pos),
7311 RSTRING_PTR(rb_iseq_label(iseq)));
7312 }
7313 VM_ASSERT(reg_cfp->pc == pc);
7314 VM_ASSERT(pc_events != 0);
7315
7316 /* check traces */
7317 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7318 /* b_call instruction running as a method. Fire call event. */
7319 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7320 }
7321 VM_TRACE_HOOK(RUBY_EVENT_CLASS | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL, Qundef);
7322 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7323 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7324 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7325 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7326 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7327 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7328 /* b_return instruction running as a method. Fire return event. */
7329 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7330 }
7331 }
7332 }
7333}
7334#undef VM_TRACE_HOOK
7335
7336#if VM_CHECK_MODE > 0
7337NORETURN( NOINLINE( COLDFUNC
7338void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7339
7340void
7341Init_vm_stack_canary(void)
7342{
7343 /* This has to be called _after_ our PRNG is properly set up. */
7344 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7345 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7346
7347 vm_stack_canary_was_born = true;
7348 VM_ASSERT(n == 0);
7349}
7350
7351void
7352rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7353{
7354 /* Because a method has already been called, why not call
7355 * another one. */
7356 const char *insn = rb_insns_name(i);
7357 VALUE inspection = rb_inspect(c);
7358 const char *str = StringValueCStr(inspection);
7359
7360 rb_bug("dead canary found at %s: %s", insn, str);
7361}
7362
7363#else
7364void Init_vm_stack_canary(void) { /* nothing to do */ }
7365#endif
7366
7367
7368/* a part of the following code is generated by this ruby script:
7369
737016.times{|i|
7371 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7372 typedef_args.prepend(", ") if i != 0
7373 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7374 call_args.prepend(", ") if i != 0
7375 puts %Q{
7376static VALUE
7377builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7378{
7379 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7380 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7381}}
7382}
7383
7384puts
7385puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
738616.times{|i|
7387 puts " builtin_invoker#{i},"
7388}
7389puts "};"
7390*/
7391
7392static VALUE
7393builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7394{
7395 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7396 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7397}
7398
7399static VALUE
7400builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7401{
7402 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7403 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7404}
7405
7406static VALUE
7407builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7408{
7409 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7410 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7411}
7412
7413static VALUE
7414builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7415{
7416 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7417 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7418}
7419
7420static VALUE
7421builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7422{
7423 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7424 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7425}
7426
7427static VALUE
7428builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7429{
7430 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7431 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7432}
7433
7434static VALUE
7435builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7436{
7437 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7438 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7439}
7440
7441static VALUE
7442builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7443{
7444 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7445 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7446}
7447
7448static VALUE
7449builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7450{
7451 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7452 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7453}
7454
7455static VALUE
7456builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7457{
7458 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7459 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7460}
7461
7462static VALUE
7463builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7464{
7465 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7466 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7467}
7468
7469static VALUE
7470builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7471{
7472 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7473 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7474}
7475
7476static VALUE
7477builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7478{
7479 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7480 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7481}
7482
7483static VALUE
7484builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7485{
7486 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7487 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7488}
7489
7490static VALUE
7491builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7492{
7493 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7494 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7495}
7496
7497static VALUE
7498builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7499{
7500 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7501 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7502}
7503
7504typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7505
7506static builtin_invoker
7507lookup_builtin_invoker(int argc)
7508{
7509 static const builtin_invoker invokers[] = {
7510 builtin_invoker0,
7511 builtin_invoker1,
7512 builtin_invoker2,
7513 builtin_invoker3,
7514 builtin_invoker4,
7515 builtin_invoker5,
7516 builtin_invoker6,
7517 builtin_invoker7,
7518 builtin_invoker8,
7519 builtin_invoker9,
7520 builtin_invoker10,
7521 builtin_invoker11,
7522 builtin_invoker12,
7523 builtin_invoker13,
7524 builtin_invoker14,
7525 builtin_invoker15,
7526 };
7527
7528 return invokers[argc];
7529}
7530
7531static inline VALUE
7532invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7533{
7534 const bool canary_p = ISEQ_BODY(reg_cfp->iseq)->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7535 SETUP_CANARY(canary_p);
7536 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7537 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7538 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7539 return ret;
7540}
7541
7542static VALUE
7543vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7544{
7545 return invoke_bf(ec, cfp, bf, argv);
7546}
7547
7548static VALUE
7549vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7550{
7551 if (0) { // debug print
7552 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7553 for (int i=0; i<bf->argc; i++) {
7554 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(cfp->iseq)->local_table[i+start_index]));
7555 }
7556 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7557 (void *)(uintptr_t)bf->func_ptr);
7558 }
7559
7560 if (bf->argc == 0) {
7561 return invoke_bf(ec, cfp, bf, NULL);
7562 }
7563 else {
7564 const VALUE *argv = cfp->ep - ISEQ_BODY(cfp->iseq)->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7565 return invoke_bf(ec, cfp, bf, argv);
7566 }
7567}
7568
7569// for __builtin_inline!()
7570
7571VALUE
7572rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7573{
7574 const rb_control_frame_t *cfp = ec->cfp;
7575 return cfp->ep[index];
7576}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
RUBY_EVENT_END
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
RUBY_EVENT_B_RETURN
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
RUBY_EVENT_CLASS
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
RUBY_EVENT_LINE
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
RUBY_EVENT_RETURN
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
RUBY_EVENT_B_CALL
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_EVENT_CALL
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
RUBY_EVENT_RESCUE
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
RB_FL_ABLE
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:440
rb_singleton_class
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:2800
rb_module_new
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1572
rb_class_inherited
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1469
rb_define_class_id
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1448
TYPE
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
RFLOAT_VALUE
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
FIXABLE
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
STATIC_SYM_P
#define STATIC_SYM_P
Old name of RB_STATIC_SYM_P.
Definition special_consts.h:57
ASSUME
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
FIX2ULONG
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
FL_TEST_RAW
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:131
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
LONG2NUM
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
rb_exc_new3
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:130
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
FL_USHIFT
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:68
FL_SET_RAW
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:129
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
rb_notimplement
void rb_notimplement(void)
Definition error.c:3840
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:653
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:475
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1431
rb_eFatal
VALUE rb_eFatal
fatal exception.
Definition error.c:1427
rb_eNoMethodError
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1439
rb_exc_fatal
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:666
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1429
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:466
rb_error_frozen_object
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4161
rb_exc_new_str
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1482
RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
rb_cClass
VALUE rb_cClass
Class class.
Definition object.c:63
rb_cArray
VALUE rb_cArray
Array class.
rb_obj_alloc
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2208
rb_cRegexp
VALUE rb_cRegexp
Regexp class.
Definition re.c:2657
rb_obj_frozen_p
VALUE rb_obj_frozen_p(VALUE obj)
Just calls RB_OBJ_FROZEN() inside.
Definition object.c:1354
rb_cHash
VALUE rb_cHash
Hash class.
Definition hash.c:109
rb_obj_class
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:264
rb_inspect
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:686
rb_cBasicObject
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:59
rb_cModule
VALUE rb_cModule
Module class.
Definition object.c:62
rb_class_real
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:255
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:923
rb_cFloat
VALUE rb_cFloat
Float class.
Definition numeric.c:197
rb_cProc
VALUE rb_cProc
Proc class.
Definition proc.c:45
rb_cString
VALUE rb_cString
String class.
Definition string.c:84
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:615
RB_OBJ_WRITE
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:603
rb_during_gc
int rb_during_gc(void)
Queries if the GC is busy.
Definition gc.c:4268
rb_ary_concat
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
rb_ary_shift
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
rb_ary_resurrect
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
rb_ary_dup
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
rb_ary_includes
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
rb_ary_plus
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *train, long len)
Destructively appends multiple elements at the end of the array.
rb_check_array_type
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
rb_ary_pop
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
rb_ary_hidden_new
VALUE rb_ary_hidden_new(long capa)
Allocates a hidden (no class) empty array.
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
rb_ary_entry
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
rb_ary_store
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
UNLIMITED_ARGUMENTS
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_proc_call_with_block
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1169
rb_reg_last_match
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:1947
rb_reg_match
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:3720
rb_reg_nth_match
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:1922
rb_reg_match_post
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2004
rb_reg_nth_defined
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:1905
rb_reg_match_pre
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:1971
rb_reg_match_last
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2037
rb_str_append
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3799
rb_str_succ
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5341
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3765
rb_str_concat
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4036
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1657
rb_str_length
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2437
rb_str_intern
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:937
rb_thread_schedule
void rb_thread_schedule(void)
Tries to switch to another thread.
Definition thread.c:1513
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3461
rb_ivar_set
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2030
rb_cvar_set
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4261
rb_cvar_find
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4324
rb_ivar_get
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1505
rb_const_set
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:3939
rb_autoload_load
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3291
rb_mod_name
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:136
rb_const_get_at
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3467
rb_set_class_path_string
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:423
rb_ivar_defined
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2109
rb_const_defined_at
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3799
rb_cvar_defined
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4346
rb_class_path
VALUE rb_class_path(VALUE mod)
Identical to rb_mod_name(), except it returns #<Class: ...> style inspection for anonymous modules.
Definition variable.c:380
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3793
rb_check_funcall
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:686
rb_get_alloc_func
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1670
rb_define_alloc_func
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
rb_method_boundp
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2252
rb_check_id
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1133
rb_sym2str
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:993
off
int off
Offset inside of ptr.
Definition io.h:5
len
int len
Length of the buffer.
Definition io.h:8
rb_ractor_shareable_p
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:249
RB_OBJ_SHAREABLE_P
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:235
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
ruby::backward::cxxanyargs::rb_ensure
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
Definition cxxanyargs.hpp:310
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:51
RARRAY_LENINT
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:281
RARRAY_PTR
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:366
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:403
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:52
RBASIC_CLASS
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
RCLASS_SUPER
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
RHASH_SIZE
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:69
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:79
ROBJECT_FIELDS
static VALUE * ROBJECT_FIELDS(VALUE obj)
Queries the instance variables.
Definition robject.h:128
StringValueCStr
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
RB_PASS_KEYWORDS
#define RB_PASS_KEYWORDS
Pass keywords, final argument should be a hash of keywords.
Definition scan_args.h:72
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
ANYARGS
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
RArray
Ruby's array.
Definition rarray.h:128
RArray::ary
const VALUE ary[1]
Embedded elements.
Definition rarray.h:188
RHash
Definition hash.h:53
iseq_catch_table_entry
Definition iseq.h:288
iseq_catch_table
Definition iseq.h:299
iseq_inline_constant_cache_entry
Definition vm_core.h:261
iseq_inline_constant_cache::segments
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:285
rb_block
Definition vm_core.h:902
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:590
rb_callable_method_entry_struct
Definition method.h:63
rb_callcache
Definition vm_callinfo.h:278
rb_callinfo_kwarg
Definition vm_callinfo.h:48
rb_callinfo
Definition vm_callinfo.h:65
rb_calling_info
Definition vm_core.h:306
rb_captured_block
Definition vm_core.h:878
rb_class_cc_entries
Definition vm_callinfo.h:595
rb_control_frame_struct
Definition vm_core.h:911
rb_cvar_class_tbl_entry
Definition class.h:37
rb_env_t
Definition vm_core.h:1300
rb_execution_context_struct
Definition vm_core.h:1033
rb_forwarding_call_data
Definition vm_insnhelper.h:69
rb_id_table
Definition id_table.c:41
rb_iseq_struct
Definition vm_core.h:572
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
rb_method_iseq_struct::iseqptr
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
rb_proc_t
Definition vm_core.h:1287
set_table
Definition set_table.h:10
vm_ifunc
IFUNC (Internal FUNCtion).
Definition imemo.h:86
vm_svar
SVAR (Special VARiable).
Definition imemo.h:50
vm_svar::cref_or_me
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:52
vm_throw_data
THROW_DATA.
Definition imemo.h:59
__attribute__
Definition vm_insnhelper.c:2399
iseq_inline_storage_entry
Definition vm_core.h:297
SIGNED_VALUE
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
SIZEOF_VALUE
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_FLOAT_TYPE_P
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
Generated by doxygen 1.17.0