[ruby-core:120459] [Ruby master Bug#20996] Embed Failure

1 Jan 2025

      Issue #20996 has been reported by cfis (Charlie Savage).

----------------------------------------
Bug #20996: Embed Failure
https://bugs.ruby-lang.org/issues/20996

* Author: cfis (Charlie Savage)
* Status: Open
* ruby -v: ruby 3.4.1 (2024-12-25 revision 48d4efcb85) +PRISM
* Backport: 3.1: UNKNOWN, 3.2: UNKNOWN, 3.3: UNKNOWN, 3.4: UNKNOWN
----------------------------------------
Rice embeds Ruby in order to run its tests. This code has worked across multiple Ruby versions (current tests are 3.1, 3.2, 3.3 and 3.4 but previously tests were run on 3.0 and 2.x). However, it fails on 3.4 in different ways on different operating systems.

The code is here - https://github.com/ruby-rice/rice/blob/master/test/embed_ruby.cpp#L10.

``` c
int argc = 0;
char* argv = nullptr;
char** pArgv = &argv;

// ruby_sysinit(&argc, &pArgv);
ruby_init();
ruby_init_loadpath();

#if RUBY_API_VERSION_MAJOR == 3 && RUBY_API_VERSION_MINOR >= 1
// Force the prelude / builtins
const char* opts[] = { "ruby", "-e;" };
ruby_options(2, (char**)opts);
#endif

```

The test suite is here:

https://github.com/ruby-rice/rice/actions/runs/12534403540/job/35007888873

With Ruby 3.4 on Fedora this fails with the following stack trace:

```c
libruby.so.3.4!str_gsub(int argc, VALUE * argv, VALUE str, int bang) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\string.c:6348)
libruby.so.3.4!vm_call_cfunc_with_frame_(rb_execution_context_t * ec, rb_control_frame_t * reg_cfp, struct rb_calling_info * calling, int argc, VALUE * argv, VALUE * stack_bottom) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_insnhelper.c:3801)
libruby.so.3.4!vm_call_cfunc_with_frame(rb_execution_context_t * ec, rb_control_frame_t * reg_cfp, struct rb_calling_info * calling) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_insnhelper.c:3847)
libruby.so.3.4!vm_sendish(struct rb_execution_context_struct * ec, struct rb_control_frame_struct * reg_cfp, struct rb_call_data * cd, VALUE block_handler) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_callinfo.h:415)
libruby.so.3.4!vm_exec_core(rb_execution_context_t * ec, rb_execution_context_t * ec@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\insns.def:851)
libruby.so.3.4!vm_exec_loop(enum ruby_tag_type state, VALUE result) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2622)
libruby.so.3.4!rb_vm_exec(rb_execution_context_t * ec) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2598)
libruby.so.3.4!vm_yield_with_cref(int is_lambda, rb_execution_context_t * ec, int argc, const VALUE * argv, int kw_splat, const rb_cref_t * cref) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:1670)
libruby.so.3.4!vm_yield(rb_execution_context_t * ec, int argc, const VALUE * argv, int kw_splat) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:1684)
libruby.so.3.4!rb_yield_0(int argc, const VALUE * argv) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_eval.c:1344)
libruby.so.3.4!rb_yield(VALUE val) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_eval.c:1360)
libruby.so.3.4!each_value_i(VALUE key, VALUE value, VALUE _) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\hash.c:3015)
libruby.so.3.4!hash_foreach_iter(st_data_t key, st_data_t key@entry, st_data_t value, st_data_t argp, st_data_t argp@entry, int error, int error@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\hash.c:1316)
libruby.so.3.4!st_general_foreach(st_table * tab, st_table * tab@entry, st_foreach_check_callback_func * func, st_data_t arg) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\st.c:1543)
libruby.so.3.4!rb_st_foreach_check(st_table * tab, st_table * tab@entry, st_foreach_check_callback_func * func, st_foreach_check_callback_func * func@entry, st_data_t arg, st_data_t arg@entry, st_data_t never, st_data_t never@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\st.c:1648)
libruby.so.3.4!hash_foreach_call(VALUE arg, VALUE arg@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\hash.c:1439)
libruby.so.3.4!rb_ensure(VALUE (*)(VALUE) b_proc, VALUE (*)(VALUE) b_proc@entry, VALUE data1, VALUE data1@entry, VALUE (*)(VALUE) e_proc, VALUE (*)(VALUE) e_proc@entry, VALUE data2, VALUE data2@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\eval.c:1074)
libruby.so.3.4!rb_hash_foreach(VALUE hash, VALUE hash@entry, rb_foreach_func * func, rb_foreach_func * func@entry, VALUE farg, VALUE farg@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\hash.c:1463)
libruby.so.3.4!rb_hash_each_value(VALUE hash) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\hash.c:3047)
libruby.so.3.4!vm_call_cfunc_with_frame_(rb_execution_context_t * ec, rb_control_frame_t * reg_cfp, struct rb_calling_info * calling, int argc, VALUE * argv, VALUE * stack_bottom) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_insnhelper.c:3801)
libruby.so.3.4!vm_sendish(struct rb_execution_context_struct * ec, struct rb_control_frame_struct * reg_cfp, struct rb_call_data * cd, VALUE block_handler) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_callinfo.h:415)
libruby.so.3.4!vm_exec_core(rb_execution_context_t * ec, rb_execution_context_t * ec@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\insns.def:851)
libruby.so.3.4!vm_exec_loop(enum ruby_tag_type state, VALUE result) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2622)
libruby.so.3.4!rb_vm_exec(rb_execution_context_t * ec) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2598)
libruby.so.3.4!rb_iseq_eval(const rb_iseq_t * iseq, const rb_iseq_t * iseq@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2850)
libruby.so.3.4!load_iseq_eval(rb_execution_context_t * ec, VALUE fname) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:789)
libruby.so.3.4!require_internal(rb_execution_context_t * ec, rb_execution_context_t * ec@entry, VALUE fname, VALUE fname@entry, int exception, int exception@entry, _Bool warn) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:1296)
libruby.so.3.4!rb_require_string_internal(VALUE fname) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:1402)
libruby.so.3.4!rb_require_string(VALUE fname) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:1388)
libruby.so.3.4!vm_call_cfunc_with_frame_(rb_execution_context_t * ec, rb_control_frame_t * reg_cfp, struct rb_calling_info * calling, int argc, VALUE * argv, VALUE * stack_bottom) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_insnhelper.c:3801)
libruby.so.3.4!vm_call_cfunc_with_frame(rb_execution_context_t * ec, rb_control_frame_t * reg_cfp, struct rb_calling_info * calling) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_insnhelper.c:3847)
libruby.so.3.4!vm_sendish() (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_callinfo.h:415)
libruby.so.3.4!vm_exec_core(rb_execution_context_t * ec, rb_execution_context_t * ec@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\insns.def:898)
libruby.so.3.4!vm_exec_loop(enum ruby_tag_type state, VALUE result) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2622)
libruby.so.3.4!rb_vm_exec(rb_execution_context_t * ec) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2598)
libruby.so.3.4!rb_iseq_eval(const rb_iseq_t * iseq, const rb_iseq_t * iseq@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2850)
libruby.so.3.4!load_iseq_eval(rb_execution_context_t * ec, VALUE fname) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:789)
libruby.so.3.4!require_internal(rb_execution_context_t * ec, rb_execution_context_t * ec@entry, VALUE fname, VALUE fname@entry, int exception, int exception@entry, _Bool warn) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:1296)
libruby.so.3.4!rb_require_string_internal(VALUE fname) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:1402)
libruby.so.3.4!rb_require_string(VALUE fname) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\load.c:1388)
libruby.so.3.4!vm_call_cfunc_with_frame_(rb_execution_context_t * ec, rb_control_frame_t * reg_cfp, struct rb_calling_info * calling, int argc, VALUE * argv, VALUE * stack_bottom) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_insnhelper.c:3801)
libruby.so.3.4!vm_sendish() (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm_callinfo.h:415)
libruby.so.3.4!vm_exec_core(rb_execution_context_t * ec, rb_execution_context_t * ec@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\insns.def:898)
libruby.so.3.4!vm_exec_loop(enum ruby_tag_type state, VALUE result) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2622)
libruby.so.3.4!rb_vm_exec(rb_execution_context_t * ec) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\vm.c:2598)
libruby.so.3.4!ruby_init_prelude() (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\ruby.c:1750)
libruby.so.3.4!ruby_opt_init(ruby_cmdline_options_t * opt) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\ruby.c:1811)
libruby.so.3.4!ruby_opt_init(ruby_cmdline_options_t * opt) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\ruby.c:1767)
libruby.so.3.4!prism_script(ruby_cmdline_options_t * opt, pm_parse_result_t * result) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\ruby.c:2199)
libruby.so.3.4!process_options(int argc, int argc@entry, char ** argv, char ** argv@entry, ruby_cmdline_options_t * opt, ruby_cmdline_options_t * opt@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\ruby.c:2538)
libruby.so.3.4!ruby_process_options(int argc, int argc@entry, char ** argv, char ** argv@entry) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\ruby.c:228)
libruby.so.3.4!ruby_options(int argc, char ** argv) (\home\cfis\.rbenv\sources\3.4.1\ruby-3.4.1\eval.c:117)
embed_ruby() (\usr\local\src\rice\test\embed_ruby.cpp:21)
fixture__Keep_Alive__setup() (\usr\local\src\rice\test\test_Keep_Alive.cpp:60)
Test_Suite::run(class Test_Suite * const this, class Test_Suite * const this@entry, class Test_Result & result) (\usr\local\src\rice\test\unittest.cpp:61)
main(int argc, char ** argv) (\usr\local\src\rice\test\unittest.cpp:124)
```

On MacOS, it fails like this (see https://github.com/ruby-rice/rice/actions/runs/12534403540/job/35007888873):

```c
-- C level backtrace information -------------------------------------------
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_vm_bugreport+0xb6c) [0x103b1f10c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_bug_for_fatal_signal+0x100) [0x103957970]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(sigsegv+0x84) [0x103a809d4]
/usr/lib/system/libsystem_platform.dylib(_sigtramp+0x38) [0x19a576584]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(callable_method_entry_or_negative+0xc0) [0x103afbac8]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_vm_search_method_slowpath+0xc8) [0x103aeefa4]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(gccct_method_search_slowpath+0x24) [0x103b0a484]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_funcallv_scope+0x17c) [0x103b002e0]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_funcall+0x88) [0x103b00750]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_obj_as_string+0x44) [0x103a927d4]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(ruby__sfvextra+0xe0) [0x103a8692c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(BSD_vfprintf+0x60c) [0x103a84a24]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(ruby_vsprintf0+0xa8) [0x103a84100]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_sprintf+0x5c) [0x103a84278]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(unexpected_type+0x64) [0x103c63c64]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_unexpected_type+0x30) [0x103c63cac]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_fstring+0x1a0) [0x103a8f77c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(require_internal+0x550) [0x1039c37f8]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_require_string_internal+0x58) [0x1039c2b6c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_f_require+0x44) [0x1039c2a3c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(vm_call_cfunc_with_frame_+0xf0) [0x103b10ce8]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(vm_exec_core+0x2468) [0x103af4cf0]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_vm_exec+0x1e8) [0x103af136c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(load_iseq_eval+0x22c) [0x1039c56ac]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(require_internal+0x354) [0x1039c35fc]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_require_string_internal+0x58) [0x1039c2b6c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_f_require+0x44) [0x1039c2a3c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(vm_call_cfunc_with_frame_+0xf0) [0x103b10ce8]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(vm_exec_core+0x2468) [0x103af4cf0]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(rb_vm_exec+0x1e8) [0x103af136c]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(ruby_opt_init+0x138) [0x103a76fb4]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(ruby_process_options+0x1150) [0x103a75a90]
/Users/runner/hostedtoolcache/Ruby/3.4.1/arm64/lib/libruby.3.4.dylib(ruby_options+0x9c) [0x1039621d8]
/Users/runner/work/rice/rice/test/unittest(_Z10embed_rubyv+0x6c) [0x100fe3ba0]
/Users/runner/work/rice/rice/test/unittest(_ZL42fixture__Address_Registration_Guard__setupv) [0x100fec360]
/Users/runner/work/rice/rice/test/unittest(_ZN10Test_Suite3runER11Test_Result) [0x10190adf8]
/Users/runner/work/rice/rice/test/unittest(main) [0x10190b748]
```

On Window, both MSVC and UCRT64 fails with a stack overflow error.

MSVC:

```C
	x64-vcruntime140-ruby340.dll!ec_stack_overflow(rb_execution_context_struct * ec, int setup) Line 70	C
 	x64-vcruntime140-ruby340.dll!rb_ec_stack_overflow(rb_execution_context_struct * ec, int crit) Line 96	C
 	[Inline Frame] x64-vcruntime140-ruby340.dll!stack_check(rb_execution_context_struct *) Line 384	C
 	x64-vcruntime140-ruby340.dll!rb_call0(rb_execution_context_struct * ec, unsigned __int64 recv, unsigned __int64 mid, int argc, const unsigned __int64 * argv, call_type call_scope, unsigned __int64 self) Line 553	C
 	[Inline Frame] x64-vcruntime140-ruby340.dll!rb_call(unsigned __int64) Line 873	C
 	x64-vcruntime140-ruby340.dll!rb_funcallv_kw(unsigned __int64 recv, unsigned __int64 mid, int argc, const unsigned __int64 * argv, int kw_splat) Line 1071	C
 	x64-vcruntime140-ruby340.dll!rb_obj_call_init_kw(unsigned __int64 obj, int argc, const unsigned __int64 * argv, int kw_splat) Line 1753	C
 	x64-vcruntime140-ruby340.dll!rb_class_new_instance_kw(int argc, const unsigned __int64 * argv, unsigned __int64 klass, int kw_splat) Line 2194	C
 	x64-vcruntime140-ruby340.dll!rb_exc_new_str(unsigned __int64 etype, unsigned __int64 str) Line 1486	C
 	x64-vcruntime140-ruby340.dll!rb_vm_register_special_exception_str(ruby_special_exceptions sp, unsigned __int64 cls, unsigned __int64 mesg) Line 3058	C
 	x64-vcruntime140-ruby340.dll!Init_eval() Line 2163	C
 	x64-vcruntime140-ruby340.dll!rb_call_inits() Line 37	C
 	x64-vcruntime140-ruby340.dll!ruby_setup() Line 87	C
 	x64-vcruntime140-ruby340.dll!ruby_init() Line 99	C
 	unittest.exe!embed_ruby() Line 15	C++
 	unittest.exe!fixture__Keep_Alive__setup() Line 60	C++
 	unittest.exe!Test_Suite::run(Test_Result & result) Line 61	C++
 	unittest.exe!main(int argc, char * * argv) Line 124	C++
 	[External Code]	
```

Mingw64 with UCRT64:

```
rb_ec_stack_overflow vm_insnhelper.c:96
stack_check vm_eval.c:384
stack_check vm_eval.c:379
rb_call0 vm_eval.c:553
rb_call vm_eval.c:873
rb_funcallv_kw vm_eval.c:1070
rb_obj_call_init_kw eval.c:1752
rb_class_new_instance_kw object.c:2191
rb_class_new_instance object.c:2199
rb_exc_new_str error.c:1485
rb_vm_register_special_exception_str vm.c:3057
Init_eval eval.c:2162
rb_call_inits inits.c:36
ruby_setup eval.c:86
<unknown> 0x0000000000000000
```

What is the recommended way to embed Ruby 3.4 in a C program?

-- 
https://bugs.ruby-lang.org/