Ruby: gc/default/default.c Source File

Ruby 3.4.9p82 (2026-03-11 revision 76cca827ab52ab1d346a728f068d5b8da3e2952b)

default.c
1#include "ruby/internal/config.h"
2
3#include <signal.h>
4
5#ifndef _WIN32
6# include <sys/mman.h>
7# include <unistd.h>
8# ifdef HAVE_SYS_PRCTL_H
9# include <sys/prctl.h>
10# endif
11#endif
12
13#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
14/* LIST_HEAD conflicts with sys/queue.h on macOS */
15# include <sys/user.h>
16#endif
17
18#include "internal/bits.h"
19#include "internal/hash.h"
20
21#include "ruby/ruby.h"
22#include "ruby/atomic.h"
23#include "ruby/debug.h"
24#include "ruby/thread.h"
25#include "ruby/util.h"
26#include "ruby/vm.h"
27#include "ruby/internal/encoding/string.h"
28#include "ccan/list/list.h"
29#include "darray.h"
30#include "gc/gc.h"
31#include "gc/gc_impl.h"
32
33#ifndef BUILDING_MODULAR_GC
34# include "probes.h"
35#endif
36
37#include "debug_counter.h"
38#include "internal/sanitizers.h"
39
40/* MALLOC_HEADERS_BEGIN */
41#ifndef HAVE_MALLOC_USABLE_SIZE
42# ifdef _WIN32
43# define HAVE_MALLOC_USABLE_SIZE
44# define malloc_usable_size(a) _msize(a)
45# elif defined HAVE_MALLOC_SIZE
46# define HAVE_MALLOC_USABLE_SIZE
47# define malloc_usable_size(a) malloc_size(a)
48# endif
49#endif
50
51#ifdef HAVE_MALLOC_USABLE_SIZE
52# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
53/* Alternative malloc header is included in ruby/missing.h */
54# elif defined(HAVE_MALLOC_H)
55# include <malloc.h>
56# elif defined(HAVE_MALLOC_NP_H)
57# include <malloc_np.h>
58# elif defined(HAVE_MALLOC_MALLOC_H)
59# include <malloc/malloc.h>
60# endif
61#endif
62
63#ifdef HAVE_MALLOC_TRIM
64# include <malloc.h>
65
66# ifdef __EMSCRIPTEN__
67/* malloc_trim is defined in emscripten/emmalloc.h on emscripten. */
68# include <emscripten/emmalloc.h>
69# endif
70#endif
71
72#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
73# include <mach/task.h>
74# include <mach/mach_init.h>
75# include <mach/mach_port.h>
76#endif
77
78#ifndef VM_CHECK_MODE
79# define VM_CHECK_MODE RUBY_DEBUG
80#endif
81
82// From ractor_core.h
83#ifndef RACTOR_CHECK_MODE
84# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
85#endif
86
87#ifndef RUBY_DEBUG_LOG
88# define RUBY_DEBUG_LOG(...)
89#endif
90
91#ifndef GC_HEAP_INIT_SLOTS
92#define GC_HEAP_INIT_SLOTS 10000
93#endif
94#ifndef GC_HEAP_FREE_SLOTS
95#define GC_HEAP_FREE_SLOTS 4096
96#endif
97#ifndef GC_HEAP_GROWTH_FACTOR
98#define GC_HEAP_GROWTH_FACTOR 1.8
99#endif
100#ifndef GC_HEAP_GROWTH_MAX_SLOTS
101#define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
102#endif
103#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
104# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
105#endif
106#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
107#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
108#endif
109
110#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
111#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
112#endif
113#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
114#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
115#endif
116#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
117#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
118#endif
119
120#ifndef GC_MALLOC_LIMIT_MIN
121#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
122#endif
123#ifndef GC_MALLOC_LIMIT_MAX
124#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
125#endif
126#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
127#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
128#endif
129
130#ifndef GC_OLDMALLOC_LIMIT_MIN
131#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
132#endif
133#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
134#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
135#endif
136#ifndef GC_OLDMALLOC_LIMIT_MAX
137#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
138#endif
139
140#ifndef GC_CAN_COMPILE_COMPACTION
141#if defined(__wasi__) /* WebAssembly doesn't support signals */
142# define GC_CAN_COMPILE_COMPACTION 0
143#else
144# define GC_CAN_COMPILE_COMPACTION 1
145#endif
146#endif
147
148#ifndef PRINT_ENTER_EXIT_TICK
149# define PRINT_ENTER_EXIT_TICK 0
150#endif
151#ifndef PRINT_ROOT_TICKS
152#define PRINT_ROOT_TICKS 0
153#endif
154
155#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
156
157#ifndef HEAP_COUNT
158# define HEAP_COUNT 5
159#endif
160
161 typedef struct ractor_newobj_heap_cache {
162 struct free_slot *freelist;
163 struct heap_page *using_page;
164} rb_ractor_newobj_heap_cache_t;
165
166 typedef struct ractor_newobj_cache {
167 size_t incremental_mark_step_allocated_slots;
168 rb_ractor_newobj_heap_cache_t heap_caches[HEAP_COUNT];
169} rb_ractor_newobj_cache_t;
170
171 typedef struct {
172 size_t heap_init_slots[HEAP_COUNT];
173 size_t heap_free_slots;
174 double growth_factor;
175 size_t growth_max_slots;
176
177 double heap_free_slots_min_ratio;
178 double heap_free_slots_goal_ratio;
179 double heap_free_slots_max_ratio;
180 double uncollectible_wb_unprotected_objects_limit_ratio;
181 double oldobject_limit_factor;
182
183 size_t malloc_limit_min;
184 size_t malloc_limit_max;
185 double malloc_limit_growth_factor;
186
187 size_t oldmalloc_limit_min;
188 size_t oldmalloc_limit_max;
189 double oldmalloc_limit_growth_factor;
190} ruby_gc_params_t;
191
192static ruby_gc_params_t gc_params = {
193 { GC_HEAP_INIT_SLOTS },
194 GC_HEAP_FREE_SLOTS,
195 GC_HEAP_GROWTH_FACTOR,
196 GC_HEAP_GROWTH_MAX_SLOTS,
197
198 GC_HEAP_FREE_SLOTS_MIN_RATIO,
199 GC_HEAP_FREE_SLOTS_GOAL_RATIO,
200 GC_HEAP_FREE_SLOTS_MAX_RATIO,
201 GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO,
202 GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
203
204 GC_MALLOC_LIMIT_MIN,
205 GC_MALLOC_LIMIT_MAX,
206 GC_MALLOC_LIMIT_GROWTH_FACTOR,
207
208 GC_OLDMALLOC_LIMIT_MIN,
209 GC_OLDMALLOC_LIMIT_MAX,
210 GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
211};
212
213/* GC_DEBUG:
214 * enable to embed GC debugging information.
215 */
216#ifndef GC_DEBUG
217#define GC_DEBUG 0
218#endif
219
220/* RGENGC_DEBUG:
221 * 1: basic information
222 * 2: remember set operation
223 * 3: mark
224 * 4:
225 * 5: sweep
226 */
227#ifndef RGENGC_DEBUG
228#ifdef RUBY_DEVEL
229#define RGENGC_DEBUG -1
230#else
231#define RGENGC_DEBUG 0
232#endif
233#endif
234#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
235# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
236#elif defined(HAVE_VA_ARGS_MACRO)
237# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
238#else
239# define RGENGC_DEBUG_ENABLED(level) 0
240#endif
241int ruby_rgengc_debug;
242
243/* RGENGC_PROFILE
244 * 0: disable RGenGC profiling
245 * 1: enable profiling for basic information
246 * 2: enable profiling for each types
247 */
248#ifndef RGENGC_PROFILE
249# define RGENGC_PROFILE 0
250#endif
251
252/* RGENGC_ESTIMATE_OLDMALLOC
253 * Enable/disable to estimate increase size of malloc'ed size by old objects.
254 * If estimation exceeds threshold, then will invoke full GC.
255 * 0: disable estimation.
256 * 1: enable estimation.
257 */
258#ifndef RGENGC_ESTIMATE_OLDMALLOC
259# define RGENGC_ESTIMATE_OLDMALLOC 1
260#endif
261
262#ifndef GC_PROFILE_MORE_DETAIL
263# define GC_PROFILE_MORE_DETAIL 0
264#endif
265#ifndef GC_PROFILE_DETAIL_MEMORY
266# define GC_PROFILE_DETAIL_MEMORY 0
267#endif
268#ifndef GC_ENABLE_LAZY_SWEEP
269# define GC_ENABLE_LAZY_SWEEP 1
270#endif
271#ifndef CALC_EXACT_MALLOC_SIZE
272# define CALC_EXACT_MALLOC_SIZE 0
273#endif
274#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
275# ifndef MALLOC_ALLOCATED_SIZE
276# define MALLOC_ALLOCATED_SIZE 0
277# endif
278#else
279# define MALLOC_ALLOCATED_SIZE 0
280#endif
281#ifndef MALLOC_ALLOCATED_SIZE_CHECK
282# define MALLOC_ALLOCATED_SIZE_CHECK 0
283#endif
284
285#ifndef GC_DEBUG_STRESS_TO_CLASS
286# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
287#endif
288
289typedef enum {
290 GPR_FLAG_NONE = 0x000,
291 /* major reason */
292 GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
293 GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
294 GPR_FLAG_MAJOR_BY_SHADY = 0x004,
295 GPR_FLAG_MAJOR_BY_FORCE = 0x008,
296#if RGENGC_ESTIMATE_OLDMALLOC
297 GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
298#endif
299 GPR_FLAG_MAJOR_MASK = 0x0ff,
300
301 /* gc reason */
302 GPR_FLAG_NEWOBJ = 0x100,
303 GPR_FLAG_MALLOC = 0x200,
304 GPR_FLAG_METHOD = 0x400,
305 GPR_FLAG_CAPI = 0x800,
306 GPR_FLAG_STRESS = 0x1000,
307
308 /* others */
309 GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
310 GPR_FLAG_HAVE_FINALIZE = 0x4000,
311 GPR_FLAG_IMMEDIATE_MARK = 0x8000,
312 GPR_FLAG_FULL_MARK = 0x10000,
313 GPR_FLAG_COMPACT = 0x20000,
314
315 GPR_DEFAULT_REASON =
316 (GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
317 GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
318} gc_profile_record_flag;
319
320 typedef struct gc_profile_record {
321 unsigned int flags;
322
323 double gc_time;
324 double gc_invoke_time;
325
326 size_t heap_total_objects;
327 size_t heap_use_size;
328 size_t heap_total_size;
329 size_t moved_objects;
330
331#if GC_PROFILE_MORE_DETAIL
332 double gc_mark_time;
333 double gc_sweep_time;
334
335 size_t heap_use_pages;
336 size_t heap_live_objects;
337 size_t heap_free_objects;
338
339 size_t allocate_increase;
340 size_t allocate_limit;
341
342 double prepare_time;
343 size_t removing_objects;
344 size_t empty_objects;
345#if GC_PROFILE_DETAIL_MEMORY
346 long maxrss;
347 long minflt;
348 long majflt;
349#endif
350#endif
351#if MALLOC_ALLOCATED_SIZE
352 size_t allocated_size;
353#endif
354
355#if RGENGC_PROFILE > 0
356 size_t old_objects;
357 size_t remembered_normal_objects;
358 size_t remembered_shady_objects;
359#endif
360} gc_profile_record;
361
362 struct RMoved {
363 VALUE flags;
364 VALUE dummy;
365 VALUE destination;
366 uint32_t original_shape_id;
367};
368
369#define RMOVED(obj) ((struct RMoved *)(obj))
370
371typedef uintptr_t bits_t;
372enum {
373 BITS_SIZE = sizeof(bits_t),
374 BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
375};
376
377 struct heap_page_header {
378 struct heap_page *page;
379};
380
381 struct heap_page_body {
382 struct heap_page_header header;
383 /* char gap[]; */
384 /* RVALUE values[]; */
385};
386
387#define STACK_CHUNK_SIZE 500
388
389 typedef struct stack_chunk {
390 VALUE data[STACK_CHUNK_SIZE];
391 struct stack_chunk *next;
392} stack_chunk_t;
393
394 typedef struct mark_stack {
395 stack_chunk_t *chunk;
396 stack_chunk_t *cache;
397 int index;
398 int limit;
399 size_t cache_size;
400 size_t unused_cache_size;
401} mark_stack_t;
402
403typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d);
404
405 typedef struct rb_heap_struct {
406 short slot_size;
407
408 /* Basic statistics */
409 size_t total_allocated_pages;
410 size_t force_major_gc_count;
411 size_t force_incremental_marking_finish_count;
412 size_t total_allocated_objects;
413 size_t total_freed_objects;
414 size_t final_slots_count;
415
416 /* Sweeping statistics */
417 size_t freed_slots;
418 size_t empty_slots;
419
420 struct heap_page *free_pages;
421 struct ccan_list_head pages;
422 struct heap_page *sweeping_page; /* iterator for .pages */
423 struct heap_page *compact_cursor;
424 uintptr_t compact_cursor_index;
425 struct heap_page *pooled_pages;
426 size_t total_pages; /* total page count in a heap */
427 size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */
428
429} rb_heap_t;
430
431enum {
432 gc_stress_no_major,
433 gc_stress_no_immediate_sweep,
434 gc_stress_full_mark_after_malloc,
435 gc_stress_max
436};
437
438enum gc_mode {
439 gc_mode_none,
440 gc_mode_marking,
441 gc_mode_sweeping,
442 gc_mode_compacting,
443};
444
445 typedef struct rb_objspace {
446 struct {
447 size_t limit;
448 size_t increase;
449#if MALLOC_ALLOCATED_SIZE
450 size_t allocated_size;
451 size_t allocations;
452#endif
453 } malloc_params;
454
455 struct rb_gc_config {
456 bool full_mark;
457 } gc_config;
458
459 struct {
460 unsigned int mode : 2;
461 unsigned int immediate_sweep : 1;
462 unsigned int dont_gc : 1;
463 unsigned int dont_incremental : 1;
464 unsigned int during_gc : 1;
465 unsigned int during_compacting : 1;
466 unsigned int during_reference_updating : 1;
467 unsigned int gc_stressful: 1;
468 unsigned int has_newobj_hook: 1;
469 unsigned int during_minor_gc : 1;
470 unsigned int during_incremental_marking : 1;
471 unsigned int measure_gc : 1;
472 } flags;
473
474 rb_event_flag_t hook_events;
475 unsigned long long next_object_id;
476
477 rb_heap_t heaps[HEAP_COUNT];
478 size_t empty_pages_count;
479 struct heap_page *empty_pages;
480
481 struct {
482 rb_atomic_t finalizing;
483 } atomic_flags;
484
485 mark_stack_t mark_stack;
486 size_t marked_slots;
487
488 struct {
489 rb_darray(struct heap_page *) sorted;
490
491 size_t allocated_pages;
492 size_t freed_pages;
493 uintptr_t range[2];
494 size_t freeable_pages;
495
496 size_t allocatable_slots;
497
498 /* final */
499 VALUE deferred_final;
500 } heap_pages;
501
502 st_table *finalizer_table;
503
504 struct {
505 int run;
506 unsigned int latest_gc_info;
507 gc_profile_record *records;
508 gc_profile_record *current_record;
509 size_t next_index;
510 size_t size;
511
512#if GC_PROFILE_MORE_DETAIL
513 double prepare_time;
514#endif
515 double invoke_time;
516
517 size_t minor_gc_count;
518 size_t major_gc_count;
519 size_t compact_count;
520 size_t read_barrier_faults;
521#if RGENGC_PROFILE > 0
522 size_t total_generated_normal_object_count;
523 size_t total_generated_shady_object_count;
524 size_t total_shade_operation_count;
525 size_t total_promoted_count;
526 size_t total_remembered_normal_object_count;
527 size_t total_remembered_shady_object_count;
528
529#if RGENGC_PROFILE >= 2
530 size_t generated_normal_object_count_types[RUBY_T_MASK];
531 size_t generated_shady_object_count_types[RUBY_T_MASK];
532 size_t shade_operation_count_types[RUBY_T_MASK];
533 size_t promoted_types[RUBY_T_MASK];
534 size_t remembered_normal_object_count_types[RUBY_T_MASK];
535 size_t remembered_shady_object_count_types[RUBY_T_MASK];
536#endif
537#endif /* RGENGC_PROFILE */
538
539 /* temporary profiling space */
540 double gc_sweep_start_time;
541 size_t total_allocated_objects_at_gc_start;
542 size_t heap_used_at_gc_start;
543
544 /* basic statistics */
545 size_t count;
546 unsigned long long marking_time_ns;
547 struct timespec marking_start_time;
548 unsigned long long sweeping_time_ns;
549 struct timespec sweeping_start_time;
550
551 /* Weak references */
552 size_t weak_references_count;
553 size_t retained_weak_references_count;
554 } profile;
555
556 VALUE gc_stress_mode;
557
558 struct {
559 VALUE parent_object;
560 int need_major_gc;
561 size_t last_major_gc;
562 size_t uncollectible_wb_unprotected_objects;
563 size_t uncollectible_wb_unprotected_objects_limit;
564 size_t old_objects;
565 size_t old_objects_limit;
566
567#if RGENGC_ESTIMATE_OLDMALLOC
568 size_t oldmalloc_increase;
569 size_t oldmalloc_increase_limit;
570#endif
571
572#if RGENGC_CHECK_MODE >= 2
573 struct st_table *allrefs_table;
574 size_t error_count;
575#endif
576 } rgengc;
577
578 struct {
579 size_t considered_count_table[T_MASK];
580 size_t moved_count_table[T_MASK];
581 size_t moved_up_count_table[T_MASK];
582 size_t moved_down_count_table[T_MASK];
583 size_t total_moved;
584
585 /* This function will be used, if set, to sort the heap prior to compaction */
586 gc_compact_compare_func compare_func;
587 } rcompactor;
588
589 struct {
590 size_t pooled_slots;
591 size_t step_slots;
592 } rincgc;
593
594 st_table *id_to_obj_tbl;
595 st_table *obj_to_id_tbl;
596
597#if GC_DEBUG_STRESS_TO_CLASS
598 VALUE stress_to_class;
599#endif
600
601 rb_darray(VALUE *) weak_references;
602 rb_postponed_job_handle_t finalize_deferred_pjob;
603
604 unsigned long live_ractor_cache_count;
605} rb_objspace_t;
606
607#ifndef HEAP_PAGE_ALIGN_LOG
608/* default tiny heap size: 64KiB */
609#define HEAP_PAGE_ALIGN_LOG 16
610#endif
611
612#if RACTOR_CHECK_MODE || GC_DEBUG
613struct rvalue_overhead {
614# if RACTOR_CHECK_MODE
615 uint32_t _ractor_belonging_id;
616# endif
617# if GC_DEBUG
618 const char *file;
619 int line;
620# endif
621};
622
623// Make sure that RVALUE_OVERHEAD aligns to sizeof(VALUE)
624# define RVALUE_OVERHEAD (sizeof(struct { \
625 union { \
626 struct rvalue_overhead overhead; \
627 VALUE value; \
628 }; \
629}))
630size_t rb_gc_impl_obj_slot_size(VALUE obj);
631# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
632#else
633# define RVALUE_OVERHEAD 0
634#endif
635
636#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
637
638#ifndef MAX
639# define MAX(a, b) (((a) > (b)) ? (a) : (b))
640#endif
641#ifndef MIN
642# define MIN(a, b) (((a) < (b)) ? (a) : (b))
643#endif
644#define roomof(x, y) (((x) + (y) - 1) / (y))
645#define CEILDIV(i, mod) roomof(i, mod)
646enum {
647 HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
648 HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
649 HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
650 HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)) / BASE_SLOT_SIZE),
651 HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH),
652 HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
653};
654#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
655#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
656
657#if !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
658# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
659#endif
660
661#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
662/* Must define either HEAP_PAGE_ALLOC_USE_MMAP or
663 * INIT_HEAP_PAGE_ALLOC_USE_MMAP. */
664
665#ifndef HAVE_MMAP
666/* We can't use mmap of course, if it is not available. */
667static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
668
669#elif defined(__wasm__)
670/* wasmtime does not have proper support for mmap.
671 * See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends
672 */
673static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
674
675#elif HAVE_CONST_PAGE_SIZE
676/* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */
677static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
678
679#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
680/* If we can use the maximum page size. */
681static const bool HEAP_PAGE_ALLOC_USE_MMAP = true;
682
683#elif defined(PAGE_SIZE)
684/* If the PAGE_SIZE macro can be used dynamically. */
685# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
686
687#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
688/* If we can use sysconf to determine the page size. */
689# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
690
691#else
692/* Otherwise we can't determine the system page size, so don't use mmap. */
693static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
694#endif
695
696#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
697/* We can determine the system page size at runtime. */
698# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
699
700static bool heap_page_alloc_use_mmap;
701#endif
702
703#define RVALUE_AGE_BIT_COUNT 2
704#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1)
705#define RVALUE_OLD_AGE 3
706
707 struct free_slot {
708 VALUE flags; /* always 0 for freed obj */
709 struct free_slot *next;
710};
711
712 struct heap_page {
713 unsigned short slot_size;
714 unsigned short total_slots;
715 unsigned short free_slots;
716 unsigned short final_slots;
717 unsigned short pinned_slots;
718 struct {
719 unsigned int before_sweep : 1;
720 unsigned int has_remembered_objects : 1;
721 unsigned int has_uncollectible_wb_unprotected_objects : 1;
722 } flags;
723
724 rb_heap_t *heap;
725
726 struct heap_page *free_next;
727 struct heap_page_body *body;
728 uintptr_t start;
729 struct free_slot *freelist;
730 struct ccan_list_node page_node;
731
732 bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
733 /* the following three bitmaps are cleared at the beginning of full GC */
734 bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
735 bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
736 bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
737
738 bits_t remembered_bits[HEAP_PAGE_BITMAP_LIMIT];
739
740 /* If set, the object is not movable */
741 bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
742 bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
743};
744
745/*
746 * When asan is enabled, this will prohibit writing to the freelist until it is unlocked
747 */
748static void
749asan_lock_freelist(struct heap_page *page)
750{
751 asan_poison_memory_region(&page->freelist, sizeof(struct free_list *));
752}
753
754/*
755 * When asan is enabled, this will enable the ability to write to the freelist
756 */
757static void
758asan_unlock_freelist(struct heap_page *page)
759{
760 asan_unpoison_memory_region(&page->freelist, sizeof(struct free_list *), false);
761}
762
763static inline bool
764heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *page)
765{
766 if (page->total_slots == 0) {
767 GC_ASSERT(page->start == 0);
768 GC_ASSERT(page->slot_size == 0);
769 GC_ASSERT(page->heap == NULL);
770 GC_ASSERT(page->free_slots == 0);
771 asan_unpoisoning_memory_region(&page->freelist, sizeof(&page->freelist)) {
772 GC_ASSERT(page->freelist == NULL);
773 }
774
775 return true;
776 }
777 else {
778 GC_ASSERT(page->start != 0);
779 GC_ASSERT(page->slot_size != 0);
780 GC_ASSERT(page->heap != NULL);
781
782 return false;
783 }
784}
785
786#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
787#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
788#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
789
790#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
791#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
792#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
793#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
794
795/* Bitmap Operations */
796#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
797#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
798#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
799
800/* getting bitmap */
801#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
802#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
803#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
804#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
805#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
806
807#define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3
808
809#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT))
810#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT)
811
812static int
813RVALUE_AGE_GET(VALUE obj)
814{
815 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
816 return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK;
817}
818
819static void
820RVALUE_AGE_SET(VALUE obj, int age)
821{
822 RUBY_ASSERT(age <= RVALUE_OLD_AGE);
823 bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
824 // clear the bits
825 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj)));
826 // shift the correct value in
827 age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj));
828 if (age == RVALUE_OLD_AGE) {
829 RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED);
830 }
831 else {
832 RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED);
833 }
834}
835
836#define malloc_limit objspace->malloc_params.limit
837#define malloc_increase objspace->malloc_params.increase
838#define malloc_allocated_size objspace->malloc_params.allocated_size
839#define heap_pages_lomem objspace->heap_pages.range[0]
840#define heap_pages_himem objspace->heap_pages.range[1]
841#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
842#define heap_pages_deferred_final objspace->heap_pages.deferred_final
843#define heaps objspace->heaps
844#define during_gc objspace->flags.during_gc
845#define finalizing objspace->atomic_flags.finalizing
846#define finalizer_table objspace->finalizer_table
847#define ruby_gc_stressful objspace->flags.gc_stressful
848#define ruby_gc_stress_mode objspace->gc_stress_mode
849#if GC_DEBUG_STRESS_TO_CLASS
850#define stress_to_class objspace->stress_to_class
851#define set_stress_to_class(c) (stress_to_class = (c))
852#else
853#define stress_to_class (objspace, 0)
854#define set_stress_to_class(c) (objspace, (c))
855#endif
856
857#if 0
858#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
859#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
860#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = (int)(b))
861#define dont_gc_val() (objspace->flags.dont_gc)
862#else
863#define dont_gc_on() (objspace->flags.dont_gc = 1)
864#define dont_gc_off() (objspace->flags.dont_gc = 0)
865#define dont_gc_set(b) (objspace->flags.dont_gc = (int)(b))
866#define dont_gc_val() (objspace->flags.dont_gc)
867#endif
868
869#define gc_config_full_mark_set(b) (objspace->gc_config.full_mark = (int)(b))
870#define gc_config_full_mark_val (objspace->gc_config.full_mark)
871
872#ifndef DURING_GC_COULD_MALLOC_REGION_START
873# define DURING_GC_COULD_MALLOC_REGION_START() \
874 assert(rb_during_gc()); \
875 bool _prev_enabled = rb_gc_impl_gc_enabled_p(objspace); \
876 rb_gc_impl_gc_disable(objspace, false)
877#endif
878
879#ifndef DURING_GC_COULD_MALLOC_REGION_END
880# define DURING_GC_COULD_MALLOC_REGION_END() \
881 if (_prev_enabled) rb_gc_impl_gc_enable(objspace)
882#endif
883
884static inline enum gc_mode
885gc_mode_verify(enum gc_mode mode)
886{
887#if RGENGC_CHECK_MODE > 0
888 switch (mode) {
889 case gc_mode_none:
890 case gc_mode_marking:
891 case gc_mode_sweeping:
892 case gc_mode_compacting:
893 break;
894 default:
895 rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
896 }
897#endif
898 return mode;
899}
900
901static inline bool
902has_sweeping_pages(rb_objspace_t *objspace)
903{
904 for (int i = 0; i < HEAP_COUNT; i++) {
905 if ((&heaps[i])->sweeping_page) {
906 return TRUE;
907 }
908 }
909 return FALSE;
910}
911
912static inline size_t
913heap_eden_total_pages(rb_objspace_t *objspace)
914{
915 size_t count = 0;
916 for (int i = 0; i < HEAP_COUNT; i++) {
917 count += (&heaps[i])->total_pages;
918 }
919 return count;
920}
921
922static inline size_t
923total_allocated_objects(rb_objspace_t *objspace)
924{
925 size_t count = 0;
926 for (int i = 0; i < HEAP_COUNT; i++) {
927 rb_heap_t *heap = &heaps[i];
928 count += heap->total_allocated_objects;
929 }
930 return count;
931}
932
933static inline size_t
934total_freed_objects(rb_objspace_t *objspace)
935{
936 size_t count = 0;
937 for (int i = 0; i < HEAP_COUNT; i++) {
938 rb_heap_t *heap = &heaps[i];
939 count += heap->total_freed_objects;
940 }
941 return count;
942}
943
944static inline size_t
945total_final_slots_count(rb_objspace_t *objspace)
946{
947 size_t count = 0;
948 for (int i = 0; i < HEAP_COUNT; i++) {
949 rb_heap_t *heap = &heaps[i];
950 count += heap->final_slots_count;
951 }
952 return count;
953}
954
955#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
956#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m))
957#define gc_needs_major_flags objspace->rgengc.need_major_gc
958
959#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
960#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
961#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
962#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
963#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
964#define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048
965#define GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT 1024
966#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
967
968#if SIZEOF_LONG == SIZEOF_VOIDP
969# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
970#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
971# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
972 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
973#else
974# error not supported
975#endif
976
977 struct RZombie {
978 struct RBasic basic;
979 VALUE next;
980 void (*dfree)(void *);
981 void *data;
982};
983
984#define RZOMBIE(o) ((struct RZombie *)(o))
985
986int ruby_enable_autocompact = 0;
987#if RGENGC_CHECK_MODE
988gc_compact_compare_func ruby_autocompact_compare_func;
989#endif
990
991static void init_mark_stack(mark_stack_t *stack);
992static int garbage_collect(rb_objspace_t *, unsigned int reason);
993
994static int gc_start(rb_objspace_t *objspace, unsigned int reason);
995static void gc_rest(rb_objspace_t *objspace);
996
997enum gc_enter_event {
998 gc_enter_event_start,
999 gc_enter_event_continue,
1000 gc_enter_event_rest,
1001 gc_enter_event_finalizer,
1002};
1003
1004static inline void gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
1005static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
1006static void gc_marking_enter(rb_objspace_t *objspace);
1007static void gc_marking_exit(rb_objspace_t *objspace);
1008static void gc_sweeping_enter(rb_objspace_t *objspace);
1009static void gc_sweeping_exit(rb_objspace_t *objspace);
1010static bool gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
1011
1012static void gc_sweep(rb_objspace_t *objspace);
1013static void gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap);
1014static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
1015
1016static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
1017static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
1018static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
1019
1020static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
1021NO_SANITIZE("memory", static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr));
1022
1023static void gc_verify_internal_consistency(void *objspace_ptr);
1024
1025static double getrusage_time(void);
1026static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason);
1027static inline void gc_prof_timer_start(rb_objspace_t *);
1028static inline void gc_prof_timer_stop(rb_objspace_t *);
1029static inline void gc_prof_mark_timer_start(rb_objspace_t *);
1030static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
1031static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
1032static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
1033static inline void gc_prof_set_malloc_info(rb_objspace_t *);
1034static inline void gc_prof_set_heap_info(rb_objspace_t *);
1035
1036#define gc_prof_record(objspace) (objspace)->profile.current_record
1037#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1038
1039#ifdef HAVE_VA_ARGS_MACRO
1040# define gc_report(level, objspace, ...) \
1041 if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1042#else
1043# define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
1044#endif
1045PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
1046
1047static void gc_finalize_deferred(void *dmy);
1048
1049#if USE_TICK_T
1050
1051/* the following code is only for internal tuning. */
1052
1053/* Source code to use RDTSC is quoted and modified from
1054 * https://www.mcs.anl.gov/~kazutomo/rdtsc.html
1055 * written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
1056 */
1057
1058#if defined(__GNUC__) && defined(__i386__)
1059typedef unsigned long long tick_t;
1060#define PRItick "llu"
1061static inline tick_t
1062tick(void)
1063{
1064 unsigned long long int x;
1065 __asm__ __volatile__ ("rdtsc" : "=A" (x));
1066 return x;
1067}
1068
1069#elif defined(__GNUC__) && defined(__x86_64__)
1070typedef unsigned long long tick_t;
1071#define PRItick "llu"
1072
1073static __inline__ tick_t
1074tick(void)
1075{
1076 unsigned long hi, lo;
1077 __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
1078 return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
1079}
1080
1081#elif defined(__powerpc64__) && (GCC_VERSION_SINCE(4,8,0) || defined(__clang__))
1082typedef unsigned long long tick_t;
1083#define PRItick "llu"
1084
1085static __inline__ tick_t
1086tick(void)
1087{
1088 unsigned long long val = __builtin_ppc_get_timebase();
1089 return val;
1090}
1091
1092/* Implementation for macOS PPC by @nobu
1093 * See: https://github.com/ruby/ruby/pull/5975#discussion_r890045558
1094 */
1095#elif defined(__POWERPC__) && defined(__APPLE__)
1096typedef unsigned long long tick_t;
1097#define PRItick "llu"
1098
1099static __inline__ tick_t
1100tick(void)
1101{
1102 unsigned long int upper, lower, tmp;
1103 # define mftbu(r) __asm__ volatile("mftbu %0" : "=r"(r))
1104 # define mftb(r) __asm__ volatile("mftb %0" : "=r"(r))
1105 do {
1106 mftbu(upper);
1107 mftb(lower);
1108 mftbu(tmp);
1109 } while (tmp != upper);
1110 return ((tick_t)upper << 32) | lower;
1111}
1112
1113#elif defined(__aarch64__) && defined(__GNUC__)
1114typedef unsigned long tick_t;
1115#define PRItick "lu"
1116
1117static __inline__ tick_t
1118tick(void)
1119{
1120 unsigned long val;
1121 __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val));
1122 return val;
1123}
1124
1125
1126#elif defined(_WIN32) && defined(_MSC_VER)
1127#include <intrin.h>
1128typedef unsigned __int64 tick_t;
1129#define PRItick "llu"
1130
1131static inline tick_t
1132tick(void)
1133{
1134 return __rdtsc();
1135}
1136
1137#else /* use clock */
1138typedef clock_t tick_t;
1139#define PRItick "llu"
1140
1141static inline tick_t
1142tick(void)
1143{
1144 return clock();
1145}
1146#endif /* TSC */
1147#else /* USE_TICK_T */
1148#define MEASURE_LINE(expr) expr
1149#endif /* USE_TICK_T */
1150
1151static inline VALUE check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj);
1152
1153#define RVALUE_MARKED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1154#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1155#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1156#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1157#define RVALUE_PINNED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1158
1159static inline int
1160RVALUE_MARKED(rb_objspace_t *objspace, VALUE obj)
1161{
1162 check_rvalue_consistency(objspace, obj);
1163 return RVALUE_MARKED_BITMAP(obj) != 0;
1164}
1165
1166static inline int
1167RVALUE_PINNED(rb_objspace_t *objspace, VALUE obj)
1168{
1169 check_rvalue_consistency(objspace, obj);
1170 return RVALUE_PINNED_BITMAP(obj) != 0;
1171}
1172
1173static inline int
1174RVALUE_WB_UNPROTECTED(rb_objspace_t *objspace, VALUE obj)
1175{
1176 check_rvalue_consistency(objspace, obj);
1177 return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1178}
1179
1180static inline int
1181RVALUE_MARKING(rb_objspace_t *objspace, VALUE obj)
1182{
1183 check_rvalue_consistency(objspace, obj);
1184 return RVALUE_MARKING_BITMAP(obj) != 0;
1185}
1186
1187static inline int
1188RVALUE_REMEMBERED(rb_objspace_t *objspace, VALUE obj)
1189{
1190 check_rvalue_consistency(objspace, obj);
1191 return MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1192}
1193
1194static inline int
1195RVALUE_UNCOLLECTIBLE(rb_objspace_t *objspace, VALUE obj)
1196{
1197 check_rvalue_consistency(objspace, obj);
1198 return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1199}
1200
1201#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1202#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1203#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1204
1205static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
1206static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
1207static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
1208
1209static int
1210check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int terminate)
1211{
1212 int err = 0;
1213
1214 int lev = rb_gc_vm_lock_no_barrier();
1215 {
1216 if (SPECIAL_CONST_P(obj)) {
1217 fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
1218 err++;
1219 }
1220 else if (!is_pointer_to_heap(objspace, (void *)obj)) {
1221 struct heap_page *empty_page = objspace->empty_pages;
1222 while (empty_page) {
1223 if ((uintptr_t)empty_page->body <= (uintptr_t)obj &&
1224 (uintptr_t)obj < (uintptr_t)empty_page->body + HEAP_PAGE_SIZE) {
1225 GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, empty_page));
1226 fprintf(stderr, "check_rvalue_consistency: %p is in an empty page (%p).\n",
1227 (void *)obj, (void *)empty_page);
1228 err++;
1229 goto skip;
1230 }
1231 }
1232 fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
1233 err++;
1234 skip:
1235 ;
1236 }
1237 else {
1238 const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1239 const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1240 const int mark_bit = RVALUE_MARKED_BITMAP(obj) != 0;
1241 const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0;
1242 const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0;
1243 const int age = RVALUE_AGE_GET((VALUE)obj);
1244
1245 if (heap_page_in_global_empty_pages_pool(objspace, GET_HEAP_PAGE(obj))) {
1246 fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", rb_obj_info(obj));
1247 err++;
1248 }
1249 if (BUILTIN_TYPE(obj) == T_NONE) {
1250 fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", rb_obj_info(obj));
1251 err++;
1252 }
1253 if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
1254 fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", rb_obj_info(obj));
1255 err++;
1256 }
1257
1258 if (BUILTIN_TYPE(obj) != T_DATA) {
1259 rb_obj_memsize_of((VALUE)obj);
1260 }
1261
1262 /* check generation
1263 *
1264 * OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
1265 */
1266 if (age > 0 && wb_unprotected_bit) {
1267 fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", rb_obj_info(obj), age);
1268 err++;
1269 }
1270
1271 if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
1272 fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", rb_obj_info(obj));
1273 err++;
1274 }
1275
1276 if (!is_full_marking(objspace)) {
1277 if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1278 fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1279 rb_obj_info(obj), age);
1280 err++;
1281 }
1282 if (remembered_bit && age != RVALUE_OLD_AGE) {
1283 fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1284 rb_obj_info(obj), age);
1285 err++;
1286 }
1287 }
1288
1289 /*
1290 * check coloring
1291 *
1292 * marking:false marking:true
1293 * marked:false white *invalid*
1294 * marked:true black grey
1295 */
1296 if (is_incremental_marking(objspace) && marking_bit) {
1297 if (!is_marking(objspace) && !mark_bit) {
1298 fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", rb_obj_info(obj));
1299 err++;
1300 }
1301 }
1302 }
1303 }
1304 rb_gc_vm_unlock_no_barrier(lev);
1305
1306 if (err > 0 && terminate) {
1307 rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
1308 }
1309 return err;
1310}
1311
1312#if RGENGC_CHECK_MODE == 0
1313static inline VALUE
1314check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
1315{
1316 return obj;
1317}
1318#else
1319static VALUE
1320check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
1321{
1322 check_rvalue_consistency_force(objspace, obj, TRUE);
1323 return obj;
1324}
1325#endif
1326
1327static inline bool
1328gc_object_moved_p(rb_objspace_t *objspace, VALUE obj)
1329{
1330 if (RB_SPECIAL_CONST_P(obj)) {
1331 return FALSE;
1332 }
1333 else {
1334 int ret;
1335 asan_unpoisoning_object(obj) {
1336 ret = BUILTIN_TYPE(obj) == T_MOVED;
1337 }
1338 return ret;
1339 }
1340}
1341
1342static inline int
1343RVALUE_OLD_P(rb_objspace_t *objspace, VALUE obj)
1344{
1345 GC_ASSERT(!RB_SPECIAL_CONST_P(obj));
1346 check_rvalue_consistency(objspace, obj);
1347 // Because this will only ever be called on GC controlled objects,
1348 // we can use the faster _RAW function here
1349 return RB_OBJ_PROMOTED_RAW(obj);
1350}
1351
1352static inline void
1353RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
1354{
1355 MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
1356 objspace->rgengc.old_objects++;
1357
1358#if RGENGC_PROFILE >= 2
1359 objspace->profile.total_promoted_count++;
1360 objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
1361#endif
1362}
1363
1364static inline void
1365RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
1366{
1367 RB_DEBUG_COUNTER_INC(obj_promote);
1368 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
1369}
1370
1371/* set age to age+1 */
1372static inline void
1373RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
1374{
1375 int age = RVALUE_AGE_GET((VALUE)obj);
1376
1377 if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
1378 rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", rb_obj_info(obj));
1379 }
1380
1381 age++;
1382 RVALUE_AGE_SET(obj, age);
1383
1384 if (age == RVALUE_OLD_AGE) {
1385 RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
1386 }
1387
1388 check_rvalue_consistency(objspace, obj);
1389}
1390
1391static inline void
1392RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
1393{
1394 check_rvalue_consistency(objspace, obj);
1395 GC_ASSERT(!RVALUE_OLD_P(objspace, obj));
1396 RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1);
1397 check_rvalue_consistency(objspace, obj);
1398}
1399
1400static inline void
1401RVALUE_AGE_RESET(VALUE obj)
1402{
1403 RVALUE_AGE_SET(obj, 0);
1404}
1405
1406static inline void
1407RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
1408{
1409 check_rvalue_consistency(objspace, obj);
1410 GC_ASSERT(RVALUE_OLD_P(objspace, obj));
1411
1412 if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(objspace, obj)) {
1413 CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj);
1414 }
1415
1416 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
1417 RVALUE_AGE_RESET(obj);
1418
1419 if (RVALUE_MARKED(objspace, obj)) {
1420 objspace->rgengc.old_objects--;
1421 }
1422
1423 check_rvalue_consistency(objspace, obj);
1424}
1425
1426static inline int
1427RVALUE_BLACK_P(rb_objspace_t *objspace, VALUE obj)
1428{
1429 return RVALUE_MARKED(objspace, obj) && !RVALUE_MARKING(objspace, obj);
1430}
1431
1432static inline int
1433RVALUE_WHITE_P(rb_objspace_t *objspace, VALUE obj)
1434{
1435 return !RVALUE_MARKED(objspace, obj);
1436}
1437
1438bool
1439rb_gc_impl_gc_enabled_p(void *objspace_ptr)
1440{
1441 rb_objspace_t *objspace = objspace_ptr;
1442 return !dont_gc_val();
1443}
1444
1445void
1446rb_gc_impl_gc_enable(void *objspace_ptr)
1447{
1448 rb_objspace_t *objspace = objspace_ptr;
1449
1450 dont_gc_off();
1451}
1452
1453void
1454rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
1455{
1456 rb_objspace_t *objspace = objspace_ptr;
1457
1458 if (finish_current_gc) {
1459 gc_rest(objspace);
1460 }
1461
1462 dont_gc_on();
1463}
1464
1465/*
1466 --------------------------- ObjectSpace -----------------------------
1467*/
1468
1469static inline void *
1470calloc1(size_t n)
1471{
1472 return calloc(1, n);
1473}
1474
1475void
1476rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
1477{
1478 rb_objspace_t *objspace = objspace_ptr;
1479 objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
1480 objspace->flags.has_newobj_hook = !!(objspace->hook_events & RUBY_INTERNAL_EVENT_NEWOBJ);
1481}
1482
1483unsigned long long
1484rb_gc_impl_get_total_time(void *objspace_ptr)
1485{
1486 rb_objspace_t *objspace = objspace_ptr;
1487
1488 unsigned long long marking_time = objspace->profile.marking_time_ns;
1489 unsigned long long sweeping_time = objspace->profile.sweeping_time_ns;
1490
1491 return marking_time + sweeping_time;
1492}
1493
1494void
1495rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag)
1496{
1497 rb_objspace_t *objspace = objspace_ptr;
1498
1499 objspace->flags.measure_gc = RTEST(flag) ? TRUE : FALSE;
1500}
1501
1502bool
1503rb_gc_impl_get_measure_total_time(void *objspace_ptr)
1504{
1505 rb_objspace_t *objspace = objspace_ptr;
1506
1507 return objspace->flags.measure_gc;
1508}
1509
1510static size_t
1511minimum_slots_for_heap(rb_objspace_t *objspace, rb_heap_t *heap)
1512{
1513 size_t heap_idx = heap - heaps;
1514 return gc_params.heap_init_slots[heap_idx];
1515}
1516
1517static int
1518object_id_cmp(st_data_t x, st_data_t y)
1519{
1520 if (RB_TYPE_P(x, T_BIGNUM)) {
1521 return !rb_big_eql(x, y);
1522 }
1523 else {
1524 return x != y;
1525 }
1526}
1527
1528static st_index_t
1529object_id_hash(st_data_t n)
1530{
1531 return FIX2LONG(rb_hash((VALUE)n));
1532}
1533
1534#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
1535#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
1536
1537static const struct st_hash_type object_id_hash_type = {
1538 object_id_cmp,
1539 object_id_hash,
1540};
1541
1542/* garbage objects will be collected soon. */
1543bool
1544rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr)
1545{
1546 rb_objspace_t *objspace = objspace_ptr;
1547
1548 bool dead = false;
1549
1550 asan_unpoisoning_object(ptr) {
1551 switch (BUILTIN_TYPE(ptr)) {
1552 case T_NONE:
1553 case T_MOVED:
1554 case T_ZOMBIE:
1555 dead = true;
1556 break;
1557 default:
1558 break;
1559 }
1560 }
1561
1562 if (dead) return true;
1563 return is_lazy_sweeping(objspace) && GET_HEAP_PAGE(ptr)->flags.before_sweep &&
1564 !RVALUE_MARKED(objspace, ptr);
1565}
1566
1567 VALUE
1568rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id)
1569{
1570 rb_objspace_t *objspace = objspace_ptr;
1571
1572 VALUE obj;
1573 if (st_lookup(objspace->id_to_obj_tbl, object_id, &obj) &&
1574 !rb_gc_impl_garbage_object_p(objspace, obj)) {
1575 return obj;
1576 }
1577
1578 if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(objspace->next_object_id))) {
1579 rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
1580 }
1581 else {
1582 rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
1583 }
1584}
1585
1586 VALUE
1587rb_gc_impl_object_id(void *objspace_ptr, VALUE obj)
1588{
1589 VALUE id;
1590 rb_objspace_t *objspace = objspace_ptr;
1591
1592 unsigned int lev = rb_gc_vm_lock();
1593 if (FL_TEST(obj, FL_SEEN_OBJ_ID)) {
1594 st_data_t val;
1595 if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &val)) {
1596 id = (VALUE)val;
1597 }
1598 else {
1599 rb_bug("rb_gc_impl_object_id: FL_SEEN_OBJ_ID flag set but not found in table");
1600 }
1601 }
1602 else {
1603 GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, NULL));
1604
1605 id = ULL2NUM(objspace->next_object_id);
1606 objspace->next_object_id += OBJ_ID_INCREMENT;
1607
1608 st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
1609 st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
1610 FL_SET(obj, FL_SEEN_OBJ_ID);
1611 }
1612 rb_gc_vm_unlock(lev);
1613
1614 return id;
1615}
1616
1617static void free_stack_chunks(mark_stack_t *);
1618static void mark_stack_free_cache(mark_stack_t *);
1619static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
1620
1621static inline void
1622heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
1623{
1624 rb_asan_unpoison_object(obj, false);
1625
1626 asan_unlock_freelist(page);
1627
1628 struct free_slot *slot = (struct free_slot *)obj;
1629 slot->flags = 0;
1630 slot->next = page->freelist;
1631 page->freelist = slot;
1632 asan_lock_freelist(page);
1633
1634 RVALUE_AGE_RESET(obj);
1635
1636 if (RGENGC_CHECK_MODE &&
1637 /* obj should belong to page */
1638 !(page->start <= (uintptr_t)obj &&
1639 (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
1640 obj % BASE_SLOT_SIZE == 0)) {
1641 rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj);
1642 }
1643
1644 rb_asan_poison_object(obj);
1645 gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
1646}
1647
1648static void
1649heap_allocatable_slots_expand(rb_objspace_t *objspace,
1650 rb_heap_t *heap, size_t free_slots, size_t total_slots)
1651{
1652 double goal_ratio = gc_params.heap_free_slots_goal_ratio;
1653 size_t target_total_slots;
1654
1655 if (goal_ratio == 0.0) {
1656 target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
1657 }
1658 else if (total_slots == 0) {
1659 target_total_slots = minimum_slots_for_heap(objspace, heap);
1660 }
1661 else {
1662 /* Find `f' where free_slots = f * total_slots * goal_ratio
1663 * => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
1664 */
1665 double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
1666
1667 if (f > gc_params.growth_factor) f = gc_params.growth_factor;
1668 if (f < 1.0) f = 1.1;
1669
1670 target_total_slots = (size_t)(f * total_slots);
1671
1672 if (0) {
1673 fprintf(stderr,
1674 "free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
1675 " G(%1.2f), f(%1.2f),"
1676 " total_slots(%8"PRIuSIZE") => target_total_slots(%8"PRIuSIZE")\n",
1677 free_slots, total_slots, free_slots/(double)total_slots,
1678 goal_ratio, f, total_slots, target_total_slots);
1679 }
1680 }
1681
1682 if (gc_params.growth_max_slots > 0) {
1683 size_t max_total_slots = (size_t)(total_slots + gc_params.growth_max_slots);
1684 if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
1685 }
1686
1687 size_t extend_slot_count = target_total_slots - total_slots;
1688 /* Extend by at least 1 page. */
1689 if (extend_slot_count == 0) extend_slot_count = 1;
1690
1691 objspace->heap_pages.allocatable_slots += extend_slot_count;
1692}
1693
1694static inline void
1695heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
1696{
1697 asan_unlock_freelist(page);
1698 GC_ASSERT(page->free_slots != 0);
1699 GC_ASSERT(page->freelist != NULL);
1700
1701 page->free_next = heap->free_pages;
1702 heap->free_pages = page;
1703
1704 RUBY_DEBUG_LOG("page:%p freelist:%p", (void *)page, (void *)page->freelist);
1705
1706 asan_lock_freelist(page);
1707}
1708
1709static inline void
1710heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
1711{
1712 asan_unlock_freelist(page);
1713 GC_ASSERT(page->free_slots != 0);
1714 GC_ASSERT(page->freelist != NULL);
1715
1716 page->free_next = heap->pooled_pages;
1717 heap->pooled_pages = page;
1718 objspace->rincgc.pooled_slots += page->free_slots;
1719
1720 asan_lock_freelist(page);
1721}
1722
1723static void
1724heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
1725{
1726 ccan_list_del(&page->page_node);
1727 heap->total_pages--;
1728 heap->total_slots -= page->total_slots;
1729}
1730
1731static void
1732gc_aligned_free(void *ptr, size_t size)
1733{
1734#if defined __MINGW32__
1735 __mingw_aligned_free(ptr);
1736#elif defined _WIN32
1737 _aligned_free(ptr);
1738#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
1739 free(ptr);
1740#else
1741 free(((void**)ptr)[-1]);
1742#endif
1743}
1744
1745static void
1746heap_page_body_free(struct heap_page_body *page_body)
1747{
1748 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
1749
1750 if (HEAP_PAGE_ALLOC_USE_MMAP) {
1751#ifdef HAVE_MMAP
1752 GC_ASSERT(HEAP_PAGE_SIZE % sysconf(_SC_PAGE_SIZE) == 0);
1753 if (munmap(page_body, HEAP_PAGE_SIZE)) {
1754 rb_bug("heap_page_body_free: munmap failed");
1755 }
1756#endif
1757 }
1758 else {
1759 gc_aligned_free(page_body, HEAP_PAGE_SIZE);
1760 }
1761}
1762
1763static void
1764heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
1765{
1766 objspace->heap_pages.freed_pages++;
1767 heap_page_body_free(page->body);
1768 free(page);
1769}
1770
1771static void
1772heap_pages_free_unused_pages(rb_objspace_t *objspace)
1773{
1774 size_t pages_to_keep_count =
1775 // Get number of pages estimated for the smallest size pool
1776 CEILDIV(objspace->heap_pages.allocatable_slots, HEAP_PAGE_OBJ_LIMIT) *
1777 // Estimate the average slot size multiple
1778 (1 << (HEAP_COUNT / 2));
1779
1780 if (objspace->empty_pages != NULL && objspace->empty_pages_count > pages_to_keep_count) {
1781 GC_ASSERT(objspace->empty_pages_count > 0);
1782 objspace->empty_pages = NULL;
1783 objspace->empty_pages_count = 0;
1784
1785 size_t i, j;
1786 for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
1787 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
1788
1789 if (heap_page_in_global_empty_pages_pool(objspace, page) && pages_to_keep_count == 0) {
1790 heap_page_free(objspace, page);
1791 }
1792 else {
1793 if (heap_page_in_global_empty_pages_pool(objspace, page) && pages_to_keep_count > 0) {
1794 page->free_next = objspace->empty_pages;
1795 objspace->empty_pages = page;
1796 objspace->empty_pages_count++;
1797 pages_to_keep_count--;
1798 }
1799
1800 if (i != j) {
1801 rb_darray_set(objspace->heap_pages.sorted, j, page);
1802 }
1803 j++;
1804 }
1805 }
1806
1807 rb_darray_pop(objspace->heap_pages.sorted, i - j);
1808 GC_ASSERT(rb_darray_size(objspace->heap_pages.sorted) == j);
1809
1810 struct heap_page *hipage = rb_darray_get(objspace->heap_pages.sorted, rb_darray_size(objspace->heap_pages.sorted) - 1);
1811 uintptr_t himem = (uintptr_t)hipage->body + HEAP_PAGE_SIZE;
1812 GC_ASSERT(himem <= heap_pages_himem);
1813 heap_pages_himem = himem;
1814
1815 struct heap_page *lopage = rb_darray_get(objspace->heap_pages.sorted, 0);
1816 uintptr_t lomem = (uintptr_t)lopage->body + sizeof(struct heap_page_header);
1817 GC_ASSERT(lomem >= heap_pages_lomem);
1818 heap_pages_lomem = lomem;
1819 }
1820}
1821
1822static void *
1823gc_aligned_malloc(size_t alignment, size_t size)
1824{
1825 /* alignment must be a power of 2 */
1826 GC_ASSERT(((alignment - 1) & alignment) == 0);
1827 GC_ASSERT(alignment % sizeof(void*) == 0);
1828
1829 void *res;
1830
1831#if defined __MINGW32__
1832 res = __mingw_aligned_malloc(size, alignment);
1833#elif defined _WIN32
1834 void *_aligned_malloc(size_t, size_t);
1835 res = _aligned_malloc(size, alignment);
1836#elif defined(HAVE_POSIX_MEMALIGN)
1837 if (posix_memalign(&res, alignment, size) != 0) {
1838 return NULL;
1839 }
1840#elif defined(HAVE_MEMALIGN)
1841 res = memalign(alignment, size);
1842#else
1843 char* aligned;
1844 res = malloc(alignment + size + sizeof(void*));
1845 aligned = (char*)res + alignment + sizeof(void*);
1846 aligned -= ((VALUE)aligned & (alignment - 1));
1847 ((void**)aligned)[-1] = res;
1848 res = (void*)aligned;
1849#endif
1850
1851 GC_ASSERT((uintptr_t)res % alignment == 0);
1852
1853 return res;
1854}
1855
1856static struct heap_page_body *
1857heap_page_body_allocate(void)
1858{
1859 struct heap_page_body *page_body;
1860
1861 if (HEAP_PAGE_ALLOC_USE_MMAP) {
1862#ifdef HAVE_MMAP
1863 GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
1864
1865 size_t mmap_size = HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE;
1866 char *ptr = mmap(NULL, mmap_size,
1867 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1868 if (ptr == MAP_FAILED) {
1869 return NULL;
1870 }
1871
1872 // If we are building `default.c` as part of the ruby executable, we
1873 // may just call `ruby_annotate_mmap`. But if we are building
1874 // `default.c` as a shared library, we will not have access to private
1875 // symbols, and we have to either call prctl directly or make our own
1876 // wrapper.
1877#if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_VMA) && defined(PR_SET_VMA_ANON_NAME)
1878 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, mmap_size, "Ruby:GC:default:heap_page_body_allocate");
1879 errno = 0;
1880#endif
1881
1882 char *aligned = ptr + HEAP_PAGE_ALIGN;
1883 aligned -= ((VALUE)aligned & (HEAP_PAGE_ALIGN - 1));
1884 GC_ASSERT(aligned > ptr);
1885 GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
1886
1887 size_t start_out_of_range_size = aligned - ptr;
1888 GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
1889 if (start_out_of_range_size > 0) {
1890 if (munmap(ptr, start_out_of_range_size)) {
1891 rb_bug("heap_page_body_allocate: munmap failed for start");
1892 }
1893 }
1894
1895 size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
1896 GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
1897 if (end_out_of_range_size > 0) {
1898 if (munmap(aligned + HEAP_PAGE_SIZE, end_out_of_range_size)) {
1899 rb_bug("heap_page_body_allocate: munmap failed for end");
1900 }
1901 }
1902
1903 page_body = (struct heap_page_body *)aligned;
1904#endif
1905 }
1906 else {
1907 page_body = gc_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
1908 }
1909
1910 GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
1911
1912 return page_body;
1913}
1914
1915static struct heap_page *
1916heap_page_resurrect(rb_objspace_t *objspace)
1917{
1918 struct heap_page *page = NULL;
1919 if (objspace->empty_pages != NULL) {
1920 GC_ASSERT(objspace->empty_pages_count > 0);
1921 objspace->empty_pages_count--;
1922 page = objspace->empty_pages;
1923 objspace->empty_pages = page->free_next;
1924 }
1925
1926 return page;
1927}
1928
1929static struct heap_page *
1930heap_page_allocate(rb_objspace_t *objspace)
1931{
1932 struct heap_page_body *page_body = heap_page_body_allocate();
1933 if (page_body == 0) {
1934 rb_memerror();
1935 }
1936
1937 struct heap_page *page = calloc1(sizeof(struct heap_page));
1938 if (page == 0) {
1939 heap_page_body_free(page_body);
1940 rb_memerror();
1941 }
1942
1943 uintptr_t start = (uintptr_t)page_body + sizeof(struct heap_page_header);
1944 uintptr_t end = (uintptr_t)page_body + HEAP_PAGE_SIZE;
1945
1946 size_t lo = 0;
1947 size_t hi = rb_darray_size(objspace->heap_pages.sorted);
1948 while (lo < hi) {
1949 struct heap_page *mid_page;
1950
1951 size_t mid = (lo + hi) / 2;
1952 mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
1953 if ((uintptr_t)mid_page->start < start) {
1954 lo = mid + 1;
1955 }
1956 else if ((uintptr_t)mid_page->start > start) {
1957 hi = mid;
1958 }
1959 else {
1960 rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
1961 }
1962 }
1963
1964 rb_darray_insert(&objspace->heap_pages.sorted, hi, page);
1965
1966 if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
1967 if (heap_pages_himem < end) heap_pages_himem = end;
1968
1969 page->body = page_body;
1970 page_body->header.page = page;
1971
1972 objspace->heap_pages.allocated_pages++;
1973
1974 return page;
1975}
1976
1977static void
1978heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
1979{
1980 /* Adding to eden heap during incremental sweeping is forbidden */
1981 GC_ASSERT(!heap->sweeping_page);
1982 GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));
1983
1984 /* adjust obj_limit (object number available in this page) */
1985 uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
1986 if (start % BASE_SLOT_SIZE != 0) {
1987 int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
1988 start = start + delta;
1989 GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
1990
1991 /* Find a num in page that is evenly divisible by `stride`.
1992 * This is to ensure that objects are aligned with bit planes.
1993 * In other words, ensure there are an even number of objects
1994 * per bit plane. */
1995 if (NUM_IN_PAGE(start) == 1) {
1996 start += heap->slot_size - BASE_SLOT_SIZE;
1997 }
1998
1999 GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % heap->slot_size == 0);
2000 }
2001
2002 int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
2003
2004 page->start = start;
2005 page->total_slots = slot_count;
2006 page->slot_size = heap->slot_size;
2007 page->heap = heap;
2008
2009 asan_unlock_freelist(page);
2010 page->freelist = NULL;
2011 asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false);
2012 for (VALUE p = (VALUE)start; p < start + (slot_count * heap->slot_size); p += heap->slot_size) {
2013 heap_page_add_freeobj(objspace, page, p);
2014 }
2015 asan_lock_freelist(page);
2016
2017 page->free_slots = slot_count;
2018
2019 heap->total_allocated_pages++;
2020
2021 ccan_list_add_tail(&heap->pages, &page->page_node);
2022 heap->total_pages++;
2023 heap->total_slots += page->total_slots;
2024}
2025
2026static int
2027heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap)
2028{
2029 if (objspace->heap_pages.allocatable_slots > 0) {
2030 gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", "
2031 "allocatable_slots: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
2032 rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_slots, heap->total_pages);
2033
2034 struct heap_page *page = heap_page_resurrect(objspace);
2035 if (page == NULL) {
2036 page = heap_page_allocate(objspace);
2037 }
2038 heap_add_page(objspace, heap, page);
2039 heap_add_freepage(heap, page);
2040
2041 if (objspace->heap_pages.allocatable_slots > (size_t)page->total_slots) {
2042 objspace->heap_pages.allocatable_slots -= page->total_slots;
2043 }
2044 else {
2045 objspace->heap_pages.allocatable_slots = 0;
2046 }
2047
2048 return true;
2049 }
2050
2051 return false;
2052}
2053
2054static void
2055heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap)
2056{
2057 size_t prev_allocatable_slots = objspace->heap_pages.allocatable_slots;
2058 // Set allocatable slots to 1 to force a page to be created.
2059 objspace->heap_pages.allocatable_slots = 1;
2060 heap_page_allocate_and_initialize(objspace, heap);
2061 GC_ASSERT(heap->free_pages != NULL);
2062 objspace->heap_pages.allocatable_slots = prev_allocatable_slots;
2063}
2064
2065static void
2066gc_continue(rb_objspace_t *objspace, rb_heap_t *heap)
2067{
2068 unsigned int lock_lev;
2069 gc_enter(objspace, gc_enter_event_continue, &lock_lev);
2070
2071 /* Continue marking if in incremental marking. */
2072 if (is_incremental_marking(objspace)) {
2073 if (gc_marks_continue(objspace, heap)) {
2074 gc_sweep(objspace);
2075 }
2076 }
2077
2078 /* Continue sweeping if in lazy sweeping or the previous incremental
2079 * marking finished and did not yield a free page. */
2080 if (heap->free_pages == NULL && is_lazy_sweeping(objspace)) {
2081 gc_sweep_continue(objspace, heap);
2082 }
2083
2084 gc_exit(objspace, gc_enter_event_continue, &lock_lev);
2085}
2086
2087static void
2088heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
2089{
2090 GC_ASSERT(heap->free_pages == NULL);
2091
2092 if (heap->total_slots < gc_params.heap_init_slots[heap - heaps] &&
2093 heap->sweeping_page == NULL) {
2094 heap_page_allocate_and_initialize_force(objspace, heap);
2095 GC_ASSERT(heap->free_pages != NULL);
2096 return;
2097 }
2098
2099 /* Continue incremental marking or lazy sweeping, if in any of those steps. */
2100 gc_continue(objspace, heap);
2101
2102 if (heap->free_pages == NULL) {
2103 heap_page_allocate_and_initialize(objspace, heap);
2104 }
2105
2106 /* If we still don't have a free page and not allowed to create a new page,
2107 * we should start a new GC cycle. */
2108 if (heap->free_pages == NULL) {
2109 if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2110 rb_memerror();
2111 }
2112 else {
2113 if (objspace->heap_pages.allocatable_slots == 0 && !gc_config_full_mark_val) {
2114 heap_allocatable_slots_expand(objspace, heap,
2115 heap->freed_slots + heap->empty_slots,
2116 heap->total_slots);
2117 GC_ASSERT(objspace->heap_pages.allocatable_slots > 0);
2118 }
2119 /* Do steps of incremental marking or lazy sweeping if the GC run permits. */
2120 gc_continue(objspace, heap);
2121
2122 /* If we're not incremental marking (e.g. a minor GC) or finished
2123 * sweeping and still don't have a free page, then
2124 * gc_sweep_finish_heap should allow us to create a new page. */
2125 if (heap->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, heap)) {
2126 if (gc_needs_major_flags == GPR_FLAG_NONE) {
2127 rb_bug("cannot create a new page after GC");
2128 }
2129 else { // Major GC is required, which will allow us to create new page
2130 if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2131 rb_memerror();
2132 }
2133 else {
2134 /* Do steps of incremental marking or lazy sweeping. */
2135 gc_continue(objspace, heap);
2136
2137 if (heap->free_pages == NULL &&
2138 !heap_page_allocate_and_initialize(objspace, heap)) {
2139 rb_bug("cannot create a new page after major GC");
2140 }
2141 }
2142 }
2143 }
2144 }
2145 }
2146
2147 GC_ASSERT(heap->free_pages != NULL);
2148}
2149
2150static inline VALUE
2151newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3)
2152{
2153 VALUE *p = (VALUE *)obj;
2154 p[2] = v1;
2155 p[3] = v2;
2156 p[4] = v3;
2157 return obj;
2158}
2159
2160#if GC_DEBUG
2161static inline const char*
2162rb_gc_impl_source_location_cstr(int *ptr)
2163{
2164 /* We could directly refer `rb_source_location_cstr()` before, but not any
2165 * longer. We have to heavy lift using our debugging API. */
2166 if (! ptr) {
2167 return NULL;
2168 }
2169 else if (! (*ptr = rb_sourceline())) {
2170 return NULL;
2171 }
2172 else {
2173 return rb_sourcefile();
2174 }
2175}
2176#endif
2177
2178static inline VALUE
2179newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
2180{
2181#if !__has_feature(memory_sanitizer)
2182 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
2183 GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2184#endif
2185 RBASIC(obj)->flags = flags;
2186 *((VALUE *)&RBASIC(obj)->klass) = klass;
2187
2188 int t = flags & RUBY_T_MASK;
2189 if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
2190 RVALUE_AGE_SET_CANDIDATE(objspace, obj);
2191 }
2192
2193#if RACTOR_CHECK_MODE
2194 void rb_ractor_setup_belonging(VALUE obj);
2195 rb_ractor_setup_belonging(obj);
2196#endif
2197
2198#if RGENGC_CHECK_MODE
2199 newobj_fill(obj, 0, 0, 0);
2200
2201 int lev = rb_gc_vm_lock_no_barrier();
2202 {
2203 check_rvalue_consistency(objspace, obj);
2204
2205 GC_ASSERT(RVALUE_MARKED(objspace, obj) == FALSE);
2206 GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
2207 GC_ASSERT(RVALUE_OLD_P(objspace, obj) == FALSE);
2208 GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, obj) == FALSE);
2209
2210 if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj));
2211 }
2212 rb_gc_vm_unlock_no_barrier(lev);
2213#endif
2214
2215 if (RB_UNLIKELY(wb_protected == FALSE)) {
2216 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
2217 }
2218
2219#if RGENGC_PROFILE
2220 if (wb_protected) {
2221 objspace->profile.total_generated_normal_object_count++;
2222#if RGENGC_PROFILE >= 2
2223 objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
2224#endif
2225 }
2226 else {
2227 objspace->profile.total_generated_shady_object_count++;
2228#if RGENGC_PROFILE >= 2
2229 objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
2230#endif
2231 }
2232#endif
2233
2234#if GC_DEBUG
2235 GET_RVALUE_OVERHEAD(obj)->file = rb_gc_impl_source_location_cstr(&GET_RVALUE_OVERHEAD(obj)->line);
2236 GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
2237#endif
2238
2239 gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj));
2240
2241 RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj));
2242 return obj;
2243}
2244
2245size_t
2246rb_gc_impl_obj_slot_size(VALUE obj)
2247{
2248 return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
2249}
2250
2251static inline size_t
2252heap_slot_size(unsigned char pool_id)
2253{
2254 GC_ASSERT(pool_id < HEAP_COUNT);
2255
2256 size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
2257
2258#if RGENGC_CHECK_MODE
2259 rb_objspace_t *objspace = rb_gc_get_objspace();
2260 GC_ASSERT(heaps[pool_id].slot_size == (short)slot_size);
2261#endif
2262
2263 slot_size -= RVALUE_OVERHEAD;
2264
2265 return slot_size;
2266}
2267
2268bool
2269rb_gc_impl_size_allocatable_p(size_t size)
2270{
2271 return size <= heap_slot_size(HEAP_COUNT - 1);
2272}
2273
2274static inline VALUE
2275ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache,
2276 size_t heap_idx)
2277{
2278 rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx];
2279 struct free_slot *p = heap_cache->freelist;
2280
2281 if (RB_UNLIKELY(is_incremental_marking(objspace))) {
2282 // Not allowed to allocate without running an incremental marking step
2283 if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
2284 return Qfalse;
2285 }
2286
2287 if (p) {
2288 cache->incremental_mark_step_allocated_slots++;
2289 }
2290 }
2291
2292 if (RB_LIKELY(p)) {
2293 VALUE obj = (VALUE)p;
2294 rb_asan_unpoison_object(obj, true);
2295 heap_cache->freelist = p->next;
2296#if RGENGC_CHECK_MODE
2297 GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
2298 // zero clear
2299 MEMZERO((char *)obj, char, heap_slot_size(heap_idx));
2300#endif
2301 return obj;
2302 }
2303 else {
2304 return Qfalse;
2305 }
2306}
2307
2308static struct heap_page *
2309heap_next_free_page(rb_objspace_t *objspace, rb_heap_t *heap)
2310{
2311 struct heap_page *page;
2312
2313 if (heap->free_pages == NULL) {
2314 heap_prepare(objspace, heap);
2315 }
2316
2317 page = heap->free_pages;
2318 heap->free_pages = page->free_next;
2319
2320 GC_ASSERT(page->free_slots != 0);
2321
2322 asan_unlock_freelist(page);
2323
2324 return page;
2325}
2326
2327static inline void
2328ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx,
2329 struct heap_page *page)
2330{
2331 gc_report(3, objspace, "ractor_set_cache: Using page %p\n", (void *)page->body);
2332
2333 rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx];
2334
2335 GC_ASSERT(heap_cache->freelist == NULL);
2336 GC_ASSERT(page->free_slots != 0);
2337 GC_ASSERT(page->freelist != NULL);
2338
2339 heap_cache->using_page = page;
2340 heap_cache->freelist = page->freelist;
2341 page->free_slots = 0;
2342 page->freelist = NULL;
2343
2344 rb_asan_unpoison_object((VALUE)heap_cache->freelist, false);
2345 GC_ASSERT(RB_TYPE_P((VALUE)heap_cache->freelist, T_NONE));
2346 rb_asan_poison_object((VALUE)heap_cache->freelist);
2347}
2348
2349static inline size_t
2350heap_idx_for_size(size_t size)
2351{
2352 size += RVALUE_OVERHEAD;
2353
2354 size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
2355
2356 /* heap_idx is ceil(log2(slot_count)) */
2357 size_t heap_idx = 64 - nlz_int64(slot_count - 1);
2358
2359 if (heap_idx >= HEAP_COUNT) {
2360 rb_bug("heap_idx_for_size: allocation size too large "
2361 "(size=%"PRIuSIZE"u, heap_idx=%"PRIuSIZE"u)", size, heap_idx);
2362 }
2363
2364#if RGENGC_CHECK_MODE
2365 rb_objspace_t *objspace = rb_gc_get_objspace();
2366 GC_ASSERT(size <= (size_t)heaps[heap_idx].slot_size);
2367 if (heap_idx > 0) GC_ASSERT(size > (size_t)heaps[heap_idx - 1].slot_size);
2368#endif
2369
2370 return heap_idx;
2371}
2372
2373size_t
2374rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
2375{
2376 return heap_idx_for_size(size);
2377}
2378
2379
2380static size_t heap_sizes[HEAP_COUNT + 1] = { 0 };
2381
2382size_t *
2383rb_gc_impl_heap_sizes(void *objspace_ptr)
2384{
2385 if (heap_sizes[0] == 0) {
2386 for (unsigned char i = 0; i < HEAP_COUNT; i++) {
2387 heap_sizes[i] = heap_slot_size(i);
2388 }
2389 }
2390
2391 return heap_sizes;
2392}
2393
2394NOINLINE(static VALUE newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked));
2395
2396static VALUE
2397newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked)
2398{
2399 rb_heap_t *heap = &heaps[heap_idx];
2400 VALUE obj = Qfalse;
2401
2402 unsigned int lev = 0;
2403 bool unlock_vm = false;
2404
2405 if (!vm_locked) {
2406 lev = rb_gc_cr_lock();
2407 vm_locked = true;
2408 unlock_vm = true;
2409 }
2410
2411 {
2412 if (is_incremental_marking(objspace)) {
2413 gc_continue(objspace, heap);
2414 cache->incremental_mark_step_allocated_slots = 0;
2415
2416 // Retry allocation after resetting incremental_mark_step_allocated_slots
2417 obj = ractor_cache_allocate_slot(objspace, cache, heap_idx);
2418 }
2419
2420 if (obj == Qfalse) {
2421 // Get next free page (possibly running GC)
2422 struct heap_page *page = heap_next_free_page(objspace, heap);
2423 ractor_cache_set_page(objspace, cache, heap_idx, page);
2424
2425 // Retry allocation after moving to new page
2426 obj = ractor_cache_allocate_slot(objspace, cache, heap_idx);
2427 }
2428 }
2429
2430 if (unlock_vm) {
2431 rb_gc_cr_unlock(lev);
2432 }
2433
2434 if (RB_UNLIKELY(obj == Qfalse)) {
2435 rb_memerror();
2436 }
2437 return obj;
2438}
2439
2440static VALUE
2441newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked)
2442{
2443 VALUE obj = ractor_cache_allocate_slot(objspace, cache, heap_idx);
2444
2445 if (RB_UNLIKELY(obj == Qfalse)) {
2446 obj = newobj_cache_miss(objspace, cache, heap_idx, vm_locked);
2447 }
2448
2449 rb_heap_t *heap = &heaps[heap_idx];
2450 heap->total_allocated_objects++;
2451 GC_ASSERT(rb_gc_multi_ractor_p() ||
2452 heap->total_slots >=
2453 (heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count));
2454
2455 return obj;
2456}
2457
2458ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t heap_idx));
2459
2460static inline VALUE
2461newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t heap_idx)
2462{
2463 VALUE obj;
2464 unsigned int lev;
2465
2466 lev = rb_gc_cr_lock();
2467 {
2468 if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
2469 if (during_gc) {
2470 dont_gc_on();
2471 during_gc = 0;
2472 if (rb_memerror_reentered()) {
2473 rb_memerror();
2474 }
2475 rb_bug("object allocation during garbage collection phase");
2476 }
2477
2478 if (ruby_gc_stressful) {
2479 if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
2480 rb_memerror();
2481 }
2482 }
2483 }
2484
2485 obj = newobj_alloc(objspace, cache, heap_idx, true);
2486 newobj_init(klass, flags, wb_protected, objspace, obj);
2487 }
2488 rb_gc_cr_unlock(lev);
2489
2490 return obj;
2491}
2492
2493NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
2494 rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx));
2495NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
2496 rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx));
2497
2498static VALUE
2499newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx)
2500{
2501 return newobj_slowpath(klass, flags, objspace, cache, TRUE, heap_idx);
2502}
2503
2504static VALUE
2505newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx)
2506{
2507 return newobj_slowpath(klass, flags, objspace, cache, FALSE, heap_idx);
2508}
2509
2510 VALUE
2511rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size)
2512{
2513 VALUE obj;
2514 rb_objspace_t *objspace = objspace_ptr;
2515
2516 RB_DEBUG_COUNTER_INC(obj_newobj);
2517 (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
2518
2519 if (RB_UNLIKELY(stress_to_class)) {
2520 long cnt = RARRAY_LEN(stress_to_class);
2521 for (long i = 0; i < cnt; i++) {
2522 if (klass == RARRAY_AREF(stress_to_class, i)) rb_memerror();
2523 }
2524 }
2525
2526 size_t heap_idx = heap_idx_for_size(alloc_size);
2527
2528 rb_ractor_newobj_cache_t *cache = (rb_ractor_newobj_cache_t *)cache_ptr;
2529
2530 if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) &&
2531 wb_protected) {
2532 obj = newobj_alloc(objspace, cache, heap_idx, false);
2533 newobj_init(klass, flags, wb_protected, objspace, obj);
2534 }
2535 else {
2536 RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
2537
2538 obj = wb_protected ?
2539 newobj_slowpath_wb_protected(klass, flags, objspace, cache, heap_idx) :
2540 newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, heap_idx);
2541 }
2542
2543 return newobj_fill(obj, v1, v2, v3);
2544}
2545
2546static int
2547ptr_in_page_body_p(const void *ptr, const void *memb)
2548{
2549 struct heap_page *page = *(struct heap_page **)memb;
2550 uintptr_t p_body = (uintptr_t)page->body;
2551
2552 if ((uintptr_t)ptr >= p_body) {
2553 return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
2554 }
2555 else {
2556 return -1;
2557 }
2558}
2559
2560PUREFUNC(static inline struct heap_page *heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);)
2561static inline struct heap_page *
2562heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr)
2563{
2564 struct heap_page **res;
2565
2566 if (ptr < (uintptr_t)heap_pages_lomem ||
2567 ptr > (uintptr_t)heap_pages_himem) {
2568 return NULL;
2569 }
2570
2571 res = bsearch((void *)ptr, rb_darray_ref(objspace->heap_pages.sorted, 0),
2572 rb_darray_size(objspace->heap_pages.sorted), sizeof(struct heap_page *),
2573 ptr_in_page_body_p);
2574
2575 if (res) {
2576 return *res;
2577 }
2578 else {
2579 return NULL;
2580 }
2581}
2582
2583PUREFUNC(static inline bool is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr);)
2584static inline bool
2585is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
2586{
2587 register uintptr_t p = (uintptr_t)ptr;
2588 register struct heap_page *page;
2589
2590 RB_DEBUG_COUNTER_INC(gc_isptr_trial);
2591
2592 if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
2593 RB_DEBUG_COUNTER_INC(gc_isptr_range);
2594
2595 if (p % BASE_SLOT_SIZE != 0) return FALSE;
2596 RB_DEBUG_COUNTER_INC(gc_isptr_align);
2597
2598 page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
2599 if (page) {
2600 RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
2601 if (heap_page_in_global_empty_pages_pool(objspace, page)) {
2602 return FALSE;
2603 }
2604 else {
2605 if (p < page->start) return FALSE;
2606 if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
2607 if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0) return FALSE;
2608
2609 return TRUE;
2610 }
2611 }
2612 return FALSE;
2613}
2614
2615bool
2616rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr)
2617{
2618 return is_pointer_to_heap(objspace_ptr, ptr);
2619}
2620
2621#define ZOMBIE_OBJ_KEPT_FLAGS (FL_SEEN_OBJ_ID | FL_FINALIZE)
2622
2623void
2624rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
2625{
2626 rb_objspace_t *objspace = objspace_ptr;
2627
2628 struct RZombie *zombie = RZOMBIE(obj);
2629 zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & ZOMBIE_OBJ_KEPT_FLAGS);
2630 zombie->dfree = dfree;
2631 zombie->data = data;
2632 VALUE prev, next = heap_pages_deferred_final;
2633 do {
2634 zombie->next = prev = next;
2635 next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj);
2636 } while (next != prev);
2637
2638 struct heap_page *page = GET_HEAP_PAGE(obj);
2639 page->final_slots++;
2640 page->heap->final_slots_count++;
2641}
2642
2643static void
2644obj_free_object_id(rb_objspace_t *objspace, VALUE obj)
2645{
2646 st_data_t o = (st_data_t)obj, id;
2647
2648 GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE || FL_TEST(obj, FL_SEEN_OBJ_ID));
2649 FL_UNSET(obj, FL_SEEN_OBJ_ID);
2650
2651 if (st_delete(objspace->obj_to_id_tbl, &o, &id)) {
2652 GC_ASSERT(id);
2653 st_delete(objspace->id_to_obj_tbl, &id, NULL);
2654 }
2655 else {
2656 rb_bug("Object ID seen, but not in mapping table: %s", rb_obj_info(obj));
2657 }
2658}
2659
2660typedef int each_obj_callback(void *, void *, size_t, void *);
2661typedef int each_page_callback(struct heap_page *, void *);
2662
2663 struct each_obj_data {
2664 rb_objspace_t *objspace;
2665 bool reenable_incremental;
2666
2667 each_obj_callback *each_obj_callback;
2668 each_page_callback *each_page_callback;
2669 void *data;
2670
2671 struct heap_page **pages[HEAP_COUNT];
2672 size_t pages_counts[HEAP_COUNT];
2673};
2674
2675static VALUE
2676objspace_each_objects_ensure(VALUE arg)
2677{
2678 struct each_obj_data *data = (struct each_obj_data *)arg;
2679 rb_objspace_t *objspace = data->objspace;
2680
2681 /* Reenable incremental GC */
2682 if (data->reenable_incremental) {
2683 objspace->flags.dont_incremental = FALSE;
2684 }
2685
2686 for (int i = 0; i < HEAP_COUNT; i++) {
2687 struct heap_page **pages = data->pages[i];
2688 free(pages);
2689 }
2690
2691 return Qnil;
2692}
2693
2694static VALUE
2695objspace_each_objects_try(VALUE arg)
2696{
2697 struct each_obj_data *data = (struct each_obj_data *)arg;
2698 rb_objspace_t *objspace = data->objspace;
2699
2700 /* Copy pages from all heaps to their respective buffers. */
2701 for (int i = 0; i < HEAP_COUNT; i++) {
2702 rb_heap_t *heap = &heaps[i];
2703 size_t size = heap->total_pages * sizeof(struct heap_page *);
2704
2705 struct heap_page **pages = malloc(size);
2706 if (!pages) rb_memerror();
2707
2708 /* Set up pages buffer by iterating over all pages in the current eden
2709 * heap. This will be a snapshot of the state of the heap before we
2710 * call the callback over each page that exists in this buffer. Thus it
2711 * is safe for the callback to allocate objects without possibly entering
2712 * an infinite loop. */
2713 struct heap_page *page = 0;
2714 size_t pages_count = 0;
2715 ccan_list_for_each(&heap->pages, page, page_node) {
2716 pages[pages_count] = page;
2717 pages_count++;
2718 }
2719 data->pages[i] = pages;
2720 data->pages_counts[i] = pages_count;
2721 GC_ASSERT(pages_count == heap->total_pages);
2722 }
2723
2724 for (int i = 0; i < HEAP_COUNT; i++) {
2725 rb_heap_t *heap = &heaps[i];
2726 size_t pages_count = data->pages_counts[i];
2727 struct heap_page **pages = data->pages[i];
2728
2729 struct heap_page *page = ccan_list_top(&heap->pages, struct heap_page, page_node);
2730 for (size_t i = 0; i < pages_count; i++) {
2731 /* If we have reached the end of the linked list then there are no
2732 * more pages, so break. */
2733 if (page == NULL) break;
2734
2735 /* If this page does not match the one in the buffer, then move to
2736 * the next page in the buffer. */
2737 if (pages[i] != page) continue;
2738
2739 uintptr_t pstart = (uintptr_t)page->start;
2740 uintptr_t pend = pstart + (page->total_slots * heap->slot_size);
2741
2742 if (data->each_obj_callback &&
2743 (*data->each_obj_callback)((void *)pstart, (void *)pend, heap->slot_size, data->data)) {
2744 break;
2745 }
2746 if (data->each_page_callback &&
2747 (*data->each_page_callback)(page, data->data)) {
2748 break;
2749 }
2750
2751 page = ccan_list_next(&heap->pages, page, page_node);
2752 }
2753 }
2754
2755 return Qnil;
2756}
2757
2758static void
2759objspace_each_exec(bool protected, struct each_obj_data *each_obj_data)
2760{
2761 /* Disable incremental GC */
2762 rb_objspace_t *objspace = each_obj_data->objspace;
2763 bool reenable_incremental = FALSE;
2764 if (protected) {
2765 reenable_incremental = !objspace->flags.dont_incremental;
2766
2767 gc_rest(objspace);
2768 objspace->flags.dont_incremental = TRUE;
2769 }
2770
2771 each_obj_data->reenable_incremental = reenable_incremental;
2772 memset(&each_obj_data->pages, 0, sizeof(each_obj_data->pages));
2773 memset(&each_obj_data->pages_counts, 0, sizeof(each_obj_data->pages_counts));
2774 rb_ensure(objspace_each_objects_try, (VALUE)each_obj_data,
2775 objspace_each_objects_ensure, (VALUE)each_obj_data);
2776}
2777
2778static void
2779objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected)
2780{
2781 struct each_obj_data each_obj_data = {
2782 .objspace = objspace,
2783 .each_obj_callback = callback,
2784 .each_page_callback = NULL,
2785 .data = data,
2786 };
2787 objspace_each_exec(protected, &each_obj_data);
2788}
2789
2790void
2791rb_gc_impl_each_objects(void *objspace_ptr, each_obj_callback *callback, void *data)
2792{
2793 objspace_each_objects(objspace_ptr, callback, data, TRUE);
2794}
2795
2796#if GC_CAN_COMPILE_COMPACTION
2797static void
2798objspace_each_pages(rb_objspace_t *objspace, each_page_callback *callback, void *data, bool protected)
2799{
2800 struct each_obj_data each_obj_data = {
2801 .objspace = objspace,
2802 .each_obj_callback = NULL,
2803 .each_page_callback = callback,
2804 .data = data,
2805 };
2806 objspace_each_exec(protected, &each_obj_data);
2807}
2808#endif
2809
2810 VALUE
2811rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
2812{
2813 rb_objspace_t *objspace = objspace_ptr;
2814 VALUE table;
2815 st_data_t data;
2816
2817 GC_ASSERT(!OBJ_FROZEN(obj));
2818
2819 RBASIC(obj)->flags |= FL_FINALIZE;
2820
2821 if (st_lookup(finalizer_table, obj, &data)) {
2822 table = (VALUE)data;
2823
2824 /* avoid duplicate block, table is usually small */
2825 {
2826 long len = RARRAY_LEN(table);
2827 long i;
2828
2829 for (i = 0; i < len; i++) {
2830 VALUE recv = RARRAY_AREF(table, i);
2831 if (rb_equal(recv, block)) {
2832 return recv;
2833 }
2834 }
2835 }
2836
2837 rb_ary_push(table, block);
2838 }
2839 else {
2840 table = rb_ary_new3(1, block);
2841 rb_obj_hide(table);
2842 st_add_direct(finalizer_table, obj, table);
2843 }
2844
2845 return block;
2846}
2847
2848void
2849rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
2850{
2851 rb_objspace_t *objspace = objspace_ptr;
2852
2853 GC_ASSERT(!OBJ_FROZEN(obj));
2854
2855 st_data_t data = obj;
2856 st_delete(finalizer_table, &data, 0);
2857 FL_UNSET(obj, FL_FINALIZE);
2858}
2859
2860void
2861rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
2862{
2863 rb_objspace_t *objspace = objspace_ptr;
2864 VALUE table;
2865 st_data_t data;
2866
2867 if (!FL_TEST(obj, FL_FINALIZE)) return;
2868
2869 if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
2870 table = (VALUE)data;
2871 st_insert(finalizer_table, dest, table);
2872 FL_SET(dest, FL_FINALIZE);
2873 }
2874 else {
2875 rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
2876 }
2877}
2878
2879static VALUE
2880get_object_id_in_finalizer(rb_objspace_t *objspace, VALUE obj)
2881{
2882 if (FL_TEST(obj, FL_SEEN_OBJ_ID)) {
2883 return rb_gc_impl_object_id(objspace, obj);
2884 }
2885 else {
2886 VALUE id = ULL2NUM(objspace->next_object_id);
2887 objspace->next_object_id += OBJ_ID_INCREMENT;
2888 return id;
2889 }
2890}
2891
2892static VALUE
2893get_final(long i, void *data)
2894{
2895 VALUE table = (VALUE)data;
2896
2897 return RARRAY_AREF(table, i);
2898}
2899
2900static void
2901run_final(rb_objspace_t *objspace, VALUE zombie)
2902{
2903 if (RZOMBIE(zombie)->dfree) {
2904 RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
2905 }
2906
2907 st_data_t key = (st_data_t)zombie;
2908 if (FL_TEST_RAW(zombie, FL_FINALIZE)) {
2909 FL_UNSET(zombie, FL_FINALIZE);
2910 st_data_t table;
2911 if (st_delete(finalizer_table, &key, &table)) {
2912 rb_gc_run_obj_finalizer(get_object_id_in_finalizer(objspace, zombie), RARRAY_LEN(table), get_final, (void *)table);
2913 }
2914 else {
2915 rb_bug("FL_FINALIZE flag is set, but finalizers are not found");
2916 }
2917 }
2918 else {
2919 GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
2920 }
2921}
2922
2923static void
2924finalize_list(rb_objspace_t *objspace, VALUE zombie)
2925{
2926 while (zombie) {
2927 VALUE next_zombie;
2928 struct heap_page *page;
2929 rb_asan_unpoison_object(zombie, false);
2930 next_zombie = RZOMBIE(zombie)->next;
2931 page = GET_HEAP_PAGE(zombie);
2932
2933 run_final(objspace, zombie);
2934
2935 int lev = rb_gc_vm_lock();
2936 {
2937 GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
2938 if (FL_TEST(zombie, FL_SEEN_OBJ_ID)) {
2939 obj_free_object_id(objspace, zombie);
2940 }
2941
2942 GC_ASSERT(page->heap->final_slots_count > 0);
2943 GC_ASSERT(page->final_slots > 0);
2944
2945 page->heap->final_slots_count--;
2946 page->final_slots--;
2947 page->free_slots++;
2948 heap_page_add_freeobj(objspace, page, zombie);
2949 page->heap->total_freed_objects++;
2950 }
2951 rb_gc_vm_unlock(lev);
2952
2953 zombie = next_zombie;
2954 }
2955}
2956
2957static void
2958finalize_deferred_heap_pages(rb_objspace_t *objspace)
2959{
2960 VALUE zombie;
2961 while ((zombie = RUBY_ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
2962 finalize_list(objspace, zombie);
2963 }
2964}
2965
2966static void
2967finalize_deferred(rb_objspace_t *objspace)
2968{
2969 rb_gc_set_pending_interrupt();
2970 finalize_deferred_heap_pages(objspace);
2971 rb_gc_unset_pending_interrupt();
2972}
2973
2974static void
2975gc_finalize_deferred(void *dmy)
2976{
2977 rb_objspace_t *objspace = dmy;
2978 if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) return;
2979
2980 finalize_deferred(objspace);
2981 RUBY_ATOMIC_SET(finalizing, 0);
2982}
2983
2984static void
2985gc_finalize_deferred_register(rb_objspace_t *objspace)
2986{
2987 /* will enqueue a call to gc_finalize_deferred */
2988 rb_postponed_job_trigger(objspace->finalize_deferred_pjob);
2989}
2990
2991static int pop_mark_stack(mark_stack_t *stack, VALUE *data);
2992
2993static void
2994gc_abort(void *objspace_ptr)
2995{
2996 rb_objspace_t *objspace = objspace_ptr;
2997
2998 if (is_incremental_marking(objspace)) {
2999 /* Remove all objects from the mark stack. */
3000 VALUE obj;
3001 while (pop_mark_stack(&objspace->mark_stack, &obj));
3002
3003 objspace->flags.during_incremental_marking = FALSE;
3004 }
3005
3006 if (is_lazy_sweeping(objspace)) {
3007 for (int i = 0; i < HEAP_COUNT; i++) {
3008 rb_heap_t *heap = &heaps[i];
3009
3010 heap->sweeping_page = NULL;
3011 struct heap_page *page = NULL;
3012
3013 ccan_list_for_each(&heap->pages, page, page_node) {
3014 page->flags.before_sweep = false;
3015 }
3016 }
3017 }
3018
3019 for (int i = 0; i < HEAP_COUNT; i++) {
3020 rb_heap_t *heap = &heaps[i];
3021 rgengc_mark_and_rememberset_clear(objspace, heap);
3022 }
3023
3024 gc_mode_set(objspace, gc_mode_none);
3025}
3026
3027void
3028rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
3029{
3030 rb_objspace_t *objspace = objspace_ptr;
3031
3032 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
3033 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
3034 short stride = page->slot_size;
3035
3036 uintptr_t p = (uintptr_t)page->start;
3037 uintptr_t pend = p + page->total_slots * stride;
3038 for (; p < pend; p += stride) {
3039 VALUE vp = (VALUE)p;
3040 asan_unpoisoning_object(vp) {
3041 if (RB_BUILTIN_TYPE(vp) != T_NONE) {
3042 rb_gc_obj_free_vm_weak_references(vp);
3043 if (rb_gc_obj_free(objspace, vp)) {
3044 RBASIC(vp)->flags = 0;
3045 }
3046 }
3047 }
3048 }
3049 }
3050}
3051
3052static int
3053rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
3054{
3055 rb_objspace_t *objspace = (rb_objspace_t *)data;
3056 VALUE obj = (VALUE)key;
3057 VALUE table = (VALUE)val;
3058
3059 GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE));
3060 GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY);
3061
3062 rb_gc_run_obj_finalizer(rb_gc_impl_object_id(objspace, obj), RARRAY_LEN(table), get_final, (void *)table);
3063
3064 FL_UNSET(obj, FL_FINALIZE);
3065
3066 return ST_DELETE;
3067}
3068
3069void
3070rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
3071{
3072 rb_objspace_t *objspace = objspace_ptr;
3073
3074#if RGENGC_CHECK_MODE >= 2
3075 gc_verify_internal_consistency(objspace);
3076#endif
3077
3078 /* prohibit incremental GC */
3079 objspace->flags.dont_incremental = 1;
3080
3081 if (RUBY_ATOMIC_EXCHANGE(finalizing, 1)) {
3082 /* Abort incremental marking and lazy sweeping to speed up shutdown. */
3083 gc_abort(objspace);
3084 dont_gc_on();
3085 return;
3086 }
3087
3088 while (finalizer_table->num_entries) {
3089 st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, (st_data_t)objspace);
3090 }
3091
3092 /* run finalizers */
3093 finalize_deferred(objspace);
3094 GC_ASSERT(heap_pages_deferred_final == 0);
3095
3096 /* Abort incremental marking and lazy sweeping to speed up shutdown. */
3097 gc_abort(objspace);
3098
3099 /* prohibit GC because force T_DATA finalizers can break an object graph consistency */
3100 dont_gc_on();
3101
3102 /* running data/file finalizers are part of garbage collection */
3103 unsigned int lock_lev;
3104 gc_enter(objspace, gc_enter_event_finalizer, &lock_lev);
3105
3106 /* run data/file object's finalizers */
3107 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
3108 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
3109 short stride = page->slot_size;
3110
3111 uintptr_t p = (uintptr_t)page->start;
3112 uintptr_t pend = p + page->total_slots * stride;
3113 for (; p < pend; p += stride) {
3114 VALUE vp = (VALUE)p;
3115 asan_unpoisoning_object(vp) {
3116 if (rb_gc_shutdown_call_finalizer_p(vp)) {
3117 rb_gc_obj_free_vm_weak_references(vp);
3118 if (rb_gc_obj_free(objspace, vp)) {
3119 RBASIC(vp)->flags = 0;
3120 }
3121 }
3122 }
3123 }
3124 }
3125
3126 gc_exit(objspace, gc_enter_event_finalizer, &lock_lev);
3127
3128 finalize_deferred_heap_pages(objspace);
3129
3130 st_free_table(finalizer_table);
3131 finalizer_table = 0;
3132 RUBY_ATOMIC_SET(finalizing, 0);
3133}
3134
3135void
3136rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data)
3137{
3138 rb_objspace_t *objspace = objspace_ptr;
3139
3140 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
3141 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
3142 short stride = page->slot_size;
3143
3144 uintptr_t p = (uintptr_t)page->start;
3145 uintptr_t pend = p + page->total_slots * stride;
3146 for (; p < pend; p += stride) {
3147 VALUE obj = (VALUE)p;
3148
3149 asan_unpoisoning_object(obj) {
3150 func(obj, data);
3151 }
3152 }
3153 }
3154}
3155
3156/*
3157 ------------------------ Garbage Collection ------------------------
3158*/
3159
3160/* Sweeping */
3161
3162static size_t
3163objspace_available_slots(rb_objspace_t *objspace)
3164{
3165 size_t total_slots = 0;
3166 for (int i = 0; i < HEAP_COUNT; i++) {
3167 rb_heap_t *heap = &heaps[i];
3168 total_slots += heap->total_slots;
3169 }
3170 return total_slots;
3171}
3172
3173static size_t
3174objspace_live_slots(rb_objspace_t *objspace)
3175{
3176 return total_allocated_objects(objspace) - total_freed_objects(objspace) - total_final_slots_count(objspace);
3177}
3178
3179static size_t
3180objspace_free_slots(rb_objspace_t *objspace)
3181{
3182 return objspace_available_slots(objspace) - objspace_live_slots(objspace) - total_final_slots_count(objspace);
3183}
3184
3185static void
3186gc_setup_mark_bits(struct heap_page *page)
3187{
3188 /* copy oldgen bitmap to mark bitmap */
3189 memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
3190}
3191
3192static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
3193static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size);
3194
3195#if defined(_WIN32)
3196enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
3197
3198static BOOL
3199protect_page_body(struct heap_page_body *body, DWORD protect)
3200{
3201 DWORD old_protect;
3202 return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
3203}
3204#else
3205enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
3206#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
3207#endif
3208
3209static void
3210lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
3211{
3212 if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
3213 rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno));
3214 }
3215 else {
3216 gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
3217 }
3218}
3219
3220static void
3221unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
3222{
3223 if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
3224 rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno));
3225 }
3226 else {
3227 gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
3228 }
3229}
3230
3231static bool
3232try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src)
3233{
3234 GC_ASSERT(gc_is_moveable_obj(objspace, src));
3235
3236 struct heap_page *src_page = GET_HEAP_PAGE(src);
3237 if (!free_page) {
3238 return false;
3239 }
3240
3241 /* We should return true if either src is successfully moved, or src is
3242 * unmoveable. A false return will cause the sweeping cursor to be
3243 * incremented to the next page, and src will attempt to move again */
3244 GC_ASSERT(RVALUE_MARKED(objspace, src));
3245
3246 asan_unlock_freelist(free_page);
3247 VALUE dest = (VALUE)free_page->freelist;
3248 asan_lock_freelist(free_page);
3249 if (dest) {
3250 rb_asan_unpoison_object(dest, false);
3251 }
3252 else {
3253 /* if we can't get something from the freelist then the page must be
3254 * full */
3255 return false;
3256 }
3257 asan_unlock_freelist(free_page);
3258 free_page->freelist = ((struct free_slot *)dest)->next;
3259 asan_lock_freelist(free_page);
3260
3261 GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE);
3262
3263 if (src_page->slot_size > free_page->slot_size) {
3264 objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++;
3265 }
3266 else if (free_page->slot_size > src_page->slot_size) {
3267 objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++;
3268 }
3269 objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
3270 objspace->rcompactor.total_moved++;
3271
3272 gc_move(objspace, src, dest, src_page->slot_size, free_page->slot_size);
3273 gc_pin(objspace, src);
3274 free_page->free_slots--;
3275
3276 return true;
3277}
3278
3279static void
3280gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
3281{
3282 struct heap_page *cursor = heap->compact_cursor;
3283
3284 while (cursor) {
3285 unlock_page_body(objspace, cursor->body);
3286 cursor = ccan_list_next(&heap->pages, cursor, page_node);
3287 }
3288}
3289
3290static void gc_update_references(rb_objspace_t *objspace);
3291#if GC_CAN_COMPILE_COMPACTION
3292static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
3293#endif
3294
3295#if defined(__MINGW32__) || defined(_WIN32)
3296# define GC_COMPACTION_SUPPORTED 1
3297#else
3298/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
3299 * the read barrier, so we must disable compaction. */
3300# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
3301#endif
3302
3303#if GC_CAN_COMPILE_COMPACTION
3304static void
3305read_barrier_handler(uintptr_t address)
3306{
3307 rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
3308
3309 struct heap_page_body *page_body = GET_PAGE_BODY(address);
3310
3311 /* If the page_body is NULL, then mprotect cannot handle it and will crash
3312 * with "Cannot allocate memory". */
3313 if (page_body == NULL) {
3314 rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address);
3315 }
3316
3317 int lev = rb_gc_vm_lock();
3318 {
3319 unlock_page_body(objspace, page_body);
3320
3321 objspace->profile.read_barrier_faults++;
3322
3323 invalidate_moved_page(objspace, GET_HEAP_PAGE(address));
3324 }
3325 rb_gc_vm_unlock(lev);
3326}
3327#endif
3328
3329#if !GC_CAN_COMPILE_COMPACTION
3330static void
3331uninstall_handlers(void)
3332{
3333 /* no-op */
3334}
3335
3336static void
3337install_handlers(void)
3338{
3339 /* no-op */
3340}
3341#elif defined(_WIN32)
3342static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
3343typedef void (*signal_handler)(int);
3344static signal_handler old_sigsegv_handler;
3345
3346static LONG WINAPI
3347read_barrier_signal(EXCEPTION_POINTERS *info)
3348{
3349 /* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
3350 if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
3351 /* > The second array element specifies the virtual address of the inaccessible data.
3352 * https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
3353 *
3354 * Use this address to invalidate the page */
3355 read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
3356 return EXCEPTION_CONTINUE_EXECUTION;
3357 }
3358 else {
3359 return EXCEPTION_CONTINUE_SEARCH;
3360 }
3361}
3362
3363static void
3364uninstall_handlers(void)
3365{
3366 signal(SIGSEGV, old_sigsegv_handler);
3367 SetUnhandledExceptionFilter(old_handler);
3368}
3369
3370static void
3371install_handlers(void)
3372{
3373 /* Remove SEGV handler so that the Unhandled Exception Filter handles it */
3374 old_sigsegv_handler = signal(SIGSEGV, NULL);
3375 /* Unhandled Exception Filter has access to the violation address similar
3376 * to si_addr from sigaction */
3377 old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
3378}
3379#else
3380static struct sigaction old_sigbus_handler;
3381static struct sigaction old_sigsegv_handler;
3382
3383#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3384static exception_mask_t old_exception_masks[32];
3385static mach_port_t old_exception_ports[32];
3386static exception_behavior_t old_exception_behaviors[32];
3387static thread_state_flavor_t old_exception_flavors[32];
3388static mach_msg_type_number_t old_exception_count;
3389
3390static void
3391disable_mach_bad_access_exc(void)
3392{
3393 old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]);
3394 task_swap_exception_ports(
3395 mach_task_self(), EXC_MASK_BAD_ACCESS,
3396 MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
3397 old_exception_masks, &old_exception_count,
3398 old_exception_ports, old_exception_behaviors, old_exception_flavors
3399 );
3400}
3401
3402static void
3403restore_mach_bad_access_exc(void)
3404{
3405 for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
3406 task_set_exception_ports(
3407 mach_task_self(),
3408 old_exception_masks[i], old_exception_ports[i],
3409 old_exception_behaviors[i], old_exception_flavors[i]
3410 );
3411 }
3412}
3413#endif
3414
3415static void
3416read_barrier_signal(int sig, siginfo_t *info, void *data)
3417{
3418 // setup SEGV/BUS handlers for errors
3419 struct sigaction prev_sigbus, prev_sigsegv;
3420 sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
3421 sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
3422
3423 // enable SIGBUS/SEGV
3424 sigset_t set, prev_set;
3425 sigemptyset(&set);
3426 sigaddset(&set, SIGBUS);
3427 sigaddset(&set, SIGSEGV);
3428 sigprocmask(SIG_UNBLOCK, &set, &prev_set);
3429#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3430 disable_mach_bad_access_exc();
3431#endif
3432 // run handler
3433 read_barrier_handler((uintptr_t)info->si_addr);
3434
3435 // reset SEGV/BUS handlers
3436#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3437 restore_mach_bad_access_exc();
3438#endif
3439 sigaction(SIGBUS, &prev_sigbus, NULL);
3440 sigaction(SIGSEGV, &prev_sigsegv, NULL);
3441 sigprocmask(SIG_SETMASK, &prev_set, NULL);
3442}
3443
3444static void
3445uninstall_handlers(void)
3446{
3447#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3448 restore_mach_bad_access_exc();
3449#endif
3450 sigaction(SIGBUS, &old_sigbus_handler, NULL);
3451 sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
3452}
3453
3454static void
3455install_handlers(void)
3456{
3457 struct sigaction action;
3458 memset(&action, 0, sizeof(struct sigaction));
3459 sigemptyset(&action.sa_mask);
3460 action.sa_sigaction = read_barrier_signal;
3461 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
3462
3463 sigaction(SIGBUS, &action, &old_sigbus_handler);
3464 sigaction(SIGSEGV, &action, &old_sigsegv_handler);
3465#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
3466 disable_mach_bad_access_exc();
3467#endif
3468}
3469#endif
3470
3471static void
3472gc_compact_finish(rb_objspace_t *objspace)
3473{
3474 for (int i = 0; i < HEAP_COUNT; i++) {
3475 rb_heap_t *heap = &heaps[i];
3476 gc_unprotect_pages(objspace, heap);
3477 }
3478
3479 uninstall_handlers();
3480
3481 gc_update_references(objspace);
3482 objspace->profile.compact_count++;
3483
3484 for (int i = 0; i < HEAP_COUNT; i++) {
3485 rb_heap_t *heap = &heaps[i];
3486 heap->compact_cursor = NULL;
3487 heap->free_pages = NULL;
3488 heap->compact_cursor_index = 0;
3489 }
3490
3491 if (gc_prof_enabled(objspace)) {
3492 gc_profile_record *record = gc_prof_record(objspace);
3493 record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
3494 }
3495 objspace->flags.during_compacting = FALSE;
3496}
3497
3498 struct gc_sweep_context {
3499 struct heap_page *page;
3500 int final_slots;
3501 int freed_slots;
3502 int empty_slots;
3503};
3504
3505static inline void
3506gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
3507{
3508 struct heap_page *sweep_page = ctx->page;
3509 short slot_size = sweep_page->slot_size;
3510 short slot_bits = slot_size / BASE_SLOT_SIZE;
3511 GC_ASSERT(slot_bits > 0);
3512
3513 do {
3514 VALUE vp = (VALUE)p;
3515 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
3516
3517 rb_asan_unpoison_object(vp, false);
3518 if (bitset & 1) {
3519 switch (BUILTIN_TYPE(vp)) {
3520 default: /* majority case */
3521 gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
3522#if RGENGC_CHECK_MODE
3523 if (!is_full_marking(objspace)) {
3524 if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
3525 if (RVALUE_REMEMBERED(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p);
3526 }
3527#endif
3528
3529 if (RVALUE_WB_UNPROTECTED(objspace, vp)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(vp), vp);
3530
3531#if RGENGC_CHECK_MODE
3532#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
3533 CHECK(RVALUE_WB_UNPROTECTED);
3534 CHECK(RVALUE_MARKED);
3535 CHECK(RVALUE_MARKING);
3536 CHECK(RVALUE_UNCOLLECTIBLE);
3537#undef CHECK
3538#endif
3539
3540 rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
3541
3542 bool has_object_id = FL_TEST(vp, FL_SEEN_OBJ_ID);
3543 rb_gc_obj_free_vm_weak_references(vp);
3544 if (rb_gc_obj_free(objspace, vp)) {
3545 if (has_object_id) {
3546 obj_free_object_id(objspace, vp);
3547 }
3548 // always add free slots back to the swept pages freelist,
3549 // so that if we're compacting, we can re-use the slots
3550 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE);
3551 heap_page_add_freeobj(objspace, sweep_page, vp);
3552 gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
3553 ctx->freed_slots++;
3554 }
3555 else {
3556 ctx->final_slots++;
3557 }
3558 break;
3559
3560 case T_MOVED:
3561 if (objspace->flags.during_compacting) {
3562 /* The sweep cursor shouldn't have made it to any
3563 * T_MOVED slots while the compact flag is enabled.
3564 * The sweep cursor and compact cursor move in
3565 * opposite directions, and when they meet references will
3566 * get updated and "during_compacting" should get disabled */
3567 rb_bug("T_MOVED shouldn't be seen until compaction is finished");
3568 }
3569 gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
3570 ctx->empty_slots++;
3571 heap_page_add_freeobj(objspace, sweep_page, vp);
3572 break;
3573 case T_ZOMBIE:
3574 /* already counted */
3575 break;
3576 case T_NONE:
3577 ctx->empty_slots++; /* already freed */
3578 break;
3579 }
3580 }
3581 p += slot_size;
3582 bitset >>= slot_bits;
3583 } while (bitset);
3584}
3585
3586static inline void
3587gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx)
3588{
3589 struct heap_page *sweep_page = ctx->page;
3590 GC_ASSERT(sweep_page->heap == heap);
3591
3592 uintptr_t p;
3593 bits_t *bits, bitset;
3594
3595 gc_report(2, objspace, "page_sweep: start.\n");
3596
3597#if RGENGC_CHECK_MODE
3598 if (!objspace->flags.immediate_sweep) {
3599 GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
3600 }
3601#endif
3602 sweep_page->flags.before_sweep = FALSE;
3603 sweep_page->free_slots = 0;
3604
3605 p = (uintptr_t)sweep_page->start;
3606 bits = sweep_page->mark_bits;
3607
3608 int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE);
3609 int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH;
3610 if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
3611 bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
3612 }
3613
3614 /* The last bitmap plane may not be used if the last plane does not
3615 * have enough space for the slot_size. In that case, the last plane must
3616 * be skipped since none of the bits will be set. */
3617 int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH);
3618 GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 ||
3619 bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT);
3620
3621 // Skip out of range slots at the head of the page
3622 bitset = ~bits[0];
3623 bitset >>= NUM_IN_PAGE(p);
3624 if (bitset) {
3625 gc_sweep_plane(objspace, heap, p, bitset, ctx);
3626 }
3627 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
3628
3629 for (int i = 1; i < bitmap_plane_count; i++) {
3630 bitset = ~bits[i];
3631 if (bitset) {
3632 gc_sweep_plane(objspace, heap, p, bitset, ctx);
3633 }
3634 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
3635 }
3636
3637 if (!heap->compact_cursor) {
3638 gc_setup_mark_bits(sweep_page);
3639 }
3640
3641#if GC_PROFILE_MORE_DETAIL
3642 if (gc_prof_enabled(objspace)) {
3643 gc_profile_record *record = gc_prof_record(objspace);
3644 record->removing_objects += ctx->final_slots + ctx->freed_slots;
3645 record->empty_objects += ctx->empty_slots;
3646 }
3647#endif
3648 if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
3649 rb_gc_count(),
3650 sweep_page->total_slots,
3651 ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
3652
3653 sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
3654 sweep_page->heap->total_freed_objects += ctx->freed_slots;
3655
3656 if (heap_pages_deferred_final && !finalizing) {
3657 gc_finalize_deferred_register(objspace);
3658 }
3659
3660#if RGENGC_CHECK_MODE
3661 short freelist_len = 0;
3662 asan_unlock_freelist(sweep_page);
3663 struct free_slot *ptr = sweep_page->freelist;
3664 while (ptr) {
3665 freelist_len++;
3666 rb_asan_unpoison_object((VALUE)ptr, false);
3667 struct free_slot *next = ptr->next;
3668 rb_asan_poison_object((VALUE)ptr);
3669 ptr = next;
3670 }
3671 asan_lock_freelist(sweep_page);
3672 if (freelist_len != sweep_page->free_slots) {
3673 rb_bug("inconsistent freelist length: expected %d but was %d", sweep_page->free_slots, freelist_len);
3674 }
3675#endif
3676
3677 gc_report(2, objspace, "page_sweep: end.\n");
3678}
3679
3680static const char *
3681gc_mode_name(enum gc_mode mode)
3682{
3683 switch (mode) {
3684 case gc_mode_none: return "none";
3685 case gc_mode_marking: return "marking";
3686 case gc_mode_sweeping: return "sweeping";
3687 case gc_mode_compacting: return "compacting";
3688 default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
3689 }
3690}
3691
3692static void
3693gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
3694{
3695#if RGENGC_CHECK_MODE
3696 enum gc_mode prev_mode = gc_mode(objspace);
3697 switch (prev_mode) {
3698 case gc_mode_none: GC_ASSERT(mode == gc_mode_marking); break;
3699 case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break;
3700 case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break;
3701 case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break;
3702 }
3703#endif
3704 if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
3705 gc_mode_set(objspace, mode);
3706}
3707
3708static void
3709heap_page_freelist_append(struct heap_page *page, struct free_slot *freelist)
3710{
3711 if (freelist) {
3712 asan_unlock_freelist(page);
3713 if (page->freelist) {
3714 struct free_slot *p = page->freelist;
3715 rb_asan_unpoison_object((VALUE)p, false);
3716 while (p->next) {
3717 struct free_slot *prev = p;
3718 p = p->next;
3719 rb_asan_poison_object((VALUE)prev);
3720 rb_asan_unpoison_object((VALUE)p, false);
3721 }
3722 p->next = freelist;
3723 rb_asan_poison_object((VALUE)p);
3724 }
3725 else {
3726 page->freelist = freelist;
3727 }
3728 asan_lock_freelist(page);
3729 }
3730}
3731
3732static void
3733gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
3734{
3735 heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
3736 heap->free_pages = NULL;
3737 heap->pooled_pages = NULL;
3738 if (!objspace->flags.immediate_sweep) {
3739 struct heap_page *page = NULL;
3740
3741 ccan_list_for_each(&heap->pages, page, page_node) {
3742 page->flags.before_sweep = TRUE;
3743 }
3744 }
3745}
3746
3747#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
3748 __attribute__((noinline))
3749#endif
3750
3751#if GC_CAN_COMPILE_COMPACTION
3752static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func);
3753static int compare_pinned_slots(const void *left, const void *right, void *d);
3754#endif
3755
3756static void
3757gc_ractor_newobj_cache_clear(void *c, void *data)
3758{
3759 rb_ractor_newobj_cache_t *newobj_cache = c;
3760
3761 newobj_cache->incremental_mark_step_allocated_slots = 0;
3762
3763 for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
3764 rb_ractor_newobj_heap_cache_t *cache = &newobj_cache->heap_caches[heap_idx];
3765
3766 struct heap_page *page = cache->using_page;
3767 struct free_slot *freelist = cache->freelist;
3768 RUBY_DEBUG_LOG("ractor using_page:%p freelist:%p", (void *)page, (void *)freelist);
3769
3770 heap_page_freelist_append(page, freelist);
3771
3772 cache->using_page = NULL;
3773 cache->freelist = NULL;
3774 }
3775}
3776
3777static void
3778gc_sweep_start(rb_objspace_t *objspace)
3779{
3780 gc_mode_transition(objspace, gc_mode_sweeping);
3781 objspace->rincgc.pooled_slots = 0;
3782 objspace->heap_pages.allocatable_slots = 0;
3783
3784#if GC_CAN_COMPILE_COMPACTION
3785 if (objspace->flags.during_compacting) {
3786 gc_sort_heap_by_compare_func(
3787 objspace,
3788 objspace->rcompactor.compare_func ? objspace->rcompactor.compare_func : compare_pinned_slots
3789 );
3790 }
3791#endif
3792
3793 for (int i = 0; i < HEAP_COUNT; i++) {
3794 rb_heap_t *heap = &heaps[i];
3795 gc_sweep_start_heap(objspace, heap);
3796
3797 /* We should call gc_sweep_finish_heap for size pools with no pages. */
3798 if (heap->sweeping_page == NULL) {
3799 GC_ASSERT(heap->total_pages == 0);
3800 GC_ASSERT(heap->total_slots == 0);
3801 gc_sweep_finish_heap(objspace, heap);
3802 }
3803 }
3804
3805 rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL);
3806}
3807
3808static void
3809gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap)
3810{
3811 size_t total_slots = heap->total_slots;
3812 size_t swept_slots = heap->freed_slots + heap->empty_slots;
3813
3814 size_t init_slots = gc_params.heap_init_slots[heap - heaps];
3815 size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
3816
3817 if (swept_slots < min_free_slots &&
3818 /* The heap is a growth heap if it freed more slots than had empty slots. */
3819 (heap->empty_slots == 0 || heap->freed_slots > heap->empty_slots)) {
3820 /* If we don't have enough slots and we have pages on the tomb heap, move
3821 * pages from the tomb heap to the eden heap. This may prevent page
3822 * creation thrashing (frequently allocating and deallocting pages) and
3823 * GC thrashing (running GC more frequently than required). */
3824 struct heap_page *resurrected_page;
3825 while (swept_slots < min_free_slots &&
3826 (resurrected_page = heap_page_resurrect(objspace))) {
3827 heap_add_page(objspace, heap, resurrected_page);
3828 heap_add_freepage(heap, resurrected_page);
3829
3830 swept_slots += resurrected_page->free_slots;
3831 }
3832
3833 if (swept_slots < min_free_slots) {
3834 /* Grow this heap if we are in a major GC or if we haven't run at least
3835 * RVALUE_OLD_AGE minor GC since the last major GC. */
3836 if (is_full_marking(objspace) ||
3837 objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
3838 heap_allocatable_slots_expand(objspace, heap, swept_slots, heap->total_slots);
3839 }
3840 else if (objspace->heap_pages.allocatable_slots < (min_free_slots - swept_slots)) {
3841 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
3842 heap->force_major_gc_count++;
3843 }
3844 }
3845 }
3846}
3847
3848static void
3849gc_sweep_finish(rb_objspace_t *objspace)
3850{
3851 gc_report(1, objspace, "gc_sweep_finish\n");
3852
3853 gc_prof_set_heap_info(objspace);
3854 heap_pages_free_unused_pages(objspace);
3855
3856 for (int i = 0; i < HEAP_COUNT; i++) {
3857 rb_heap_t *heap = &heaps[i];
3858
3859 heap->freed_slots = 0;
3860 heap->empty_slots = 0;
3861
3862 if (!will_be_incremental_marking(objspace)) {
3863 struct heap_page *end_page = heap->free_pages;
3864 if (end_page) {
3865 while (end_page->free_next) end_page = end_page->free_next;
3866 end_page->free_next = heap->pooled_pages;
3867 }
3868 else {
3869 heap->free_pages = heap->pooled_pages;
3870 }
3871 heap->pooled_pages = NULL;
3872 objspace->rincgc.pooled_slots = 0;
3873 }
3874 }
3875
3876 rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP);
3877 gc_mode_transition(objspace, gc_mode_none);
3878
3879#if RGENGC_CHECK_MODE >= 2
3880 gc_verify_internal_consistency(objspace);
3881#endif
3882}
3883
3884static int
3885gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
3886{
3887 struct heap_page *sweep_page = heap->sweeping_page;
3888 int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP;
3889 int swept_slots = 0;
3890 int pooled_slots = 0;
3891
3892 if (sweep_page == NULL) return FALSE;
3893
3894#if GC_ENABLE_LAZY_SWEEP
3895 gc_prof_sweep_timer_start(objspace);
3896#endif
3897
3898 do {
3899 RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page);
3900
3901 struct gc_sweep_context ctx = {
3902 .page = sweep_page,
3903 .final_slots = 0,
3904 .freed_slots = 0,
3905 .empty_slots = 0,
3906 };
3907 gc_sweep_page(objspace, heap, &ctx);
3908 int free_slots = ctx.freed_slots + ctx.empty_slots;
3909
3910 heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
3911
3912 if (free_slots == sweep_page->total_slots &&
3913 heap_pages_freeable_pages > 0 &&
3914 unlink_limit > 0) {
3915 heap_pages_freeable_pages--;
3916 unlink_limit--;
3917 /* There are no living objects, so move this page to the global empty pages. */
3918 heap_unlink_page(objspace, heap, sweep_page);
3919
3920 sweep_page->start = 0;
3921 sweep_page->total_slots = 0;
3922 sweep_page->slot_size = 0;
3923 sweep_page->heap = NULL;
3924 sweep_page->free_slots = 0;
3925
3926 asan_unlock_freelist(sweep_page);
3927 sweep_page->freelist = NULL;
3928 asan_lock_freelist(sweep_page);
3929
3930 asan_poison_memory_region(sweep_page->body, HEAP_PAGE_SIZE);
3931
3932 objspace->empty_pages_count++;
3933 sweep_page->free_next = objspace->empty_pages;
3934 objspace->empty_pages = sweep_page;
3935 }
3936 else if (free_slots > 0) {
3937 heap->freed_slots += ctx.freed_slots;
3938 heap->empty_slots += ctx.empty_slots;
3939
3940 if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) {
3941 heap_add_poolpage(objspace, heap, sweep_page);
3942 pooled_slots += free_slots;
3943 }
3944 else {
3945 heap_add_freepage(heap, sweep_page);
3946 swept_slots += free_slots;
3947 if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) {
3948 break;
3949 }
3950 }
3951 }
3952 else {
3953 sweep_page->free_next = NULL;
3954 }
3955 } while ((sweep_page = heap->sweeping_page));
3956
3957 if (!heap->sweeping_page) {
3958 gc_sweep_finish_heap(objspace, heap);
3959
3960 if (!has_sweeping_pages(objspace)) {
3961 gc_sweep_finish(objspace);
3962 }
3963 }
3964
3965#if GC_ENABLE_LAZY_SWEEP
3966 gc_prof_sweep_timer_stop(objspace);
3967#endif
3968
3969 return heap->free_pages != NULL;
3970}
3971
3972static void
3973gc_sweep_rest(rb_objspace_t *objspace)
3974{
3975 for (int i = 0; i < HEAP_COUNT; i++) {
3976 rb_heap_t *heap = &heaps[i];
3977
3978 while (heap->sweeping_page) {
3979 gc_sweep_step(objspace, heap);
3980 }
3981 }
3982}
3983
3984static void
3985gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap)
3986{
3987 GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
3988 if (!GC_ENABLE_LAZY_SWEEP) return;
3989
3990 gc_sweeping_enter(objspace);
3991
3992 for (int i = 0; i < HEAP_COUNT; i++) {
3993 rb_heap_t *heap = &heaps[i];
3994 if (!gc_sweep_step(objspace, heap)) {
3995 /* sweep_heap requires a free slot but sweeping did not yield any
3996 * and we cannot allocate a new page. */
3997 if (heap == sweep_heap && objspace->heap_pages.allocatable_slots == 0) {
3998 /* Not allowed to create a new page so finish sweeping. */
3999 gc_sweep_rest(objspace);
4000 break;
4001 }
4002 }
4003 }
4004
4005 gc_sweeping_exit(objspace);
4006}
4007
4008 VALUE
4009rb_gc_impl_location(void *objspace_ptr, VALUE value)
4010{
4011 VALUE destination;
4012
4013 asan_unpoisoning_object(value) {
4014 if (BUILTIN_TYPE(value) == T_MOVED) {
4015 destination = (VALUE)RMOVED(value)->destination;
4016 GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
4017 }
4018 else {
4019 destination = value;
4020 }
4021 }
4022
4023 return destination;
4024}
4025
4026#if GC_CAN_COMPILE_COMPACTION
4027static void
4028invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset)
4029{
4030 if (bitset) {
4031 do {
4032 if (bitset & 1) {
4033 VALUE forwarding_object = (VALUE)p;
4034 VALUE object;
4035
4036 if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
4037 GC_ASSERT(RVALUE_PINNED(objspace, forwarding_object));
4038 GC_ASSERT(!RVALUE_MARKED(objspace, forwarding_object));
4039
4040 CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
4041
4042 object = rb_gc_impl_location(objspace, forwarding_object);
4043
4044 uint32_t original_shape_id = 0;
4045 if (RB_TYPE_P(object, T_OBJECT)) {
4046 original_shape_id = RMOVED(forwarding_object)->original_shape_id;
4047 }
4048
4049 gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object)->slot_size, page->slot_size);
4050 /* forwarding_object is now our actual object, and "object"
4051 * is the free slot for the original page */
4052
4053 if (original_shape_id) {
4054 rb_gc_set_shape(forwarding_object, original_shape_id);
4055 }
4056
4057 struct heap_page *orig_page = GET_HEAP_PAGE(object);
4058 orig_page->free_slots++;
4059 heap_page_add_freeobj(objspace, orig_page, object);
4060
4061 GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object));
4062 GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
4063 GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
4064 }
4065 }
4066 p += BASE_SLOT_SIZE;
4067 bitset >>= 1;
4068 } while (bitset);
4069 }
4070}
4071
4072static void
4073invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
4074{
4075 int i;
4076 bits_t *mark_bits, *pin_bits;
4077 bits_t bitset;
4078
4079 mark_bits = page->mark_bits;
4080 pin_bits = page->pinned_bits;
4081
4082 uintptr_t p = page->start;
4083
4084 // Skip out of range slots at the head of the page
4085 bitset = pin_bits[0] & ~mark_bits[0];
4086 bitset >>= NUM_IN_PAGE(p);
4087 invalidate_moved_plane(objspace, page, p, bitset);
4088 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
4089
4090 for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
4091 /* Moved objects are pinned but never marked. We reuse the pin bits
4092 * to indicate there is a moved object in this slot. */
4093 bitset = pin_bits[i] & ~mark_bits[i];
4094
4095 invalidate_moved_plane(objspace, page, p, bitset);
4096 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
4097 }
4098}
4099#endif
4100
4101static void
4102gc_compact_start(rb_objspace_t *objspace)
4103{
4104 struct heap_page *page = NULL;
4105 gc_mode_transition(objspace, gc_mode_compacting);
4106
4107 for (int i = 0; i < HEAP_COUNT; i++) {
4108 rb_heap_t *heap = &heaps[i];
4109 ccan_list_for_each(&heap->pages, page, page_node) {
4110 page->flags.before_sweep = TRUE;
4111 }
4112
4113 heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node);
4114 heap->compact_cursor_index = 0;
4115 }
4116
4117 if (gc_prof_enabled(objspace)) {
4118 gc_profile_record *record = gc_prof_record(objspace);
4119 record->moved_objects = objspace->rcompactor.total_moved;
4120 }
4121
4122 memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
4123 memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
4124 memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t));
4125 memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t));
4126
4127 /* Set up read barrier for pages containing MOVED objects */
4128 install_handlers();
4129}
4130
4131static void gc_sweep_compact(rb_objspace_t *objspace);
4132
4133static void
4134gc_sweep(rb_objspace_t *objspace)
4135{
4136 gc_sweeping_enter(objspace);
4137
4138 const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
4139
4140 gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
4141
4142 gc_sweep_start(objspace);
4143 if (objspace->flags.during_compacting) {
4144 gc_sweep_compact(objspace);
4145 }
4146
4147 if (immediate_sweep) {
4148#if !GC_ENABLE_LAZY_SWEEP
4149 gc_prof_sweep_timer_start(objspace);
4150#endif
4151 gc_sweep_rest(objspace);
4152#if !GC_ENABLE_LAZY_SWEEP
4153 gc_prof_sweep_timer_stop(objspace);
4154#endif
4155 }
4156 else {
4157
4158 /* Sweep every size pool. */
4159 for (int i = 0; i < HEAP_COUNT; i++) {
4160 rb_heap_t *heap = &heaps[i];
4161 gc_sweep_step(objspace, heap);
4162 }
4163 }
4164
4165 gc_sweeping_exit(objspace);
4166}
4167
4168/* Marking - Marking stack */
4169
4170static stack_chunk_t *
4171stack_chunk_alloc(void)
4172{
4173 stack_chunk_t *res;
4174
4175 res = malloc(sizeof(stack_chunk_t));
4176 if (!res)
4177 rb_memerror();
4178
4179 return res;
4180}
4181
4182static inline int
4183is_mark_stack_empty(mark_stack_t *stack)
4184{
4185 return stack->chunk == NULL;
4186}
4187
4188static size_t
4189mark_stack_size(mark_stack_t *stack)
4190{
4191 size_t size = stack->index;
4192 stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
4193
4194 while (chunk) {
4195 size += stack->limit;
4196 chunk = chunk->next;
4197 }
4198 return size;
4199}
4200
4201static void
4202add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
4203{
4204 chunk->next = stack->cache;
4205 stack->cache = chunk;
4206 stack->cache_size++;
4207}
4208
4209static void
4210shrink_stack_chunk_cache(mark_stack_t *stack)
4211{
4212 stack_chunk_t *chunk;
4213
4214 if (stack->unused_cache_size > (stack->cache_size/2)) {
4215 chunk = stack->cache;
4216 stack->cache = stack->cache->next;
4217 stack->cache_size--;
4218 free(chunk);
4219 }
4220 stack->unused_cache_size = stack->cache_size;
4221}
4222
4223static void
4224push_mark_stack_chunk(mark_stack_t *stack)
4225{
4226 stack_chunk_t *next;
4227
4228 GC_ASSERT(stack->index == stack->limit);
4229
4230 if (stack->cache_size > 0) {
4231 next = stack->cache;
4232 stack->cache = stack->cache->next;
4233 stack->cache_size--;
4234 if (stack->unused_cache_size > stack->cache_size)
4235 stack->unused_cache_size = stack->cache_size;
4236 }
4237 else {
4238 next = stack_chunk_alloc();
4239 }
4240 next->next = stack->chunk;
4241 stack->chunk = next;
4242 stack->index = 0;
4243}
4244
4245static void
4246pop_mark_stack_chunk(mark_stack_t *stack)
4247{
4248 stack_chunk_t *prev;
4249
4250 prev = stack->chunk->next;
4251 GC_ASSERT(stack->index == 0);
4252 add_stack_chunk_cache(stack, stack->chunk);
4253 stack->chunk = prev;
4254 stack->index = stack->limit;
4255}
4256
4257static void
4258mark_stack_chunk_list_free(stack_chunk_t *chunk)
4259{
4260 stack_chunk_t *next = NULL;
4261
4262 while (chunk != NULL) {
4263 next = chunk->next;
4264 free(chunk);
4265 chunk = next;
4266 }
4267}
4268
4269static void
4270free_stack_chunks(mark_stack_t *stack)
4271{
4272 mark_stack_chunk_list_free(stack->chunk);
4273}
4274
4275static void
4276mark_stack_free_cache(mark_stack_t *stack)
4277{
4278 mark_stack_chunk_list_free(stack->cache);
4279 stack->cache_size = 0;
4280 stack->unused_cache_size = 0;
4281}
4282
4283static void
4284push_mark_stack(mark_stack_t *stack, VALUE obj)
4285{
4286 switch (BUILTIN_TYPE(obj)) {
4287 case T_OBJECT:
4288 case T_CLASS:
4289 case T_MODULE:
4290 case T_FLOAT:
4291 case T_STRING:
4292 case T_REGEXP:
4293 case T_ARRAY:
4294 case T_HASH:
4295 case T_STRUCT:
4296 case T_BIGNUM:
4297 case T_FILE:
4298 case T_DATA:
4299 case T_MATCH:
4300 case T_COMPLEX:
4301 case T_RATIONAL:
4302 case T_TRUE:
4303 case T_FALSE:
4304 case T_SYMBOL:
4305 case T_IMEMO:
4306 case T_ICLASS:
4307 if (stack->index == stack->limit) {
4308 push_mark_stack_chunk(stack);
4309 }
4310 stack->chunk->data[stack->index++] = obj;
4311 return;
4312
4313 case T_NONE:
4314 case T_NIL:
4315 case T_FIXNUM:
4316 case T_MOVED:
4317 case T_ZOMBIE:
4318 case T_UNDEF:
4319 case T_MASK:
4320 rb_bug("push_mark_stack() called for broken object");
4321 break;
4322
4323 case T_NODE:
4324 rb_bug("push_mark_stack: unexpected T_NODE object");
4325 break;
4326 }
4327
4328 rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
4329 BUILTIN_TYPE(obj), (void *)obj,
4330 is_pointer_to_heap((rb_objspace_t *)rb_gc_get_objspace(), (void *)obj) ? "corrupted object" : "non object");
4331}
4332
4333static int
4334pop_mark_stack(mark_stack_t *stack, VALUE *data)
4335{
4336 if (is_mark_stack_empty(stack)) {
4337 return FALSE;
4338 }
4339 if (stack->index == 1) {
4340 *data = stack->chunk->data[--stack->index];
4341 pop_mark_stack_chunk(stack);
4342 }
4343 else {
4344 *data = stack->chunk->data[--stack->index];
4345 }
4346 return TRUE;
4347}
4348
4349static void
4350init_mark_stack(mark_stack_t *stack)
4351{
4352 int i;
4353
4354 MEMZERO(stack, mark_stack_t, 1);
4355 stack->index = stack->limit = STACK_CHUNK_SIZE;
4356
4357 for (i=0; i < 4; i++) {
4358 add_stack_chunk_cache(stack, stack_chunk_alloc());
4359 }
4360 stack->unused_cache_size = stack->cache_size;
4361}
4362
4363/* Marking */
4364
4365static void
4366rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
4367{
4368 const VALUE old_parent = objspace->rgengc.parent_object;
4369
4370 if (old_parent) { /* parent object is old */
4371 if (RVALUE_WB_UNPROTECTED(objspace, obj) || !RVALUE_OLD_P(objspace, obj)) {
4372 rgengc_remember(objspace, old_parent);
4373 }
4374 }
4375
4376 GC_ASSERT(old_parent == objspace->rgengc.parent_object);
4377}
4378
4379static inline int
4380gc_mark_set(rb_objspace_t *objspace, VALUE obj)
4381{
4382 if (RVALUE_MARKED(objspace, obj)) return 0;
4383 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
4384 return 1;
4385}
4386
4387static void
4388gc_aging(rb_objspace_t *objspace, VALUE obj)
4389{
4390 /* Disable aging if Major GC's are disabled. This will prevent longish lived
4391 * objects filling up the heap at the expense of marking many more objects.
4392 *
4393 * We should always pre-warm our process when disabling majors, by running
4394 * GC manually several times so that most objects likely to become oldgen
4395 * are already oldgen.
4396 */
4397 if(!gc_config_full_mark_val)
4398 return;
4399
4400 struct heap_page *page = GET_HEAP_PAGE(obj);
4401
4402 GC_ASSERT(RVALUE_MARKING(objspace, obj) == FALSE);
4403 check_rvalue_consistency(objspace, obj);
4404
4405 if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
4406 if (!RVALUE_OLD_P(objspace, obj)) {
4407 gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj));
4408 RVALUE_AGE_INC(objspace, obj);
4409 }
4410 else if (is_full_marking(objspace)) {
4411 GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
4412 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
4413 }
4414 }
4415 check_rvalue_consistency(objspace, obj);
4416
4417 objspace->marked_slots++;
4418}
4419
4420static void
4421gc_grey(rb_objspace_t *objspace, VALUE obj)
4422{
4423#if RGENGC_CHECK_MODE
4424 if (RVALUE_MARKED(objspace, obj) == FALSE) rb_bug("gc_grey: %s is not marked.", rb_obj_info(obj));
4425 if (RVALUE_MARKING(objspace, obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", rb_obj_info(obj));
4426#endif
4427
4428 if (is_incremental_marking(objspace)) {
4429 MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
4430 }
4431
4432 push_mark_stack(&objspace->mark_stack, obj);
4433}
4434
4435static void
4436gc_mark(rb_objspace_t *objspace, VALUE obj)
4437{
4438 GC_ASSERT(during_gc);
4439
4440 rgengc_check_relation(objspace, obj);
4441 if (!gc_mark_set(objspace, obj)) return; /* already marked */
4442
4443 if (0) { // for debug GC marking miss
4444 if (objspace->rgengc.parent_object) {
4445 RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
4446 (void *)obj, obj_type_name(obj),
4447 (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
4448 }
4449 else {
4450 RUBY_DEBUG_LOG("%p (%s)", (void *)obj, obj_type_name(obj));
4451 }
4452 }
4453
4454 if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
4455 rb_obj_info_dump(obj);
4456 rb_bug("try to mark T_NONE object"); /* check here will help debugging */
4457 }
4458
4459 gc_aging(objspace, obj);
4460 gc_grey(objspace, obj);
4461}
4462
4463static inline void
4464gc_pin(rb_objspace_t *objspace, VALUE obj)
4465{
4466 GC_ASSERT(!SPECIAL_CONST_P(obj));
4467 if (RB_UNLIKELY(objspace->flags.during_compacting)) {
4468 if (RB_LIKELY(during_gc)) {
4469 if (!RVALUE_PINNED(objspace, obj)) {
4470 GC_ASSERT(GET_HEAP_PAGE(obj)->pinned_slots <= GET_HEAP_PAGE(obj)->total_slots);
4471 GET_HEAP_PAGE(obj)->pinned_slots++;
4472 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
4473 }
4474 }
4475 }
4476}
4477
4478static inline void
4479gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
4480{
4481 gc_pin(objspace, obj);
4482 gc_mark(objspace, obj);
4483}
4484
4485void
4486rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
4487{
4488 rb_objspace_t *objspace = objspace_ptr;
4489
4490 if (RB_UNLIKELY(objspace->flags.during_reference_updating)) {
4491 GC_ASSERT(objspace->flags.during_compacting);
4492 GC_ASSERT(during_gc);
4493
4494 *ptr = rb_gc_impl_location(objspace, *ptr);
4495 }
4496 else {
4497 gc_mark(objspace, *ptr);
4498 }
4499}
4500
4501void
4502rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
4503{
4504 rb_objspace_t *objspace = objspace_ptr;
4505
4506 gc_mark(objspace, obj);
4507}
4508
4509void
4510rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
4511{
4512 rb_objspace_t *objspace = objspace_ptr;
4513
4514 gc_mark_and_pin(objspace, obj);
4515}
4516
4517void
4518rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
4519{
4520 rb_objspace_t *objspace = objspace_ptr;
4521
4522 (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
4523
4524 if (is_pointer_to_heap(objspace, (void *)obj)) {
4525 asan_unpoisoning_object(obj) {
4526 /* Garbage can live on the stack, so do not mark or pin */
4527 switch (BUILTIN_TYPE(obj)) {
4528 case T_ZOMBIE:
4529 case T_NONE:
4530 break;
4531 default:
4532 gc_mark_and_pin(objspace, obj);
4533 break;
4534 }
4535 }
4536 }
4537}
4538
4539void
4540rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr)
4541{
4542 rb_objspace_t *objspace = objspace_ptr;
4543
4544 GC_ASSERT(objspace->rgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED));
4545
4546 VALUE obj = *ptr;
4547
4548 if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
4549 rb_obj_info_dump(obj);
4550 rb_bug("try to mark T_NONE object");
4551 }
4552
4553 /* If we are in a minor GC and the other object is old, then obj should
4554 * already be marked and cannot be reclaimed in this GC cycle so we don't
4555 * need to add it to the weak references list. */
4556 if (!is_full_marking(objspace) && RVALUE_OLD_P(objspace, obj)) {
4557 GC_ASSERT(RVALUE_MARKED(objspace, obj));
4558 GC_ASSERT(!objspace->flags.during_compacting);
4559
4560 return;
4561 }
4562
4563 rgengc_check_relation(objspace, obj);
4564
4565 DURING_GC_COULD_MALLOC_REGION_START();
4566 {
4567 rb_darray_append(&objspace->weak_references, ptr);
4568 }
4569 DURING_GC_COULD_MALLOC_REGION_END();
4570
4571 objspace->profile.weak_references_count++;
4572}
4573
4574void
4575rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr)
4576{
4577 rb_objspace_t *objspace = objspace_ptr;
4578
4579 /* If we're not incremental marking, then the state of the objects can't
4580 * change so we don't need to do anything. */
4581 if (!is_incremental_marking(objspace)) return;
4582 /* If parent_obj has not been marked, then ptr has not yet been marked
4583 * weak, so we don't need to do anything. */
4584 if (!RVALUE_MARKED(objspace, parent_obj)) return;
4585
4586 VALUE **ptr_ptr;
4587 rb_darray_foreach(objspace->weak_references, i, ptr_ptr) {
4588 if (*ptr_ptr == ptr) {
4589 *ptr_ptr = NULL;
4590 break;
4591 }
4592 }
4593}
4594
4595static int
4596pin_value(st_data_t key, st_data_t value, st_data_t data)
4597{
4598 rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
4599
4600 return ST_CONTINUE;
4601}
4602
4603static void
4604mark_roots(rb_objspace_t *objspace, const char **categoryp)
4605{
4606#define MARK_CHECKPOINT(category) do { \
4607 if (categoryp) *categoryp = category; \
4608} while (0)
4609
4610 MARK_CHECKPOINT("objspace");
4611 objspace->rgengc.parent_object = Qfalse;
4612
4613 if (finalizer_table != NULL) {
4614 st_foreach(finalizer_table, pin_value, (st_data_t)objspace);
4615 }
4616
4617 st_foreach(objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)objspace);
4618
4619 if (stress_to_class) rb_gc_mark(stress_to_class);
4620
4621 rb_gc_save_machine_context();
4622 rb_gc_mark_roots(objspace, categoryp);
4623}
4624
4625static inline void
4626gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
4627{
4628 if (RVALUE_OLD_P(objspace, obj)) {
4629 objspace->rgengc.parent_object = obj;
4630 }
4631 else {
4632 objspace->rgengc.parent_object = Qfalse;
4633 }
4634}
4635
4636static void
4637gc_mark_children(rb_objspace_t *objspace, VALUE obj)
4638{
4639 gc_mark_set_parent(objspace, obj);
4640 rb_gc_mark_children(objspace, obj);
4641}
4642
4647static inline int
4648gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
4649{
4650 mark_stack_t *mstack = &objspace->mark_stack;
4651 VALUE obj;
4652 size_t marked_slots_at_the_beginning = objspace->marked_slots;
4653 size_t popped_count = 0;
4654
4655 while (pop_mark_stack(mstack, &obj)) {
4656 if (obj == Qundef) continue; /* skip */
4657
4658 if (RGENGC_CHECK_MODE && !RVALUE_MARKED(objspace, obj)) {
4659 rb_bug("gc_mark_stacked_objects: %s is not marked.", rb_obj_info(obj));
4660 }
4661 gc_mark_children(objspace, obj);
4662
4663 if (incremental) {
4664 if (RGENGC_CHECK_MODE && !RVALUE_MARKING(objspace, obj)) {
4665 rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
4666 }
4667 CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
4668 popped_count++;
4669
4670 if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
4671 break;
4672 }
4673 }
4674 else {
4675 /* just ignore marking bits */
4676 }
4677 }
4678
4679 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
4680
4681 if (is_mark_stack_empty(mstack)) {
4682 shrink_stack_chunk_cache(mstack);
4683 return TRUE;
4684 }
4685 else {
4686 return FALSE;
4687 }
4688}
4689
4690static int
4691gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
4692{
4693 return gc_mark_stacked_objects(objspace, TRUE, count);
4694}
4695
4696static int
4697gc_mark_stacked_objects_all(rb_objspace_t *objspace)
4698{
4699 return gc_mark_stacked_objects(objspace, FALSE, 0);
4700}
4701
4702#if RGENGC_CHECK_MODE >= 4
4703
4704#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
4705#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
4706#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
4707
4708struct reflist {
4709 VALUE *list;
4710 int pos;
4711 int size;
4712};
4713
4714static struct reflist *
4715reflist_create(VALUE obj)
4716{
4717 struct reflist *refs = xmalloc(sizeof(struct reflist));
4718 refs->size = 1;
4719 refs->list = ALLOC_N(VALUE, refs->size);
4720 refs->list[0] = obj;
4721 refs->pos = 1;
4722 return refs;
4723}
4724
4725static void
4726reflist_destruct(struct reflist *refs)
4727{
4728 xfree(refs->list);
4729 xfree(refs);
4730}
4731
4732static void
4733reflist_add(struct reflist *refs, VALUE obj)
4734{
4735 if (refs->pos == refs->size) {
4736 refs->size *= 2;
4737 SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
4738 }
4739
4740 refs->list[refs->pos++] = obj;
4741}
4742
4743static void
4744reflist_dump(struct reflist *refs)
4745{
4746 int i;
4747 for (i=0; i<refs->pos; i++) {
4748 VALUE obj = refs->list[i];
4749 if (IS_ROOTSIG(obj)) { /* root */
4750 fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
4751 }
4752 else {
4753 fprintf(stderr, "<%s>", rb_obj_info(obj));
4754 }
4755 if (i+1 < refs->pos) fprintf(stderr, ", ");
4756 }
4757}
4758
4759static int
4760reflist_referred_from_machine_context(struct reflist *refs)
4761{
4762 int i;
4763 for (i=0; i<refs->pos; i++) {
4764 VALUE obj = refs->list[i];
4765 if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
4766 }
4767 return 0;
4768}
4769
4770struct allrefs {
4771 rb_objspace_t *objspace;
4772 /* a -> obj1
4773 * b -> obj1
4774 * c -> obj1
4775 * c -> obj2
4776 * d -> obj3
4777 * #=> {obj1 => [a, b, c], obj2 => [c, d]}
4778 */
4779 struct st_table *references;
4780 const char *category;
4781 VALUE root_obj;
4782 mark_stack_t mark_stack;
4783};
4784
4785static int
4786allrefs_add(struct allrefs *data, VALUE obj)
4787{
4788 struct reflist *refs;
4789 st_data_t r;
4790
4791 if (st_lookup(data->references, obj, &r)) {
4792 refs = (struct reflist *)r;
4793 reflist_add(refs, data->root_obj);
4794 return 0;
4795 }
4796 else {
4797 refs = reflist_create(data->root_obj);
4798 st_insert(data->references, obj, (st_data_t)refs);
4799 return 1;
4800 }
4801}
4802
4803static void
4804allrefs_i(VALUE obj, void *ptr)
4805{
4806 struct allrefs *data = (struct allrefs *)ptr;
4807
4808 if (allrefs_add(data, obj)) {
4809 push_mark_stack(&data->mark_stack, obj);
4810 }
4811}
4812
4813static void
4814allrefs_roots_i(VALUE obj, void *ptr)
4815{
4816 struct allrefs *data = (struct allrefs *)ptr;
4817 if (strlen(data->category) == 0) rb_bug("!!!");
4818 data->root_obj = MAKE_ROOTSIG(data->category);
4819
4820 if (allrefs_add(data, obj)) {
4821 push_mark_stack(&data->mark_stack, obj);
4822 }
4823}
4824#define PUSH_MARK_FUNC_DATA(v) do { \
4825 struct gc_mark_func_data_struct *prev_mark_func_data = GET_VM()->gc.mark_func_data; \
4826 GET_VM()->gc.mark_func_data = (v);
4827
4828#define POP_MARK_FUNC_DATA() GET_VM()->gc.mark_func_data = prev_mark_func_data;} while (0)
4829
4830static st_table *
4831objspace_allrefs(rb_objspace_t *objspace)
4832{
4833 struct allrefs data;
4834 struct gc_mark_func_data_struct mfd;
4835 VALUE obj;
4836 int prev_dont_gc = dont_gc_val();
4837 dont_gc_on();
4838
4839 data.objspace = objspace;
4840 data.references = st_init_numtable();
4841 init_mark_stack(&data.mark_stack);
4842
4843 mfd.mark_func = allrefs_roots_i;
4844 mfd.data = &data;
4845
4846 /* traverse root objects */
4847 PUSH_MARK_FUNC_DATA(&mfd);
4848 GET_VM()->gc.mark_func_data = &mfd;
4849 mark_roots(objspace, &data.category);
4850 POP_MARK_FUNC_DATA();
4851
4852 /* traverse rest objects reachable from root objects */
4853 while (pop_mark_stack(&data.mark_stack, &obj)) {
4854 rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data);
4855 }
4856 free_stack_chunks(&data.mark_stack);
4857
4858 dont_gc_set(prev_dont_gc);
4859 return data.references;
4860}
4861
4862static int
4863objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
4864{
4865 struct reflist *refs = (struct reflist *)value;
4866 reflist_destruct(refs);
4867 return ST_CONTINUE;
4868}
4869
4870static void
4871objspace_allrefs_destruct(struct st_table *refs)
4872{
4873 st_foreach(refs, objspace_allrefs_destruct_i, 0);
4874 st_free_table(refs);
4875}
4876
4877#if RGENGC_CHECK_MODE >= 5
4878static int
4879allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
4880{
4881 VALUE obj = (VALUE)k;
4882 struct reflist *refs = (struct reflist *)v;
4883 fprintf(stderr, "[allrefs_dump_i] %s <- ", rb_obj_info(obj));
4884 reflist_dump(refs);
4885 fprintf(stderr, "\n");
4886 return ST_CONTINUE;
4887}
4888
4889static void
4890allrefs_dump(rb_objspace_t *objspace)
4891{
4892 VALUE size = objspace->rgengc.allrefs_table->num_entries;
4893 fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size);
4894 st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
4895}
4896#endif
4897
4898static int
4899gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
4900{
4901 VALUE obj = k;
4902 struct reflist *refs = (struct reflist *)v;
4903 rb_objspace_t *objspace = (rb_objspace_t *)ptr;
4904
4905 /* object should be marked or oldgen */
4906 if (!RVALUE_MARKED(objspace, obj)) {
4907 fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", rb_obj_info(obj));
4908 fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
4909 reflist_dump(refs);
4910
4911 if (reflist_referred_from_machine_context(refs)) {
4912 fprintf(stderr, " (marked from machine stack).\n");
4913 /* marked from machine context can be false positive */
4914 }
4915 else {
4916 objspace->rgengc.error_count++;
4917 fprintf(stderr, "\n");
4918 }
4919 }
4920 return ST_CONTINUE;
4921}
4922
4923static void
4924gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
4925{
4926 size_t saved_malloc_increase = objspace->malloc_params.increase;
4927#if RGENGC_ESTIMATE_OLDMALLOC
4928 size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase;
4929#endif
4930 VALUE already_disabled = rb_objspace_gc_disable(objspace);
4931
4932 objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
4933
4934 if (checker_func) {
4935 st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
4936 }
4937
4938 if (objspace->rgengc.error_count > 0) {
4939#if RGENGC_CHECK_MODE >= 5
4940 allrefs_dump(objspace);
4941#endif
4942 if (checker_name) rb_bug("%s: GC has problem.", checker_name);
4943 }
4944
4945 objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
4946 objspace->rgengc.allrefs_table = 0;
4947
4948 if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
4949 objspace->malloc_params.increase = saved_malloc_increase;
4950#if RGENGC_ESTIMATE_OLDMALLOC
4951 objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
4952#endif
4953}
4954#endif /* RGENGC_CHECK_MODE >= 4 */
4955
4956 struct verify_internal_consistency_struct {
4957 rb_objspace_t *objspace;
4958 int err_count;
4959 size_t live_object_count;
4960 size_t zombie_object_count;
4961
4962 VALUE parent;
4963 size_t old_object_count;
4964 size_t remembered_shady_count;
4965};
4966
4967static void
4968check_generation_i(const VALUE child, void *ptr)
4969{
4970 struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
4971 const VALUE parent = data->parent;
4972
4973 if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(data->objspace, parent));
4974
4975 if (!RVALUE_OLD_P(data->objspace, child)) {
4976 if (!RVALUE_REMEMBERED(data->objspace, parent) &&
4977 !RVALUE_REMEMBERED(data->objspace, child) &&
4978 !RVALUE_UNCOLLECTIBLE(data->objspace, child)) {
4979 fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", rb_obj_info(parent), rb_obj_info(child));
4980 data->err_count++;
4981 }
4982 }
4983}
4984
4985static void
4986check_color_i(const VALUE child, void *ptr)
4987{
4988 struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
4989 const VALUE parent = data->parent;
4990
4991 if (!RVALUE_WB_UNPROTECTED(data->objspace, parent) && RVALUE_WHITE_P(data->objspace, child)) {
4992 fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
4993 rb_obj_info(parent), rb_obj_info(child));
4994 data->err_count++;
4995 }
4996}
4997
4998static void
4999check_children_i(const VALUE child, void *ptr)
5000{
5001 struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
5002 if (check_rvalue_consistency_force(data->objspace, child, FALSE) != 0) {
5003 fprintf(stderr, "check_children_i: %s has error (referenced from %s)",
5004 rb_obj_info(child), rb_obj_info(data->parent));
5005
5006 data->err_count++;
5007 }
5008}
5009
5010static int
5011verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
5012 struct verify_internal_consistency_struct *data)
5013{
5014 VALUE obj;
5015 rb_objspace_t *objspace = data->objspace;
5016
5017 for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
5018 asan_unpoisoning_object(obj) {
5019 if (!rb_gc_impl_garbage_object_p(objspace, obj)) {
5020 /* count objects */
5021 data->live_object_count++;
5022 data->parent = obj;
5023
5024 /* Normally, we don't expect T_MOVED objects to be in the heap.
5025 * But they can stay alive on the stack, */
5026 if (!gc_object_moved_p(objspace, obj)) {
5027 /* moved slots don't have children */
5028 rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data);
5029 }
5030
5031 /* check health of children */
5032 if (RVALUE_OLD_P(objspace, obj)) data->old_object_count++;
5033 if (RVALUE_WB_UNPROTECTED(objspace, obj) && RVALUE_UNCOLLECTIBLE(objspace, obj)) data->remembered_shady_count++;
5034
5035 if (!is_marking(objspace) && RVALUE_OLD_P(objspace, obj)) {
5036 /* reachable objects from an oldgen object should be old or (young with remember) */
5037 data->parent = obj;
5038 rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data);
5039 }
5040
5041 if (is_incremental_marking(objspace)) {
5042 if (RVALUE_BLACK_P(objspace, obj)) {
5043 /* reachable objects from black objects should be black or grey objects */
5044 data->parent = obj;
5045 rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data);
5046 }
5047 }
5048 }
5049 else {
5050 if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
5051 data->zombie_object_count++;
5052
5053 if ((RBASIC(obj)->flags & ~ZOMBIE_OBJ_KEPT_FLAGS) != T_ZOMBIE) {
5054 fprintf(stderr, "verify_internal_consistency_i: T_ZOMBIE has extra flags set: %s\n",
5055 rb_obj_info(obj));
5056 data->err_count++;
5057 }
5058
5059 if (!!FL_TEST(obj, FL_FINALIZE) != !!st_is_member(finalizer_table, obj)) {
5060 fprintf(stderr, "verify_internal_consistency_i: FL_FINALIZE %s but %s finalizer_table: %s\n",
5061 FL_TEST(obj, FL_FINALIZE) ? "set" : "not set", st_is_member(finalizer_table, obj) ? "in" : "not in",
5062 rb_obj_info(obj));
5063 data->err_count++;
5064 }
5065 }
5066 }
5067 }
5068 }
5069
5070 return 0;
5071}
5072
5073static int
5074gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
5075{
5076 unsigned int has_remembered_shady = FALSE;
5077 unsigned int has_remembered_old = FALSE;
5078 int remembered_old_objects = 0;
5079 int free_objects = 0;
5080 int zombie_objects = 0;
5081
5082 short slot_size = page->slot_size;
5083 uintptr_t start = (uintptr_t)page->start;
5084 uintptr_t end = start + page->total_slots * slot_size;
5085
5086 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
5087 VALUE val = (VALUE)ptr;
5088 asan_unpoisoning_object(val) {
5089 enum ruby_value_type type = BUILTIN_TYPE(val);
5090
5091 if (type == T_NONE) free_objects++;
5092 if (type == T_ZOMBIE) zombie_objects++;
5093 if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
5094 has_remembered_shady = TRUE;
5095 }
5096 if (RVALUE_PAGE_MARKING(page, val)) {
5097 has_remembered_old = TRUE;
5098 remembered_old_objects++;
5099 }
5100 }
5101 }
5102
5103 if (!is_incremental_marking(objspace) &&
5104 page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
5105
5106 for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
5107 VALUE val = (VALUE)ptr;
5108 if (RVALUE_PAGE_MARKING(page, val)) {
5109 fprintf(stderr, "marking -> %s\n", rb_obj_info(val));
5110 }
5111 }
5112 rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
5113 (void *)page, remembered_old_objects, obj ? rb_obj_info(obj) : "");
5114 }
5115
5116 if (page->flags.has_uncollectible_wb_unprotected_objects == FALSE && has_remembered_shady == TRUE) {
5117 rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
5118 (void *)page, obj ? rb_obj_info(obj) : "");
5119 }
5120
5121 if (0) {
5122 /* free_slots may not equal to free_objects */
5123 if (page->free_slots != free_objects) {
5124 rb_bug("page %p's free_slots should be %d, but %d", (void *)page, page->free_slots, free_objects);
5125 }
5126 }
5127 if (page->final_slots != zombie_objects) {
5128 rb_bug("page %p's final_slots should be %d, but %d", (void *)page, page->final_slots, zombie_objects);
5129 }
5130
5131 return remembered_old_objects;
5132}
5133
5134static int
5135gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head)
5136{
5137 int remembered_old_objects = 0;
5138 struct heap_page *page = 0;
5139
5140 ccan_list_for_each(head, page, page_node) {
5141 asan_unlock_freelist(page);
5142 struct free_slot *p = page->freelist;
5143 while (p) {
5144 VALUE vp = (VALUE)p;
5145 VALUE prev = vp;
5146 rb_asan_unpoison_object(vp, false);
5147 if (BUILTIN_TYPE(vp) != T_NONE) {
5148 fprintf(stderr, "freelist slot expected to be T_NONE but was: %s\n", rb_obj_info(vp));
5149 }
5150 p = p->next;
5151 rb_asan_poison_object(prev);
5152 }
5153 asan_lock_freelist(page);
5154
5155 if (page->flags.has_remembered_objects == FALSE) {
5156 remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
5157 }
5158 }
5159
5160 return remembered_old_objects;
5161}
5162
5163static int
5164gc_verify_heap_pages(rb_objspace_t *objspace)
5165{
5166 int remembered_old_objects = 0;
5167 for (int i = 0; i < HEAP_COUNT; i++) {
5168 remembered_old_objects += gc_verify_heap_pages_(objspace, &((&heaps[i])->pages));
5169 }
5170 return remembered_old_objects;
5171}
5172
5173static void
5174gc_verify_internal_consistency_(rb_objspace_t *objspace)
5175{
5176 struct verify_internal_consistency_struct data = {0};
5177
5178 data.objspace = objspace;
5179 gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
5180
5181 /* check relations */
5182 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
5183 struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
5184 short slot_size = page->slot_size;
5185
5186 uintptr_t start = (uintptr_t)page->start;
5187 uintptr_t end = start + page->total_slots * slot_size;
5188
5189 verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data);
5190 }
5191
5192 if (data.err_count != 0) {
5193#if RGENGC_CHECK_MODE >= 5
5194 objspace->rgengc.error_count = data.err_count;
5195 gc_marks_check(objspace, NULL, NULL);
5196 allrefs_dump(objspace);
5197#endif
5198 rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
5199 }
5200
5201 /* check heap_page status */
5202 gc_verify_heap_pages(objspace);
5203
5204 /* check counters */
5205
5206 if (!is_lazy_sweeping(objspace) &&
5207 !finalizing &&
5208 !rb_gc_multi_ractor_p()) {
5209 if (objspace_live_slots(objspace) != data.live_object_count) {
5210 fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", total_freed_objects: %"PRIdSIZE"\n",
5211 total_final_slots_count(objspace), total_freed_objects(objspace));
5212 rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
5213 objspace_live_slots(objspace), data.live_object_count);
5214 }
5215 }
5216
5217 if (!is_marking(objspace)) {
5218 if (objspace->rgengc.old_objects != data.old_object_count) {
5219 rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
5220 objspace->rgengc.old_objects, data.old_object_count);
5221 }
5222 if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
5223 rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".",
5224 objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
5225 }
5226 }
5227
5228 if (!finalizing) {
5229 size_t list_count = 0;
5230
5231 {
5232 VALUE z = heap_pages_deferred_final;
5233 while (z) {
5234 list_count++;
5235 z = RZOMBIE(z)->next;
5236 }
5237 }
5238
5239 if (total_final_slots_count(objspace) != data.zombie_object_count ||
5240 total_final_slots_count(objspace) != list_count) {
5241
5242 rb_bug("inconsistent finalizing object count:\n"
5243 " expect %"PRIuSIZE"\n"
5244 " but %"PRIuSIZE" zombies\n"
5245 " heap_pages_deferred_final list has %"PRIuSIZE" items.",
5246 total_final_slots_count(objspace),
5247 data.zombie_object_count,
5248 list_count);
5249 }
5250 }
5251
5252 gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
5253}
5254
5255static void
5256gc_verify_internal_consistency(void *objspace_ptr)
5257{
5258 rb_objspace_t *objspace = objspace_ptr;
5259
5260 unsigned int lev = rb_gc_vm_lock();
5261 {
5262 rb_gc_vm_barrier(); // stop other ractors
5263
5264 unsigned int prev_during_gc = during_gc;
5265 during_gc = FALSE; // stop gc here
5266 {
5267 gc_verify_internal_consistency_(objspace);
5268 }
5269 during_gc = prev_during_gc;
5270 }
5271 rb_gc_vm_unlock(lev);
5272}
5273
5274static void
5275heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
5276{
5277 if (heap->pooled_pages) {
5278 if (heap->free_pages) {
5279 struct heap_page *free_pages_tail = heap->free_pages;
5280 while (free_pages_tail->free_next) {
5281 free_pages_tail = free_pages_tail->free_next;
5282 }
5283 free_pages_tail->free_next = heap->pooled_pages;
5284 }
5285 else {
5286 heap->free_pages = heap->pooled_pages;
5287 }
5288
5289 heap->pooled_pages = NULL;
5290 }
5291}
5292
5293static int
5294gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
5295{
5296 struct heap_page *page = GET_HEAP_PAGE(obj);
5297 bits_t *uncollectible_bits = &page->uncollectible_bits[0];
5298
5299 if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
5300 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
5301 MARK_IN_BITMAP(uncollectible_bits, obj);
5302 objspace->rgengc.uncollectible_wb_unprotected_objects++;
5303
5304#if RGENGC_PROFILE > 0
5305 objspace->profile.total_remembered_shady_object_count++;
5306#if RGENGC_PROFILE >= 2
5307 objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
5308#endif
5309#endif
5310 return TRUE;
5311 }
5312 else {
5313 return FALSE;
5314 }
5315}
5316
5317static inline void
5318gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits)
5319{
5320 if (bits) {
5321 do {
5322 if (bits & 1) {
5323 gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", rb_obj_info((VALUE)p));
5324 GC_ASSERT(RVALUE_WB_UNPROTECTED(objspace, (VALUE)p));
5325 GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p));
5326 gc_mark_children(objspace, (VALUE)p);
5327 }
5328 p += BASE_SLOT_SIZE;
5329 bits >>= 1;
5330 } while (bits);
5331 }
5332}
5333
5334static void
5335gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
5336{
5337 struct heap_page *page = 0;
5338
5339 ccan_list_for_each(&heap->pages, page, page_node) {
5340 bits_t *mark_bits = page->mark_bits;
5341 bits_t *wbun_bits = page->wb_unprotected_bits;
5342 uintptr_t p = page->start;
5343 size_t j;
5344
5345 bits_t bits = mark_bits[0] & wbun_bits[0];
5346 bits >>= NUM_IN_PAGE(p);
5347 gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
5348 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5349
5350 for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
5351 bits_t bits = mark_bits[j] & wbun_bits[j];
5352
5353 gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
5354 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5355 }
5356 }
5357
5358 gc_mark_stacked_objects_all(objspace);
5359}
5360
5361static void
5362gc_update_weak_references(rb_objspace_t *objspace)
5363{
5364 size_t retained_weak_references_count = 0;
5365 VALUE **ptr_ptr;
5366 rb_darray_foreach(objspace->weak_references, i, ptr_ptr) {
5367 if (!*ptr_ptr) continue;
5368
5369 VALUE obj = **ptr_ptr;
5370
5371 if (RB_SPECIAL_CONST_P(obj)) continue;
5372
5373 if (!RVALUE_MARKED(objspace, obj)) {
5374 **ptr_ptr = Qundef;
5375 }
5376 else {
5377 retained_weak_references_count++;
5378 }
5379 }
5380
5381 objspace->profile.retained_weak_references_count = retained_weak_references_count;
5382
5383 rb_darray_clear(objspace->weak_references);
5384 DURING_GC_COULD_MALLOC_REGION_START();
5385 {
5386 rb_darray_resize_capa(&objspace->weak_references, retained_weak_references_count);
5387 }
5388 DURING_GC_COULD_MALLOC_REGION_END();
5389}
5390
5391static void
5392gc_marks_finish(rb_objspace_t *objspace)
5393{
5394 /* finish incremental GC */
5395 if (is_incremental_marking(objspace)) {
5396 if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
5397 rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").",
5398 mark_stack_size(&objspace->mark_stack));
5399 }
5400
5401 mark_roots(objspace, NULL);
5402 while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false);
5403
5404#if RGENGC_CHECK_MODE >= 2
5405 if (gc_verify_heap_pages(objspace) != 0) {
5406 rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
5407 }
5408#endif
5409
5410 objspace->flags.during_incremental_marking = FALSE;
5411 /* check children of all marked wb-unprotected objects */
5412 for (int i = 0; i < HEAP_COUNT; i++) {
5413 gc_marks_wb_unprotected_objects(objspace, &heaps[i]);
5414 }
5415 }
5416
5417 gc_update_weak_references(objspace);
5418
5419#if RGENGC_CHECK_MODE >= 2
5420 gc_verify_internal_consistency(objspace);
5421#endif
5422
5423#if RGENGC_CHECK_MODE >= 4
5424 during_gc = FALSE;
5425 gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
5426 during_gc = TRUE;
5427#endif
5428
5429 {
5430 const unsigned long r_mul = objspace->live_ractor_cache_count > 8 ? 8 : objspace->live_ractor_cache_count; // upto 8
5431
5432 size_t total_slots = objspace_available_slots(objspace);
5433 size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
5434 size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
5435 size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
5436 if (min_free_slots < gc_params.heap_free_slots * r_mul) {
5437 min_free_slots = gc_params.heap_free_slots * r_mul;
5438 }
5439
5440 int full_marking = is_full_marking(objspace);
5441
5442 GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots);
5443
5444 /* Setup freeable slots. */
5445 size_t total_init_slots = 0;
5446 for (int i = 0; i < HEAP_COUNT; i++) {
5447 total_init_slots += gc_params.heap_init_slots[i] * r_mul;
5448 }
5449
5450 if (max_free_slots < total_init_slots) {
5451 max_free_slots = total_init_slots;
5452 }
5453
5454 if (sweep_slots > max_free_slots) {
5455 heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
5456 }
5457 else {
5458 heap_pages_freeable_pages = 0;
5459 }
5460
5461 if (objspace->heap_pages.allocatable_slots == 0 && sweep_slots < min_free_slots) {
5462 if (!full_marking) {
5463 if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
5464 full_marking = TRUE;
5465 }
5466 else {
5467 gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
5468 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
5469 }
5470 }
5471 }
5472
5473 if (full_marking) {
5474 /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
5475 const double r = gc_params.oldobject_limit_factor;
5476 objspace->rgengc.uncollectible_wb_unprotected_objects_limit = MAX(
5477 (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r),
5478 (size_t)(objspace->rgengc.old_objects * gc_params.uncollectible_wb_unprotected_objects_limit_ratio)
5479 );
5480 objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
5481 }
5482
5483 if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
5484 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_SHADY;
5485 }
5486 if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
5487 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDGEN;
5488 }
5489
5490 gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
5491 "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
5492 "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" slots, next GC: %s)\n",
5493 objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots,
5494 gc_needs_major_flags ? "major" : "minor");
5495 }
5496
5497 // TODO: refactor so we don't need to call this
5498 rb_ractor_finish_marking();
5499
5500 rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_MARK);
5501}
5502
5503static bool
5504gc_compact_heap_cursors_met_p(rb_heap_t *heap)
5505{
5506 return heap->sweeping_page == heap->compact_cursor;
5507}
5508
5509
5510static rb_heap_t *
5511gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE obj)
5512{
5513 size_t obj_size = rb_gc_obj_optimal_size(obj);
5514 if (obj_size == 0) {
5515 return src_pool;
5516 }
5517
5518 size_t idx = 0;
5519 if (rb_gc_impl_size_allocatable_p(obj_size)) {
5520 idx = heap_idx_for_size(obj_size);
5521 }
5522
5523 return &heaps[idx];
5524}
5525
5526static bool
5527gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src)
5528{
5529 GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED);
5530 GC_ASSERT(gc_is_moveable_obj(objspace, src));
5531
5532 rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src);
5533 uint32_t orig_shape = 0;
5534 uint32_t new_shape = 0;
5535
5536 if (gc_compact_heap_cursors_met_p(dest_pool)) {
5537 return dest_pool != heap;
5538 }
5539
5540 if (RB_TYPE_P(src, T_OBJECT)) {
5541 orig_shape = rb_gc_get_shape(src);
5542
5543 if (dest_pool != heap) {
5544 new_shape = rb_gc_rebuild_shape(src, dest_pool - heaps);
5545
5546 if (new_shape == 0) {
5547 dest_pool = heap;
5548 }
5549 }
5550 }
5551
5552 while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) {
5553 struct gc_sweep_context ctx = {
5554 .page = dest_pool->sweeping_page,
5555 .final_slots = 0,
5556 .freed_slots = 0,
5557 .empty_slots = 0,
5558 };
5559
5560 /* The page of src could be partially compacted, so it may contain
5561 * T_MOVED. Sweeping a page may read objects on this page, so we
5562 * need to lock the page. */
5563 lock_page_body(objspace, GET_PAGE_BODY(src));
5564 gc_sweep_page(objspace, dest_pool, &ctx);
5565 unlock_page_body(objspace, GET_PAGE_BODY(src));
5566
5567 if (dest_pool->sweeping_page->free_slots > 0) {
5568 heap_add_freepage(dest_pool, dest_pool->sweeping_page);
5569 }
5570
5571 dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node);
5572 if (gc_compact_heap_cursors_met_p(dest_pool)) {
5573 return dest_pool != heap;
5574 }
5575 }
5576
5577 if (orig_shape != 0) {
5578 if (new_shape != 0) {
5579 VALUE dest = rb_gc_impl_location(objspace, src);
5580 rb_gc_set_shape(dest, new_shape);
5581 }
5582 RMOVED(src)->original_shape_id = orig_shape;
5583 }
5584
5585 return true;
5586}
5587
5588static bool
5589gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
5590{
5591 short slot_size = page->slot_size;
5592 short slot_bits = slot_size / BASE_SLOT_SIZE;
5593 GC_ASSERT(slot_bits > 0);
5594
5595 do {
5596 VALUE vp = (VALUE)p;
5597 GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
5598
5599 if (bitset & 1) {
5600 objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
5601
5602 if (gc_is_moveable_obj(objspace, vp)) {
5603 if (!gc_compact_move(objspace, heap, vp)) {
5604 //the cursors met. bubble up
5605 return false;
5606 }
5607 }
5608 }
5609 p += slot_size;
5610 bitset >>= slot_bits;
5611 } while (bitset);
5612
5613 return true;
5614}
5615
5616// Iterate up all the objects in page, moving them to where they want to go
5617static bool
5618gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
5619{
5620 GC_ASSERT(page == heap->compact_cursor);
5621
5622 bits_t *mark_bits, *pin_bits;
5623 bits_t bitset;
5624 uintptr_t p = page->start;
5625
5626 mark_bits = page->mark_bits;
5627 pin_bits = page->pinned_bits;
5628
5629 // objects that can be moved are marked and not pinned
5630 bitset = (mark_bits[0] & ~pin_bits[0]);
5631 bitset >>= NUM_IN_PAGE(p);
5632 if (bitset) {
5633 if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
5634 return false;
5635 }
5636 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5637
5638 for (int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
5639 bitset = (mark_bits[j] & ~pin_bits[j]);
5640 if (bitset) {
5641 if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
5642 return false;
5643 }
5644 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5645 }
5646
5647 return true;
5648}
5649
5650static bool
5651gc_compact_all_compacted_p(rb_objspace_t *objspace)
5652{
5653 for (int i = 0; i < HEAP_COUNT; i++) {
5654 rb_heap_t *heap = &heaps[i];
5655
5656 if (heap->total_pages > 0 &&
5657 !gc_compact_heap_cursors_met_p(heap)) {
5658 return false;
5659 }
5660 }
5661
5662 return true;
5663}
5664
5665static void
5666gc_sweep_compact(rb_objspace_t *objspace)
5667{
5668 gc_compact_start(objspace);
5669#if RGENGC_CHECK_MODE >= 2
5670 gc_verify_internal_consistency(objspace);
5671#endif
5672
5673 while (!gc_compact_all_compacted_p(objspace)) {
5674 for (int i = 0; i < HEAP_COUNT; i++) {
5675 rb_heap_t *heap = &heaps[i];
5676
5677 if (gc_compact_heap_cursors_met_p(heap)) {
5678 continue;
5679 }
5680
5681 struct heap_page *start_page = heap->compact_cursor;
5682
5683 if (!gc_compact_page(objspace, heap, start_page)) {
5684 lock_page_body(objspace, start_page->body);
5685
5686 continue;
5687 }
5688
5689 // If we get here, we've finished moving all objects on the compact_cursor page
5690 // So we can lock it and move the cursor on to the next one.
5691 lock_page_body(objspace, start_page->body);
5692 heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
5693 }
5694 }
5695
5696 gc_compact_finish(objspace);
5697
5698#if RGENGC_CHECK_MODE >= 2
5699 gc_verify_internal_consistency(objspace);
5700#endif
5701}
5702
5703static void
5704gc_marks_rest(rb_objspace_t *objspace)
5705{
5706 gc_report(1, objspace, "gc_marks_rest\n");
5707
5708 for (int i = 0; i < HEAP_COUNT; i++) {
5709 (&heaps[i])->pooled_pages = NULL;
5710 }
5711
5712 if (is_incremental_marking(objspace)) {
5713 while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
5714 }
5715 else {
5716 gc_mark_stacked_objects_all(objspace);
5717 }
5718
5719 gc_marks_finish(objspace);
5720}
5721
5722static bool
5723gc_marks_step(rb_objspace_t *objspace, size_t slots)
5724{
5725 bool marking_finished = false;
5726
5727 GC_ASSERT(is_marking(objspace));
5728 if (gc_mark_stacked_objects_incremental(objspace, slots)) {
5729 gc_marks_finish(objspace);
5730
5731 marking_finished = true;
5732 }
5733
5734 return marking_finished;
5735}
5736
5737static bool
5738gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap)
5739{
5740 GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD);
5741 bool marking_finished = true;
5742
5743 gc_marking_enter(objspace);
5744
5745 if (heap->free_pages) {
5746 gc_report(2, objspace, "gc_marks_continue: has pooled pages");
5747
5748 marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots);
5749 }
5750 else {
5751 gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n",
5752 mark_stack_size(&objspace->mark_stack));
5753 heap->force_incremental_marking_finish_count++;
5754 gc_marks_rest(objspace);
5755 }
5756
5757 gc_marking_exit(objspace);
5758
5759 return marking_finished;
5760}
5761
5762static void
5763gc_marks_start(rb_objspace_t *objspace, int full_mark)
5764{
5765 /* start marking */
5766 gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
5767 gc_mode_transition(objspace, gc_mode_marking);
5768
5769 if (full_mark) {
5770 size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
5771 objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps;
5772
5773 if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", "
5774 "objspace->rincgc.pooled_page_num: %"PRIdSIZE", "
5775 "objspace->rincgc.step_slots: %"PRIdSIZE", \n",
5776 objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
5777 objspace->flags.during_minor_gc = FALSE;
5778 if (ruby_enable_autocompact) {
5779 objspace->flags.during_compacting |= TRUE;
5780 }
5781 objspace->profile.major_gc_count++;
5782 objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
5783 objspace->rgengc.old_objects = 0;
5784 objspace->rgengc.last_major_gc = objspace->profile.count;
5785 objspace->marked_slots = 0;
5786
5787 for (int i = 0; i < HEAP_COUNT; i++) {
5788 rb_heap_t *heap = &heaps[i];
5789 rgengc_mark_and_rememberset_clear(objspace, heap);
5790 heap_move_pooled_pages_to_free_pages(heap);
5791
5792 if (objspace->flags.during_compacting) {
5793 struct heap_page *page = NULL;
5794
5795 ccan_list_for_each(&heap->pages, page, page_node) {
5796 page->pinned_slots = 0;
5797 }
5798 }
5799 }
5800 }
5801 else {
5802 objspace->flags.during_minor_gc = TRUE;
5803 objspace->marked_slots =
5804 objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
5805 objspace->profile.minor_gc_count++;
5806
5807 for (int i = 0; i < HEAP_COUNT; i++) {
5808 rgengc_rememberset_mark(objspace, &heaps[i]);
5809 }
5810 }
5811
5812 mark_roots(objspace, NULL);
5813
5814 gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n",
5815 full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack));
5816}
5817
5818static bool
5819gc_marks(rb_objspace_t *objspace, int full_mark)
5820{
5821 gc_prof_mark_timer_start(objspace);
5822 gc_marking_enter(objspace);
5823
5824 bool marking_finished = false;
5825
5826 /* setup marking */
5827
5828 gc_marks_start(objspace, full_mark);
5829 if (!is_incremental_marking(objspace)) {
5830 gc_marks_rest(objspace);
5831 marking_finished = true;
5832 }
5833
5834#if RGENGC_PROFILE > 0
5835 if (gc_prof_record(objspace)) {
5836 gc_profile_record *record = gc_prof_record(objspace);
5837 record->old_objects = objspace->rgengc.old_objects;
5838 }
5839#endif
5840
5841 gc_marking_exit(objspace);
5842 gc_prof_mark_timer_stop(objspace);
5843
5844 return marking_finished;
5845}
5846
5847/* RGENGC */
5848
5849static void
5850gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
5851{
5852 if (level <= RGENGC_DEBUG) {
5853 char buf[1024];
5854 FILE *out = stderr;
5855 va_list args;
5856 const char *status = " ";
5857
5858 if (during_gc) {
5859 status = is_full_marking(objspace) ? "+" : "-";
5860 }
5861 else {
5862 if (is_lazy_sweeping(objspace)) {
5863 status = "S";
5864 }
5865 if (is_incremental_marking(objspace)) {
5866 status = "M";
5867 }
5868 }
5869
5870 va_start(args, fmt);
5871 vsnprintf(buf, 1024, fmt, args);
5872 va_end(args);
5873
5874 fprintf(out, "%s|", status);
5875 fputs(buf, out);
5876 }
5877}
5878
5879/* bit operations */
5880
5881static int
5882rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
5883{
5884 struct heap_page *page = GET_HEAP_PAGE(obj);
5885 bits_t *bits = &page->remembered_bits[0];
5886
5887 if (MARKED_IN_BITMAP(bits, obj)) {
5888 return FALSE;
5889 }
5890 else {
5891 page->flags.has_remembered_objects = TRUE;
5892 MARK_IN_BITMAP(bits, obj);
5893 return TRUE;
5894 }
5895}
5896
5897/* wb, etc */
5898
5899/* return FALSE if already remembered */
5900static int
5901rgengc_remember(rb_objspace_t *objspace, VALUE obj)
5902{
5903 gc_report(6, objspace, "rgengc_remember: %s %s\n", rb_obj_info(obj),
5904 RVALUE_REMEMBERED(objspace, obj) ? "was already remembered" : "is remembered now");
5905
5906 check_rvalue_consistency(objspace, obj);
5907
5908 if (RGENGC_CHECK_MODE) {
5909 if (RVALUE_WB_UNPROTECTED(objspace, obj)) rb_bug("rgengc_remember: %s is not wb protected.", rb_obj_info(obj));
5910 }
5911
5912#if RGENGC_PROFILE > 0
5913 if (!RVALUE_REMEMBERED(objspace, obj)) {
5914 if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0) {
5915 objspace->profile.total_remembered_normal_object_count++;
5916#if RGENGC_PROFILE >= 2
5917 objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
5918#endif
5919 }
5920 }
5921#endif /* RGENGC_PROFILE > 0 */
5922
5923 return rgengc_remembersetbits_set(objspace, obj);
5924}
5925
5926#ifndef PROFILE_REMEMBERSET_MARK
5927#define PROFILE_REMEMBERSET_MARK 0
5928#endif
5929
5930static inline void
5931rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset)
5932{
5933 if (bitset) {
5934 do {
5935 if (bitset & 1) {
5936 VALUE obj = (VALUE)p;
5937 gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", rb_obj_info(obj));
5938 GC_ASSERT(RVALUE_UNCOLLECTIBLE(objspace, obj));
5939 GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj));
5940
5941 gc_mark_children(objspace, obj);
5942 }
5943 p += BASE_SLOT_SIZE;
5944 bitset >>= 1;
5945 } while (bitset);
5946 }
5947}
5948
5949static void
5950rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
5951{
5952 size_t j;
5953 struct heap_page *page = 0;
5954#if PROFILE_REMEMBERSET_MARK
5955 int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
5956#endif
5957 gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
5958
5959 ccan_list_for_each(&heap->pages, page, page_node) {
5960 if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
5961 uintptr_t p = page->start;
5962 bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
5963 bits_t *remembered_bits = page->remembered_bits;
5964 bits_t *uncollectible_bits = page->uncollectible_bits;
5965 bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
5966#if PROFILE_REMEMBERSET_MARK
5967 if (page->flags.has_remembered_objects && page->flags.has_uncollectible_wb_unprotected_objects) has_both++;
5968 else if (page->flags.has_remembered_objects) has_old++;
5969 else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
5970#endif
5971 for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
5972 bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
5973 remembered_bits[j] = 0;
5974 }
5975 page->flags.has_remembered_objects = FALSE;
5976
5977 bitset = bits[0];
5978 bitset >>= NUM_IN_PAGE(p);
5979 rgengc_rememberset_mark_plane(objspace, p, bitset);
5980 p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
5981
5982 for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
5983 bitset = bits[j];
5984 rgengc_rememberset_mark_plane(objspace, p, bitset);
5985 p += BITS_BITLENGTH * BASE_SLOT_SIZE;
5986 }
5987 }
5988#if PROFILE_REMEMBERSET_MARK
5989 else {
5990 skip++;
5991 }
5992#endif
5993 }
5994
5995#if PROFILE_REMEMBERSET_MARK
5996 fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
5997#endif
5998 gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
5999}
6000
6001static void
6002rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap)
6003{
6004 struct heap_page *page = 0;
6005
6006 ccan_list_for_each(&heap->pages, page, page_node) {
6007 memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6008 memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6009 memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6010 memset(&page->remembered_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6011 memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6012 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
6013 page->flags.has_remembered_objects = FALSE;
6014 }
6015}
6016
6017/* RGENGC: APIs */
6018
6019NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
6020
6021static void
6022gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
6023{
6024 if (RGENGC_CHECK_MODE) {
6025 if (!RVALUE_OLD_P(objspace, a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", rb_obj_info(a));
6026 if ( RVALUE_OLD_P(objspace, b)) rb_bug("gc_writebarrier_generational: %s is an old object.", rb_obj_info(b));
6027 if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", rb_obj_info(a), rb_obj_info(b));
6028 }
6029
6030 /* mark `a' and remember (default behavior) */
6031 if (!RVALUE_REMEMBERED(objspace, a)) {
6032 int lev = rb_gc_vm_lock_no_barrier();
6033 {
6034 rgengc_remember(objspace, a);
6035 }
6036 rb_gc_vm_unlock_no_barrier(lev);
6037
6038 gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
6039 }
6040
6041 check_rvalue_consistency(objspace, a);
6042 check_rvalue_consistency(objspace, b);
6043}
6044
6045static void
6046gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
6047{
6048 gc_mark_set_parent(objspace, parent);
6049 rgengc_check_relation(objspace, obj);
6050 if (gc_mark_set(objspace, obj) == FALSE) return;
6051 gc_aging(objspace, obj);
6052 gc_grey(objspace, obj);
6053}
6054
6055NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
6056
6057static void
6058gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
6059{
6060 gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, rb_obj_info(b));
6061
6062 if (RVALUE_BLACK_P(objspace, a)) {
6063 if (RVALUE_WHITE_P(objspace, b)) {
6064 if (!RVALUE_WB_UNPROTECTED(objspace, a)) {
6065 gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, rb_obj_info(b));
6066 gc_mark_from(objspace, b, a);
6067 }
6068 }
6069 else if (RVALUE_OLD_P(objspace, a) && !RVALUE_OLD_P(objspace, b)) {
6070 rgengc_remember(objspace, a);
6071 }
6072
6073 if (RB_UNLIKELY(objspace->flags.during_compacting)) {
6074 MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
6075 }
6076 }
6077}
6078
6079void
6080rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
6081{
6082 rb_objspace_t *objspace = objspace_ptr;
6083
6084 if (RGENGC_CHECK_MODE) {
6085 if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
6086 if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
6087 }
6088
6089 GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE);
6090 GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED);
6091 GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE);
6092 GC_ASSERT(RB_BUILTIN_TYPE(b) != T_NONE);
6093 GC_ASSERT(RB_BUILTIN_TYPE(b) != T_MOVED);
6094 GC_ASSERT(RB_BUILTIN_TYPE(b) != T_ZOMBIE);
6095
6096 retry:
6097 if (!is_incremental_marking(objspace)) {
6098 if (!RVALUE_OLD_P(objspace, a) || RVALUE_OLD_P(objspace, b)) {
6099 // do nothing
6100 }
6101 else {
6102 gc_writebarrier_generational(a, b, objspace);
6103 }
6104 }
6105 else {
6106 bool retry = false;
6107 /* slow path */
6108 int lev = rb_gc_vm_lock_no_barrier();
6109 {
6110 if (is_incremental_marking(objspace)) {
6111 gc_writebarrier_incremental(a, b, objspace);
6112 }
6113 else {
6114 retry = true;
6115 }
6116 }
6117 rb_gc_vm_unlock_no_barrier(lev);
6118
6119 if (retry) goto retry;
6120 }
6121 return;
6122}
6123
6124void
6125rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
6126{
6127 rb_objspace_t *objspace = objspace_ptr;
6128
6129 if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
6130 return;
6131 }
6132 else {
6133 gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
6134 RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : "");
6135
6136 unsigned int lev = rb_gc_vm_lock_no_barrier();
6137 {
6138 if (RVALUE_OLD_P(objspace, obj)) {
6139 gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
6140 RVALUE_DEMOTE(objspace, obj);
6141 gc_mark_set(objspace, obj);
6142 gc_remember_unprotected(objspace, obj);
6143
6144#if RGENGC_PROFILE
6145 objspace->profile.total_shade_operation_count++;
6146#if RGENGC_PROFILE >= 2
6147 objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
6148#endif /* RGENGC_PROFILE >= 2 */
6149#endif /* RGENGC_PROFILE */
6150 }
6151 else {
6152 RVALUE_AGE_RESET(obj);
6153 }
6154
6155 RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
6156 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
6157 }
6158 rb_gc_vm_unlock_no_barrier(lev);
6159 }
6160}
6161
6162void
6163rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
6164{
6165 rb_objspace_t *objspace = objspace_ptr;
6166
6167 if (RVALUE_WB_UNPROTECTED(objspace, obj)) {
6168 rb_gc_impl_writebarrier_unprotect(objspace, dest);
6169 }
6170 rb_gc_impl_copy_finalizer(objspace, dest, obj);
6171}
6172
6173const char *
6174rb_gc_impl_active_gc_name(void)
6175{
6176 return "default";
6177}
6178
6179void
6180rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
6181{
6182 rb_objspace_t *objspace = objspace_ptr;
6183
6184 gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
6185
6186 if (is_incremental_marking(objspace)) {
6187 if (RVALUE_BLACK_P(objspace, obj)) {
6188 gc_grey(objspace, obj);
6189 }
6190 }
6191 else {
6192 if (RVALUE_OLD_P(objspace, obj)) {
6193 rgengc_remember(objspace, obj);
6194 }
6195 }
6196}
6197
6198// TODO: rearchitect this function to work for a generic GC
6199size_t
6200rb_gc_impl_obj_flags(void *objspace_ptr, VALUE obj, ID* flags, size_t max)
6201{
6202 rb_objspace_t *objspace = objspace_ptr;
6203 size_t n = 0;
6204 static ID ID_marked;
6205 static ID ID_wb_protected, ID_old, ID_marking, ID_uncollectible, ID_pinned;
6206
6207 if (!ID_marked) {
6208#define I(s) ID_##s = rb_intern(#s);
6209 I(marked);
6210 I(wb_protected);
6211 I(old);
6212 I(marking);
6213 I(uncollectible);
6214 I(pinned);
6215#undef I
6216 }
6217
6218 if (RVALUE_WB_UNPROTECTED(objspace, obj) == 0 && n < max) flags[n++] = ID_wb_protected;
6219 if (RVALUE_OLD_P(objspace, obj) && n < max) flags[n++] = ID_old;
6220 if (RVALUE_UNCOLLECTIBLE(objspace, obj) && n < max) flags[n++] = ID_uncollectible;
6221 if (RVALUE_MARKING(objspace, obj) && n < max) flags[n++] = ID_marking;
6222 if (RVALUE_MARKED(objspace, obj) && n < max) flags[n++] = ID_marked;
6223 if (RVALUE_PINNED(objspace, obj) && n < max) flags[n++] = ID_pinned;
6224 return n;
6225}
6226
6227void *
6228rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
6229{
6230 rb_objspace_t *objspace = objspace_ptr;
6231
6232 objspace->live_ractor_cache_count++;
6233
6234 return calloc1(sizeof(rb_ractor_newobj_cache_t));
6235}
6236
6237void
6238rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache)
6239{
6240 rb_objspace_t *objspace = objspace_ptr;
6241
6242 objspace->live_ractor_cache_count--;
6243
6244 gc_ractor_newobj_cache_clear(cache, NULL);
6245 free(cache);
6246}
6247
6248static void
6249heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap)
6250{
6251 if (!heap->free_pages) {
6252 if (!heap_page_allocate_and_initialize(objspace, heap)) {
6253 objspace->heap_pages.allocatable_slots = 1;
6254 heap_page_allocate_and_initialize(objspace, heap);
6255 }
6256 }
6257}
6258
6259static int
6260ready_to_gc(rb_objspace_t *objspace)
6261{
6262 if (dont_gc_val() || during_gc) {
6263 for (int i = 0; i < HEAP_COUNT; i++) {
6264 rb_heap_t *heap = &heaps[i];
6265 heap_ready_to_gc(objspace, heap);
6266 }
6267 return FALSE;
6268 }
6269 else {
6270 return TRUE;
6271 }
6272}
6273
6274static void
6275gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
6276{
6277 gc_prof_set_malloc_info(objspace);
6278 {
6279 size_t inc = RUBY_ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
6280 size_t old_limit = malloc_limit;
6281
6282 if (inc > malloc_limit) {
6283 malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
6284 if (malloc_limit > gc_params.malloc_limit_max) {
6285 malloc_limit = gc_params.malloc_limit_max;
6286 }
6287 }
6288 else {
6289 malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
6290 if (malloc_limit < gc_params.malloc_limit_min) {
6291 malloc_limit = gc_params.malloc_limit_min;
6292 }
6293 }
6294
6295 if (0) {
6296 if (old_limit != malloc_limit) {
6297 fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
6298 rb_gc_count(), old_limit, malloc_limit);
6299 }
6300 else {
6301 fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
6302 rb_gc_count(), malloc_limit);
6303 }
6304 }
6305 }
6306
6307 /* reset oldmalloc info */
6308#if RGENGC_ESTIMATE_OLDMALLOC
6309 if (!full_mark) {
6310 if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
6311 gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
6312 objspace->rgengc.oldmalloc_increase_limit =
6313 (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
6314
6315 if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
6316 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
6317 }
6318 }
6319
6320 if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRIuSIZE"\t%"PRIuSIZE"\t%"PRIdSIZE"\n",
6321 rb_gc_count(),
6322 gc_needs_major_flags,
6323 objspace->rgengc.oldmalloc_increase,
6324 objspace->rgengc.oldmalloc_increase_limit,
6325 gc_params.oldmalloc_limit_max);
6326 }
6327 else {
6328 /* major GC */
6329 objspace->rgengc.oldmalloc_increase = 0;
6330
6331 if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
6332 objspace->rgengc.oldmalloc_increase_limit =
6333 (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
6334 if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
6335 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
6336 }
6337 }
6338 }
6339#endif
6340}
6341
6342static int
6343garbage_collect(rb_objspace_t *objspace, unsigned int reason)
6344{
6345 int ret;
6346
6347 int lev = rb_gc_vm_lock();
6348 {
6349#if GC_PROFILE_MORE_DETAIL
6350 objspace->profile.prepare_time = getrusage_time();
6351#endif
6352
6353 gc_rest(objspace);
6354
6355#if GC_PROFILE_MORE_DETAIL
6356 objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
6357#endif
6358
6359 ret = gc_start(objspace, reason);
6360 }
6361 rb_gc_vm_unlock(lev);
6362
6363 return ret;
6364}
6365
6366static int
6367gc_start(rb_objspace_t *objspace, unsigned int reason)
6368{
6369 unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
6370
6371 /* reason may be clobbered, later, so keep set immediate_sweep here */
6372 objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
6373
6374 if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */
6375 if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
6376
6377 GC_ASSERT(gc_mode(objspace) == gc_mode_none);
6378 GC_ASSERT(!is_lazy_sweeping(objspace));
6379 GC_ASSERT(!is_incremental_marking(objspace));
6380
6381 unsigned int lock_lev;
6382 gc_enter(objspace, gc_enter_event_start, &lock_lev);
6383
6384#if RGENGC_CHECK_MODE >= 2
6385 gc_verify_internal_consistency(objspace);
6386#endif
6387
6388 if (ruby_gc_stressful) {
6389 int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0;
6390
6391 if ((flag & (1 << gc_stress_no_major)) == 0) {
6392 do_full_mark = TRUE;
6393 }
6394
6395 objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
6396 }
6397
6398 if (gc_needs_major_flags) {
6399 reason |= gc_needs_major_flags;
6400 do_full_mark = TRUE;
6401 }
6402
6403 /* if major gc has been disabled, never do a full mark */
6404 if (!gc_config_full_mark_val) {
6405 do_full_mark = FALSE;
6406 }
6407 gc_needs_major_flags = GPR_FLAG_NONE;
6408
6409 if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
6410 reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
6411 }
6412
6413 if (objspace->flags.dont_incremental ||
6414 reason & GPR_FLAG_IMMEDIATE_MARK ||
6415 ruby_gc_stressful) {
6416 objspace->flags.during_incremental_marking = FALSE;
6417 }
6418 else {
6419 objspace->flags.during_incremental_marking = do_full_mark;
6420 }
6421
6422 /* Explicitly enable compaction (GC.compact) */
6423 if (do_full_mark && ruby_enable_autocompact) {
6424 objspace->flags.during_compacting = TRUE;
6425#if RGENGC_CHECK_MODE
6426 objspace->rcompactor.compare_func = ruby_autocompact_compare_func;
6427#endif
6428 }
6429 else {
6430 objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
6431 }
6432
6433 if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
6434 objspace->flags.immediate_sweep = TRUE;
6435 }
6436
6437 if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
6438
6439 gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n",
6440 reason,
6441 do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
6442
6443#if USE_DEBUG_COUNTER
6444 RB_DEBUG_COUNTER_INC(gc_count);
6445
6446 if (reason & GPR_FLAG_MAJOR_MASK) {
6447 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
6448 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
6449 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
6450 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
6451#if RGENGC_ESTIMATE_OLDMALLOC
6452 (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
6453#endif
6454 }
6455 else {
6456 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
6457 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
6458 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
6459 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
6460 (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
6461 }
6462#endif
6463
6464 objspace->profile.count++;
6465 objspace->profile.latest_gc_info = reason;
6466 objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace);
6467 objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted);
6468 objspace->profile.weak_references_count = 0;
6469 objspace->profile.retained_weak_references_count = 0;
6470 gc_prof_setup_new_record(objspace, reason);
6471 gc_reset_malloc_info(objspace, do_full_mark);
6472
6473 rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START);
6474
6475 GC_ASSERT(during_gc);
6476
6477 gc_prof_timer_start(objspace);
6478 {
6479 if (gc_marks(objspace, do_full_mark)) {
6480 gc_sweep(objspace);
6481 }
6482 }
6483 gc_prof_timer_stop(objspace);
6484
6485 gc_exit(objspace, gc_enter_event_start, &lock_lev);
6486 return TRUE;
6487}
6488
6489static void
6490gc_rest(rb_objspace_t *objspace)
6491{
6492 if (is_incremental_marking(objspace) || is_lazy_sweeping(objspace)) {
6493 unsigned int lock_lev;
6494 gc_enter(objspace, gc_enter_event_rest, &lock_lev);
6495
6496 if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace);
6497
6498 if (is_incremental_marking(objspace)) {
6499 gc_marking_enter(objspace);
6500 gc_marks_rest(objspace);
6501 gc_marking_exit(objspace);
6502
6503 gc_sweep(objspace);
6504 }
6505
6506 if (is_lazy_sweeping(objspace)) {
6507 gc_sweeping_enter(objspace);
6508 gc_sweep_rest(objspace);
6509 gc_sweeping_exit(objspace);
6510 }
6511
6512 gc_exit(objspace, gc_enter_event_rest, &lock_lev);
6513 }
6514}
6515
6516 struct objspace_and_reason {
6517 rb_objspace_t *objspace;
6518 unsigned int reason;
6519};
6520
6521static void
6522gc_current_status_fill(rb_objspace_t *objspace, char *buff)
6523{
6524 int i = 0;
6525 if (is_marking(objspace)) {
6526 buff[i++] = 'M';
6527 if (is_full_marking(objspace)) buff[i++] = 'F';
6528 if (is_incremental_marking(objspace)) buff[i++] = 'I';
6529 }
6530 else if (is_sweeping(objspace)) {
6531 buff[i++] = 'S';
6532 if (is_lazy_sweeping(objspace)) buff[i++] = 'L';
6533 }
6534 else {
6535 buff[i++] = 'N';
6536 }
6537 buff[i] = '\0';
6538}
6539
6540static const char *
6541gc_current_status(rb_objspace_t *objspace)
6542{
6543 static char buff[0x10];
6544 gc_current_status_fill(objspace, buff);
6545 return buff;
6546}
6547
6548#if PRINT_ENTER_EXIT_TICK
6549
6550static tick_t last_exit_tick;
6551static tick_t enter_tick;
6552static int enter_count = 0;
6553static char last_gc_status[0x10];
6554
6555static inline void
6556gc_record(rb_objspace_t *objspace, int direction, const char *event)
6557{
6558 if (direction == 0) { /* enter */
6559 enter_count++;
6560 enter_tick = tick();
6561 gc_current_status_fill(objspace, last_gc_status);
6562 }
6563 else { /* exit */
6564 tick_t exit_tick = tick();
6565 char current_gc_status[0x10];
6566 gc_current_status_fill(objspace, current_gc_status);
6567#if 1
6568 /* [last mutator time] [gc time] [event] */
6569 fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
6570 enter_tick - last_exit_tick,
6571 exit_tick - enter_tick,
6572 event,
6573 last_gc_status, current_gc_status,
6574 (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
6575 last_exit_tick = exit_tick;
6576#else
6577 /* [enter_tick] [gc time] [event] */
6578 fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
6579 enter_tick,
6580 exit_tick - enter_tick,
6581 event,
6582 last_gc_status, current_gc_status,
6583 (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
6584#endif
6585 }
6586}
6587#else /* PRINT_ENTER_EXIT_TICK */
6588static inline void
6589gc_record(rb_objspace_t *objspace, int direction, const char *event)
6590{
6591 /* null */
6592}
6593#endif /* PRINT_ENTER_EXIT_TICK */
6594
6595static const char *
6596gc_enter_event_cstr(enum gc_enter_event event)
6597{
6598 switch (event) {
6599 case gc_enter_event_start: return "start";
6600 case gc_enter_event_continue: return "continue";
6601 case gc_enter_event_rest: return "rest";
6602 case gc_enter_event_finalizer: return "finalizer";
6603 }
6604 return NULL;
6605}
6606
6607static void
6608gc_enter_count(enum gc_enter_event event)
6609{
6610 switch (event) {
6611 case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
6612 case gc_enter_event_continue: RB_DEBUG_COUNTER_INC(gc_enter_continue); break;
6613 case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest); break;
6614 case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break;
6615 }
6616}
6617
6618static bool current_process_time(struct timespec *ts);
6619
6620static void
6621gc_clock_start(struct timespec *ts)
6622{
6623 if (!current_process_time(ts)) {
6624 ts->tv_sec = 0;
6625 ts->tv_nsec = 0;
6626 }
6627}
6628
6629static unsigned long long
6630gc_clock_end(struct timespec *ts)
6631{
6632 struct timespec end_time;
6633
6634 if ((ts->tv_sec > 0 || ts->tv_nsec > 0) &&
6635 current_process_time(&end_time) &&
6636 end_time.tv_sec >= ts->tv_sec) {
6637 return (unsigned long long)(end_time.tv_sec - ts->tv_sec) * (1000 * 1000 * 1000) +
6638 (end_time.tv_nsec - ts->tv_nsec);
6639 }
6640
6641 return 0;
6642}
6643
6644static inline void
6645gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
6646{
6647 *lock_lev = rb_gc_vm_lock();
6648
6649 switch (event) {
6650 case gc_enter_event_rest:
6651 if (!is_marking(objspace)) break;
6652 // fall through
6653 case gc_enter_event_start:
6654 case gc_enter_event_continue:
6655 // stop other ractors
6656 rb_gc_vm_barrier();
6657 break;
6658 default:
6659 break;
6660 }
6661
6662 gc_enter_count(event);
6663 if (RB_UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0");
6664 if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
6665
6666 during_gc = TRUE;
6667 RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace));
6668 gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
6669 gc_record(objspace, 0, gc_enter_event_cstr(event));
6670
6671 rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_ENTER);
6672}
6673
6674static inline void
6675gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
6676{
6677 GC_ASSERT(during_gc != 0);
6678
6679 rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_EXIT);
6680
6681 gc_record(objspace, 1, gc_enter_event_cstr(event));
6682 RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace));
6683 gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
6684 during_gc = FALSE;
6685
6686 rb_gc_vm_unlock(*lock_lev);
6687}
6688
6689#ifndef MEASURE_GC
6690#define MEASURE_GC (objspace->flags.measure_gc)
6691#endif
6692
6693static void
6694gc_marking_enter(rb_objspace_t *objspace)
6695{
6696 GC_ASSERT(during_gc != 0);
6697
6698 if (MEASURE_GC) {
6699 gc_clock_start(&objspace->profile.marking_start_time);
6700 }
6701}
6702
6703static void
6704gc_marking_exit(rb_objspace_t *objspace)
6705{
6706 GC_ASSERT(during_gc != 0);
6707
6708 if (MEASURE_GC) {
6709 objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time);
6710 }
6711}
6712
6713static void
6714gc_sweeping_enter(rb_objspace_t *objspace)
6715{
6716 GC_ASSERT(during_gc != 0);
6717
6718 if (MEASURE_GC) {
6719 gc_clock_start(&objspace->profile.sweeping_start_time);
6720 }
6721}
6722
6723static void
6724gc_sweeping_exit(rb_objspace_t *objspace)
6725{
6726 GC_ASSERT(during_gc != 0);
6727
6728 if (MEASURE_GC) {
6729 objspace->profile.sweeping_time_ns += gc_clock_end(&objspace->profile.sweeping_start_time);
6730 }
6731}
6732
6733static void *
6734gc_with_gvl(void *ptr)
6735{
6736 struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
6737 return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
6738}
6739
6740int ruby_thread_has_gvl_p(void);
6741
6742static int
6743garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason)
6744{
6745 if (dont_gc_val()) {
6746 return TRUE;
6747 }
6748 else if (!ruby_native_thread_p()) {
6749 return TRUE;
6750 }
6751 else if (!ruby_thread_has_gvl_p()) {
6752 void *ret;
6753 struct objspace_and_reason oar;
6754 oar.objspace = objspace;
6755 oar.reason = reason;
6756 ret = rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
6757
6758 return !!ret;
6759 }
6760 else {
6761 return garbage_collect(objspace, reason);
6762 }
6763}
6764
6765static int
6766gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data)
6767{
6768 rb_objspace_t *objspace = (rb_objspace_t *)data;
6769
6770 VALUE v = (VALUE)vstart;
6771 for (; v != (VALUE)vend; v += stride) {
6772 asan_unpoisoning_object(v) {
6773 switch (BUILTIN_TYPE(v)) {
6774 case T_NONE:
6775 case T_ZOMBIE:
6776 break;
6777 default:
6778 rb_gc_prepare_heap_process_object(v);
6779 if (!RVALUE_OLD_P(objspace, v) && !RVALUE_WB_UNPROTECTED(objspace, v)) {
6780 RVALUE_AGE_SET_CANDIDATE(objspace, v);
6781 }
6782 }
6783 }
6784 }
6785
6786 return 0;
6787}
6788
6789void
6790rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact)
6791{
6792 rb_objspace_t *objspace = objspace_ptr;
6793 unsigned int reason = (GPR_FLAG_FULL_MARK |
6794 GPR_FLAG_IMMEDIATE_MARK |
6795 GPR_FLAG_IMMEDIATE_SWEEP |
6796 GPR_FLAG_METHOD);
6797
6798 int full_marking_p = gc_config_full_mark_val;
6799 gc_config_full_mark_set(TRUE);
6800
6801 /* For now, compact implies full mark / sweep, so ignore other flags */
6802 if (compact) {
6803 GC_ASSERT(GC_COMPACTION_SUPPORTED);
6804
6805 reason |= GPR_FLAG_COMPACT;
6806 }
6807 else {
6808 if (!full_mark) reason &= ~GPR_FLAG_FULL_MARK;
6809 if (!immediate_mark) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
6810 if (!immediate_sweep) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
6811 }
6812
6813 garbage_collect(objspace, reason);
6814 gc_finalize_deferred(objspace);
6815
6816 gc_config_full_mark_set(full_marking_p);
6817}
6818
6819void
6820rb_gc_impl_prepare_heap(void *objspace_ptr)
6821{
6822 rb_objspace_t *objspace = objspace_ptr;
6823
6824 size_t orig_total_slots = objspace_available_slots(objspace);
6825 size_t orig_allocatable_slots = objspace->heap_pages.allocatable_slots;
6826
6827 rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr);
6828
6829 double orig_max_free_slots = gc_params.heap_free_slots_max_ratio;
6830 /* Ensure that all empty pages are moved onto empty_pages. */
6831 gc_params.heap_free_slots_max_ratio = 0.0;
6832 rb_gc_impl_start(objspace, true, true, true, true);
6833 gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
6834
6835 objspace->heap_pages.allocatable_slots = 0;
6836 heap_pages_free_unused_pages(objspace_ptr);
6837 GC_ASSERT(objspace->empty_pages_count == 0);
6838 objspace->heap_pages.allocatable_slots = orig_allocatable_slots;
6839
6840 size_t total_slots = objspace_available_slots(objspace);
6841 if (orig_total_slots > total_slots) {
6842 objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots;
6843 }
6844
6845#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
6846 malloc_trim(0);
6847#endif
6848}
6849
6850static int
6851gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
6852{
6853 GC_ASSERT(!SPECIAL_CONST_P(obj));
6854
6855 switch (BUILTIN_TYPE(obj)) {
6856 case T_NONE:
6857 case T_MOVED:
6858 case T_ZOMBIE:
6859 return FALSE;
6860 case T_SYMBOL:
6861 // TODO: restore original behavior
6862 // if (RSYMBOL(obj)->id & ~ID_SCOPE_MASK) {
6863 // return FALSE;
6864 // }
6865 return false;
6866 /* fall through */
6867 case T_STRING:
6868 case T_OBJECT:
6869 case T_FLOAT:
6870 case T_IMEMO:
6871 case T_ARRAY:
6872 case T_BIGNUM:
6873 case T_ICLASS:
6874 case T_MODULE:
6875 case T_REGEXP:
6876 case T_DATA:
6877 case T_MATCH:
6878 case T_STRUCT:
6879 case T_HASH:
6880 case T_FILE:
6881 case T_COMPLEX:
6882 case T_RATIONAL:
6883 case T_NODE:
6884 case T_CLASS:
6885 if (FL_TEST(obj, FL_FINALIZE)) {
6886 /* The finalizer table is a numtable. It looks up objects by address.
6887 * We can't mark the keys in the finalizer table because that would
6888 * prevent the objects from being collected. This check prevents
6889 * objects that are keys in the finalizer table from being moved
6890 * without directly pinning them. */
6891 GC_ASSERT(st_is_member(finalizer_table, obj));
6892
6893 return FALSE;
6894 }
6895 GC_ASSERT(RVALUE_MARKED(objspace, obj));
6896 GC_ASSERT(!RVALUE_PINNED(objspace, obj));
6897
6898 return TRUE;
6899
6900 default:
6901 rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
6902 break;
6903 }
6904
6905 return FALSE;
6906}
6907
6908void rb_mv_generic_ivar(VALUE src, VALUE dst);
6909
6910static VALUE
6911gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, size_t slot_size)
6912{
6913 int marked;
6914 int wb_unprotected;
6915 int uncollectible;
6916 int age;
6917
6918 gc_report(4, objspace, "Moving object: %p -> %p\n", (void *)src, (void *)dest);
6919
6920 GC_ASSERT(BUILTIN_TYPE(src) != T_NONE);
6921 GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
6922
6923 GC_ASSERT(!RVALUE_MARKING(objspace, src));
6924
6925 /* Save off bits for current object. */
6926 marked = RVALUE_MARKED(objspace, src);
6927 wb_unprotected = RVALUE_WB_UNPROTECTED(objspace, src);
6928 uncollectible = RVALUE_UNCOLLECTIBLE(objspace, src);
6929 bool remembered = RVALUE_REMEMBERED(objspace, src);
6930 age = RVALUE_AGE_GET(src);
6931
6932 /* Clear bits for eventual T_MOVED */
6933 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(src), src);
6934 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(src), src);
6935 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
6936 CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
6937
6938 if (FL_TEST(src, FL_EXIVAR)) {
6939 /* Resizing the st table could cause a malloc */
6940 DURING_GC_COULD_MALLOC_REGION_START();
6941 {
6942 rb_mv_generic_ivar(src, dest);
6943 }
6944 DURING_GC_COULD_MALLOC_REGION_END();
6945 }
6946
6947 if (FL_TEST(src, FL_SEEN_OBJ_ID)) {
6948 /* If the source object's object_id has been seen, we need to update
6949 * the object to object id mapping. */
6950 st_data_t srcid = (st_data_t)src, id;
6951
6952 gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest);
6953 /* Resizing the st table could cause a malloc */
6954 DURING_GC_COULD_MALLOC_REGION_START();
6955 {
6956 if (!st_delete(objspace->obj_to_id_tbl, &srcid, &id)) {
6957 rb_bug("gc_move: object ID seen, but not in mapping table: %s", rb_obj_info((VALUE)src));
6958 }
6959
6960 st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id);
6961 }
6962 DURING_GC_COULD_MALLOC_REGION_END();
6963 }
6964 else {
6965 GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)src, NULL));
6966 }
6967
6968 /* Move the object */
6969 memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size));
6970
6971 if (RVALUE_OVERHEAD > 0) {
6972 void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
6973 void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
6974
6975 memcpy(dest_overhead, src_overhead, RVALUE_OVERHEAD);
6976 }
6977
6978 memset((void *)src, 0, src_slot_size);
6979 RVALUE_AGE_RESET(src);
6980
6981 /* Set bits for object in new location */
6982 if (remembered) {
6983 MARK_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
6984 }
6985 else {
6986 CLEAR_IN_BITMAP(GET_HEAP_PAGE(dest)->remembered_bits, dest);
6987 }
6988
6989 if (marked) {
6990 MARK_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
6991 }
6992 else {
6993 CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest);
6994 }
6995
6996 if (wb_unprotected) {
6997 MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
6998 }
6999 else {
7000 CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
7001 }
7002
7003 if (uncollectible) {
7004 MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
7005 }
7006 else {
7007 CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(dest), dest);
7008 }
7009
7010 RVALUE_AGE_SET(dest, age);
7011 /* Assign forwarding address */
7012 RMOVED(src)->flags = T_MOVED;
7013 RMOVED(src)->dummy = Qundef;
7014 RMOVED(src)->destination = dest;
7015 GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE);
7016
7017 GET_HEAP_PAGE(src)->heap->total_freed_objects++;
7018 GET_HEAP_PAGE(dest)->heap->total_allocated_objects++;
7019
7020 return src;
7021}
7022
7023#if GC_CAN_COMPILE_COMPACTION
7024static int
7025compare_pinned_slots(const void *left, const void *right, void *dummy)
7026{
7027 struct heap_page *left_page;
7028 struct heap_page *right_page;
7029
7030 left_page = *(struct heap_page * const *)left;
7031 right_page = *(struct heap_page * const *)right;
7032
7033 return left_page->pinned_slots - right_page->pinned_slots;
7034}
7035
7036static int
7037compare_free_slots(const void *left, const void *right, void *dummy)
7038{
7039 struct heap_page *left_page;
7040 struct heap_page *right_page;
7041
7042 left_page = *(struct heap_page * const *)left;
7043 right_page = *(struct heap_page * const *)right;
7044
7045 return left_page->free_slots - right_page->free_slots;
7046}
7047
7048static void
7049gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func)
7050{
7051 for (int j = 0; j < HEAP_COUNT; j++) {
7052 rb_heap_t *heap = &heaps[j];
7053
7054 size_t total_pages = heap->total_pages;
7055 size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
7056 struct heap_page *page = 0, **page_list = malloc(size);
7057 size_t i = 0;
7058
7059 heap->free_pages = NULL;
7060 ccan_list_for_each(&heap->pages, page, page_node) {
7061 page_list[i++] = page;
7062 GC_ASSERT(page);
7063 }
7064
7065 GC_ASSERT((size_t)i == total_pages);
7066
7067 /* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
7068 * head of the list, so empty pages will end up at the start of the heap */
7069 ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL);
7070
7071 /* Reset the eden heap */
7072 ccan_list_head_init(&heap->pages);
7073
7074 for (i = 0; i < total_pages; i++) {
7075 ccan_list_add(&heap->pages, &page_list[i]->page_node);
7076 if (page_list[i]->free_slots != 0) {
7077 heap_add_freepage(heap, page_list[i]);
7078 }
7079 }
7080
7081 free(page_list);
7082 }
7083}
7084#endif
7085
7086bool
7087rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
7088{
7089 return gc_object_moved_p(objspace_ptr, obj);
7090}
7091
7092static int
7093gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t *objspace, struct heap_page *page)
7094{
7095 VALUE v = (VALUE)vstart;
7096
7097 page->flags.has_uncollectible_wb_unprotected_objects = FALSE;
7098 page->flags.has_remembered_objects = FALSE;
7099
7100 /* For each object on the page */
7101 for (; v != (VALUE)vend; v += stride) {
7102 asan_unpoisoning_object(v) {
7103 switch (BUILTIN_TYPE(v)) {
7104 case T_NONE:
7105 case T_MOVED:
7106 case T_ZOMBIE:
7107 break;
7108 default:
7109 if (RVALUE_WB_UNPROTECTED(objspace, v)) {
7110 page->flags.has_uncollectible_wb_unprotected_objects = TRUE;
7111 }
7112 if (RVALUE_REMEMBERED(objspace, v)) {
7113 page->flags.has_remembered_objects = TRUE;
7114 }
7115 if (page->flags.before_sweep) {
7116 if (RVALUE_MARKED(objspace, v)) {
7117 rb_gc_update_object_references(objspace, v);
7118 }
7119 }
7120 else {
7121 rb_gc_update_object_references(objspace, v);
7122 }
7123 }
7124 }
7125 }
7126
7127 return 0;
7128}
7129
7130static void
7131gc_update_references(rb_objspace_t *objspace)
7132{
7133 objspace->flags.during_reference_updating = true;
7134
7135 struct heap_page *page = NULL;
7136
7137 for (int i = 0; i < HEAP_COUNT; i++) {
7138 bool should_set_mark_bits = TRUE;
7139 rb_heap_t *heap = &heaps[i];
7140
7141 ccan_list_for_each(&heap->pages, page, page_node) {
7142 uintptr_t start = (uintptr_t)page->start;
7143 uintptr_t end = start + (page->total_slots * heap->slot_size);
7144
7145 gc_ref_update((void *)start, (void *)end, heap->slot_size, objspace, page);
7146 if (page == heap->sweeping_page) {
7147 should_set_mark_bits = FALSE;
7148 }
7149 if (should_set_mark_bits) {
7150 gc_setup_mark_bits(page);
7151 }
7152 }
7153 }
7154 gc_ref_update_table_values_only(objspace->obj_to_id_tbl);
7155 gc_update_table_refs(objspace->id_to_obj_tbl);
7156 gc_update_table_refs(finalizer_table);
7157
7158 rb_gc_update_vm_references((void *)objspace);
7159
7160 objspace->flags.during_reference_updating = false;
7161}
7162
7163#if GC_CAN_COMPILE_COMPACTION
7164static void
7165root_obj_check_moved_i(const char *category, VALUE obj, void *data)
7166{
7167 rb_objspace_t *objspace = data;
7168
7169 if (gc_object_moved_p(objspace, obj)) {
7170 rb_bug("ROOT %s points to MOVED: %p -> %s", category, (void *)obj, rb_obj_info(rb_gc_impl_location(objspace, obj)));
7171 }
7172}
7173
7174static void
7175reachable_object_check_moved_i(VALUE ref, void *data)
7176{
7177 VALUE parent = (VALUE)data;
7178 if (gc_object_moved_p(rb_gc_get_objspace(), ref)) {
7179 rb_bug("Object %s points to MOVED: %p -> %s", rb_obj_info(parent), (void *)ref, rb_obj_info(rb_gc_impl_location(rb_gc_get_objspace(), ref)));
7180 }
7181}
7182
7183static int
7184heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
7185{
7186 rb_objspace_t *objspace = data;
7187
7188 VALUE v = (VALUE)vstart;
7189 for (; v != (VALUE)vend; v += stride) {
7190 if (gc_object_moved_p(objspace, v)) {
7191 /* Moved object still on the heap, something may have a reference. */
7192 }
7193 else {
7194 asan_unpoisoning_object(v) {
7195 switch (BUILTIN_TYPE(v)) {
7196 case T_NONE:
7197 case T_ZOMBIE:
7198 break;
7199 default:
7200 if (!rb_gc_impl_garbage_object_p(objspace, v)) {
7201 rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v);
7202 }
7203 }
7204 }
7205 }
7206 }
7207
7208 return 0;
7209}
7210#endif
7211
7212bool
7213rb_gc_impl_during_gc_p(void *objspace_ptr)
7214{
7215 rb_objspace_t *objspace = objspace_ptr;
7216
7217 return during_gc;
7218}
7219
7220#if RGENGC_PROFILE >= 2
7221
7222static const char*
7223type_name(int type, VALUE obj)
7224{
7225 switch ((enum ruby_value_type)type) {
7226 case RUBY_T_NONE: return "T_NONE";
7227 case RUBY_T_OBJECT: return "T_OBJECT";
7228 case RUBY_T_CLASS: return "T_CLASS";
7229 case RUBY_T_MODULE: return "T_MODULE";
7230 case RUBY_T_FLOAT: return "T_FLOAT";
7231 case RUBY_T_STRING: return "T_STRING";
7232 case RUBY_T_REGEXP: return "T_REGEXP";
7233 case RUBY_T_ARRAY: return "T_ARRAY";
7234 case RUBY_T_HASH: return "T_HASH";
7235 case RUBY_T_STRUCT: return "T_STRUCT";
7236 case RUBY_T_BIGNUM: return "T_BIGNUM";
7237 case RUBY_T_FILE: return "T_FILE";
7238 case RUBY_T_DATA: return "T_DATA";
7239 case RUBY_T_MATCH: return "T_MATCH";
7240 case RUBY_T_COMPLEX: return "T_COMPLEX";
7241 case RUBY_T_RATIONAL: return "T_RATIONAL";
7242 case RUBY_T_NIL: return "T_NIL";
7243 case RUBY_T_TRUE: return "T_TRUE";
7244 case RUBY_T_FALSE: return "T_FALSE";
7245 case RUBY_T_SYMBOL: return "T_SYMBOL";
7246 case RUBY_T_FIXNUM: return "T_FIXNUM";
7247 case RUBY_T_UNDEF: return "T_UNDEF";
7248 case RUBY_T_IMEMO: return "T_IMEMO";
7249 case RUBY_T_NODE: return "T_NODE";
7250 case RUBY_T_ICLASS: return "T_ICLASS";
7251 case RUBY_T_ZOMBIE: return "T_ZOMBIE";
7252 case RUBY_T_MOVED: return "T_MOVED";
7253 default: return "unknown";
7254 }
7255}
7256
7257static void
7258gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
7259{
7260 VALUE result = rb_hash_new_with_size(T_MASK);
7261 int i;
7262 for (i=0; i<T_MASK; i++) {
7263 const char *type = type_name(i, 0);
7264 rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i]));
7265 }
7266 rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
7267}
7268#endif
7269
7270size_t
7271rb_gc_impl_gc_count(void *objspace_ptr)
7272{
7273 rb_objspace_t *objspace = objspace_ptr;
7274
7275 return objspace->profile.count;
7276}
7277
7278static VALUE
7279gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags)
7280{
7281 static VALUE sym_major_by = Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state, sym_need_major_by;
7282 static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
7283#if RGENGC_ESTIMATE_OLDMALLOC
7284 static VALUE sym_oldmalloc;
7285#endif
7286 static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
7287 static VALUE sym_none, sym_marking, sym_sweeping;
7288 static VALUE sym_weak_references_count, sym_retained_weak_references_count;
7289 VALUE hash = Qnil, key = Qnil;
7290 VALUE major_by, need_major_by;
7291 unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
7292
7293 if (SYMBOL_P(hash_or_key)) {
7294 key = hash_or_key;
7295 }
7296 else if (RB_TYPE_P(hash_or_key, T_HASH)) {
7297 hash = hash_or_key;
7298 }
7299 else {
7300 rb_bug("gc_info_decode: non-hash or symbol given");
7301 }
7302
7303 if (NIL_P(sym_major_by)) {
7304#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
7305 S(major_by);
7306 S(gc_by);
7307 S(immediate_sweep);
7308 S(have_finalizer);
7309 S(state);
7310 S(need_major_by);
7311
7312 S(stress);
7313 S(nofree);
7314 S(oldgen);
7315 S(shady);
7316 S(force);
7317#if RGENGC_ESTIMATE_OLDMALLOC
7318 S(oldmalloc);
7319#endif
7320 S(newobj);
7321 S(malloc);
7322 S(method);
7323 S(capi);
7324
7325 S(none);
7326 S(marking);
7327 S(sweeping);
7328
7329 S(weak_references_count);
7330 S(retained_weak_references_count);
7331#undef S
7332 }
7333
7334#define SET(name, attr) \
7335 if (key == sym_##name) \
7336 return (attr); \
7337 else if (hash != Qnil) \
7338 rb_hash_aset(hash, sym_##name, (attr));
7339
7340 major_by =
7341 (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
7342 (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
7343 (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
7344 (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
7345#if RGENGC_ESTIMATE_OLDMALLOC
7346 (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
7347#endif
7348 Qnil;
7349 SET(major_by, major_by);
7350
7351 if (orig_flags == 0) { /* set need_major_by only if flags not set explicitly */
7352 unsigned int need_major_flags = gc_needs_major_flags;
7353 need_major_by =
7354 (need_major_flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
7355 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
7356 (need_major_flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
7357 (need_major_flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
7358#if RGENGC_ESTIMATE_OLDMALLOC
7359 (need_major_flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
7360#endif
7361 Qnil;
7362 SET(need_major_by, need_major_by);
7363 }
7364
7365 SET(gc_by,
7366 (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
7367 (flags & GPR_FLAG_MALLOC) ? sym_malloc :
7368 (flags & GPR_FLAG_METHOD) ? sym_method :
7369 (flags & GPR_FLAG_CAPI) ? sym_capi :
7370 (flags & GPR_FLAG_STRESS) ? sym_stress :
7371 Qnil
7372 );
7373
7374 SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
7375 SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ? Qtrue : Qfalse);
7376
7377 if (orig_flags == 0) {
7378 SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
7379 gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
7380 }
7381
7382 SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count));
7383 SET(retained_weak_references_count, LONG2FIX(objspace->profile.retained_weak_references_count));
7384#undef SET
7385
7386 if (!NIL_P(key)) {
7387 // Matched key should return above
7388 return Qundef;
7389 }
7390
7391 return hash;
7392}
7393
7394 VALUE
7395rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key)
7396{
7397 rb_objspace_t *objspace = objspace_ptr;
7398
7399 return gc_info_decode(objspace, key, 0);
7400}
7401
7402
7403enum gc_stat_sym {
7404 gc_stat_sym_count,
7405 gc_stat_sym_time,
7406 gc_stat_sym_marking_time,
7407 gc_stat_sym_sweeping_time,
7408 gc_stat_sym_heap_allocated_pages,
7409 gc_stat_sym_heap_empty_pages,
7410 gc_stat_sym_heap_allocatable_slots,
7411 gc_stat_sym_heap_available_slots,
7412 gc_stat_sym_heap_live_slots,
7413 gc_stat_sym_heap_free_slots,
7414 gc_stat_sym_heap_final_slots,
7415 gc_stat_sym_heap_marked_slots,
7416 gc_stat_sym_heap_eden_pages,
7417 gc_stat_sym_total_allocated_pages,
7418 gc_stat_sym_total_freed_pages,
7419 gc_stat_sym_total_allocated_objects,
7420 gc_stat_sym_total_freed_objects,
7421 gc_stat_sym_malloc_increase_bytes,
7422 gc_stat_sym_malloc_increase_bytes_limit,
7423 gc_stat_sym_minor_gc_count,
7424 gc_stat_sym_major_gc_count,
7425 gc_stat_sym_compact_count,
7426 gc_stat_sym_read_barrier_faults,
7427 gc_stat_sym_total_moved_objects,
7428 gc_stat_sym_remembered_wb_unprotected_objects,
7429 gc_stat_sym_remembered_wb_unprotected_objects_limit,
7430 gc_stat_sym_old_objects,
7431 gc_stat_sym_old_objects_limit,
7432#if RGENGC_ESTIMATE_OLDMALLOC
7433 gc_stat_sym_oldmalloc_increase_bytes,
7434 gc_stat_sym_oldmalloc_increase_bytes_limit,
7435#endif
7436 gc_stat_sym_weak_references_count,
7437#if RGENGC_PROFILE
7438 gc_stat_sym_total_generated_normal_object_count,
7439 gc_stat_sym_total_generated_shady_object_count,
7440 gc_stat_sym_total_shade_operation_count,
7441 gc_stat_sym_total_promoted_count,
7442 gc_stat_sym_total_remembered_normal_object_count,
7443 gc_stat_sym_total_remembered_shady_object_count,
7444#endif
7445 gc_stat_sym_last
7446};
7447
7448static VALUE gc_stat_symbols[gc_stat_sym_last];
7449
7450static void
7451setup_gc_stat_symbols(void)
7452{
7453 if (gc_stat_symbols[0] == 0) {
7454#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
7455 S(count);
7456 S(time);
7457 S(marking_time),
7458 S(sweeping_time),
7459 S(heap_allocated_pages);
7460 S(heap_empty_pages);
7461 S(heap_allocatable_slots);
7462 S(heap_available_slots);
7463 S(heap_live_slots);
7464 S(heap_free_slots);
7465 S(heap_final_slots);
7466 S(heap_marked_slots);
7467 S(heap_eden_pages);
7468 S(total_allocated_pages);
7469 S(total_freed_pages);
7470 S(total_allocated_objects);
7471 S(total_freed_objects);
7472 S(malloc_increase_bytes);
7473 S(malloc_increase_bytes_limit);
7474 S(minor_gc_count);
7475 S(major_gc_count);
7476 S(compact_count);
7477 S(read_barrier_faults);
7478 S(total_moved_objects);
7479 S(remembered_wb_unprotected_objects);
7480 S(remembered_wb_unprotected_objects_limit);
7481 S(old_objects);
7482 S(old_objects_limit);
7483#if RGENGC_ESTIMATE_OLDMALLOC
7484 S(oldmalloc_increase_bytes);
7485 S(oldmalloc_increase_bytes_limit);
7486#endif
7487 S(weak_references_count);
7488#if RGENGC_PROFILE
7489 S(total_generated_normal_object_count);
7490 S(total_generated_shady_object_count);
7491 S(total_shade_operation_count);
7492 S(total_promoted_count);
7493 S(total_remembered_normal_object_count);
7494 S(total_remembered_shady_object_count);
7495#endif /* RGENGC_PROFILE */
7496#undef S
7497 }
7498}
7499
7500static uint64_t
7501ns_to_ms(uint64_t ns)
7502{
7503 return ns / (1000 * 1000);
7504}
7505
7506 VALUE
7507rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
7508{
7509 rb_objspace_t *objspace = objspace_ptr;
7510 VALUE hash = Qnil, key = Qnil;
7511
7512 setup_gc_stat_symbols();
7513
7514 if (RB_TYPE_P(hash_or_sym, T_HASH)) {
7515 hash = hash_or_sym;
7516 }
7517 else if (SYMBOL_P(hash_or_sym)) {
7518 key = hash_or_sym;
7519 }
7520 else {
7521 rb_bug("non-hash or symbol given");
7522 }
7523
7524#define SET(name, attr) \
7525 if (key == gc_stat_symbols[gc_stat_sym_##name]) \
7526 return SIZET2NUM(attr); \
7527 else if (hash != Qnil) \
7528 rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
7529
7530 SET(count, objspace->profile.count);
7531 SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM
7532 SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns));
7533 SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns));
7534
7535 /* implementation dependent counters */
7536 SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted));
7537 SET(heap_empty_pages, objspace->empty_pages_count)
7538 SET(heap_allocatable_slots, objspace->heap_pages.allocatable_slots);
7539 SET(heap_available_slots, objspace_available_slots(objspace));
7540 SET(heap_live_slots, objspace_live_slots(objspace));
7541 SET(heap_free_slots, objspace_free_slots(objspace));
7542 SET(heap_final_slots, total_final_slots_count(objspace));
7543 SET(heap_marked_slots, objspace->marked_slots);
7544 SET(heap_eden_pages, heap_eden_total_pages(objspace));
7545 SET(total_allocated_pages, objspace->heap_pages.allocated_pages);
7546 SET(total_freed_pages, objspace->heap_pages.freed_pages);
7547 SET(total_allocated_objects, total_allocated_objects(objspace));
7548 SET(total_freed_objects, total_freed_objects(objspace));
7549 SET(malloc_increase_bytes, malloc_increase);
7550 SET(malloc_increase_bytes_limit, malloc_limit);
7551 SET(minor_gc_count, objspace->profile.minor_gc_count);
7552 SET(major_gc_count, objspace->profile.major_gc_count);
7553 SET(compact_count, objspace->profile.compact_count);
7554 SET(read_barrier_faults, objspace->profile.read_barrier_faults);
7555 SET(total_moved_objects, objspace->rcompactor.total_moved);
7556 SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
7557 SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
7558 SET(old_objects, objspace->rgengc.old_objects);
7559 SET(old_objects_limit, objspace->rgengc.old_objects_limit);
7560#if RGENGC_ESTIMATE_OLDMALLOC
7561 SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase);
7562 SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
7563#endif
7564
7565#if RGENGC_PROFILE
7566 SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
7567 SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
7568 SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
7569 SET(total_promoted_count, objspace->profile.total_promoted_count);
7570 SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
7571 SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
7572#endif /* RGENGC_PROFILE */
7573#undef SET
7574
7575 if (!NIL_P(key)) {
7576 // Matched key should return above
7577 return Qundef;
7578 }
7579
7580#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
7581 if (hash != Qnil) {
7582 gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
7583 gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
7584 gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
7585 gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
7586 gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
7587 gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
7588 }
7589#endif
7590
7591 return hash;
7592}
7593
7594enum gc_stat_heap_sym {
7595 gc_stat_heap_sym_slot_size,
7596 gc_stat_heap_sym_heap_eden_pages,
7597 gc_stat_heap_sym_heap_eden_slots,
7598 gc_stat_heap_sym_total_allocated_pages,
7599 gc_stat_heap_sym_force_major_gc_count,
7600 gc_stat_heap_sym_force_incremental_marking_finish_count,
7601 gc_stat_heap_sym_total_allocated_objects,
7602 gc_stat_heap_sym_total_freed_objects,
7603 gc_stat_heap_sym_last
7604};
7605
7606static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
7607
7608static void
7609setup_gc_stat_heap_symbols(void)
7610{
7611 if (gc_stat_heap_symbols[0] == 0) {
7612#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
7613 S(slot_size);
7614 S(heap_eden_pages);
7615 S(heap_eden_slots);
7616 S(total_allocated_pages);
7617 S(force_major_gc_count);
7618 S(force_incremental_marking_finish_count);
7619 S(total_allocated_objects);
7620 S(total_freed_objects);
7621#undef S
7622 }
7623}
7624
7625static VALUE
7626stat_one_heap(rb_heap_t *heap, VALUE hash, VALUE key)
7627{
7628#define SET(name, attr) \
7629 if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
7630 return SIZET2NUM(attr); \
7631 else if (hash != Qnil) \
7632 rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
7633
7634 SET(slot_size, heap->slot_size);
7635 SET(heap_eden_pages, heap->total_pages);
7636 SET(heap_eden_slots, heap->total_slots);
7637 SET(total_allocated_pages, heap->total_allocated_pages);
7638 SET(force_major_gc_count, heap->force_major_gc_count);
7639 SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
7640 SET(total_allocated_objects, heap->total_allocated_objects);
7641 SET(total_freed_objects, heap->total_freed_objects);
7642#undef SET
7643
7644 if (!NIL_P(key)) {
7645 // Matched key should return above
7646 return Qundef;
7647 }
7648
7649 return hash;
7650}
7651
7652 VALUE
7653rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
7654{
7655 rb_objspace_t *objspace = objspace_ptr;
7656
7657 setup_gc_stat_heap_symbols();
7658
7659 if (NIL_P(heap_name)) {
7660 if (!RB_TYPE_P(hash_or_sym, T_HASH)) {
7661 rb_bug("non-hash given");
7662 }
7663
7664 for (int i = 0; i < HEAP_COUNT; i++) {
7665 VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i));
7666 if (NIL_P(hash)) {
7667 hash = rb_hash_new();
7668 rb_hash_aset(hash_or_sym, INT2FIX(i), hash);
7669 }
7670
7671 stat_one_heap(&heaps[i], hash, Qnil);
7672 }
7673 }
7674 else if (FIXNUM_P(heap_name)) {
7675 int heap_idx = FIX2INT(heap_name);
7676
7677 if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
7678 rb_raise(rb_eArgError, "size pool index out of range");
7679 }
7680
7681 if (SYMBOL_P(hash_or_sym)) {
7682 return stat_one_heap(&heaps[heap_idx], Qnil, hash_or_sym);
7683 }
7684 else if (RB_TYPE_P(hash_or_sym, T_HASH)) {
7685 return stat_one_heap(&heaps[heap_idx], hash_or_sym, Qnil);
7686 }
7687 else {
7688 rb_bug("non-hash or symbol given");
7689 }
7690 }
7691 else {
7692 rb_bug("heap_name must be nil or an Integer");
7693 }
7694
7695 return hash_or_sym;
7696}
7697
7698/* I could include internal.h for this, but doing so undefines some Array macros
7699 * necessary for initialising objects, and I don't want to include all the array
7700 * headers to get them back
7701 * TODO: Investigate why RARRAY_AREF gets undefined in internal.h
7702 */
7703#ifndef RBOOL
7704#define RBOOL(v) (v ? Qtrue : Qfalse)
7705#endif
7706
7707 VALUE
7708rb_gc_impl_config_get(void *objspace_ptr)
7709{
7710#define sym(name) ID2SYM(rb_intern_const(name))
7711 rb_objspace_t *objspace = objspace_ptr;
7712 VALUE hash = rb_hash_new();
7713
7714 rb_hash_aset(hash, sym("rgengc_allow_full_mark"), RBOOL(gc_config_full_mark_val));
7715
7716 return hash;
7717}
7718
7719static int
7720gc_config_set_key(st_data_t key, st_data_t value, st_data_t data)
7721{
7722 rb_objspace_t *objspace = (rb_objspace_t *)data;
7723 if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) {
7724 gc_rest(objspace);
7725 gc_config_full_mark_set(RTEST(value));
7726 }
7727 return ST_CONTINUE;
7728}
7729
7730void
7731rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
7732{
7733 rb_objspace_t *objspace = objspace_ptr;
7734
7735 if (!RB_TYPE_P(hash, T_HASH)) {
7736 rb_raise(rb_eArgError, "expected keyword arguments");
7737 }
7738
7739 rb_hash_stlike_foreach(hash, gc_config_set_key, (st_data_t)objspace);
7740}
7741
7742 VALUE
7743rb_gc_impl_stress_get(void *objspace_ptr)
7744{
7745 rb_objspace_t *objspace = objspace_ptr;
7746 return ruby_gc_stress_mode;
7747}
7748
7749void
7750rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag)
7751{
7752 rb_objspace_t *objspace = objspace_ptr;
7753
7754 objspace->flags.gc_stressful = RTEST(flag);
7755 objspace->gc_stress_mode = flag;
7756}
7757
7758static int
7759get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
7760{
7761 const char *ptr = getenv(name);
7762 ssize_t val;
7763
7764 if (ptr != NULL && *ptr) {
7765 size_t unit = 0;
7766 char *end;
7767#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
7768 val = strtoll(ptr, &end, 0);
7769#else
7770 val = strtol(ptr, &end, 0);
7771#endif
7772 switch (*end) {
7773 case 'k': case 'K':
7774 unit = 1024;
7775 ++end;
7776 break;
7777 case 'm': case 'M':
7778 unit = 1024*1024;
7779 ++end;
7780 break;
7781 case 'g': case 'G':
7782 unit = 1024*1024*1024;
7783 ++end;
7784 break;
7785 }
7786 while (*end && isspace((unsigned char)*end)) end++;
7787 if (*end) {
7788 if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
7789 return 0;
7790 }
7791 if (unit > 0) {
7792 if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
7793 if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
7794 return 0;
7795 }
7796 val *= unit;
7797 }
7798 if (val > 0 && (size_t)val > lower_bound) {
7799 if (RTEST(ruby_verbose)) {
7800 fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
7801 }
7802 *default_value = (size_t)val;
7803 return 1;
7804 }
7805 else {
7806 if (RTEST(ruby_verbose)) {
7807 fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
7808 name, val, *default_value, lower_bound);
7809 }
7810 return 0;
7811 }
7812 }
7813 return 0;
7814}
7815
7816static int
7817get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
7818{
7819 const char *ptr = getenv(name);
7820 double val;
7821
7822 if (ptr != NULL && *ptr) {
7823 char *end;
7824 val = strtod(ptr, &end);
7825 if (!*ptr || *end) {
7826 if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
7827 return 0;
7828 }
7829
7830 if (accept_zero && val == 0.0) {
7831 goto accept;
7832 }
7833 else if (val <= lower_bound) {
7834 if (RTEST(ruby_verbose)) {
7835 fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
7836 name, val, *default_value, lower_bound);
7837 }
7838 }
7839 else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
7840 val > upper_bound) {
7841 if (RTEST(ruby_verbose)) {
7842 fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
7843 name, val, *default_value, upper_bound);
7844 }
7845 }
7846 else {
7847 goto accept;
7848 }
7849 }
7850 return 0;
7851
7852 accept:
7853 if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
7854 *default_value = val;
7855 return 1;
7856}
7857
7858/*
7859 * GC tuning environment variables
7860 *
7861 * * RUBY_GC_HEAP_FREE_SLOTS
7862 * - Prepare at least this amount of slots after GC.
7863 * - Allocate slots if there are not enough slots.
7864 * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
7865 * - Allocate slots by this factor.
7866 * - (next slots number) = (current slots number) * (this factor)
7867 * * RUBY_GC_HEAP_GROWTH_MAX_SLOTS (new from 2.1)
7868 * - Allocation rate is limited to this number of slots.
7869 * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
7870 * - Allocate additional pages when the number of free slots is
7871 * lower than the value (total_slots * (this ratio)).
7872 * * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
7873 * - Allocate slots to satisfy this formula:
7874 * free_slots = total_slots * goal_ratio
7875 * - In other words, prepare (total_slots * goal_ratio) free slots.
7876 * - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
7877 * * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
7878 * - Allow to free pages when the number of free slots is
7879 * greater than the value (total_slots * (this ratio)).
7880 * * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
7881 * - Do full GC when the number of old objects is more than R * N
7882 * where R is this factor and
7883 * N is the number of old objects just after last full GC.
7884 *
7885 * * obsolete
7886 * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
7887 * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1)
7888 *
7889 * * RUBY_GC_MALLOC_LIMIT
7890 * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
7891 * * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
7892 *
7893 * * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
7894 * * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
7895 * * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
7896 */
7897
7898void
7899rb_gc_impl_set_params(void *objspace_ptr)
7900{
7901 rb_objspace_t *objspace = objspace_ptr;
7902 /* RUBY_GC_HEAP_FREE_SLOTS */
7903 if (get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) {
7904 /* ok */
7905 }
7906
7907 for (int i = 0; i < HEAP_COUNT; i++) {
7908 char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)];
7909 snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i);
7910
7911 get_envparam_size(env_key, &gc_params.heap_init_slots[i], 0);
7912 }
7913
7914 get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
7915 get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
7916 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
7917 0.0, 1.0, FALSE);
7918 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
7919 gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
7920 get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
7921 gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
7922 get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
7923 get_envparam_double("RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO", &gc_params.uncollectible_wb_unprotected_objects_limit_ratio, 0.0, 0.0, TRUE);
7924
7925 if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
7926 malloc_limit = gc_params.malloc_limit_min;
7927 }
7928 get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
7929 if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
7930 gc_params.malloc_limit_max = SIZE_MAX;
7931 }
7932 get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
7933
7934#if RGENGC_ESTIMATE_OLDMALLOC
7935 if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
7936 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
7937 }
7938 get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
7939 get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
7940#endif
7941}
7942
7943static inline size_t
7944objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
7945{
7946#ifdef HAVE_MALLOC_USABLE_SIZE
7947 return malloc_usable_size(ptr);
7948#else
7949 return hint;
7950#endif
7951}
7952
7953enum memop_type {
7954 MEMOP_TYPE_MALLOC = 0,
7955 MEMOP_TYPE_FREE,
7956 MEMOP_TYPE_REALLOC
7957};
7958
7959static inline void
7960atomic_sub_nounderflow(size_t *var, size_t sub)
7961{
7962 if (sub == 0) return;
7963
7964 while (1) {
7965 size_t val = *var;
7966 if (val < sub) sub = val;
7967 if (RUBY_ATOMIC_SIZE_CAS(*var, val, val-sub) == val) break;
7968 }
7969}
7970
7971#define gc_stress_full_mark_after_malloc_p() \
7972 (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
7973
7974static void
7975objspace_malloc_gc_stress(rb_objspace_t *objspace)
7976{
7977 if (ruby_gc_stressful && ruby_native_thread_p()) {
7978 unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
7979 GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
7980
7981 if (gc_stress_full_mark_after_malloc_p()) {
7982 reason |= GPR_FLAG_FULL_MARK;
7983 }
7984 garbage_collect_with_gvl(objspace, reason);
7985 }
7986}
7987
7988static inline bool
7989objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
7990{
7991 if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
7992 mem,
7993 type == MEMOP_TYPE_MALLOC ? "malloc" :
7994 type == MEMOP_TYPE_FREE ? "free " :
7995 type == MEMOP_TYPE_REALLOC ? "realloc": "error",
7996 new_size, old_size);
7997 return false;
7998}
7999
8000static bool
8001objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
8002{
8003 if (new_size > old_size) {
8004 RUBY_ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size);
8005#if RGENGC_ESTIMATE_OLDMALLOC
8006 RUBY_ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size);
8007#endif
8008 }
8009 else {
8010 atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
8011#if RGENGC_ESTIMATE_OLDMALLOC
8012 atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size);
8013#endif
8014 }
8015
8016 if (type == MEMOP_TYPE_MALLOC) {
8017 retry:
8018 if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) {
8019 if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
8020 gc_rest(objspace); /* gc_rest can reduce malloc_increase */
8021 goto retry;
8022 }
8023 garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
8024 }
8025 }
8026
8027#if MALLOC_ALLOCATED_SIZE
8028 if (new_size >= old_size) {
8029 RUBY_ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
8030 }
8031 else {
8032 size_t dec_size = old_size - new_size;
8033
8034#if MALLOC_ALLOCATED_SIZE_CHECK
8035 size_t allocated_size = objspace->malloc_params.allocated_size;
8036 if (allocated_size < dec_size) {
8037 rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
8038 }
8039#endif
8040 atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
8041 }
8042
8043 switch (type) {
8044 case MEMOP_TYPE_MALLOC:
8045 RUBY_ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
8046 break;
8047 case MEMOP_TYPE_FREE:
8048 {
8049 size_t allocations = objspace->malloc_params.allocations;
8050 if (allocations > 0) {
8051 atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
8052 }
8053#if MALLOC_ALLOCATED_SIZE_CHECK
8054 else {
8055 GC_ASSERT(objspace->malloc_params.allocations > 0);
8056 }
8057#endif
8058 }
8059 break;
8060 case MEMOP_TYPE_REALLOC: /* ignore */ break;
8061 }
8062#endif
8063 return true;
8064}
8065
8066#define objspace_malloc_increase(...) \
8067 for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
8068 !malloc_increase_done; \
8069 malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
8070
8071 struct malloc_obj_info { /* 4 words */
8072 size_t size;
8073};
8074
8075static inline size_t
8076objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
8077{
8078 if (size == 0) size = 1;
8079
8080#if CALC_EXACT_MALLOC_SIZE
8081 size += sizeof(struct malloc_obj_info);
8082#endif
8083
8084 return size;
8085}
8086
8087static bool
8088malloc_during_gc_p(rb_objspace_t *objspace)
8089{
8090 /* malloc is not allowed during GC when we're not using multiple ractors
8091 * (since ractors can run while another thread is sweeping) and when we
8092 * have the GVL (since if we don't have the GVL, we'll try to acquire the
8093 * GVL which will block and ensure the other thread finishes GC). */
8094 return during_gc && !dont_gc_val() && !rb_gc_multi_ractor_p() && ruby_thread_has_gvl_p();
8095}
8096
8097static inline void *
8098objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
8099{
8100 size = objspace_malloc_size(objspace, mem, size);
8101 objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {}
8102
8103#if CALC_EXACT_MALLOC_SIZE
8104 {
8105 struct malloc_obj_info *info = mem;
8106 info->size = size;
8107 mem = info + 1;
8108 }
8109#endif
8110
8111 return mem;
8112}
8113
8114#if defined(__GNUC__) && RUBY_DEBUG
8115#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
8116#endif
8117
8118#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
8119# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
8120#endif
8121
8122#define GC_MEMERROR(...) \
8123 ((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : (void)0)
8124
8125#define TRY_WITH_GC(siz, expr) do { \
8126 const gc_profile_record_flag gpr = \
8127 GPR_FLAG_FULL_MARK | \
8128 GPR_FLAG_IMMEDIATE_MARK | \
8129 GPR_FLAG_IMMEDIATE_SWEEP | \
8130 GPR_FLAG_MALLOC; \
8131 objspace_malloc_gc_stress(objspace); \
8132 \
8133 if (RB_LIKELY((expr))) { \
8134 /* Success on 1st try */ \
8135 } \
8136 else if (!garbage_collect_with_gvl(objspace, gpr)) { \
8137 /* @shyouhei thinks this doesn't happen */ \
8138 GC_MEMERROR("TRY_WITH_GC: could not GC"); \
8139 } \
8140 else if ((expr)) { \
8141 /* Success on 2nd try */ \
8142 } \
8143 else { \
8144 GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
8145 "%"PRIdSIZE" bytes for %s", \
8146 siz, # expr); \
8147 } \
8148 } while (0)
8149
8150static void
8151check_malloc_not_in_gc(rb_objspace_t *objspace, const char *msg)
8152{
8153 if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
8154 dont_gc_on();
8155 during_gc = false;
8156 rb_bug("Cannot %s during GC", msg);
8157 }
8158}
8159
8160void
8161rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
8162{
8163 rb_objspace_t *objspace = objspace_ptr;
8164
8165 if (!ptr) {
8166 /*
8167 * ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
8168 * its first version. We would better follow.
8169 */
8170 return;
8171 }
8172#if CALC_EXACT_MALLOC_SIZE
8173 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
8174 ptr = info;
8175 old_size = info->size;
8176#endif
8177 old_size = objspace_malloc_size(objspace, ptr, old_size);
8178
8179 objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE) {
8180 free(ptr);
8181 ptr = NULL;
8182 RB_DEBUG_COUNTER_INC(heap_xfree);
8183 }
8184}
8185
8186void *
8187rb_gc_impl_malloc(void *objspace_ptr, size_t size)
8188{
8189 rb_objspace_t *objspace = objspace_ptr;
8190 check_malloc_not_in_gc(objspace, "malloc");
8191
8192 void *mem;
8193
8194 size = objspace_malloc_prepare(objspace, size);
8195 TRY_WITH_GC(size, mem = malloc(size));
8196 RB_DEBUG_COUNTER_INC(heap_xmalloc);
8197 if (!mem) return mem;
8198 return objspace_malloc_fixup(objspace, mem, size);
8199}
8200
8201void *
8202rb_gc_impl_calloc(void *objspace_ptr, size_t size)
8203{
8204 rb_objspace_t *objspace = objspace_ptr;
8205
8206 if (RB_UNLIKELY(malloc_during_gc_p(objspace))) {
8207 rb_warn("calloc during GC detected, this could cause crashes if it triggers another GC");
8208#if RGENGC_CHECK_MODE || RUBY_DEBUG
8209 rb_bug("Cannot calloc during GC");
8210#endif
8211 }
8212
8213 void *mem;
8214
8215 size = objspace_malloc_prepare(objspace, size);
8216 TRY_WITH_GC(size, mem = calloc1(size));
8217 if (!mem) return mem;
8218 return objspace_malloc_fixup(objspace, mem, size);
8219}
8220
8221void *
8222rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size)
8223{
8224 rb_objspace_t *objspace = objspace_ptr;
8225
8226 check_malloc_not_in_gc(objspace, "realloc");
8227
8228 void *mem;
8229
8230 if (!ptr) return rb_gc_impl_malloc(objspace, new_size);
8231
8232 /*
8233 * The behavior of realloc(ptr, 0) is implementation defined.
8234 * Therefore we don't use realloc(ptr, 0) for portability reason.
8235 * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
8236 */
8237 if (new_size == 0) {
8238 if ((mem = rb_gc_impl_malloc(objspace, 0)) != NULL) {
8239 /*
8240 * - OpenBSD's malloc(3) man page says that when 0 is passed, it
8241 * returns a non-NULL pointer to an access-protected memory page.
8242 * The returned pointer cannot be read / written at all, but
8243 * still be a valid argument of free().
8244 *
8245 * https://man.openbsd.org/malloc.3
8246 *
8247 * - Linux's malloc(3) man page says that it _might_ perhaps return
8248 * a non-NULL pointer when its argument is 0. That return value
8249 * is safe (and is expected) to be passed to free().
8250 *
8251 * https://man7.org/linux/man-pages/man3/malloc.3.html
8252 *
8253 * - As I read the implementation jemalloc's malloc() returns fully
8254 * normal 16 bytes memory region when its argument is 0.
8255 *
8256 * - As I read the implementation musl libc's malloc() returns
8257 * fully normal 32 bytes memory region when its argument is 0.
8258 *
8259 * - Other malloc implementations can also return non-NULL.
8260 */
8261 rb_gc_impl_free(objspace, ptr, old_size);
8262 return mem;
8263 }
8264 else {
8265 /*
8266 * It is dangerous to return NULL here, because that could lead to
8267 * RCE. Fallback to 1 byte instead of zero.
8268 *
8269 * https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
8270 */
8271 new_size = 1;
8272 }
8273 }
8274
8275#if CALC_EXACT_MALLOC_SIZE
8276 {
8277 struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
8278 new_size += sizeof(struct malloc_obj_info);
8279 ptr = info;
8280 old_size = info->size;
8281 }
8282#endif
8283
8284 old_size = objspace_malloc_size(objspace, ptr, old_size);
8285 TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size)));
8286 if (!mem) return mem;
8287 new_size = objspace_malloc_size(objspace, mem, new_size);
8288
8289#if CALC_EXACT_MALLOC_SIZE
8290 {
8291 struct malloc_obj_info *info = mem;
8292 info->size = new_size;
8293 mem = info + 1;
8294 }
8295#endif
8296
8297 objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
8298
8299 RB_DEBUG_COUNTER_INC(heap_xrealloc);
8300 return mem;
8301}
8302
8303void
8304rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff)
8305{
8306 rb_objspace_t *objspace = objspace_ptr;
8307
8308 if (diff > 0) {
8309 objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
8310 }
8311 else if (diff < 0) {
8312 objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
8313 }
8314}
8315
8316// TODO: move GC profiler stuff back into gc.c
8317/*
8318 ------------------------------ GC profiler ------------------------------
8319*/
8320
8321#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
8322
8323static bool
8324current_process_time(struct timespec *ts)
8325{
8326#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
8327 {
8328 static int try_clock_gettime = 1;
8329 if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
8330 return true;
8331 }
8332 else {
8333 try_clock_gettime = 0;
8334 }
8335 }
8336#endif
8337
8338#ifdef RUSAGE_SELF
8339 {
8340 struct rusage usage;
8341 struct timeval time;
8342 if (getrusage(RUSAGE_SELF, &usage) == 0) {
8343 time = usage.ru_utime;
8344 ts->tv_sec = time.tv_sec;
8345 ts->tv_nsec = (int32_t)time.tv_usec * 1000;
8346 return true;
8347 }
8348 }
8349#endif
8350
8351#ifdef _WIN32
8352 {
8353 FILETIME creation_time, exit_time, kernel_time, user_time;
8354 ULARGE_INTEGER ui;
8355
8356 if (GetProcessTimes(GetCurrentProcess(),
8357 &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
8358 memcpy(&ui, &user_time, sizeof(FILETIME));
8359#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
8360 ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
8361 ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
8362 return true;
8363 }
8364 }
8365#endif
8366
8367 return false;
8368}
8369
8370static double
8371getrusage_time(void)
8372{
8373 struct timespec ts;
8374 if (current_process_time(&ts)) {
8375 return ts.tv_sec + ts.tv_nsec * 1e-9;
8376 }
8377 else {
8378 return 0.0;
8379 }
8380}
8381
8382
8383static inline void
8384gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason)
8385{
8386 if (objspace->profile.run) {
8387 size_t index = objspace->profile.next_index;
8388 gc_profile_record *record;
8389
8390 /* create new record */
8391 objspace->profile.next_index++;
8392
8393 if (!objspace->profile.records) {
8394 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
8395 objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
8396 }
8397 if (index >= objspace->profile.size) {
8398 void *ptr;
8399 objspace->profile.size += 1000;
8400 ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
8401 if (!ptr) rb_memerror();
8402 objspace->profile.records = ptr;
8403 }
8404 if (!objspace->profile.records) {
8405 rb_bug("gc_profile malloc or realloc miss");
8406 }
8407 record = objspace->profile.current_record = &objspace->profile.records[objspace->profile.next_index - 1];
8408 MEMZERO(record, gc_profile_record, 1);
8409
8410 /* setup before-GC parameter */
8411 record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
8412#if MALLOC_ALLOCATED_SIZE
8413 record->allocated_size = malloc_allocated_size;
8414#endif
8415#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
8416#ifdef RUSAGE_SELF
8417 {
8418 struct rusage usage;
8419 if (getrusage(RUSAGE_SELF, &usage) == 0) {
8420 record->maxrss = usage.ru_maxrss;
8421 record->minflt = usage.ru_minflt;
8422 record->majflt = usage.ru_majflt;
8423 }
8424 }
8425#endif
8426#endif
8427 }
8428}
8429
8430static inline void
8431gc_prof_timer_start(rb_objspace_t *objspace)
8432{
8433 if (gc_prof_enabled(objspace)) {
8434 gc_profile_record *record = gc_prof_record(objspace);
8435#if GC_PROFILE_MORE_DETAIL
8436 record->prepare_time = objspace->profile.prepare_time;
8437#endif
8438 record->gc_time = 0;
8439 record->gc_invoke_time = getrusage_time();
8440 }
8441}
8442
8443static double
8444elapsed_time_from(double time)
8445{
8446 double now = getrusage_time();
8447 if (now > time) {
8448 return now - time;
8449 }
8450 else {
8451 return 0;
8452 }
8453}
8454
8455static inline void
8456gc_prof_timer_stop(rb_objspace_t *objspace)
8457{
8458 if (gc_prof_enabled(objspace)) {
8459 gc_profile_record *record = gc_prof_record(objspace);
8460 record->gc_time = elapsed_time_from(record->gc_invoke_time);
8461 record->gc_invoke_time -= objspace->profile.invoke_time;
8462 }
8463}
8464
8465#ifdef BUILDING_MODULAR_GC
8466# define RUBY_DTRACE_GC_HOOK(name)
8467#else
8468# define RUBY_DTRACE_GC_HOOK(name) \
8469 do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0)
8470#endif
8471
8472static inline void
8473gc_prof_mark_timer_start(rb_objspace_t *objspace)
8474{
8475 RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
8476#if GC_PROFILE_MORE_DETAIL
8477 if (gc_prof_enabled(objspace)) {
8478 gc_prof_record(objspace)->gc_mark_time = getrusage_time();
8479 }
8480#endif
8481}
8482
8483static inline void
8484gc_prof_mark_timer_stop(rb_objspace_t *objspace)
8485{
8486 RUBY_DTRACE_GC_HOOK(MARK_END);
8487#if GC_PROFILE_MORE_DETAIL
8488 if (gc_prof_enabled(objspace)) {
8489 gc_profile_record *record = gc_prof_record(objspace);
8490 record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
8491 }
8492#endif
8493}
8494
8495static inline void
8496gc_prof_sweep_timer_start(rb_objspace_t *objspace)
8497{
8498 RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
8499 if (gc_prof_enabled(objspace)) {
8500 gc_profile_record *record = gc_prof_record(objspace);
8501
8502 if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
8503 objspace->profile.gc_sweep_start_time = getrusage_time();
8504 }
8505 }
8506}
8507
8508static inline void
8509gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
8510{
8511 RUBY_DTRACE_GC_HOOK(SWEEP_END);
8512
8513 if (gc_prof_enabled(objspace)) {
8514 double sweep_time;
8515 gc_profile_record *record = gc_prof_record(objspace);
8516
8517 if (record->gc_time > 0) {
8518 sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
8519 /* need to accumulate GC time for lazy sweep after gc() */
8520 record->gc_time += sweep_time;
8521 }
8522 else if (GC_PROFILE_MORE_DETAIL) {
8523 sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
8524 }
8525
8526#if GC_PROFILE_MORE_DETAIL
8527 record->gc_sweep_time += sweep_time;
8528 if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
8529#endif
8530 if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
8531 }
8532}
8533
8534static inline void
8535gc_prof_set_malloc_info(rb_objspace_t *objspace)
8536{
8537#if GC_PROFILE_MORE_DETAIL
8538 if (gc_prof_enabled(objspace)) {
8539 gc_profile_record *record = gc_prof_record(objspace);
8540 record->allocate_increase = malloc_increase;
8541 record->allocate_limit = malloc_limit;
8542 }
8543#endif
8544}
8545
8546static inline void
8547gc_prof_set_heap_info(rb_objspace_t *objspace)
8548{
8549 if (gc_prof_enabled(objspace)) {
8550 gc_profile_record *record = gc_prof_record(objspace);
8551 size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
8552 size_t total = objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
8553
8554#if GC_PROFILE_MORE_DETAIL
8555 record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
8556 record->heap_live_objects = live;
8557 record->heap_free_objects = total - live;
8558#endif
8559
8560 record->heap_total_objects = total;
8561 record->heap_use_size = live * BASE_SLOT_SIZE;
8562 record->heap_total_size = total * BASE_SLOT_SIZE;
8563 }
8564}
8565
8566/*
8567 * call-seq:
8568 * GC::Profiler.clear -> nil
8569 *
8570 * Clears the \GC profiler data.
8571 *
8572 */
8573
8574static VALUE
8575gc_profile_clear(VALUE _)
8576{
8577 rb_objspace_t *objspace = rb_gc_get_objspace();
8578 void *p = objspace->profile.records;
8579 objspace->profile.records = NULL;
8580 objspace->profile.size = 0;
8581 objspace->profile.next_index = 0;
8582 objspace->profile.current_record = 0;
8583 free(p);
8584 return Qnil;
8585}
8586
8587/*
8588 * call-seq:
8589 * GC::Profiler.raw_data -> [Hash, ...]
8590 *
8591 * Returns an Array of individual raw profile data Hashes ordered
8592 * from earliest to latest by +:GC_INVOKE_TIME+.
8593 *
8594 * For example:
8595 *
8596 * [
8597 * {
8598 * :GC_TIME=>1.3000000000000858e-05,
8599 * :GC_INVOKE_TIME=>0.010634999999999999,
8600 * :HEAP_USE_SIZE=>289640,
8601 * :HEAP_TOTAL_SIZE=>588960,
8602 * :HEAP_TOTAL_OBJECTS=>14724,
8603 * :GC_IS_MARKED=>false
8604 * },
8605 * # ...
8606 * ]
8607 *
8608 * The keys mean:
8609 *
8610 * +:GC_TIME+::
8611 * Time elapsed in seconds for this GC run
8612 * +:GC_INVOKE_TIME+::
8613 * Time elapsed in seconds from startup to when the GC was invoked
8614 * +:HEAP_USE_SIZE+::
8615 * Total bytes of heap used
8616 * +:HEAP_TOTAL_SIZE+::
8617 * Total size of heap in bytes
8618 * +:HEAP_TOTAL_OBJECTS+::
8619 * Total number of objects
8620 * +:GC_IS_MARKED+::
8621 * Returns +true+ if the GC is in mark phase
8622 *
8623 * If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
8624 * to the following hash keys:
8625 *
8626 * +:GC_MARK_TIME+::
8627 * +:GC_SWEEP_TIME+::
8628 * +:ALLOCATE_INCREASE+::
8629 * +:ALLOCATE_LIMIT+::
8630 * +:HEAP_USE_PAGES+::
8631 * +:HEAP_LIVE_OBJECTS+::
8632 * +:HEAP_FREE_OBJECTS+::
8633 * +:HAVE_FINALIZE+::
8634 *
8635 */
8636
8637static VALUE
8638gc_profile_record_get(VALUE _)
8639{
8640 VALUE prof;
8641 VALUE gc_profile = rb_ary_new();
8642 size_t i;
8643 rb_objspace_t *objspace = rb_gc_get_objspace();
8644
8645 if (!objspace->profile.run) {
8646 return Qnil;
8647 }
8648
8649 for (i =0; i < objspace->profile.next_index; i++) {
8650 gc_profile_record *record = &objspace->profile.records[i];
8651
8652 prof = rb_hash_new();
8653 rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(objspace, rb_hash_new(), record->flags));
8654 rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
8655 rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
8656 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
8657 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
8658 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
8659 rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
8660 rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
8661#if GC_PROFILE_MORE_DETAIL
8662 rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
8663 rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
8664 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
8665 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
8666 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
8667 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
8668 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
8669
8670 rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
8671 rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
8672
8673 rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
8674#endif
8675
8676#if RGENGC_PROFILE > 0
8677 rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
8678 rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
8679 rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
8680#endif
8681 rb_ary_push(gc_profile, prof);
8682 }
8683
8684 return gc_profile;
8685}
8686
8687#if GC_PROFILE_MORE_DETAIL
8688#define MAJOR_REASON_MAX 0x10
8689
8690static char *
8691gc_profile_dump_major_reason(unsigned int flags, char *buff)
8692{
8693 unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
8694 int i = 0;
8695
8696 if (reason == GPR_FLAG_NONE) {
8697 buff[0] = '-';
8698 buff[1] = 0;
8699 }
8700 else {
8701#define C(x, s) \
8702 if (reason & GPR_FLAG_MAJOR_BY_##x) { \
8703 buff[i++] = #x[0]; \
8704 if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
8705 buff[i] = 0; \
8706 }
8707 C(NOFREE, N);
8708 C(OLDGEN, O);
8709 C(SHADY, S);
8710#if RGENGC_ESTIMATE_OLDMALLOC
8711 C(OLDMALLOC, M);
8712#endif
8713#undef C
8714 }
8715 return buff;
8716}
8717#endif
8718
8719
8720
8721static void
8722gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
8723{
8724 rb_objspace_t *objspace = rb_gc_get_objspace();
8725 size_t count = objspace->profile.next_index;
8726#ifdef MAJOR_REASON_MAX
8727 char reason_str[MAJOR_REASON_MAX];
8728#endif
8729
8730 if (objspace->profile.run && count /* > 1 */) {
8731 size_t i;
8732 const gc_profile_record *record;
8733
8734 append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
8735 append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
8736
8737 for (i = 0; i < count; i++) {
8738 record = &objspace->profile.records[i];
8739 append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
8740 i+1, record->gc_invoke_time, record->heap_use_size,
8741 record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
8742 }
8743
8744#if GC_PROFILE_MORE_DETAIL
8745 const char *str = "\n\n" \
8746 "More detail.\n" \
8747 "Prepare Time = Previously GC's rest sweep time\n"
8748 "Index Flags Allocate Inc. Allocate Limit"
8749#if CALC_EXACT_MALLOC_SIZE
8750 " Allocated Size"
8751#endif
8752 " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
8753#if RGENGC_PROFILE
8754 " OldgenObj RemNormObj RemShadObj"
8755#endif
8756#if GC_PROFILE_DETAIL_MEMORY
8757 " MaxRSS(KB) MinorFLT MajorFLT"
8758#endif
8759 "\n";
8760 append(out, rb_str_new_cstr(str));
8761
8762 for (i = 0; i < count; i++) {
8763 record = &objspace->profile.records[i];
8764 append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
8765#if CALC_EXACT_MALLOC_SIZE
8766 " %15"PRIuSIZE
8767#endif
8768 " %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
8769#if RGENGC_PROFILE
8770 "%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
8771#endif
8772#if GC_PROFILE_DETAIL_MEMORY
8773 "%11ld %8ld %8ld"
8774#endif
8775
8776 "\n",
8777 i+1,
8778 gc_profile_dump_major_reason(record->flags, reason_str),
8779 (record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
8780 (record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
8781 (record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
8782 (record->flags & GPR_FLAG_METHOD) ? "METHOD" :
8783 (record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????",
8784 (record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
8785 record->allocate_increase, record->allocate_limit,
8786#if CALC_EXACT_MALLOC_SIZE
8787 record->allocated_size,
8788#endif
8789 record->heap_use_pages,
8790 record->gc_mark_time*1000,
8791 record->gc_sweep_time*1000,
8792 record->prepare_time*1000,
8793
8794 record->heap_live_objects,
8795 record->heap_free_objects,
8796 record->removing_objects,
8797 record->empty_objects
8798#if RGENGC_PROFILE
8799 ,
8800 record->old_objects,
8801 record->remembered_normal_objects,
8802 record->remembered_shady_objects
8803#endif
8804#if GC_PROFILE_DETAIL_MEMORY
8805 ,
8806 record->maxrss / 1024,
8807 record->minflt,
8808 record->majflt
8809#endif
8810
8811 ));
8812 }
8813#endif
8814 }
8815}
8816
8817/*
8818 * call-seq:
8819 * GC::Profiler.result -> String
8820 *
8821 * Returns a profile data report such as:
8822 *
8823 * GC 1 invokes.
8824 * Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms)
8825 * 1 0.012 159240 212940 10647 0.00000000000001530000
8826 */
8827
8828static VALUE
8829gc_profile_result(VALUE _)
8830{
8831 VALUE str = rb_str_buf_new(0);
8832 gc_profile_dump_on(str, rb_str_buf_append);
8833 return str;
8834}
8835
8836/*
8837 * call-seq:
8838 * GC::Profiler.report
8839 * GC::Profiler.report(io)
8840 *
8841 * Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
8842 *
8843 */
8844
8845static VALUE
8846gc_profile_report(int argc, VALUE *argv, VALUE self)
8847{
8848 VALUE out;
8849
8850 out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
8851 gc_profile_dump_on(out, rb_io_write);
8852
8853 return Qnil;
8854}
8855
8856/*
8857 * call-seq:
8858 * GC::Profiler.total_time -> float
8859 *
8860 * The total time used for garbage collection in seconds
8861 */
8862
8863static VALUE
8864gc_profile_total_time(VALUE self)
8865{
8866 double time = 0;
8867 rb_objspace_t *objspace = rb_gc_get_objspace();
8868
8869 if (objspace->profile.run && objspace->profile.next_index > 0) {
8870 size_t i;
8871 size_t count = objspace->profile.next_index;
8872
8873 for (i = 0; i < count; i++) {
8874 time += objspace->profile.records[i].gc_time;
8875 }
8876 }
8877 return DBL2NUM(time);
8878}
8879
8880/*
8881 * call-seq:
8882 * GC::Profiler.enabled? -> true or false
8883 *
8884 * The current status of \GC profile mode.
8885 */
8886
8887static VALUE
8888gc_profile_enable_get(VALUE self)
8889{
8890 rb_objspace_t *objspace = rb_gc_get_objspace();
8891 return objspace->profile.run ? Qtrue : Qfalse;
8892}
8893
8894/*
8895 * call-seq:
8896 * GC::Profiler.enable -> nil
8897 *
8898 * Starts the \GC profiler.
8899 *
8900 */
8901
8902static VALUE
8903gc_profile_enable(VALUE _)
8904{
8905 rb_objspace_t *objspace = rb_gc_get_objspace();
8906 objspace->profile.run = TRUE;
8907 objspace->profile.current_record = 0;
8908 return Qnil;
8909}
8910
8911/*
8912 * call-seq:
8913 * GC::Profiler.disable -> nil
8914 *
8915 * Stops the \GC profiler.
8916 *
8917 */
8918
8919static VALUE
8920gc_profile_disable(VALUE _)
8921{
8922 rb_objspace_t *objspace = rb_gc_get_objspace();
8923
8924 objspace->profile.run = FALSE;
8925 objspace->profile.current_record = 0;
8926 return Qnil;
8927}
8928
8929/*
8930 * call-seq:
8931 * GC.verify_internal_consistency -> nil
8932 *
8933 * Verify internal consistency.
8934 *
8935 * This method is implementation specific.
8936 * Now this method checks generational consistency
8937 * if RGenGC is supported.
8938 */
8939static VALUE
8940gc_verify_internal_consistency_m(VALUE dummy)
8941{
8942 gc_verify_internal_consistency(rb_gc_get_objspace());
8943 return Qnil;
8944}
8945
8946#if GC_CAN_COMPILE_COMPACTION
8947/*
8948 * call-seq:
8949 * GC.auto_compact = flag
8950 *
8951 * Updates automatic compaction mode.
8952 *
8953 * When enabled, the compactor will execute on every major collection.
8954 *
8955 * Enabling compaction will degrade performance on major collections.
8956 */
8957static VALUE
8958gc_set_auto_compact(VALUE _, VALUE v)
8959{
8960 GC_ASSERT(GC_COMPACTION_SUPPORTED);
8961
8962 ruby_enable_autocompact = RTEST(v);
8963
8964#if RGENGC_CHECK_MODE
8965 ruby_autocompact_compare_func = NULL;
8966
8967 if (SYMBOL_P(v)) {
8968 ID id = RB_SYM2ID(v);
8969 if (id == rb_intern("empty")) {
8970 ruby_autocompact_compare_func = compare_free_slots;
8971 }
8972 }
8973#endif
8974
8975 return v;
8976}
8977#else
8978# define gc_set_auto_compact rb_f_notimplement
8979#endif
8980
8981#if GC_CAN_COMPILE_COMPACTION
8982/*
8983 * call-seq:
8984 * GC.auto_compact -> true or false
8985 *
8986 * Returns whether or not automatic compaction has been enabled.
8987 */
8988static VALUE
8989gc_get_auto_compact(VALUE _)
8990{
8991 return ruby_enable_autocompact ? Qtrue : Qfalse;
8992}
8993#else
8994# define gc_get_auto_compact rb_f_notimplement
8995#endif
8996
8997#if GC_CAN_COMPILE_COMPACTION
8998/*
8999 * call-seq:
9000 * GC.latest_compact_info -> hash
9001 *
9002 * Returns information about object moved in the most recent \GC compaction.
9003 *
9004 * The returned +hash+ contains the following keys:
9005 *
9006 * [considered]
9007 * Hash containing the type of the object as the key and the number of
9008 * objects of that type that were considered for movement.
9009 * [moved]
9010 * Hash containing the type of the object as the key and the number of
9011 * objects of that type that were actually moved.
9012 * [moved_up]
9013 * Hash containing the type of the object as the key and the number of
9014 * objects of that type that were increased in size.
9015 * [moved_down]
9016 * Hash containing the type of the object as the key and the number of
9017 * objects of that type that were decreased in size.
9018 *
9019 * Some objects can't be moved (due to pinning) so these numbers can be used to
9020 * calculate compaction efficiency.
9021 */
9022static VALUE
9023gc_compact_stats(VALUE self)
9024{
9025 rb_objspace_t *objspace = rb_gc_get_objspace();
9026 VALUE h = rb_hash_new();
9027 VALUE considered = rb_hash_new();
9028 VALUE moved = rb_hash_new();
9029 VALUE moved_up = rb_hash_new();
9030 VALUE moved_down = rb_hash_new();
9031
9032 for (size_t i = 0; i < T_MASK; i++) {
9033 if (objspace->rcompactor.considered_count_table[i]) {
9034 rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
9035 }
9036
9037 if (objspace->rcompactor.moved_count_table[i]) {
9038 rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
9039 }
9040
9041 if (objspace->rcompactor.moved_up_count_table[i]) {
9042 rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i]));
9043 }
9044
9045 if (objspace->rcompactor.moved_down_count_table[i]) {
9046 rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i]));
9047 }
9048 }
9049
9050 rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
9051 rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
9052 rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up);
9053 rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down);
9054
9055 return h;
9056}
9057#else
9058# define gc_compact_stats rb_f_notimplement
9059#endif
9060
9061#if GC_CAN_COMPILE_COMPACTION
9062/*
9063 * call-seq:
9064 * GC.compact -> hash
9065 *
9066 * This function compacts objects together in Ruby's heap. It eliminates
9067 * unused space (or fragmentation) in the heap by moving objects in to that
9068 * unused space.
9069 *
9070 * The returned +hash+ contains statistics about the objects that were moved;
9071 * see GC.latest_compact_info.
9072 *
9073 * This method is only expected to work on CRuby.
9074 *
9075 * To test whether \GC compaction is supported, use the idiom:
9076 *
9077 * GC.respond_to?(:compact)
9078 */
9079static VALUE
9080gc_compact(VALUE self)
9081{
9082 rb_objspace_t *objspace = rb_gc_get_objspace();
9083 int full_marking_p = gc_config_full_mark_val;
9084 gc_config_full_mark_set(TRUE);
9085
9086 /* Run GC with compaction enabled */
9087 rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true);
9088 gc_config_full_mark_set(full_marking_p);
9089
9090 return gc_compact_stats(self);
9091}
9092#else
9093# define gc_compact rb_f_notimplement
9094#endif
9095
9096#if GC_CAN_COMPILE_COMPACTION
9097struct desired_compaction_pages_i_data {
9098 rb_objspace_t *objspace;
9099 size_t required_slots[HEAP_COUNT];
9100};
9101
9102static int
9103desired_compaction_pages_i(struct heap_page *page, void *data)
9104{
9105 struct desired_compaction_pages_i_data *tdata = data;
9106 rb_objspace_t *objspace = tdata->objspace;
9107 VALUE vstart = (VALUE)page->start;
9108 VALUE vend = vstart + (VALUE)(page->total_slots * page->heap->slot_size);
9109
9110
9111 for (VALUE v = vstart; v != vend; v += page->heap->slot_size) {
9112 asan_unpoisoning_object(v) {
9113 /* skip T_NONEs; they won't be moved */
9114 if (BUILTIN_TYPE(v) != T_NONE) {
9115 rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, page->heap, v);
9116 size_t dest_pool_idx = dest_pool - heaps;
9117 tdata->required_slots[dest_pool_idx]++;
9118 }
9119 }
9120 }
9121
9122 return 0;
9123}
9124
9125/* call-seq:
9126 * GC.verify_compaction_references(toward: nil, double_heap: false) -> hash
9127 *
9128 * Verify compaction reference consistency.
9129 *
9130 * This method is implementation specific. During compaction, objects that
9131 * were moved are replaced with T_MOVED objects. No object should have a
9132 * reference to a T_MOVED object after compaction.
9133 *
9134 * This function expands the heap to ensure room to move all objects,
9135 * compacts the heap to make sure everything moves, updates all references,
9136 * then performs a full \GC. If any object contains a reference to a T_MOVED
9137 * object, that object should be pushed on the mark stack, and will
9138 * make a SEGV.
9139 */
9140static VALUE
9141gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
9142{
9143 static ID keywords[3] = {0};
9144 if (!keywords[0]) {
9145 keywords[0] = rb_intern("toward");
9146 keywords[1] = rb_intern("double_heap");
9147 keywords[2] = rb_intern("expand_heap");
9148 }
9149
9150 VALUE options;
9151 rb_scan_args_kw(rb_keyword_given_p(), argc, argv, ":", &options);
9152
9153 VALUE arguments[3] = { Qnil, Qfalse, Qfalse };
9154 int kwarg_count = rb_get_kwargs(options, keywords, 0, 3, arguments);
9155 bool toward_empty = kwarg_count > 0 && SYMBOL_P(arguments[0]) && SYM2ID(arguments[0]) == rb_intern("empty");
9156 bool expand_heap = (kwarg_count > 1 && RTEST(arguments[1])) || (kwarg_count > 2 && RTEST(arguments[2]));
9157
9158 rb_objspace_t *objspace = rb_gc_get_objspace();
9159
9160 /* Clear the heap. */
9161 rb_gc_impl_start(objspace, true, true, true, false);
9162
9163 unsigned int lev = rb_gc_vm_lock();
9164 {
9165 gc_rest(objspace);
9166
9167 /* if both double_heap and expand_heap are set, expand_heap takes precedence */
9168 if (expand_heap) {
9169 struct desired_compaction_pages_i_data desired_compaction = {
9170 .objspace = objspace,
9171 .required_slots = {0},
9172 };
9173 /* Work out how many objects want to be in each size pool, taking account of moves */
9174 objspace_each_pages(objspace, desired_compaction_pages_i, &desired_compaction, TRUE);
9175
9176 /* Find out which pool has the most pages */
9177 size_t max_existing_pages = 0;
9178 for (int i = 0; i < HEAP_COUNT; i++) {
9179 rb_heap_t *heap = &heaps[i];
9180 max_existing_pages = MAX(max_existing_pages, heap->total_pages);
9181 }
9182
9183 /* Add pages to each size pool so that compaction is guaranteed to move every object */
9184 for (int i = 0; i < HEAP_COUNT; i++) {
9185 rb_heap_t *heap = &heaps[i];
9186
9187 size_t pages_to_add = 0;
9188 /*
9189 * Step 1: Make sure every pool has the same number of pages, by adding empty pages
9190 * to smaller pools. This is required to make sure the compact cursor can advance
9191 * through all of the pools in `gc_sweep_compact` without hitting the "sweep &
9192 * compact cursors met" condition on some pools before fully compacting others
9193 */
9194 pages_to_add += max_existing_pages - heap->total_pages;
9195 /*
9196 * Step 2: Now add additional free pages to each size pool sufficient to hold all objects
9197 * that want to be in that size pool, whether moved into it or moved within it
9198 */
9199 objspace->heap_pages.allocatable_slots = desired_compaction.required_slots[i];
9200 while (objspace->heap_pages.allocatable_slots > 0) {
9201 heap_page_allocate_and_initialize(objspace, heap);
9202 }
9203 /*
9204 * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects
9205 * have been moved, and not on the last iteration of the `gc_sweep_compact` loop
9206 */
9207 pages_to_add += 2;
9208
9209 for (; pages_to_add > 0; pages_to_add--) {
9210 heap_page_allocate_and_initialize_force(objspace, heap);
9211 }
9212 }
9213 }
9214
9215 if (toward_empty) {
9216 objspace->rcompactor.compare_func = compare_free_slots;
9217 }
9218 }
9219 rb_gc_vm_unlock(lev);
9220
9221 rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true);
9222
9223 rb_objspace_reachable_objects_from_root(root_obj_check_moved_i, objspace);
9224 objspace_each_objects(objspace, heap_check_moved_i, objspace, TRUE);
9225
9226 objspace->rcompactor.compare_func = NULL;
9227
9228 return gc_compact_stats(self);
9229}
9230#else
9231# define gc_verify_compaction_references rb_f_notimplement
9232#endif
9233
9234void
9235rb_gc_impl_objspace_free(void *objspace_ptr)
9236{
9237 rb_objspace_t *objspace = objspace_ptr;
9238
9239 if (is_lazy_sweeping(objspace))
9240 rb_bug("lazy sweeping underway when freeing object space");
9241
9242 free(objspace->profile.records);
9243 objspace->profile.records = NULL;
9244
9245 for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
9246 heap_page_free(objspace, rb_darray_get(objspace->heap_pages.sorted, i));
9247 }
9248 rb_darray_free(objspace->heap_pages.sorted);
9249 heap_pages_lomem = 0;
9250 heap_pages_himem = 0;
9251
9252 for (int i = 0; i < HEAP_COUNT; i++) {
9253 rb_heap_t *heap = &heaps[i];
9254 heap->total_pages = 0;
9255 heap->total_slots = 0;
9256 }
9257
9258 st_free_table(objspace->id_to_obj_tbl);
9259 st_free_table(objspace->obj_to_id_tbl);
9260
9261 free_stack_chunks(&objspace->mark_stack);
9262 mark_stack_free_cache(&objspace->mark_stack);
9263
9264 rb_darray_free(objspace->weak_references);
9265
9266 free(objspace);
9267}
9268
9269#if MALLOC_ALLOCATED_SIZE
9270/*
9271 * call-seq:
9272 * GC.malloc_allocated_size -> Integer
9273 *
9274 * Returns the size of memory allocated by malloc().
9275 *
9276 * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
9277 */
9278
9279static VALUE
9280gc_malloc_allocated_size(VALUE self)
9281{
9282 rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
9283 return ULL2NUM(objspace->malloc_params.allocated_size);
9284}
9285
9286/*
9287 * call-seq:
9288 * GC.malloc_allocations -> Integer
9289 *
9290 * Returns the number of malloc() allocations.
9291 *
9292 * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
9293 */
9294
9295static VALUE
9296gc_malloc_allocations(VALUE self)
9297{
9298 rb_objspace_t *objspace = (rb_objspace_t *)rb_gc_get_objspace();
9299 return ULL2NUM(objspace->malloc_params.allocations);
9300}
9301#endif
9302
9303void rb_gc_impl_before_fork(void *objspace_ptr) { /* no-op */ }
9304void rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) { /* no-op */ }
9305
9306void *
9307rb_gc_impl_objspace_alloc(void)
9308{
9309 rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
9310
9311 return objspace;
9312}
9313
9314void
9315rb_gc_impl_objspace_init(void *objspace_ptr)
9316{
9317 rb_objspace_t *objspace = objspace_ptr;
9318
9319 gc_config_full_mark_set(TRUE);
9320
9321 objspace->flags.measure_gc = true;
9322 malloc_limit = gc_params.malloc_limit_min;
9323 objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace);
9324 if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
9325 rb_bug("Could not preregister postponed job for GC");
9326 }
9327
9328 for (int i = 0; i < HEAP_COUNT; i++) {
9329 rb_heap_t *heap = &heaps[i];
9330
9331 heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
9332
9333 ccan_list_head_init(&heap->pages);
9334 }
9335
9336 rb_darray_make(&objspace->heap_pages.sorted, 0);
9337 rb_darray_make(&objspace->weak_references, 0);
9338
9339 // TODO: debug why on Windows Ruby crashes on boot when GC is on.
9340#ifdef _WIN32
9341 dont_gc_on();
9342#endif
9343
9344#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
9345 /* Need to determine if we can use mmap at runtime. */
9346 heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
9347#endif
9348 objspace->next_object_id = OBJ_ID_INITIAL;
9349 objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
9350 objspace->obj_to_id_tbl = st_init_numtable();
9351#if RGENGC_ESTIMATE_OLDMALLOC
9352 objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
9353#endif
9354 /* Set size pools allocatable pages. */
9355 for (int i = 0; i < HEAP_COUNT; i++) {
9356 /* Set the default value of heap_init_slots. */
9357 gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS;
9358 }
9359
9360 init_mark_stack(&objspace->mark_stack);
9361
9362 objspace->profile.invoke_time = getrusage_time();
9363 finalizer_table = st_init_numtable();
9364}
9365
9366void
9367rb_gc_impl_init(void)
9368{
9369 VALUE gc_constants = rb_hash_new();
9370 rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
9371 rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
9372 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
9373 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
9374 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
9375 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
9376 rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT));
9377 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(heap_slot_size(HEAP_COUNT - 1)));
9378 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE));
9379 if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) {
9380 rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue);
9381 }
9382 OBJ_FREEZE(gc_constants);
9383 /* Internal constants in the garbage collector. */
9384 rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
9385
9386 if (GC_COMPACTION_SUPPORTED) {
9387 rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0);
9388 rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0);
9389 rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1);
9390 rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0);
9391 rb_define_singleton_method(rb_mGC, "verify_compaction_references", gc_verify_compaction_references, -1);
9392 }
9393 else {
9394 rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
9395 rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
9396 rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
9397 rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
9398 rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
9399 }
9400
9401 /* internal methods */
9402 rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
9403
9404#if MALLOC_ALLOCATED_SIZE
9405 rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
9406 rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
9407#endif
9408
9409 VALUE rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
9410 rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
9411 rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
9412 rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, 0);
9413 rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
9414 rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
9415 rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
9416 rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
9417 rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
9418
9419 {
9420 VALUE opts;
9421 /* \GC build options */
9422 rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
9423#define OPT(o) if (o) rb_ary_push(opts, rb_interned_str(#o, sizeof(#o) - 1))
9424 OPT(GC_DEBUG);
9425 OPT(USE_RGENGC);
9426 OPT(RGENGC_DEBUG);
9427 OPT(RGENGC_CHECK_MODE);
9428 OPT(RGENGC_PROFILE);
9429 OPT(RGENGC_ESTIMATE_OLDMALLOC);
9430 OPT(GC_PROFILE_MORE_DETAIL);
9431 OPT(GC_ENABLE_LAZY_SWEEP);
9432 OPT(CALC_EXACT_MALLOC_SIZE);
9433 OPT(MALLOC_ALLOCATED_SIZE);
9434 OPT(MALLOC_ALLOCATED_SIZE_CHECK);
9435 OPT(GC_PROFILE_DETAIL_MEMORY);
9436 OPT(GC_COMPACTION_SUPPORTED);
9437#undef OPT
9438 OBJ_FREEZE(opts);
9439 }
9440}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
atomic.h
Atomic operations.
RUBY_ATOMIC_VALUE_CAS
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are VALUE.
Definition atomic.h:343
RUBY_ATOMIC_SIZE_EXCHANGE
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are size_t.
Definition atomic.h:233
RUBY_ATOMIC_SIZE_INC
#define RUBY_ATOMIC_SIZE_INC(var)
Identical to RUBY_ATOMIC_INC, except it expects its argument is size_t.
Definition atomic.h:209
RUBY_ATOMIC_SIZE_CAS
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval)
Identical to RUBY_ATOMIC_CAS, except it expects its arguments are size_t.
Definition atomic.h:247
rb_atomic_t
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
RUBY_ATOMIC_SIZE_ADD
#define RUBY_ATOMIC_SIZE_ADD(var, val)
Identical to RUBY_ATOMIC_ADD, except it expects its arguments are size_t.
Definition atomic.h:260
RUBY_ATOMIC_VALUE_EXCHANGE
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are VALUE.
Definition atomic.h:329
RUBY_ATOMIC_SET
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:160
RUBY_ATOMIC_EXCHANGE
#define RUBY_ATOMIC_EXCHANGE(var, val)
Atomically replaces the value pointed by var with val.
Definition atomic.h:127
rb_define_singleton_method
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
Definition cxxanyargs.hpp:685
debug.h
rb_postponed_job_handle_t
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
Definition debug.h:665
rb_postponed_job_trigger
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
Definition vm_trace.c:1787
rb_postponed_job_preregister
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
Definition vm_trace.c:1753
RUBY_INTERNAL_EVENT_GC_EXIT
#define RUBY_INTERNAL_EVENT_GC_EXIT
gc_exit() is called.
Definition event.h:99
RUBY_INTERNAL_EVENT_GC_ENTER
#define RUBY_INTERNAL_EVENT_GC_ENTER
gc_enter() is called.
Definition event.h:98
RUBY_INTERNAL_EVENT_GC_END_SWEEP
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
GC ended sweep phase.
Definition event.h:97
RUBY_INTERNAL_EVENT_GC_END_MARK
#define RUBY_INTERNAL_EVENT_GC_END_MARK
GC ended mark phase.
Definition event.h:96
RUBY_INTERNAL_EVENT_OBJSPACE_MASK
#define RUBY_INTERNAL_EVENT_OBJSPACE_MASK
Bitmask of GC events.
Definition event.h:100
RUBY_INTERNAL_EVENT_FREEOBJ
#define RUBY_INTERNAL_EVENT_FREEOBJ
Object swept.
Definition event.h:94
RUBY_INTERNAL_EVENT_GC_START
#define RUBY_INTERNAL_EVENT_GC_START
GC started.
Definition event.h:95
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_INTERNAL_EVENT_NEWOBJ
#define RUBY_INTERNAL_EVENT_NEWOBJ
Object allocated.
Definition event.h:93
RB_FL_TEST
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
Definition fl_type.h:495
RB_FL_SET_RAW
static void RB_FL_SET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_SET().
Definition fl_type.h:606
RB_FL_UNSET_RAW
static void RB_FL_UNSET_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_UNSET().
Definition fl_type.h:666
RUBY_FL_PROMOTED
@ RUBY_FL_PROMOTED
Ruby objects are "generational".
Definition fl_type.h:218
rb_define_module_under
VALUE rb_define_module_under(VALUE outer, const char *name)
Defines a module under the namespace of outer.
Definition class.c:1119
rb_scan_args_kw
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
Definition class.c:2653
rb_keyword_given_p
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
Definition eval.c:950
rb_get_kwargs
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
Definition class.c:2429
T_COMPLEX
#define T_COMPLEX
Old name of RUBY_T_COMPLEX.
Definition value_type.h:59
T_FILE
#define T_FILE
Old name of RUBY_T_FILE.
Definition value_type.h:62
FL_EXIVAR
#define FL_EXIVAR
Old name of RUBY_FL_EXIVAR.
Definition fl_type.h:66
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
xfree
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
T_MASK
#define T_MASK
Old name of RUBY_T_MASK.
Definition value_type.h:68
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
OBJ_FROZEN
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
Definition fl_type.h:137
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
OBJ_FREEZE
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
Definition fl_type.h:135
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
T_DATA
#define T_DATA
Old name of RUBY_T_DATA.
Definition value_type.h:60
FL_SEEN_OBJ_ID
#define FL_SEEN_OBJ_ID
Old name of RUBY_FL_SEEN_OBJ_ID.
Definition fl_type.h:65
T_NONE
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
T_NODE
#define T_NODE
Old name of RUBY_T_NODE.
Definition value_type.h:73
SIZET2NUM
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
xmalloc
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
FL_FINALIZE
#define FL_FINALIZE
Old name of RUBY_FL_FINALIZE.
Definition fl_type.h:61
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_RATIONAL
#define T_RATIONAL
Old name of RUBY_T_RATIONAL.
Definition value_type.h:76
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
ALLOC_N
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
FL_TEST_RAW
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:132
FL_SET
#define FL_SET
Old name of RB_FL_SET.
Definition fl_type.h:129
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
ULL2NUM
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
T_UNDEF
#define T_UNDEF
Old name of RUBY_T_UNDEF.
Definition value_type.h:82
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
T_ZOMBIE
#define T_ZOMBIE
Old name of RUBY_T_ZOMBIE.
Definition value_type.h:83
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
FL_WB_PROTECTED
#define FL_WB_PROTECTED
Old name of RUBY_FL_WB_PROTECTED.
Definition fl_type.h:59
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_MATCH
#define T_MATCH
Old name of RUBY_T_MATCH.
Definition value_type.h:69
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
T_MOVED
#define T_MOVED
Old name of RUBY_T_MOVED.
Definition value_type.h:71
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:131
FL_UNSET
#define FL_UNSET
Old name of RB_FL_UNSET.
Definition fl_type.h:133
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
T_REGEXP
#define T_REGEXP
Old name of RUBY_T_REGEXP.
Definition value_type.h:77
rb_eRangeError
VALUE rb_eRangeError
RangeError exception.
Definition error.c:1434
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:475
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1428
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:466
rb_obj_hide
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
Definition object.c:104
rb_mGC
VALUE rb_mGC
GC module.
Definition gc.c:431
rb_equal
VALUE rb_equal(VALUE lhs, VALUE rhs)
This function is an optimised version of calling #==.
Definition object.c:179
rb_stdout
VALUE rb_stdout
STDOUT constant.
Definition io.c:201
RB_GNUC_EXTENSION_BLOCK
#define RB_GNUC_EXTENSION_BLOCK(x)
This is expanded to the passed token for non-GCC compilers.
Definition defines.h:91
string.h
Routines to manipulate encodings of strings.
rb_funcall
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
Definition vm_eval.c:1099
rb_memerror
void rb_memerror(void)
Triggers out-of-memory error.
Definition gc.c:4501
RB_OBJ_PROMOTED_RAW
static bool RB_OBJ_PROMOTED_RAW(VALUE obj)
This is the implementation of RB_OBJ_PROMOTED().
Definition gc.h:706
USE_RGENGC
#define USE_RGENGC
Definition gc.h:428
rb_gc_count
size_t rb_gc_count(void)
Identical to rb_gc_stat(), with "count" parameter.
Definition gc.c:3708
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
Definition array.c:741
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_hash_new
VALUE rb_hash_new(void)
Creates a new, empty hash object.
Definition hash.c:1477
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3651
rb_str_buf_new
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
Definition string.c:1650
rb_str_new_cstr
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1514
rb_sourcefile
const char * rb_sourcefile(void)
Resembles __FILE__.
Definition vm.c:1872
rb_f_notimplement
VALUE rb_f_notimplement(int argc, const VALUE *argv, VALUE obj, VALUE marker)
Raises rb_eNotImpError.
Definition vm_method.c:481
rb_sourceline
int rb_sourceline(void)
Resembles __LINE__.
Definition vm.c:1886
RB_SYM2ID
#define RB_SYM2ID
Just another name of rb_sym2id.
Definition symbol.h:43
rb_sym2id
ID rb_sym2id(VALUE obj)
Converts an instance of rb_cSymbol into an ID.
Definition symbol.c:918
rb_define_const
void rb_define_const(VALUE klass, const char *name, VALUE val)
Defines a Ruby level constant under a namespace.
Definition variable.c:3797
len
int len
Length of the buffer.
Definition io.h:8
thread.h
rb_thread_call_with_gvl
void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
(Re-)acquires the GVL.
Definition thread.c:1904
util.h
strtod
#define strtod(s, e)
Just another name of ruby_strtod.
Definition util.h:223
ruby_qsort
void ruby_qsort(void *, const size_t, const size_t, int(*)(const void *, const void *, void *), void *)
Reentrant implementation of quick sort.
DECIMAL_SIZE_OF_BITS
#define DECIMAL_SIZE_OF_BITS(n)
an approximation of ceil(n * log10(2)), up to 1,048,576 (1<<20) without overflow within 32-bit calcul...
Definition util.h:39
vm.h
MEMZERO
#define MEMZERO(p, type, n)
Handy macro to erase a region of memory.
Definition memory.h:360
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
ruby::backward::cxxanyargs::st_foreach
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
Definition cxxanyargs.hpp:432
ruby::backward::cxxanyargs::rb_ensure
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
Definition cxxanyargs.hpp:310
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:51
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:403
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
ruby.h
errno
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
ruby_native_thread_p
int ruby_native_thread_p(void)
Queries if the thread which calls this function is a ruby's thread.
Definition thread.c:5557
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
_
#define _(args)
This was a transition path from K&R to ANSI.
Definition stdarg.h:35
RBasic
Ruby object's base components.
Definition rbasic.h:63
RBasic::flags
VALUE flags
Per-object flags.
Definition rbasic.h:75
RMoved
Definition default.c:362
RZombie
Definition default.c:977
each_obj_data
Definition default.c:2663
free_slot
Definition default.c:707
gc_profile_record
Definition default.c:320
gc_sweep_context
Definition default.c:3498
heap_page_body
Definition default.c:381
heap_page_header
Definition default.c:377
heap_page
Definition default.c:712
malloc_obj_info
Definition default.c:8071
mark_stack
Definition default.c:394
objspace_and_reason
Definition default.c:6516
objspace
Definition mmtk.c:19
ractor_newobj_cache
Definition default.c:166
ractor_newobj_heap_cache
Definition default.c:161
rb_heap_struct
Definition default.c:405
rb_objspace::rb_gc_config
Definition default.c:455
rb_objspace
Definition default.c:445
ruby_gc_params_t
Definition default.c:171
stack_chunk
Definition default.c:389
timespec
Definition missing.h:74
timeval
Definition missing.h:63
verify_internal_consistency_struct
Definition default.c:4956
__attribute__
Definition vm_insnhelper.c:2285
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
ruby_value_type
ruby_value_type
C-level type of an object.
Definition value_type.h:113
RUBY_T_SYMBOL
@ RUBY_T_SYMBOL
Definition value_type.h:135
RUBY_T_MATCH
@ RUBY_T_MATCH
Definition value_type.h:128
RUBY_T_MODULE
@ RUBY_T_MODULE
Definition value_type.h:118
RUBY_T_ICLASS
@ RUBY_T_ICLASS
Hidden classes known as IClasses.
Definition value_type.h:141
RUBY_T_MOVED
@ RUBY_T_MOVED
Definition value_type.h:143
RUBY_T_FIXNUM
@ RUBY_T_FIXNUM
Integers formerly known as Fixnums.
Definition value_type.h:136
RUBY_T_IMEMO
@ RUBY_T_IMEMO
Definition value_type.h:139
RUBY_T_NODE
@ RUBY_T_NODE
Definition value_type.h:140
RUBY_T_OBJECT
@ RUBY_T_OBJECT
Definition value_type.h:116
RUBY_T_DATA
@ RUBY_T_DATA
Definition value_type.h:127
RUBY_T_FALSE
@ RUBY_T_FALSE
Definition value_type.h:134
RUBY_T_UNDEF
@ RUBY_T_UNDEF
Definition value_type.h:137
RUBY_T_COMPLEX
@ RUBY_T_COMPLEX
Definition value_type.h:129
RUBY_T_STRING
@ RUBY_T_STRING
Definition value_type.h:120
RUBY_T_HASH
@ RUBY_T_HASH
Definition value_type.h:123
RUBY_T_NIL
@ RUBY_T_NIL
Definition value_type.h:132
RUBY_T_CLASS
@ RUBY_T_CLASS
Definition value_type.h:117
RUBY_T_ARRAY
@ RUBY_T_ARRAY
Definition value_type.h:122
RUBY_T_MASK
@ RUBY_T_MASK
Bitmask of ruby_value_type.
Definition value_type.h:145
RUBY_T_RATIONAL
@ RUBY_T_RATIONAL
Definition value_type.h:130
RUBY_T_ZOMBIE
@ RUBY_T_ZOMBIE
Definition value_type.h:142
RUBY_T_BIGNUM
@ RUBY_T_BIGNUM
Definition value_type.h:125
RUBY_T_TRUE
@ RUBY_T_TRUE
Definition value_type.h:133
RUBY_T_FLOAT
@ RUBY_T_FLOAT
Definition value_type.h:119
RUBY_T_STRUCT
@ RUBY_T_STRUCT
Definition value_type.h:124
RUBY_T_NONE
@ RUBY_T_NONE
Non-object (swept etc.).
Definition value_type.h:114
RUBY_T_REGEXP
@ RUBY_T_REGEXP
Definition value_type.h:121
RUBY_T_FILE
@ RUBY_T_FILE
Definition value_type.h:126