Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion error.c
Original file line number Diff line number Diff line change
Expand Up @@ -4243,7 +4243,7 @@ rb_warn_unchilled_literal(VALUE obj)

VALUE str = obj;
if (STR_SHARED_P(str)) {
str = RSTRING(obj)->as.heap.aux.shared;
str = RSTRING(obj)->as.shared;
}
VALUE created = get_created_info(str, &line);
if (NIL_P(created)) {
Expand Down
12 changes: 6 additions & 6 deletions ext/-test-/string/cstr.c
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ bug_str_unterminated_substring(VALUE str, VALUE vbeg, VALUE vlen)
if (RSTRING_LEN(str) < beg) rb_raise(rb_eIndexError, "beg: %ld", beg);
if (RSTRING_LEN(str) < beg + len) rb_raise(rb_eIndexError, "end: %ld", beg + len);
str = rb_str_new_shared(str);
RSTRING(str)->len = len;
RSTRING(str)->len = (uint32_t)len;
if (STR_EMBED_P(str)) {
memmove(RSTRING(str)->as.embed.ary, RSTRING(str)->as.embed.ary + beg, len);
memmove(RSTRING(str)->as.ary, RSTRING(str)->as.ary + beg, len);
}
else {
RSTRING(str)->as.heap.ptr += beg;
RSTRING(str)->capa += (uint32_t)beg;
}
return str;
}
Expand Down Expand Up @@ -111,9 +111,9 @@ bug_str_s_cstr_noembed(VALUE self, VALUE str)
FL_SET((str2), STR_NOEMBED);
memcpy(buf, RSTRING_PTR(str), capacity);
RBASIC(str2)->flags &= ~(STR_SHARED | FL_USER5 | FL_USER6);
RSTRING(str2)->as.heap.aux.capa = RSTRING_LEN(str);
RSTRING(str2)->as.heap.ptr = buf;
RSTRING(str2)->len = RSTRING_LEN(str);
RSTRING(str2)->capa = (uint32_t)RSTRING_LEN(str);
RSTRING(str2)->as.ptr = buf;
RSTRING(str2)->len = (uint32_t)RSTRING_LEN(str);

TERM_FILL(RSTRING_END(str2), TERM_LEN(str));
return str2;
Expand Down
2 changes: 1 addition & 1 deletion ext/-test-/string/rb_str_dup.c
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ bug_sharing_with_shared_p(VALUE self, VALUE str)
{
rb_check_type(str, T_STRING);
if (bug_shared_string_p(self, str)) {
return bug_shared_string_p(self, RSTRING(str)->as.heap.aux.shared);
return bug_shared_string_p(self, RSTRING(str)->as.shared);
}
return Qfalse;
}
Expand Down
9 changes: 5 additions & 4 deletions gc.c
Original file line number Diff line number Diff line change
Expand Up @@ -3376,15 +3376,16 @@ rb_gc_mark_children(void *objspace, VALUE obj)

case T_STRING:
if (STR_SHARED_P(obj)) {
if (STR_EMBED_P(RSTRING(obj)->as.heap.aux.shared)) {
VALUE shared_root = RSTRING(obj)->as.shared;
if (STR_EMBED_P(shared_root)) {
/* Embedded shared strings cannot be moved because this string
* points into the slot of the shared string. There may be code
* using the RSTRING_PTR on the stack, which would pin this
* string but not pin the shared string, causing it to move. */
gc_mark_and_pin_internal(RSTRING(obj)->as.heap.aux.shared);
gc_mark_and_pin_internal(shared_root);
}
else {
gc_mark_internal(RSTRING(obj)->as.heap.aux.shared);
gc_mark_internal(shared_root);
}
}
break;
Expand Down Expand Up @@ -4366,7 +4367,7 @@ rb_gc_update_object_references(void *objspace, VALUE obj)
case T_STRING:
{
if (STR_SHARED_P(obj)) {
UPDATE_IF_MOVED(objspace, RSTRING(obj)->as.heap.aux.shared);
UPDATE_IF_MOVED(objspace, RSTRING(obj)->as.shared);
}

/* If, after move the string is not embedded, and can fit in the
Expand Down
14 changes: 7 additions & 7 deletions gc.rb
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ def self.stat hash_or_key = nil
# GC.stat_heap
# # =>
# {0 =>
# {slot_size: 40,
# {slot_size: 64,
# heap_eden_pages: 246,
# heap_eden_slots: 402802,
# total_allocated_pages: 246,
Expand All @@ -278,7 +278,7 @@ def self.stat hash_or_key = nil
# total_allocated_objects: 33867152,
# total_freed_objects: 33520523},
# 1 =>
# {slot_size: 80,
# {slot_size: 128,
# heap_eden_pages: 84,
# heap_eden_slots: 68746,
# total_allocated_pages: 84,
Expand All @@ -287,7 +287,7 @@ def self.stat hash_or_key = nil
# total_allocated_objects: 147491,
# total_freed_objects: 90699},
# 2 =>
# {slot_size: 160,
# {slot_size: 256,
# heap_eden_pages: 157,
# heap_eden_slots: 64182,
# total_allocated_pages: 157,
Expand All @@ -296,7 +296,7 @@ def self.stat hash_or_key = nil
# total_allocated_objects: 211460,
# total_freed_objects: 190075},
# 3 =>
# {slot_size: 320,
# {slot_size: 512,
# heap_eden_pages: 8,
# heap_eden_slots: 1631,
# total_allocated_pages: 8,
Expand All @@ -305,7 +305,7 @@ def self.stat hash_or_key = nil
# total_allocated_objects: 1422,
# total_freed_objects: 700},
# 4 =>
# {slot_size: 640,
# {slot_size: 1024,
# heap_eden_pages: 16,
# heap_eden_slots: 1628,
# total_allocated_pages: 16,
Expand All @@ -326,7 +326,7 @@ def self.stat hash_or_key = nil
#
# GC.stat_heap(2)
# # =>
# {slot_size: 160,
# {slot_size: 256,
# heap_eden_pages: 157,
# heap_eden_slots: 64182,
# total_allocated_pages: 157,
Expand All @@ -338,7 +338,7 @@ def self.stat hash_or_key = nil
# With arguments +heap_id+ and +key+ given,
# returns the value for the given key in the given heap:
#
# GC.stat_heap(2, :slot_size) # => 160
# GC.stat_heap(2, :slot_size) # => 256
#
# With arguments +nil+ and +hash+ given,
# merges the statistics for all heaps into the given hash:
Expand Down
46 changes: 26 additions & 20 deletions gc/default/default.c
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)

#ifndef HEAP_COUNT
# define HEAP_COUNT 5
# define HEAP_COUNT 6
#endif

typedef struct ractor_newobj_heap_cache {
Expand Down Expand Up @@ -687,7 +687,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj);
# endif
#endif

#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
#if SIZEOF_VALUE >= 8
#define BASE_SLOT_SIZE_LOG2 5
#else
#define BASE_SLOT_SIZE_LOG2 4
#endif
#define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2)

#ifndef MAX
# define MAX(a, b) (((a) > (b)) ? (a) : (b))
Expand Down Expand Up @@ -764,7 +769,7 @@ struct free_slot {

struct heap_page {
unsigned short slot_size;
uint32_t slot_div_magic;
unsigned char slot_size_log2;
unsigned short total_slots;
unsigned short free_slots;
unsigned short final_slots;
Expand Down Expand Up @@ -841,15 +846,13 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)

static uint32_t slot_div_magics[HEAP_COUNT];

static inline size_t
slot_index_for_offset(size_t offset, uint32_t div_magic)
slot_index_for_offset(size_t offset, unsigned char slot_size_log2)
{
return (size_t)(((uint64_t)offset * div_magic) >> 32);
return offset >> slot_size_log2;
}

#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_div_magic)
#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_log2)
#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH)
#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
Expand Down Expand Up @@ -1977,19 +1980,16 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
GC_ASSERT(!heap->sweeping_page);
GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));

/* Align start to the first slot_size boundary after the page header */
/* Align start to slot_size boundary (both are powers of 2) */
uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
size_t remainder = start % heap->slot_size;
if (remainder != 0) {
start += heap->slot_size - remainder;
}
start = (start + heap->slot_size - 1) & ~((uintptr_t)heap->slot_size - 1);

int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);

page->start = start;
page->total_slots = slot_count;
page->slot_size = heap->slot_size;
page->slot_div_magic = slot_div_magics[heap - heaps];
page->slot_size_log2 = BASE_SLOT_SIZE_LOG2 + (unsigned char)(heap - heaps);
page->heap = heap;

memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
Expand Down Expand Up @@ -2241,7 +2241,7 @@ heap_slot_size(unsigned char pool_id)
{
GC_ASSERT(pool_id < HEAP_COUNT);

size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
size_t slot_size = BASE_SLOT_SIZE << pool_id;

#if RGENGC_CHECK_MODE
rb_objspace_t *objspace = rb_gc_get_objspace();
Expand Down Expand Up @@ -2360,10 +2360,10 @@ heap_idx_for_size(size_t size)
{
size += RVALUE_OVERHEAD;

size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
if (size <= BASE_SLOT_SIZE) return 0;

/* heap_idx is ceil(log2(slot_count)) */
size_t heap_idx = 64 - nlz_int64(slot_count - 1);
/* ceil(log2(size)) - BASE_SLOT_SIZE_LOG2 */
size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2;

if (heap_idx >= HEAP_COUNT) {
rb_bug("heap_idx_for_size: allocation size too large "
Expand Down Expand Up @@ -9511,11 +9511,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
rb_bug("Could not preregister postponed job for GC");
}

GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= (BASE_SLOT_SIZE << 1));

for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];

heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
slot_div_magics[i] = (uint32_t)((uint64_t)UINT32_MAX / heap->slot_size + 1);
heap->slot_size = BASE_SLOT_SIZE << i;

ccan_list_head_init(&heap->pages);
}
Expand Down Expand Up @@ -9553,6 +9554,11 @@ rb_gc_impl_init(void)
VALUE gc_constants = rb_hash_new();
rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
/* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */
size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD;
size_t rvalue_slot = BASE_SLOT_SIZE;
while (rvalue_slot < rvalue_min) rvalue_slot <<= 1;
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
Expand Down
15 changes: 11 additions & 4 deletions gc/mmtk/mmtk.c
Original file line number Diff line number Diff line change
Expand Up @@ -618,17 +618,24 @@ void rb_gc_impl_set_params(void *objspace_ptr) { }
static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; }

#define MMTK_HEAP_COUNT 6
#define MMTK_MAX_OBJ_SIZE 640

#if SIZEOF_VALUE >= 8
#define MMTK_MAX_OBJ_SIZE 1024
static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
32, 40, 80, 160, 320, MMTK_MAX_OBJ_SIZE, 0
32, 64, 128, 256, 512, MMTK_MAX_OBJ_SIZE, 0
};
#else
#define MMTK_MAX_OBJ_SIZE 512
static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
16, 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0
};
#endif

void
rb_gc_impl_init(void)
{
VALUE gc_constants = rb_hash_new();
rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE));
Expand Down
2 changes: 1 addition & 1 deletion include/ruby/internal/abi.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
* In released versions of Ruby, this number is not defined since teeny
* versions of Ruby should guarantee ABI compatibility.
*/
#define RUBY_ABI_VERSION 1
#define RUBY_ABI_VERSION 2

/* Windows does not support weak symbols so ruby_abi_version will not exist
* in the shared library. */
Expand Down
Loading
Loading