diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index bbf1ac93e2bb5e..0bed0bdb7518a1 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -3443,13 +3443,14 @@ size_t get_region_size (heap_segment* region_info) bool region_allocator::init (uint8_t* start, uint8_t* end, size_t alignment, uint8_t** lowest, uint8_t** highest) { - actual_start = start; + uint8_t* actual_start = start; region_alignment = alignment; large_region_alignment = LARGE_REGION_FACTOR * alignment; global_region_start = (uint8_t*)align_region_up ((size_t)actual_start); uint8_t* actual_end = end; global_region_end = (uint8_t*)align_region_down ((size_t)actual_end); - global_region_used = global_region_start; + global_region_left_used = global_region_start; + global_region_right_used = global_region_end; // Note: I am allocating a map that covers the whole reserved range. // We can optimize it to only cover the current heap range. @@ -3460,8 +3461,11 @@ bool region_allocator::init (uint8_t* start, uint8_t* end, size_t alignment, uin if (unit_map) { memset (unit_map, 0, sizeof (uint32_t) * total_num_units); - region_map_start = unit_map; - region_map_end = region_map_start; + region_map_left_start = unit_map; + region_map_left_end = region_map_left_start; + + region_map_right_start = unit_map + total_num_units; + region_map_right_end = region_map_right_start; dprintf (REGIONS_LOG, ("start: %Ix, end: %Ix, total %Idmb(alignment: %Idmb), map units %d", (size_t)start, (size_t)end, @@ -3479,44 +3483,33 @@ bool region_allocator::init (uint8_t* start, uint8_t* end, size_t alignment, uin inline uint8_t* region_allocator::region_address_of (uint32_t* map_index) { - return (global_region_start + ((map_index - region_map_start) * region_alignment)); + return (global_region_start + ((map_index - region_map_left_start) * region_alignment)); } inline uint32_t* region_allocator::region_map_index_of (uint8_t* address) { - return (region_map_start + ((address - global_region_start) / region_alignment)); + return (region_map_left_start + ((address - global_region_start) / region_alignment)); } void region_allocator::make_busy_block (uint32_t* index_start, uint32_t num_units) { #ifdef _DEBUG - dprintf (REGIONS_LOG, ("MBB[B: %Id] %d->%d", (size_t)num_units, (int)(index_start - region_map_start), (int)(index_start - region_map_start + num_units))); + dprintf (REGIONS_LOG, ("MBB[B: %Id] %d->%d", (size_t)num_units, (int)(index_start - region_map_left_start), (int)(index_start - region_map_left_start + num_units))); #endif //_DEBUG ASSERT_HOLDING_SPIN_LOCK (®ion_allocator_lock); - *index_start = num_units; + uint32_t* index_end = index_start + (num_units - 1); + *index_start = *index_end = num_units; } void region_allocator::make_free_block (uint32_t* index_start, uint32_t num_units) { #ifdef _DEBUG - dprintf (REGIONS_LOG, ("MFB[F: %Id] %d->%d", (size_t)num_units, (int)(index_start - region_map_start), (int)(index_start - region_map_start + num_units))); + dprintf (REGIONS_LOG, ("MFB[F: %Id] %d->%d", (size_t)num_units, (int)(index_start - region_map_left_start), (int)(index_start - region_map_left_start + num_units))); #endif //_DEBUG ASSERT_HOLDING_SPIN_LOCK (®ion_allocator_lock); - *index_start = region_alloc_free_bit | num_units; -} - -// make [current_free_index_start, [current_free_index_start + num_units into a busy block. -// make [current_free_index_start + num_units, [current_free_index_start + num_contiguous_free_units -// into a new free block. -void region_allocator::adjust_map (uint32_t* current_free_index_start, - uint32_t num_contiguous_free_units, uint32_t num_busy_units) -{ - make_busy_block (current_free_index_start, num_busy_units); - if ((num_contiguous_free_units - num_busy_units) > 0) - { - make_free_block ((current_free_index_start + num_busy_units), (num_contiguous_free_units - num_busy_units)); - } + uint32_t* index_end = index_start + (num_units - 1); + *index_start = *index_end = region_alloc_free_bit | num_units; } void region_allocator::print_map (const char* msg) @@ -3526,45 +3519,69 @@ void region_allocator::print_map (const char* msg) const char* heap_type = "UH"; dprintf (REGIONS_LOG, ("[%s]-----printing----%s", heap_type, msg)); - uint32_t* current_index = region_map_start; - uint32_t* end_index = region_map_end; - uint32_t* map_start = current_index; + uint32_t* current_index = region_map_left_start; + uint32_t* end_index = region_map_left_end; + uint32_t count_free_units = 0; - while (current_index < end_index) + for (int i = 0; i < 2; i++) { - uint32_t current_val = *current_index; - uint32_t current_num_units = get_num_units (current_val); - bool free_p = is_unit_memory_free (current_val); + while (current_index < end_index) + { + uint32_t current_val = *current_index; + uint32_t current_num_units = get_num_units (current_val); + bool free_p = is_unit_memory_free (current_val); - dprintf (REGIONS_LOG, ("[%s][%s: %Id]%d->%d", heap_type, (free_p ? "F" : "B"), (size_t)current_num_units, - (int)(current_index - map_start), - (int)(current_index - map_start + current_num_units))); + dprintf (REGIONS_LOG, ("[%s][%s: %Id]%d->%d", heap_type, (free_p ? "F" : "B"), (size_t)current_num_units, + (int)(current_index - region_map_left_start), + (int)(current_index - region_map_left_start + current_num_units))); - current_index += current_num_units; + if (free_p) + { + count_free_units += current_num_units; + } + + current_index += current_num_units; + } + current_index = region_map_right_start; + end_index = region_map_right_end; } + count_free_units += (uint32_t)(region_map_right_start - region_map_left_end); + assert(count_free_units == total_free_units); + uint32_t total_regions = (uint32_t)((global_region_end - global_region_start) / region_alignment); - dprintf (REGIONS_LOG, ("[%s]-----end printing----[%d total, used %d]\n", heap_type, total_regions, (end_index - map_start))); + dprintf (REGIONS_LOG, ("[%s]-----end printing----[%d total, left used %d, right used %d]\n", heap_type, total_regions, (region_map_left_end - region_map_left_start), (region_map_right_end - region_map_right_start))); #endif //_DEBUG } -uint8_t* region_allocator::allocate_end (uint32_t num_units) +uint8_t* region_allocator::allocate_end (uint32_t num_units, allocate_direction direction) { uint8_t* alloc = NULL; ASSERT_HOLDING_SPIN_LOCK (®ion_allocator_lock); - if (global_region_used < global_region_end) + if (global_region_left_used < global_region_right_used) { - size_t end_remaining = global_region_end - global_region_used; + size_t end_remaining = global_region_right_used - global_region_left_used; if ((end_remaining / region_alignment) >= num_units) { - make_busy_block (region_map_end, num_units); - region_map_end += num_units; - alloc = global_region_used; - global_region_used += num_units * region_alignment; + if (direction == allocate_forward) + { + make_busy_block (region_map_left_end, num_units); + region_map_left_end += num_units; + alloc = global_region_left_used; + global_region_left_used += num_units * region_alignment; + } + else + { + assert(direction == allocate_backward); + region_map_right_start -= num_units; + make_busy_block (region_map_right_start, num_units); + global_region_right_used -= num_units * region_alignment; + alloc = global_region_right_used; + } } } @@ -3596,81 +3613,85 @@ void region_allocator::leave_spin_lock() #endif //_DEBUG } -uint8_t* region_allocator::allocate (uint32_t num_units) +uint8_t* region_allocator::allocate (uint32_t num_units, allocate_direction direction) { enter_spin_lock(); - uint32_t* current_index = region_map_start; - uint32_t* end_index = region_map_end; + uint32_t* current_index; + uint32_t* end_index; + if (direction == allocate_forward) + { + current_index = region_map_left_start; + end_index = region_map_left_end; + } + else + { + assert(direction == allocate_backward); + current_index = region_map_right_end; + end_index = region_map_right_start; + } - dprintf (REGIONS_LOG, ("searching %d->%d", (int)(current_index - region_map_start), (int)(end_index - region_map_start))); - uint32_t* current_free_index_start = 0; - uint32_t num_contiguous_free_units = 0; - uint32_t last_num_free_units = 0; + dprintf (REGIONS_LOG, ("searching %d->%d", (int)(current_index - region_map_left_start), (int)(end_index - region_map_left_start))); print_map ("before alloc"); - while (current_index < end_index) + while (((direction == allocate_forward) && (current_index < end_index)) || + ((direction == allocate_backward) && (current_index > end_index))) { - uint32_t current_val = *current_index; + uint32_t current_val = *(current_index - ((direction == -1) ? 1 : 0)); uint32_t current_num_units = get_num_units (current_val); bool free_p = is_unit_memory_free (current_val); dprintf (REGIONS_LOG, ("ALLOC[%s: %Id]%d->%d", (free_p ? "F" : "B"), (size_t)current_num_units, - (int)(current_index - region_map_start), (int)(current_index + current_num_units - region_map_start))); + (int)(current_index - region_map_left_start), (int)(current_index + current_num_units - region_map_left_start))); if (free_p) { - if (!current_free_index_start) - { - current_free_index_start = current_index; - } - - last_num_free_units = current_num_units; - num_contiguous_free_units += current_num_units; - - if (num_contiguous_free_units >= num_units) + if (current_num_units >= num_units) { dprintf (REGIONS_LOG, ("found %Id contiguous free units(%d->%d), sufficient", - (size_t)num_contiguous_free_units, - (int)(current_free_index_start - region_map_start), - (int)(current_free_index_start - region_map_start + num_contiguous_free_units))); + (size_t)current_val, + (int)(current_index - region_map_left_start), + (int)(current_index - region_map_left_start + current_val))); - adjust_map (current_free_index_start, num_contiguous_free_units, num_units); + uint32_t* busy_block; + uint32_t* free_block; + if (direction == 1) + { + busy_block = current_index; + free_block = current_index + num_units; + } + else + { + busy_block = current_index - num_units; + free_block = current_index - current_num_units; + } + + make_busy_block (busy_block, num_units); + if ((current_num_units - num_units) > 0) + { + make_free_block (free_block, (current_num_units - num_units)); + } total_free_units -= num_units; print_map ("alloc: found in free"); leave_spin_lock(); - return region_address_of (current_free_index_start); + return region_address_of (busy_block); } } + + if (direction == allocate_forward) + { + current_index += current_num_units; + } else { - // Take this opportunity to coalesce free blocks. - if (num_contiguous_free_units > last_num_free_units) - { - dprintf (REGIONS_LOG, ("Observed %Id free units in multiple blocks(%Id), coalescing", - (size_t)num_contiguous_free_units, (size_t)last_num_free_units)); - make_free_block (current_free_index_start, num_contiguous_free_units); - } - current_free_index_start = 0; - num_contiguous_free_units = 0; + current_index -= current_num_units; } - - current_index += current_num_units; } - // If at this point current_free_index_start is not 0, it means we had - // free units at the end. Simply adjust used to be where the last free - // block starts. - if (current_free_index_start != 0) - { - global_region_used = region_address_of (current_free_index_start); - region_map_end = current_free_index_start; - } - - uint8_t* alloc = allocate_end (num_units); + uint8_t* alloc = allocate_end (num_units, direction); if (alloc) { @@ -3679,7 +3700,7 @@ uint8_t* region_allocator::allocate (uint32_t num_units) } else { - dprintf (REGIONS_LOG, ("couldn't find memory at the end! only %Id bytes left", (global_region_end - global_region_used))); + dprintf (REGIONS_LOG, ("couldn't find memory at the end! only %Id bytes left", (global_region_right_used - global_region_left_used))); } leave_spin_lock(); @@ -3689,9 +3710,8 @@ uint8_t* region_allocator::allocate (uint32_t num_units) // ETW TODO: need to fire create seg events for these methods. // FIRE_EVENT(GCCreateSegment_V1 -bool region_allocator::allocate_region (size_t size, uint8_t** start, uint8_t** end) +bool region_allocator::allocate_region (size_t size, uint8_t** start, uint8_t** end, allocate_direction direction) { - uint32_t* map_start = region_map_start; size_t alignment = region_alignment; size_t alloc_size = align_region_up (size); @@ -3700,7 +3720,7 @@ bool region_allocator::allocate_region (size_t size, uint8_t** start, uint8_t** uint8_t* alloc = NULL; dprintf (REGIONS_LOG, ("----GET %d-----", num_units)); - alloc = allocate (num_units); + alloc = allocate (num_units, direction); *start = alloc; *end = alloc + alloc_size; ret = (alloc != NULL); @@ -3710,44 +3730,75 @@ bool region_allocator::allocate_region (size_t size, uint8_t** start, uint8_t** bool region_allocator::allocate_basic_region (uint8_t** start, uint8_t** end) { - return allocate_region (region_alignment, start, end); + return allocate_region (region_alignment, start, end, allocate_forward); } // Large regions are 8x basic region sizes by default. If you need a larger region than that, // call allocate_region with the size. -bool region_allocator::allocate_large_region (uint8_t** start, uint8_t** end) +bool region_allocator::allocate_large_region (uint8_t** start, uint8_t** end, allocate_direction direction) { - return allocate_region (large_region_alignment, start, end); + return allocate_region (large_region_alignment, start, end, direction); } -void region_allocator::delete_region (uint8_t* start) +void region_allocator::delete_region (uint8_t* region_start) { enter_spin_lock(); - assert (is_region_aligned (start)); + assert (is_region_aligned (region_start)); print_map ("before delete"); - uint32_t* current_index = region_map_index_of (start); + uint32_t* current_index = region_map_index_of (region_start); uint32_t current_val = *current_index; assert (!is_unit_memory_free (current_val)); - dprintf (REGIONS_LOG, ("----DEL %d-----", (current_index - region_map_start))); + dprintf (REGIONS_LOG, ("----DEL %d-----", (current_index - region_map_left_start))); + uint32_t* region_end_index = current_index + current_val; + uint8_t* region_end = region_address_of (region_end_index); - uint8_t* region_end = region_address_of (current_index + current_val); - if (region_end == global_region_used) + int free_block_size = current_val; + uint32_t* free_index = current_index; + if ((current_index != region_map_left_start) && (current_index != region_map_right_start)) + { + uint32_t previous_val = *(current_index - 1); + if (is_unit_memory_free(previous_val)) + { + uint32_t previous_size = get_num_units (previous_val); + free_index -= previous_size; + free_block_size += previous_size; + } + } + if ((region_end != global_region_left_used) && (region_end != global_region_end)) { - region_map_end = current_index; - dprintf (REGIONS_LOG, ("adjust global used from %Ix to %Ix", - global_region_used, region_address_of (current_index))); - global_region_used = region_address_of (current_index); + uint32_t next_val = *region_end_index; + if (is_unit_memory_free(next_val)) + { + uint32_t next_size = get_num_units (next_val); + free_block_size += next_size; + region_end += next_size; + } + } + if (region_end == global_region_left_used) + { + region_map_left_end = free_index; + dprintf (REGIONS_LOG, ("adjust global left used from %Ix to %Ix", + global_region_left_used, region_address_of (free_index))); + global_region_left_used = region_address_of (free_index); + } + else if (region_start == global_region_right_used) + { + region_map_right_start = free_index + free_block_size; + dprintf (REGIONS_LOG, ("adjust global right used from %Ix to %Ix", + global_region_right_used, region_address_of (free_index + free_block_size))); + global_region_right_used = region_address_of (free_index + free_block_size); } else { - make_free_block (current_index, current_val); + make_free_block (free_index, free_block_size); } total_free_units += current_val; + print_map ("after delete"); leave_spin_lock(); @@ -10535,7 +10586,8 @@ bool gc_heap::initial_make_uoh_regions (int gen, gc_heap* hp) uint8_t* region_start; uint8_t* region_end; - if (!global_region_allocator.allocate_large_region (®ion_start, ®ion_end)) + // REGIONS TODO: allocate POH regions on the right + if (!global_region_allocator.allocate_large_region (®ion_start, ®ion_end, allocate_forward)) return false; size_t region_size = region_end - region_start; @@ -27551,8 +27603,9 @@ heap_segment* gc_heap::allocate_new_region (gc_heap* hp, int gen_num, bool uoh_p { uint8_t* start = 0; uint8_t* end = 0; + // REGIONS TODO: allocate POH regions on the right bool allocated_p = (uoh_p ? - global_region_allocator.allocate_large_region (&start, &end) : + global_region_allocator.allocate_large_region (&start, &end, allocate_forward) : global_region_allocator.allocate_basic_region (&start, &end)); if (!allocated_p) diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 0a1f76efbcf138..ad48fd19b8c0f0 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -5288,6 +5288,12 @@ class heap_segment #define region_alloc_free_bit (1 << (sizeof (uint32_t) * 8 - 1)) +enum allocate_direction +{ + allocate_forward = 1, + allocate_backward = -1, +}; + // The big space we reserve for regions is divided into units of region_alignment. // // SOH regions are all basic regions, meaning their size is the same as alignment. UOH regions @@ -5321,12 +5327,11 @@ class heap_segment class region_allocator { private: - // We need to start from an aligned address. This is the actual address of the reserved range. - uint8_t* actual_start; uint8_t* global_region_start; uint8_t* global_region_end; - uint8_t* global_region_used; + uint8_t* global_region_left_used; + uint8_t* global_region_right_used; uint32_t total_free_units; @@ -5337,22 +5342,21 @@ class region_allocator void enter_spin_lock(); void leave_spin_lock(); + uint32_t* region_map_left_start; + uint32_t* region_map_left_end; - uint32_t* region_map_start; - uint32_t* region_map_end; + uint32_t* region_map_right_start; + uint32_t* region_map_right_end; uint8_t* region_address_of (uint32_t* map_index); uint32_t* region_map_index_of (uint8_t* address); - uint8_t* allocate (uint32_t num_units); - uint8_t* allocate_end (uint32_t num_units); + uint8_t* allocate (uint32_t num_units, allocate_direction direction); + uint8_t* allocate_end (uint32_t num_units, allocate_direction direction); void make_busy_block (uint32_t* index_start, uint32_t num_units); void make_free_block (uint32_t* index_start, uint32_t num_units); - void adjust_map (uint32_t* current_free_index_start, - uint32_t num_contiguous_free_units, uint32_t num_busy_units); - void print_map (const char* msg); size_t align_region_up (size_t size) @@ -5382,13 +5386,13 @@ class region_allocator public: bool init (uint8_t* start, uint8_t* end, size_t alignment, uint8_t** lowest, uint8_t** highest); - bool allocate_region (size_t size, uint8_t** start, uint8_t** end); + bool allocate_region (size_t size, uint8_t** start, uint8_t** end, allocate_direction direction); bool allocate_basic_region (uint8_t** start, uint8_t** end); - bool allocate_large_region (uint8_t** start, uint8_t** end); + bool allocate_large_region (uint8_t** start, uint8_t** end, allocate_direction direction); void delete_region (uint8_t* start); uint32_t get_va_memory_load() { - return (uint32_t)((global_region_used - global_region_start) * 100.0 + return (uint32_t)(((global_region_left_used - global_region_start) + ((global_region_end - global_region_right_used)))* 100.0 / (global_region_end - global_region_start)); } // Returns the amount of free space + end of unused region space