< prev index next >
src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp
Print this page
rev 10654 : [backport] Disable ShHeapUncommit on too-large large-pages
rev 10657 : [backport] In update-refs, update all code-roots when in degen-gc
rev 10658 : [backport] Single marking bitmap
rev 10659 : [backport] Make sure bitmap is marked incomplete before bitmap resets
rev 10660 : [backport] Refactor bitmap cleaning
rev 10661 : [backport] Cleanup unused bitmap methods
rev 10667 : [backport] Print "process weakrefs" to disambiguate vs CM-with-UR
rev 10670 : [backport] Remove Parallel Cleanup counters
rev 10671 : [backport] Clean up declarations and uses of marked_object_iterate
rev 10673 : [backport] Homogenize unimplemented stubs handling
rev 10674 : [backport] Move ShenandoahAllocType and ShenandoahAllocRequest to separate file
rev 10675 : [backport] Inline ShHeap::prepare_concurrent_evacuation
rev 10676 : [backport] Remove ShHeap::region_in_collection_set in favor of SHR::in_cset
rev 10678 : [backport] Rename concurrentMark -> concurrent_mark
rev 10679 : [backport] Remove ShHeap::_heap_expansion_count
rev 10680 : [backport] Remove useless ShHeap::heap_region_iterate
rev 10681 : [backport] Inline trivial ShHeap methods right in header
rev 10684 : [backport] Remove unused ShenandoahEvacuateRootsClosure
rev 10685 : [backport] Rename ShHeap::shenandoahPolicy -> ShHeap::shenandoah_policy
rev 10686 : [backport] Handle ShHeap::time_since_last_millis for RMI users
rev 10687 : [backport] Inline ShHeap::monitoring_support into header
rev 10688 : [backport] Sort ShenandoahHeap methods/fields into logical groups
rev 10694 : [backport] Common liveness cache in ShHeap
rev 10695 : [backport] Inline/rename confusingly named methods in ShConcurrentMark
rev 10698 : [backport] Fix inconsistent "process weakrefs" logging
rev 10704 : [backport] Renaming ShenandoahParallelEvacuationTask to ShenandoahConcurrentEvacuationTask
rev 10705 : Fix shutdown deadlock due to blocking SATB flush periodic task
rev 10711 : [backport] Precleaning should use GC workers to do actual work
rev 10712 : [backport] Fix compilation errors due to missing spaces between string literal and macro
rev 10715 : [backport] Cleanup up superfluous newlines
rev 10718 : [backport] Partial infrastructure for suspendible workers
rev 10724 : [backport] Add JFR parallel and concurrent events (infrastructure)
rev 10737 : [backport] Support VerifyBeforeGC and VerifyAfterGC VM options
rev 10738 : [backport] Skip pre-evac verification and pacing on shortcut cycle path
rev 10740 : [backport] Protect more internal code from false sharing
rev 10743 : [backport] Clean up ShHeap::heap_region_iterate uses
rev 10744 : [backport] Parallel heap region iteration
rev 10745 : [backport] Parallelise "Clear Liveness"
rev 10746 : [backport] Parallelise "Complete Liveness"
rev 10748 : [backport] Handle metadata induced GC
rev 10750 : [backport] Remove fix-roots pass in ShHeap
rev 10753 : [backport] Protect risky conversion in ShenandoahHeap::millis_since_last_gc
rev 10755 : [backport] Make heuristics tell if we can process references or unload classes
rev 10758 : [backport] Initial committed regions count can be more than max
rev 10764 : [backport] Rename BrooksPointer to ShenandoahBrooksPointer
rev 10772 : [backport] Update copyrights
rev 10801 : [backport] Rename vm_operations_shenandoah.* to shenandoahVMOperations.*
*** 1,7 ****
/*
! * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
--- 1,7 ----
/*
! * Copyright (c) 2013, 2018, Red Hat, Inc. All rights reserved.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
*** 26,36 ****
#include "gc_implementation/shared/gcTimer.hpp"
#include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
#include "gc_implementation/shared/parallelCleaning.hpp"
! #include "gc_implementation/shenandoah/brooksPointer.hpp"
#include "gc_implementation/shenandoah/shenandoahAllocTracker.hpp"
#include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc_implementation/shenandoah/shenandoahConcurrentMark.hpp"
--- 26,36 ----
#include "gc_implementation/shared/gcTimer.hpp"
#include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
#include "gc_implementation/shared/parallelCleaning.hpp"
! #include "gc_implementation/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc_implementation/shenandoah/shenandoahAllocTracker.hpp"
#include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc_implementation/shenandoah/shenandoahConcurrentMark.hpp"
*** 50,62 ****
#include "gc_implementation/shenandoah/shenandoahPacer.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
#include "gc_implementation/shenandoah/shenandoahUtils.hpp"
#include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
#include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
#include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp"
#include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
- #include "gc_implementation/shenandoah/vm_operations_shenandoah.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
--- 50,62 ----
#include "gc_implementation/shenandoah/shenandoahPacer.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
#include "gc_implementation/shenandoah/shenandoahUtils.hpp"
#include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
#include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
+ #include "gc_implementation/shenandoah/shenandoahVMOperations.hpp"
#include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp"
#include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp"
#include "gc_implementation/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
*** 79,129 ****
void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_nv(p); }
#endif
- const char* ShenandoahHeap::name() const {
- return "Shenandoah";
- }
-
class ShenandoahPretouchTask : public AbstractGangTask {
private:
ShenandoahRegionIterator _regions;
const size_t _bitmap_size;
const size_t _page_size;
! char* _bitmap0_base;
! char* _bitmap1_base;
public:
! ShenandoahPretouchTask(char* bitmap0_base, char* bitmap1_base, size_t bitmap_size,
! size_t page_size) :
AbstractGangTask("Shenandoah PreTouch"),
_bitmap_size(bitmap_size),
_page_size(page_size),
! _bitmap0_base(bitmap0_base),
! _bitmap1_base(bitmap1_base) {}
virtual void work(uint worker_id) {
ShenandoahHeapRegion* r = _regions.next();
while (r != NULL) {
os::pretouch_memory((char*) r->bottom(), (char*) r->end());
size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
assert (end <= _bitmap_size, err_msg("end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size));
! os::pretouch_memory(_bitmap0_base + start, _bitmap0_base + end);
! os::pretouch_memory(_bitmap1_base + start, _bitmap1_base + end);
r = _regions.next();
}
}
};
jint ShenandoahHeap::initialize() {
CollectedHeap::pre_initialize();
! BrooksPointer::initial_checks();
initialize_heuristics();
size_t init_byte_size = collector_policy()->initial_heap_byte_size();
size_t max_byte_size = collector_policy()->max_heap_byte_size();
--- 79,121 ----
void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_nv(p); }
#endif
class ShenandoahPretouchTask : public AbstractGangTask {
private:
ShenandoahRegionIterator _regions;
const size_t _bitmap_size;
const size_t _page_size;
! char* _bitmap_base;
public:
! ShenandoahPretouchTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
AbstractGangTask("Shenandoah PreTouch"),
_bitmap_size(bitmap_size),
_page_size(page_size),
! _bitmap_base(bitmap_base) {}
virtual void work(uint worker_id) {
ShenandoahHeapRegion* r = _regions.next();
while (r != NULL) {
os::pretouch_memory((char*) r->bottom(), (char*) r->end());
size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
assert (end <= _bitmap_size, err_msg("end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size));
! os::pretouch_memory(_bitmap_base + start, _bitmap_base + end);
r = _regions.next();
}
}
};
jint ShenandoahHeap::initialize() {
CollectedHeap::pre_initialize();
! ShenandoahBrooksPointer::initial_checks();
initialize_heuristics();
size_t init_byte_size = collector_policy()->initial_heap_byte_size();
size_t max_byte_size = collector_policy()->max_heap_byte_size();
*** 150,164 ****
set_barrier_set(new ShenandoahBarrierSet(this));
ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size);
_num_regions = ShenandoahHeapRegion::region_count();
size_t num_committed_regions = init_byte_size / ShenandoahHeapRegion::region_size_bytes();
_initial_size = num_committed_regions * ShenandoahHeapRegion::region_size_bytes();
_committed = _initial_size;
! log_info(gc, heap)("Initialize Shenandoah heap with initial size " SIZE_FORMAT " bytes", init_byte_size);
if (!os::commit_memory(pgc_rs.base(), _initial_size, false)) {
vm_exit_out_of_memory(_initial_size, OOM_MMAP_ERROR, "Shenandoah failed to initialize heap");
}
size_t reg_size_words = ShenandoahHeapRegion::region_size_words();
--- 142,161 ----
set_barrier_set(new ShenandoahBarrierSet(this));
ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size);
_num_regions = ShenandoahHeapRegion::region_count();
+
size_t num_committed_regions = init_byte_size / ShenandoahHeapRegion::region_size_bytes();
+ num_committed_regions = MIN2(num_committed_regions, _num_regions);
+ assert(num_committed_regions <= _num_regions, "sanity");
+
_initial_size = num_committed_regions * ShenandoahHeapRegion::region_size_bytes();
_committed = _initial_size;
! log_info(gc, heap)("Initialize Shenandoah heap with initial size " SIZE_FORMAT "%s",
! byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size));
if (!os::commit_memory(pgc_rs.base(), _initial_size, false)) {
vm_exit_out_of_memory(_initial_size, OOM_MMAP_ERROR, "Shenandoah failed to initialize heap");
}
size_t reg_size_words = ShenandoahHeapRegion::region_size_words();
*** 175,185 ****
} else {
_pacer = NULL;
}
assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
! err_msg("misaligned heap: "PTR_FORMAT, p2i(base())));
// The call below uses stuff (the SATB* things) that are in G1, but probably
// belong into a shared location.
JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
SATB_Q_FL_lock,
--- 172,182 ----
} else {
_pacer = NULL;
}
assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
! err_msg("misaligned heap: " PTR_FORMAT, p2i(base())));
// The call below uses stuff (the SATB* things) that are in G1, but probably
// belong into a shared location.
JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
SATB_Q_FL_lock,
*** 215,236 ****
err_msg("Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
_bitmap_bytes_per_slice, bitmap_page_size));
ReservedSpace bitmap0(_bitmap_size, bitmap_page_size);
MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
! _bitmap0_region = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
!
! ReservedSpace bitmap1(_bitmap_size, bitmap_page_size);
! MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
! _bitmap1_region = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
size_t bitmap_init_commit = _bitmap_bytes_per_slice *
align_size_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
! os::commit_memory_or_exit((char *) (_bitmap0_region.start()), bitmap_init_commit, false,
! "couldn't allocate initial bitmap");
! os::commit_memory_or_exit((char *) (_bitmap1_region.start()), bitmap_init_commit, false,
"couldn't allocate initial bitmap");
size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
if (ShenandoahVerify) {
--- 212,227 ----
err_msg("Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
_bitmap_bytes_per_slice, bitmap_page_size));
ReservedSpace bitmap0(_bitmap_size, bitmap_page_size);
MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
! _bitmap_region = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
size_t bitmap_init_commit = _bitmap_bytes_per_slice *
align_size_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
! os::commit_memory_or_exit((char *) (_bitmap_region.start()), bitmap_init_commit, false,
"couldn't allocate initial bitmap");
size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
if (ShenandoahVerify) {
*** 241,268 ****
MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
_verification_bit_map.initialize(_heap_region, verify_bitmap_region);
_verifier = new ShenandoahVerifier(this, &_verification_bit_map);
}
! _complete_marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap0_region, _num_regions);
! _next_marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap1_region, _num_regions);
{
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < _num_regions; i++) {
ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this,
(HeapWord*) pgc_rs.base() + reg_size_words * i,
reg_size_words,
i,
i < num_committed_regions);
! _complete_marking_context->set_top_at_mark_start(i, r->bottom());
! _next_marking_context->set_top_at_mark_start(i, r->bottom());
_regions[i] = r;
assert(!collection_set()->is_in(i), "New region should not be in collection set");
}
_free_set->rebuild();
}
if (ShenandoahAlwaysPreTouch) {
assert (!AlwaysPreTouch, "Should have been overridden");
--- 232,260 ----
MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
_verification_bit_map.initialize(_heap_region, verify_bitmap_region);
_verifier = new ShenandoahVerifier(this, &_verification_bit_map);
}
! _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
{
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < _num_regions; i++) {
ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this,
(HeapWord*) pgc_rs.base() + reg_size_words * i,
reg_size_words,
i,
i < num_committed_regions);
! _marking_context->initialize_top_at_mark_start(r);
_regions[i] = r;
assert(!collection_set()->is_in(i), "New region should not be in collection set");
}
+ // Initialize to complete
+ _marking_context->mark_complete();
+
_free_set->rebuild();
}
if (ShenandoahAlwaysPreTouch) {
assert (!AlwaysPreTouch, "Should have been overridden");
*** 272,286 ****
// we touch the region and the corresponding bitmaps from the same thread.
ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
_num_regions, page_size);
! ShenandoahPretouchTask cl(bitmap0.base(), bitmap1.base(), _bitmap_size, page_size);
_workers->run_task(&cl);
}
-
// Reserve aux bitmap for use in object_iterate(). We don't commit it here.
ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
_aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
_aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
--- 264,277 ----
// we touch the region and the corresponding bitmaps from the same thread.
ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
_num_regions, page_size);
! ShenandoahPretouchTask cl(bitmap0.base(), _bitmap_size, page_size);
_workers->run_task(&cl);
}
// Reserve aux bitmap for use in object_iterate(). We don't commit it here.
ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
_aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
_aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
*** 297,306 ****
--- 288,303 ----
_control_thread = new ShenandoahControlThread();
ShenandoahCodeRoots::initialize();
+ _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC);
+ for (uint worker = 0; worker < _max_workers; worker++) {
+ _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC);
+ Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort));
+ }
+
return JNI_OK;
}
#ifdef _MSC_VER
#pragma warning( push )
*** 348,365 ****
_collection_set(NULL),
_update_refs_iterator(this),
_bytes_allocated_since_gc_start(0),
_max_workers((uint)MAX2(ConcGCThreads, ParallelGCThreads)),
_ref_processor(NULL),
! _complete_marking_context(NULL),
! _next_marking_context(NULL),
_aux_bit_map(),
_verifier(NULL),
_pacer(NULL),
- #ifdef ASSERT
- _heap_expansion_count(0),
- #endif
_gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
_phase_timings(NULL),
_alloc_tracker(NULL)
{
log_info(gc, init)("GC threads: " UINTX_FORMAT " parallel, " UINTX_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
--- 345,358 ----
_collection_set(NULL),
_update_refs_iterator(this),
_bytes_allocated_since_gc_start(0),
_max_workers((uint)MAX2(ConcGCThreads, ParallelGCThreads)),
_ref_processor(NULL),
! _marking_context(NULL),
_aux_bit_map(),
_verifier(NULL),
_pacer(NULL),
_gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
_phase_timings(NULL),
_alloc_tracker(NULL)
{
log_info(gc, init)("GC threads: " UINTX_FORMAT " parallel, " UINTX_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
*** 382,421 ****
#ifdef _MSC_VER
#pragma warning( pop )
#endif
! class ShenandoahResetNextBitmapTask : public AbstractGangTask {
private:
ShenandoahRegionIterator _regions;
public:
! ShenandoahResetNextBitmapTask() :
AbstractGangTask("Parallel Reset Bitmap Task") {}
void work(uint worker_id) {
ShenandoahHeapRegion* region = _regions.next();
ShenandoahHeap* heap = ShenandoahHeap::heap();
! ShenandoahMarkingContext* const ctx = heap->next_marking_context();
while (region != NULL) {
if (heap->is_bitmap_slice_committed(region)) {
! HeapWord* bottom = region->bottom();
! HeapWord* top = ctx->top_at_mark_start(region->region_number());
! if (top > bottom) {
! ctx->clear_bitmap(bottom, top);
! }
! assert(ctx->is_bitmap_clear_range(bottom, region->end()), "must be clear");
}
region = _regions.next();
}
}
};
! void ShenandoahHeap::reset_next_mark_bitmap() {
assert_gc_workers(_workers->active_workers());
! ShenandoahResetNextBitmapTask task;
_workers->run_task(&task);
}
void ShenandoahHeap::print_on(outputStream* st) const {
st->print_cr("Shenandoah Heap");
--- 375,410 ----
#ifdef _MSC_VER
#pragma warning( pop )
#endif
! class ShenandoahResetBitmapTask : public AbstractGangTask {
private:
ShenandoahRegionIterator _regions;
public:
! ShenandoahResetBitmapTask() :
AbstractGangTask("Parallel Reset Bitmap Task") {}
void work(uint worker_id) {
ShenandoahHeapRegion* region = _regions.next();
ShenandoahHeap* heap = ShenandoahHeap::heap();
! ShenandoahMarkingContext* const ctx = heap->marking_context();
while (region != NULL) {
if (heap->is_bitmap_slice_committed(region)) {
! ctx->clear_bitmap(region);
}
region = _regions.next();
}
}
};
! void ShenandoahHeap::reset_mark_bitmap() {
assert_gc_workers(_workers->active_workers());
+ mark_incomplete_marking_context();
! ShenandoahResetBitmapTask task;
_workers->run_task(&task);
}
void ShenandoahHeap::print_on(outputStream* st) const {
st->print_cr("Shenandoah Heap");
*** 532,546 ****
size_t ShenandoahHeap::capacity() const {
return num_regions() * ShenandoahHeapRegion::region_size_bytes();
}
- bool ShenandoahHeap::is_maximal_no_gc() const {
- Unimplemented();
- return true;
- }
-
size_t ShenandoahHeap::max_capacity() const {
return _num_regions * ShenandoahHeapRegion::region_size_bytes();
}
size_t ShenandoahHeap::initial_capacity() const {
--- 521,530 ----
*** 551,569 ****
HeapWord* heap_base = (HeapWord*) base();
HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
return p >= heap_base && p < last_region_end;
}
- bool ShenandoahHeap::is_in_partial_collection(const void* p ) {
- Unimplemented();
- return false;
- }
-
- bool ShenandoahHeap::is_scavengable(const void* p) {
- return true;
- }
-
void ShenandoahHeap::op_uncommit(double shrink_before) {
assert (ShenandoahUncommit, "should be enabled");
size_t count = 0;
for (size_t i = 0; i < num_regions(); i++) {
--- 535,544 ----
*** 633,650 ****
thread->gclab().fill(obj, obj + size, actual_size);
return obj;
}
HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
! ShenandoahAllocationRequest req = ShenandoahAllocationRequest::for_tlab(word_size);
return allocate_memory(req);
}
HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
size_t word_size,
size_t* actual_size) {
! ShenandoahAllocationRequest req = ShenandoahAllocationRequest::for_gclab(min_size, word_size);
HeapWord* res = allocate_memory(req);
if (res != NULL) {
*actual_size = req.actual_size();
} else {
*actual_size = 0;
--- 608,625 ----
thread->gclab().fill(obj, obj + size, actual_size);
return obj;
}
HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
! ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(word_size);
return allocate_memory(req);
}
HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
size_t word_size,
size_t* actual_size) {
! ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
HeapWord* res = allocate_memory(req);
if (res != NULL) {
*actual_size = req.actual_size();
} else {
*actual_size = 0;
*** 662,672 ****
ShenandoahHeap* ShenandoahHeap::heap_no_check() {
CollectedHeap* heap = Universe::heap();
return (ShenandoahHeap*) heap;
}
! HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocationRequest& req) {
ShenandoahAllocTrace trace_alloc(req.size(), req.type());
intptr_t pacer_epoch = 0;
bool in_new_region = false;
HeapWord* result = NULL;
--- 637,647 ----
ShenandoahHeap* ShenandoahHeap::heap_no_check() {
CollectedHeap* heap = Universe::heap();
return (ShenandoahHeap*) heap;
}
! HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
ShenandoahAllocTrace trace_alloc(req.size(), req.type());
intptr_t pacer_epoch = 0;
bool in_new_region = false;
HeapWord* result = NULL;
*** 691,701 ****
// Then, we need to make sure the allocation was retried after at least one
// Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
size_t tries = 0;
! while (result == NULL && last_gc_made_progress()) {
tries++;
control_thread()->handle_alloc_failure(req.size());
result = allocate_memory_under_lock(req, in_new_region);
}
--- 666,676 ----
// Then, we need to make sure the allocation was retried after at least one
// Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
size_t tries = 0;
! while (result == NULL && _progress_last_gc.is_set()) {
tries++;
control_thread()->handle_alloc_failure(req.size());
result = allocate_memory_under_lock(req, in_new_region);
}
*** 720,730 ****
size_t requested = req.size();
size_t actual = req.actual_size();
assert (req.is_lab_alloc() || (requested == actual),
err_msg("Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
! alloc_type_to_string(req.type()), requested, actual));
if (req.is_mutator_alloc()) {
notify_mutator_alloc_words(actual, false);
// If we requested more than we were granted, give the rest back to pacer.
--- 695,705 ----
size_t requested = req.size();
size_t actual = req.actual_size();
assert (req.is_lab_alloc() || (requested == actual),
err_msg("Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
! ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual));
if (req.is_mutator_alloc()) {
notify_mutator_alloc_words(actual, false);
// If we requested more than we were granted, give the rest back to pacer.
*** 739,760 ****
}
return result;
}
! HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocationRequest& req, bool& in_new_region) {
ShenandoahHeapLocker locker(lock());
return _free_set->allocate(req, in_new_region);
}
HeapWord* ShenandoahHeap::mem_allocate(size_t size,
bool* gc_overhead_limit_was_exceeded) {
! ShenandoahAllocationRequest req = ShenandoahAllocationRequest::for_shared(size + BrooksPointer::word_size());
HeapWord* filler = allocate_memory(req);
! HeapWord* result = filler + BrooksPointer::word_size();
if (filler != NULL) {
! BrooksPointer::initialize(oop(result));
assert(! in_collection_set(result), "never allocate in targetted region");
return result;
} else {
return NULL;
--- 714,735 ----
}
return result;
}
! HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
ShenandoahHeapLocker locker(lock());
return _free_set->allocate(req, in_new_region);
}
HeapWord* ShenandoahHeap::mem_allocate(size_t size,
bool* gc_overhead_limit_was_exceeded) {
! ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size + ShenandoahBrooksPointer::word_size());
HeapWord* filler = allocate_memory(req);
! HeapWord* result = filler + ShenandoahBrooksPointer::word_size();
if (filler != NULL) {
! ShenandoahBrooksPointer::initialize(oop(result));
assert(! in_collection_set(result), "never allocate in targetted region");
return result;
} else {
return NULL;
*** 777,787 ****
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
if (_heap->in_collection_set(obj)) {
! shenandoah_assert_marked_complete(p, obj);
oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
if (oopDesc::unsafe_equals(resolved, obj)) {
bool evac;
resolved = _heap->evacuate_object(obj, _thread, evac);
}
--- 752,762 ----
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
if (_heap->in_collection_set(obj)) {
! shenandoah_assert_marked(p, obj);
oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
if (oopDesc::unsafe_equals(resolved, obj)) {
bool evac;
resolved = _heap->evacuate_object(obj, _thread, evac);
}
*** 797,874 ****
void do_oop(narrowOop* p) {
do_oop_work(p);
}
};
! class ShenandoahEvacuateRootsClosure: public ExtendedOopClosure {
! private:
! ShenandoahHeap* _heap;
! Thread* _thread;
! public:
! ShenandoahEvacuateRootsClosure() :
! _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
! }
!
! private:
! template <class T>
! void do_oop_work(T* p) {
! T o = oopDesc::load_heap_oop(p);
! if (! oopDesc::is_null(o)) {
! oop obj = oopDesc::decode_heap_oop_not_null(o);
! if (_heap->in_collection_set(obj)) {
! oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
! if (oopDesc::unsafe_equals(resolved, obj)) {
! bool evac;
! _heap->evacuate_object(obj, _thread, evac);
! }
! }
! }
! }
!
! public:
! void do_oop(oop* p) {
! do_oop_work(p);
! }
! void do_oop(narrowOop* p) {
! do_oop_work(p);
! }
! };
!
! class ShenandoahParallelEvacuateRegionObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
Thread* const _thread;
public:
! ShenandoahParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
_heap(heap), _thread(Thread::current()) {}
void do_object(oop p) {
! shenandoah_assert_marked_complete(NULL, p);
if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_forwarded_not_null(p))) {
bool evac;
_heap->evacuate_object(p, _thread, evac);
}
}
};
! class ShenandoahParallelEvacuationTask : public AbstractGangTask {
private:
ShenandoahHeap* const _sh;
ShenandoahCollectionSet* const _cs;
!
public:
! ShenandoahParallelEvacuationTask(ShenandoahHeap* sh,
! ShenandoahCollectionSet* cs) :
AbstractGangTask("Parallel Evacuation Task"),
_sh(sh),
! _cs(cs) {}
void work(uint worker_id) {
- ShenandoahWorkerSession worker_session(worker_id);
ShenandoahEvacOOMScope oom_evac_scope;
! ShenandoahParallelEvacuateRegionObjectClosure cl(_sh);
ShenandoahHeapRegion* r;
while ((r =_cs->claim_next()) != NULL) {
assert(r->has_live(), "all-garbage regions are reclaimed early");
_sh->marked_object_iterate(r, &cl);
--- 772,827 ----
void do_oop(narrowOop* p) {
do_oop_work(p);
}
};
! class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
Thread* const _thread;
public:
! ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
_heap(heap), _thread(Thread::current()) {}
void do_object(oop p) {
! shenandoah_assert_marked(NULL, p);
if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_forwarded_not_null(p))) {
bool evac;
_heap->evacuate_object(p, _thread, evac);
}
}
};
! class ShenandoahEvacuationTask : public AbstractGangTask {
private:
ShenandoahHeap* const _sh;
ShenandoahCollectionSet* const _cs;
! bool _concurrent;
public:
! ShenandoahEvacuationTask(ShenandoahHeap* sh,
! ShenandoahCollectionSet* cs,
! bool concurrent) :
AbstractGangTask("Parallel Evacuation Task"),
_sh(sh),
! _cs(cs),
! _concurrent(concurrent)
! {}
void work(uint worker_id) {
ShenandoahEvacOOMScope oom_evac_scope;
+ if (_concurrent) {
+ ShenandoahConcurrentWorkerSession worker_session(worker_id);
+ do_work();
+ } else {
+ ShenandoahParallelWorkerSession worker_session(worker_id);
+ do_work();
+ }
+ }
! private:
! void do_work() {
! ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
ShenandoahHeapRegion* r;
while ((r =_cs->claim_next()) != NULL) {
assert(r->has_live(), "all-garbage regions are reclaimed early");
_sh->marked_object_iterate(r, &cl);
*** 907,918 ****
}
void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
assert(start->is_humongous_start(), "reclaim regions starting with the first one");
! oop humongous_obj = oop(start->bottom() + BrooksPointer::word_size());
! size_t size = humongous_obj->size() + BrooksPointer::word_size();
size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
size_t index = start->region_number() + required_regions - 1;
assert(!start->has_live(), "liveness must be zero");
--- 860,871 ----
}
void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
assert(start->is_humongous_start(), "reclaim regions starting with the first one");
! oop humongous_obj = oop(start->bottom() + ShenandoahBrooksPointer::word_size());
! size_t size = humongous_obj->size() + ShenandoahBrooksPointer::word_size();
size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
size_t index = start->region_number() + required_regions - 1;
assert(!start->has_live(), "liveness must be zero");
*** 920,984 ****
// Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
// as it expects that every region belongs to a humongous region starting with a humongous start region.
ShenandoahHeapRegion* region = get_region(index --);
assert(region->is_humongous(), "expect correct humongous start or continuation");
! assert(!in_collection_set(region), "Humongous region should not be in collection set");
! region->make_trash();
}
}
- #ifdef ASSERT
- class ShenandoahCheckCollectionSetClosure: public ShenandoahHeapRegionClosure {
- bool heap_region_do(ShenandoahHeapRegion* r) {
- assert(! ShenandoahHeap::heap()->in_collection_set(r), "Should have been cleared by now");
- return false;
- }
- };
- #endif
-
- void ShenandoahHeap::prepare_for_concurrent_evacuation() {
- if (!cancelled_gc()) {
- make_parsable(true);
-
- if (ShenandoahVerify) {
- verifier()->verify_after_concmark();
- }
-
- trash_cset_regions();
-
- // NOTE: This needs to be done during a stop the world pause, because
- // putting regions into the collection set concurrently with Java threads
- // will create a race. In particular, acmp could fail because when we
- // resolve the first operand, the containing region might not yet be in
- // the collection set, and thus return the original oop. When the 2nd
- // operand gets resolved, the region could be in the collection set
- // and the oop gets evacuated. If both operands have originally been
- // the same, we get false negatives.
-
- {
- ShenandoahHeapLocker locker(lock());
- _collection_set->clear();
- _free_set->clear();
-
- #ifdef ASSERT
- ShenandoahCheckCollectionSetClosure ccsc;
- heap_region_iterate(&ccsc);
- #endif
-
- heuristics()->choose_collection_set(_collection_set);
- _free_set->rebuild();
- }
-
- if (ShenandoahVerify) {
- verifier()->verify_before_evacuation();
- }
- }
- }
-
-
class ShenandoahRetireGCLABClosure : public ThreadClosure {
private:
bool _retire;
public:
ShenandoahRetireGCLABClosure(bool retire) : _retire(retire) {};
--- 873,888 ----
// Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
// as it expects that every region belongs to a humongous region starting with a humongous start region.
ShenandoahHeapRegion* region = get_region(index --);
assert(region->is_humongous(), "expect correct humongous start or continuation");
! assert(!region->is_cset(), "Humongous region should not be in collection set");
! region->make_trash_immediate();
}
}
class ShenandoahRetireGCLABClosure : public ThreadClosure {
private:
bool _retire;
public:
ShenandoahRetireGCLABClosure(bool retire) : _retire(retire) {};
*** 997,1048 ****
_workers->threads_do(&cl);
}
}
class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
ShenandoahRootEvacuator* _rp;
- public:
ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
AbstractGangTask("Shenandoah evacuate and update roots"),
! _rp(rp)
! {
! // Nothing else to do.
! }
void work(uint worker_id) {
! ShenandoahWorkerSession worker_session(worker_id);
ShenandoahEvacOOMScope oom_evac_scope;
ShenandoahEvacuateUpdateRootsClosure cl;
MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
_rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
}
};
- class ShenandoahFixRootsTask : public AbstractGangTask {
- ShenandoahRootEvacuator* _rp;
- public:
-
- ShenandoahFixRootsTask(ShenandoahRootEvacuator* rp) :
- AbstractGangTask("Shenandoah update roots"),
- _rp(rp)
- {
- // Nothing else to do.
- }
-
- void work(uint worker_id) {
- ShenandoahWorkerSession worker_session(worker_id);
- ShenandoahEvacOOMScope oom_evac_scope;
- ShenandoahUpdateRefsClosure cl;
- MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
-
- _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
- }
- };
void ShenandoahHeap::evacuate_and_update_roots() {
-
COMPILER2_PRESENT(DerivedPointerTable::clear());
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
{
--- 901,929 ----
_workers->threads_do(&cl);
}
}
class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
+ private:
ShenandoahRootEvacuator* _rp;
+ public:
ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
AbstractGangTask("Shenandoah evacuate and update roots"),
! _rp(rp) {}
void work(uint worker_id) {
! ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahEvacOOMScope oom_evac_scope;
ShenandoahEvacuateUpdateRootsClosure cl;
MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
_rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
}
};
void ShenandoahHeap::evacuate_and_update_roots() {
COMPILER2_PRESENT(DerivedPointerTable::clear());
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
{
*** 1050,1091 ****
ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
workers()->run_task(&roots_task);
}
COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
-
- if (cancelled_gc()) {
- // If initial evacuation has been cancelled, we need to update all references
- // after all workers have finished. Otherwise we might run into the following problem:
- // GC thread 1 cannot allocate anymore, thus evacuation fails, leaves from-space ptr of object X.
- // GC thread 2 evacuates the same object X to to-space
- // which leaves a truly dangling from-space reference in the first root oop*. This must not happen.
- // clear() and update_pointers() must always be called in pairs,
- // cannot nest with above clear()/update_pointers().
- COMPILER2_PRESENT(DerivedPointerTable::clear());
- ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
- ShenandoahFixRootsTask update_roots_task(&rp);
- workers()->run_task(&update_roots_task);
- COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
- }
}
-
void ShenandoahHeap::roots_iterate(OopClosure* cl) {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
CodeBlobToOopClosure blobsCl(cl, false);
CLDToOopClosure cldCl(cl);
ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
rp.process_all_roots(cl, NULL, &cldCl, &blobsCl, NULL, 0);
}
- bool ShenandoahHeap::supports_tlab_allocation() const {
- return true;
- }
-
size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
// Returns size in bytes
return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
}
--- 931,952 ----
*** 1123,1167 ****
ShenandoahAccumulateStatisticsGCLABClosure cl;
Threads::java_threads_do(&cl);
_workers->threads_do(&cl);
}
- bool ShenandoahHeap::can_elide_tlab_store_barriers() const {
- return true;
- }
-
- oop ShenandoahHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) {
- // Overridden to do nothing.
- return new_obj;
- }
-
- bool ShenandoahHeap::can_elide_initializing_store_barrier(oop new_obj) {
- return true;
- }
-
- bool ShenandoahHeap::card_mark_must_follow_store() const {
- return false;
- }
-
- bool ShenandoahHeap::supports_heap_inspection() const {
- return false;
- }
-
void ShenandoahHeap::collect(GCCause::Cause cause) {
! _control_thread->handle_explicit_gc(cause);
}
void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
//assert(false, "Shouldn't need to do full collections");
}
- AdaptiveSizePolicy* ShenandoahHeap::size_policy() {
- Unimplemented();
- return NULL;
-
- }
-
CollectorPolicy* ShenandoahHeap::collector_policy() const {
return _shenandoah_policy;
}
void ShenandoahHeap::resize_tlabs() {
--- 984,1001 ----
ShenandoahAccumulateStatisticsGCLABClosure cl;
Threads::java_threads_do(&cl);
_workers->threads_do(&cl);
}
void ShenandoahHeap::collect(GCCause::Cause cause) {
! _control_thread->request_gc(cause);
}
void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
//assert(false, "Shouldn't need to do full collections");
}
CollectorPolicy* ShenandoahHeap::collector_policy() const {
return _shenandoah_policy;
}
void ShenandoahHeap::resize_tlabs() {
*** 1190,1200 ****
Space* sp = heap_region_containing(addr);
return sp->block_is_obj(addr);
}
jlong ShenandoahHeap::millis_since_last_gc() {
! return 0;
}
void ShenandoahHeap::prepare_for_verify() {
if (SafepointSynchronize::is_at_safepoint()) {
make_parsable(false);
--- 1024,1036 ----
Space* sp = heap_region_containing(addr);
return sp->block_is_obj(addr);
}
jlong ShenandoahHeap::millis_since_last_gc() {
! double v = heuristics()->time_since_last_gc() * 1000;
! assert(0 <= v && v <= max_jlong, err_msg("value should fit: %f", v));
! return (jlong)v;
}
void ShenandoahHeap::prepare_for_verify() {
if (SafepointSynchronize::is_at_safepoint()) {
make_parsable(false);
*** 1222,1232 ****
phase_timings()->print_on(out);
out->cr();
out->cr();
! shenandoahPolicy()->print_gc_stats(out);
out->cr();
out->cr();
if (ShenandoahPacing) {
--- 1058,1068 ----
phase_timings()->print_on(out);
out->cr();
out->cr();
! shenandoah_policy()->print_gc_stats(out);
out->cr();
out->cr();
if (ShenandoahPacing) {
*** 1314,1323 ****
--- 1150,1162 ----
if (!os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
return;
}
+ // Reset bitmap
+ _aux_bit_map.clear();
+
Stack<oop,mtGC> oop_stack;
// First, we process all GC roots. This populates the work stack with initial objects.
ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
*** 1352,1364 ****
class ShenandoahSpaceClosureRegionClosure: public ShenandoahHeapRegionClosure {
SpaceClosure* _cl;
public:
ShenandoahSpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}
! bool heap_region_do(ShenandoahHeapRegion* r) {
_cl->do_space(r);
- return false;
}
};
void ShenandoahHeap::space_iterate(SpaceClosure* cl) {
ShenandoahSpaceClosureRegionClosure blk(cl);
--- 1191,1202 ----
class ShenandoahSpaceClosureRegionClosure: public ShenandoahHeapRegionClosure {
SpaceClosure* _cl;
public:
ShenandoahSpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}
! void heap_region_do(ShenandoahHeapRegion* r) {
_cl->do_space(r);
}
};
void ShenandoahHeap::space_iterate(SpaceClosure* cl) {
ShenandoahSpaceClosureRegionClosure blk(cl);
*** 1376,1447 ****
void ShenandoahHeap::gc_epilogue(bool b) {
Unimplemented();
}
! // Apply blk->heap_region_do() on all committed regions in address order,
! // terminating the iteration early if heap_region_do() returns true.
! void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_cset_regions, bool skip_humongous_continuation) const {
for (size_t i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion* current = get_region(i);
! if (skip_humongous_continuation && current->is_humongous_continuation()) {
! continue;
}
- if (skip_cset_regions && in_collection_set(current)) {
- continue;
}
- if (blk->heap_region_do(current)) {
- return;
}
}
}
class ShenandoahClearLivenessClosure : public ShenandoahHeapRegionClosure {
private:
! ShenandoahHeap* sh;
public:
! ShenandoahClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) {}
! bool heap_region_do(ShenandoahHeapRegion* r) {
r->clear_live_data();
! sh->next_marking_context()->set_top_at_mark_start(r->region_number(), r->top());
! return false;
}
- };
void ShenandoahHeap::op_init_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
! assert(next_marking_context()->is_bitmap_clear(), "need clear marking bitmap");
if (ShenandoahVerify) {
verifier()->verify_before_concmark();
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::accumulate_stats);
accumulate_statistics_tlabs();
}
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
make_parsable(true);
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::clear_liveness);
! ShenandoahClearLivenessClosure clc(this);
! heap_region_iterate(&clc);
}
// Make above changes visible to worker threads
OrderAccess::fence();
! concurrentMark()->init_mark_roots();
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
resize_tlabs();
}
--- 1214,1330 ----
void ShenandoahHeap::gc_epilogue(bool b) {
Unimplemented();
}
! void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
for (size_t i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion* current = get_region(i);
! blk->heap_region_do(current);
! }
! }
!
! class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
! private:
! ShenandoahHeap* const _heap;
! ShenandoahHeapRegionClosure* const _blk;
!
! char _pad0[DEFAULT_CACHE_LINE_SIZE];
! volatile jint _index;
! char _pad1[DEFAULT_CACHE_LINE_SIZE];
!
! public:
! ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
! AbstractGangTask("Parallel Region Task"),
! _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
!
! void work(uint worker_id) {
! jint stride = (jint)ShenandoahParallelRegionStride;
!
! jint max = (jint)_heap->num_regions();
! while (_index < max) {
! jint cur = Atomic::add(stride, &_index) - stride;
! jint start = cur;
! jint end = MIN2(cur + stride, max);
! if (start >= max) break;
!
! for (jint i = cur; i < end; i++) {
! ShenandoahHeapRegion* current = _heap->get_region((size_t)i);
! _blk->heap_region_do(current);
}
}
}
+ };
+
+ void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
+ assert(blk->is_thread_safe(), "Only thread-safe closures here");
+ if (num_regions() > ShenandoahParallelRegionStride) {
+ ShenandoahParallelHeapRegionTask task(blk);
+ workers()->run_task(&task);
+ } else {
+ heap_region_iterate(blk);
}
}
class ShenandoahClearLivenessClosure : public ShenandoahHeapRegionClosure {
private:
! ShenandoahMarkingContext* const _ctx;
public:
! ShenandoahClearLivenessClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
! void heap_region_do(ShenandoahHeapRegion* r) {
! if (r->is_active()) {
r->clear_live_data();
! _ctx->capture_top_at_mark_start(r);
! } else {
! assert(!r->has_live(),
! err_msg("Region " SIZE_FORMAT " should have no live data", r->region_number()));
! assert(_ctx->top_at_mark_start(r) == r->top(),
! err_msg("Region " SIZE_FORMAT " should already have correct TAMS", r->region_number()));
! }
}
+ bool is_thread_safe() { return true; }
+ };
void ShenandoahHeap::op_init_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
+ assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
! assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
! assert(!marking_context()->is_complete(), "should not be complete");
if (ShenandoahVerify) {
verifier()->verify_before_concmark();
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::accumulate_stats);
accumulate_statistics_tlabs();
}
+ if (VerifyBeforeGC) {
+ Universe::verify();
+ }
+
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
make_parsable(true);
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::clear_liveness);
! ShenandoahClearLivenessClosure clc;
! parallel_heap_region_iterate(&clc);
}
// Make above changes visible to worker threads
OrderAccess::fence();
! concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
resize_tlabs();
}
*** 1450,1511 ****
pacer()->setup_for_mark();
}
}
void ShenandoahHeap::op_mark() {
! concurrentMark()->mark_from_roots();
}
void ShenandoahHeap::op_final_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
// It is critical that we
// evacuate roots right after finishing marking, so that we don't
// get unmarked objects in the roots.
if (!cancelled_gc()) {
! concurrentMark()->finish_mark_from_roots();
stop_concurrent_marking();
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
// All allocations past TAMS are implicitly live, adjust the region data.
// Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
! for (size_t i = 0; i < num_regions(); i++) {
! ShenandoahHeapRegion* r = get_region(i);
! if (!r->is_active()) continue;
!
! HeapWord* tams = complete_marking_context()->top_at_mark_start(r->region_number());
! HeapWord* top = r->top();
! if (top > tams) {
! r->increase_live_data_alloc_words(pointer_delta(top, tams));
! }
! }
}
{
ShenandoahGCPhase prepare_evac(ShenandoahPhaseTimings::prepare_evac);
! prepare_for_concurrent_evacuation();
}
// If collection set has candidates, start evacuation.
// Otherwise, bypass the rest of the cycle.
if (!collection_set()->is_empty()) {
set_evacuation_in_progress(true);
// From here on, we need to update references.
set_has_forwarded_objects(true);
- ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
evacuate_and_update_roots();
- }
if (ShenandoahPacing) {
pacer()->setup_for_evac();
}
} else {
! concurrentMark()->cancel();
stop_concurrent_marking();
if (process_references()) {
// Abandon reference processing right away: pre-cleaning must have failed.
ReferenceProcessor *rp = ref_processor();
--- 1333,1445 ----
pacer()->setup_for_mark();
}
}
void ShenandoahHeap::op_mark() {
! concurrent_mark()->mark_from_roots();
}
+ class ShenandoahCompleteLivenessClosure : public ShenandoahHeapRegionClosure {
+ private:
+ ShenandoahMarkingContext* const _ctx;
+ public:
+ ShenandoahCompleteLivenessClosure() : _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) {
+ if (r->is_active()) {
+ HeapWord *tams = _ctx->top_at_mark_start(r);
+ HeapWord *top = r->top();
+ if (top > tams) {
+ r->increase_live_data_alloc_words(pointer_delta(top, tams));
+ }
+ } else {
+ assert(!r->has_live(),
+ err_msg("Region " SIZE_FORMAT " should have no live data", r->region_number()));
+ assert(_ctx->top_at_mark_start(r) == r->top(),
+ err_msg("Region " SIZE_FORMAT " should have correct TAMS", r->region_number()));
+ }
+ }
+
+ bool is_thread_safe() { return true; }
+ };
+
void ShenandoahHeap::op_final_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
// It is critical that we
// evacuate roots right after finishing marking, so that we don't
// get unmarked objects in the roots.
if (!cancelled_gc()) {
! concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
!
! TASKQUEUE_STATS_ONLY(concurrent_mark()->task_queues()->reset_taskqueue_stats());
!
! if (has_forwarded_objects()) {
! concurrent_mark()->update_roots(ShenandoahPhaseTimings::update_roots);
! }
!
! TASKQUEUE_STATS_ONLY(concurrent_mark()->task_queues()->print_taskqueue_stats());
!
stop_concurrent_marking();
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
// All allocations past TAMS are implicitly live, adjust the region data.
// Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
! ShenandoahCompleteLivenessClosure cl;
! parallel_heap_region_iterate(&cl);
}
{
ShenandoahGCPhase prepare_evac(ShenandoahPhaseTimings::prepare_evac);
!
! make_parsable(true);
!
! trash_cset_regions();
!
! {
! ShenandoahHeapLocker locker(lock());
! _collection_set->clear();
! _free_set->clear();
!
! heuristics()->choose_collection_set(_collection_set);
! _free_set->rebuild();
! }
}
// If collection set has candidates, start evacuation.
// Otherwise, bypass the rest of the cycle.
if (!collection_set()->is_empty()) {
+ ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
+
+ if (ShenandoahVerify) {
+ verifier()->verify_before_evacuation();
+ }
+
set_evacuation_in_progress(true);
// From here on, we need to update references.
set_has_forwarded_objects(true);
evacuate_and_update_roots();
if (ShenandoahPacing) {
pacer()->setup_for_evac();
}
} else {
! if (ShenandoahVerify) {
! verifier()->verify_after_concmark();
! }
!
! if (VerifyAfterGC) {
! Universe::verify();
! }
! }
!
! } else {
! concurrent_mark()->cancel();
stop_concurrent_marking();
if (process_references()) {
// Abandon reference processing right away: pre-cleaning must have failed.
ReferenceProcessor *rp = ref_processor();
*** 1521,1555 ****
set_evacuation_in_progress(false);
if (ShenandoahVerify) {
verifier()->verify_after_evacuation();
}
}
! void ShenandoahHeap::op_evac() {
! ShenandoahParallelEvacuationTask task(this, _collection_set);
workers()->run_task(&task);
}
void ShenandoahHeap::op_updaterefs() {
update_heap_references(true);
}
void ShenandoahHeap::op_cleanup() {
- ShenandoahGCPhase phase_recycle(ShenandoahPhaseTimings::conc_cleanup_recycle);
free_set()->recycle_trash();
}
! void ShenandoahHeap::op_cleanup_bitmaps() {
! op_cleanup();
!
! ShenandoahGCPhase phase_reset(ShenandoahPhaseTimings::conc_cleanup_reset_bitmaps);
! reset_next_mark_bitmap();
}
void ShenandoahHeap::op_preclean() {
! concurrentMark()->preclean_weak_refs();
}
void ShenandoahHeap::op_full(GCCause::Cause cause) {
ShenandoahMetricsSnapshot metrics;
metrics.snap_before();
--- 1455,1494 ----
set_evacuation_in_progress(false);
if (ShenandoahVerify) {
verifier()->verify_after_evacuation();
}
+
+ if (VerifyAfterGC) {
+ Universe::verify();
+ }
}
! void ShenandoahHeap::op_conc_evac() {
! ShenandoahEvacuationTask task(this, _collection_set, true);
! workers()->run_task(&task);
! }
!
! void ShenandoahHeap::op_stw_evac() {
! ShenandoahEvacuationTask task(this, _collection_set, false);
workers()->run_task(&task);
}
void ShenandoahHeap::op_updaterefs() {
update_heap_references(true);
}
void ShenandoahHeap::op_cleanup() {
free_set()->recycle_trash();
}
! void ShenandoahHeap::op_reset() {
! reset_mark_bitmap();
}
void ShenandoahHeap::op_preclean() {
! concurrent_mark()->preclean_weak_refs();
}
void ShenandoahHeap::op_full(GCCause::Cause cause) {
ShenandoahMetricsSnapshot metrics;
metrics.snap_before();
*** 1590,1601 ****
// we can do the most aggressive degen cycle, which includes processing references and
// class unloading, unless those features are explicitly disabled.
//
// Note that we can only do this for "outside-cycle" degens, otherwise we would risk
// changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
! set_process_references(ShenandoahRefProcFrequency != 0);
! set_unload_classes(ClassUnloading);
op_init_mark();
if (cancelled_gc()) {
op_degenerated_fail();
return;
--- 1529,1542 ----
// we can do the most aggressive degen cycle, which includes processing references and
// class unloading, unless those features are explicitly disabled.
//
// Note that we can only do this for "outside-cycle" degens, otherwise we would risk
// changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
! set_process_references(heuristics()->can_process_references());
! set_unload_classes(heuristics()->can_unload_classes());
!
! op_reset();
op_init_mark();
if (cancelled_gc()) {
op_degenerated_fail();
return;
*** 1621,1631 ****
// it would be a simple check, which is supposed to be fast. This is also
// safe to do even without degeneration, as CSet iterator is at beginning
// in preparation for evacuation anyway.
collection_set()->clear_current_index();
! op_evac();
if (cancelled_gc()) {
op_degenerated_fail();
return;
}
}
--- 1562,1572 ----
// it would be a simple check, which is supposed to be fast. This is also
// safe to do even without degeneration, as CSet iterator is at beginning
// in preparation for evacuation anyway.
collection_set()->clear_current_index();
! op_stw_evac();
if (cancelled_gc()) {
op_degenerated_fail();
return;
}
}
*** 1647,1667 ****
op_degenerated_fail();
return;
}
}
! op_cleanup_bitmaps();
break;
default:
ShouldNotReachHere();
}
if (ShenandoahVerify) {
verifier()->verify_after_degenerated();
}
metrics.snap_after();
metrics.print();
// Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
// because that probably means the heap is overloaded and/or fragmented.
--- 1588,1612 ----
op_degenerated_fail();
return;
}
}
! op_cleanup();
break;
default:
ShouldNotReachHere();
}
if (ShenandoahVerify) {
verifier()->verify_after_degenerated();
}
+ if (VerifyAfterGC) {
+ Universe::verify();
+ }
+
metrics.snap_after();
metrics.print();
// Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
// because that probably means the heap is overloaded and/or fragmented.
*** 1674,1724 ****
}
}
void ShenandoahHeap::op_degenerated_fail() {
log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
! shenandoahPolicy()->record_degenerated_upgrade_to_full();
op_full(GCCause::_shenandoah_upgrade_to_full_gc);
}
void ShenandoahHeap::op_degenerated_futile() {
! shenandoahPolicy()->record_degenerated_upgrade_to_full();
op_full(GCCause::_shenandoah_upgrade_to_full_gc);
}
- void ShenandoahHeap::swap_mark_contexts() {
- ShenandoahMarkingContext* tmp = _complete_marking_context;
- _complete_marking_context = _next_marking_context;
- _next_marking_context = tmp;
- }
-
void ShenandoahHeap::stop_concurrent_marking() {
assert(is_concurrent_mark_in_progress(), "How else could we get here?");
if (!cancelled_gc()) {
// If we needed to update refs, and concurrent marking has been cancelled,
// we need to finish updating references.
set_has_forwarded_objects(false);
! swap_mark_contexts();
}
set_concurrent_mark_in_progress(false);
}
void ShenandoahHeap::force_satb_flush_all_threads() {
if (!is_concurrent_mark_in_progress()) {
// No need to flush SATBs
return;
}
! MutexLocker ml(Threads_lock);
JavaThread::set_force_satb_flush_all_threads(true);
// The threads are not "acquiring" their thread-local data, but it does not
// hurt to "release" the updates here anyway.
OrderAccess::fence();
}
-
void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
_gc_state.set_cond(mask, value);
JavaThread::set_gc_state_all_threads(_gc_state.raw_value());
}
--- 1619,1672 ----
}
}
void ShenandoahHeap::op_degenerated_fail() {
log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
! shenandoah_policy()->record_degenerated_upgrade_to_full();
op_full(GCCause::_shenandoah_upgrade_to_full_gc);
}
void ShenandoahHeap::op_degenerated_futile() {
! shenandoah_policy()->record_degenerated_upgrade_to_full();
op_full(GCCause::_shenandoah_upgrade_to_full_gc);
}
void ShenandoahHeap::stop_concurrent_marking() {
assert(is_concurrent_mark_in_progress(), "How else could we get here?");
if (!cancelled_gc()) {
// If we needed to update refs, and concurrent marking has been cancelled,
// we need to finish updating references.
set_has_forwarded_objects(false);
! mark_complete_marking_context();
}
set_concurrent_mark_in_progress(false);
}
void ShenandoahHeap::force_satb_flush_all_threads() {
if (!is_concurrent_mark_in_progress()) {
// No need to flush SATBs
return;
}
! // Do not block if Threads lock is busy. This avoids the potential deadlock
! // when this code is called from the periodic task, and something else is
! // expecting the periodic task to complete without blocking. On the off-chance
! // Threads lock is busy momentarily, try to acquire several times.
! for (int t = 0; t < 10; t++) {
! if (Threads_lock->try_lock()) {
JavaThread::set_force_satb_flush_all_threads(true);
+ Threads_lock->unlock();
// The threads are not "acquiring" their thread-local data, but it does not
// hurt to "release" the updates here anyway.
OrderAccess::fence();
+ break;
+ }
+ os::naked_short_sleep(1);
+ }
}
void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
_gc_state.set_cond(mask, value);
JavaThread::set_gc_state_all_threads(_gc_state.raw_value());
}
*** 1733,1757 ****
set_gc_state_mask(EVACUATION, in_progress);
}
HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
// Initialize Brooks pointer for the next object
! HeapWord* result = obj + BrooksPointer::word_size();
! BrooksPointer::initialize(oop(result));
return result;
}
uint ShenandoahHeap::oop_extra_words() {
! return BrooksPointer::word_size();
}
ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
! _mark_context(ShenandoahHeap::heap()->next_marking_context()) {
}
ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
! _mark_context(ShenandoahHeap::heap()->next_marking_context()) {
}
bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
if (oopDesc::is_null(obj)) {
return false;
--- 1681,1705 ----
set_gc_state_mask(EVACUATION, in_progress);
}
HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
// Initialize Brooks pointer for the next object
! HeapWord* result = obj + ShenandoahBrooksPointer::word_size();
! ShenandoahBrooksPointer::initialize(oop(result));
return result;
}
uint ShenandoahHeap::oop_extra_words() {
! return ShenandoahBrooksPointer::word_size();
}
ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
! _mark_context(ShenandoahHeap::heap()->marking_context()) {
}
ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
! _mark_context(ShenandoahHeap::heap()->marking_context()) {
}
bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
if (oopDesc::is_null(obj)) {
return false;
*** 1793,1803 ****
void ShenandoahHeap::release_pending_refs_lock() {
_control_thread->slt()->manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL);
}
GCTracer* ShenandoahHeap::tracer() {
! return shenandoahPolicy()->tracer();
}
size_t ShenandoahHeap::tlab_used(Thread* thread) const {
return _free_set->used();
}
--- 1741,1751 ----
void ShenandoahHeap::release_pending_refs_lock() {
_control_thread->slt()->manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL);
}
GCTracer* ShenandoahHeap::tracer() {
! return shenandoah_policy()->tracer();
}
size_t ShenandoahHeap::tlab_used(Thread* thread) const {
return _free_set->used();
}
*** 1836,1942 ****
ShenandoahStringDedup::stop();
}
}
void ShenandoahHeap::unload_classes_and_cleanup_tables(bool full_gc) {
! assert(ClassUnloading || full_gc, "Class unloading should be enabled");
! ShenandoahPhaseTimings::Phase phase_root =
! full_gc ?
ShenandoahPhaseTimings::full_gc_purge :
! ShenandoahPhaseTimings::purge;
!
! ShenandoahPhaseTimings::Phase phase_unload =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_class_unload :
! ShenandoahPhaseTimings::purge_class_unload;
!
! ShenandoahPhaseTimings::Phase phase_cldg =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_cldg :
! ShenandoahPhaseTimings::purge_cldg;
!
! ShenandoahPhaseTimings::Phase phase_par =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par :
! ShenandoahPhaseTimings::purge_par;
!
! ShenandoahPhaseTimings::Phase phase_par_classes =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par_classes :
! ShenandoahPhaseTimings::purge_par_classes;
!
! ShenandoahPhaseTimings::Phase phase_par_codecache =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par_codecache :
! ShenandoahPhaseTimings::purge_par_codecache;
!
! ShenandoahPhaseTimings::Phase phase_par_symbstring =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par_symbstring :
! ShenandoahPhaseTimings::purge_par_symbstring;
!
! ShenandoahPhaseTimings::Phase phase_par_sync =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par_sync :
! ShenandoahPhaseTimings::purge_par_sync;
!
! ShenandoahGCPhase root_phase(phase_root);
ShenandoahIsAliveSelector alive;
BoolObjectClosure* is_alive = alive.is_alive_closure();
bool purged_class;
// Unload classes and purge SystemDictionary.
{
! ShenandoahGCPhase phase(phase_unload);
purged_class = SystemDictionary::do_unloading(is_alive,
full_gc /* do_cleaning*/ );
}
{
! ShenandoahGCPhase phase(phase_par);
uint active = _workers->active_workers();
ParallelCleaningTask unlink_task(is_alive, true, true, active, purged_class);
_workers->run_task(&unlink_task);
-
- ShenandoahPhaseTimings* p = phase_timings();
- ParallelCleaningTimes times = unlink_task.times();
-
- // "times" report total time, phase_tables_cc reports wall time. Divide total times
- // by active workers to get average time per worker, that would add up to wall time.
- p->record_phase_time(phase_par_classes, times.klass_work_us() / active);
- p->record_phase_time(phase_par_codecache, times.codecache_work_us() / active);
- p->record_phase_time(phase_par_symbstring, times.tables_work_us() / active);
- p->record_phase_time(phase_par_sync, times.sync_us() / active);
}
if (ShenandoahStringDedup::is_enabled()) {
! ShenandoahPhaseTimings::Phase phase_par_string_dedup =
! full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par_string_dedup :
! ShenandoahPhaseTimings::purge_par_string_dedup;
! ShenandoahGCPhase phase(phase_par_string_dedup);
ShenandoahStringDedup::parallel_cleanup();
}
-
{
! ShenandoahGCPhase phase(phase_cldg);
ClassLoaderDataGraph::purge();
}
}
void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
set_gc_state_mask(HAS_FORWARDED, cond);
}
- bool ShenandoahHeap::last_gc_made_progress() const {
- return _progress_last_gc.is_set();
- }
-
void ShenandoahHeap::set_process_references(bool pr) {
_process_references.set_cond(pr);
}
void ShenandoahHeap::set_unload_classes(bool uc) {
--- 1784,1841 ----
ShenandoahStringDedup::stop();
}
}
void ShenandoahHeap::unload_classes_and_cleanup_tables(bool full_gc) {
! assert(heuristics()->can_unload_classes(), "Class unloading should be enabled");
! ShenandoahGCPhase root_phase(full_gc ?
ShenandoahPhaseTimings::full_gc_purge :
! ShenandoahPhaseTimings::purge);
ShenandoahIsAliveSelector alive;
BoolObjectClosure* is_alive = alive.is_alive_closure();
bool purged_class;
// Unload classes and purge SystemDictionary.
{
! ShenandoahGCPhase phase(full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_class_unload :
! ShenandoahPhaseTimings::purge_class_unload);
purged_class = SystemDictionary::do_unloading(is_alive,
full_gc /* do_cleaning*/ );
}
{
! ShenandoahGCPhase phase(full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_par :
! ShenandoahPhaseTimings::purge_par);
uint active = _workers->active_workers();
ParallelCleaningTask unlink_task(is_alive, true, true, active, purged_class);
_workers->run_task(&unlink_task);
}
if (ShenandoahStringDedup::is_enabled()) {
! ShenandoahGCPhase phase(full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_string_dedup :
! ShenandoahPhaseTimings::purge_string_dedup);
ShenandoahStringDedup::parallel_cleanup();
}
{
! ShenandoahGCPhase phase(full_gc ?
! ShenandoahPhaseTimings::full_gc_purge_cldg :
! ShenandoahPhaseTimings::purge_cldg);
ClassLoaderDataGraph::purge();
}
}
void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
set_gc_state_mask(HAS_FORWARDED, cond);
}
void ShenandoahHeap::set_process_references(bool pr) {
_process_references.set_cond(pr);
}
void ShenandoahHeap::set_unload_classes(bool uc) {
*** 1949,1975 ****
bool ShenandoahHeap::unload_classes() const {
return _unload_classes.is_set();
}
- //fixme this should be in heapregionset
- ShenandoahHeapRegion* ShenandoahHeap::next_compaction_region(const ShenandoahHeapRegion* r) {
- size_t region_idx = r->region_number() + 1;
- ShenandoahHeapRegion* next = get_region(region_idx);
- guarantee(next->region_number() == region_idx, "region number must match");
- while (next->is_humongous()) {
- region_idx = next->region_number() + 1;
- next = get_region(region_idx);
- guarantee(next->region_number() == region_idx, "region number must match");
- }
- return next;
- }
-
- ShenandoahMonitoringSupport* ShenandoahHeap::monitoring_support() {
- return _monitoring_support;
- }
-
address ShenandoahHeap::in_cset_fast_test_addr() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
assert(heap->collection_set() != NULL, "Sanity");
return (address) heap->collection_set()->biased_map_address();
}
--- 1848,1857 ----
*** 1981,2006 ****
address ShenandoahHeap::gc_state_addr() {
return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
}
size_t ShenandoahHeap::conservative_max_heap_alignment() {
! return ShenandoahMaxRegionSize;
}
size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
return OrderAccess::load_acquire(&_bytes_allocated_since_gc_start);
}
void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
OrderAccess::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
}
- ShenandoahPacer* ShenandoahHeap::pacer() const {
- assert (_pacer != NULL, "sanity");
- return _pacer;
- }
-
void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
_degenerated_gc_in_progress.set_cond(in_progress);
}
void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
--- 1863,1887 ----
address ShenandoahHeap::gc_state_addr() {
return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
}
size_t ShenandoahHeap::conservative_max_heap_alignment() {
! size_t align = ShenandoahMaxRegionSize;
! if (UseLargePages) {
! align = MAX2(align, os::large_page_size());
! }
! return align;
}
size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
return OrderAccess::load_acquire(&_bytes_allocated_since_gc_start);
}
void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
OrderAccess::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
}
void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
_degenerated_gc_in_progress.set_cond(in_progress);
}
void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
*** 2063,2081 ****
}
}
}
#endif
- ShenandoahUpdateHeapRefsClosure::ShenandoahUpdateHeapRefsClosure() :
- _heap(ShenandoahHeap::heap()) {}
-
ShenandoahVerifier* ShenandoahHeap::verifier() {
guarantee(ShenandoahVerify, "Should be enabled");
assert (_verifier != NULL, "sanity");
return _verifier;
}
class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
private:
ShenandoahHeap* _heap;
ShenandoahRegionIterator* _regions;
bool _concurrent;
--- 1944,1962 ----
}
}
}
#endif
ShenandoahVerifier* ShenandoahHeap::verifier() {
guarantee(ShenandoahVerify, "Should be enabled");
assert (_verifier != NULL, "sanity");
return _verifier;
}
+ ShenandoahUpdateHeapRefsClosure::ShenandoahUpdateHeapRefsClosure() :
+ _heap(ShenandoahHeap::heap()) {}
+
class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
private:
ShenandoahHeap* _heap;
ShenandoahRegionIterator* _regions;
bool _concurrent;
*** 2087,2115 ****
_regions(regions),
_concurrent(concurrent) {
}
void work(uint worker_id) {
! ShenandoahWorkerSession worker_session(worker_id);
ShenandoahUpdateHeapRefsClosure cl;
ShenandoahHeapRegion* r = _regions->next();
ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
while (r != NULL) {
- if (_heap->in_collection_set(r)) {
- HeapWord* bottom = r->bottom();
- HeapWord* top = ctx->top_at_mark_start(r->region_number());
- if (top > bottom) {
- ctx->clear_bitmap(bottom, top);
- }
- } else {
- if (r->is_active()) {
- _heap->marked_object_oop_safe_iterate(r, &cl);
- }
- }
- if (ShenandoahPacing) {
HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
assert (top_at_start_ur >= r->bottom(), "sanity");
_heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
}
if (_heap->cancelled_gc()) {
return;
}
--- 1968,1988 ----
_regions(regions),
_concurrent(concurrent) {
}
void work(uint worker_id) {
! ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahUpdateHeapRefsClosure cl;
ShenandoahHeapRegion* r = _regions->next();
ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
while (r != NULL) {
HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
assert (top_at_start_ur >= r->bottom(), "sanity");
+ if (r->is_active() && !r->is_cset()) {
+ _heap->marked_object_oop_iterate(r, &cl, top_at_start_ur);
+ }
+ if (ShenandoahPacing) {
_heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
}
if (_heap->cancelled_gc()) {
return;
}
*** 2164,2174 ****
if (cancelled_gc()) {
clear_cancelled_gc();
}
assert(!cancelled_gc(), "Should have been done right before");
! concurrentMark()->update_roots(ShenandoahPhaseTimings::final_update_refs_roots);
ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
trash_cset_regions();
set_has_forwarded_objects(false);
--- 2037,2049 ----
if (cancelled_gc()) {
clear_cancelled_gc();
}
assert(!cancelled_gc(), "Should have been done right before");
! concurrent_mark()->update_roots(is_degenerated_gc_in_progress() ?
! ShenandoahPhaseTimings::degen_gc_update_roots:
! ShenandoahPhaseTimings::final_update_refs_roots);
ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
trash_cset_regions();
set_has_forwarded_objects(false);
*** 2176,2185 ****
--- 2051,2064 ----
if (ShenandoahVerify) {
verifier()->verify_after_updaterefs();
}
+ if (VerifyAfterGC) {
+ Universe::verify();
+ }
+
{
ShenandoahHeapLocker locker(lock());
_free_set->rebuild();
}
}
*** 2229,2242 ****
// Commit the bitmap slice:
size_t slice = r->region_number() / _bitmap_regions_per_slice;
size_t off = _bitmap_bytes_per_slice * slice;
size_t len = _bitmap_bytes_per_slice;
! if (!os::commit_memory((char*)_bitmap0_region.start() + off, len, false)) {
! return false;
! }
! if (!os::commit_memory((char*)_bitmap1_region.start() + off, len, false)) {
return false;
}
return true;
}
--- 2108,2118 ----
// Commit the bitmap slice:
size_t slice = r->region_number() / _bitmap_regions_per_slice;
size_t off = _bitmap_bytes_per_slice * slice;
size_t len = _bitmap_bytes_per_slice;
! if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
return false;
}
return true;
}
*** 2251,2264 ****
// Uncommit the bitmap slice:
size_t slice = r->region_number() / _bitmap_regions_per_slice;
size_t off = _bitmap_bytes_per_slice * slice;
size_t len = _bitmap_bytes_per_slice;
! if (!os::uncommit_memory((char*)_bitmap0_region.start() + off, len)) {
! return false;
! }
! if (!os::uncommit_memory((char*)_bitmap1_region.start() + off, len)) {
return false;
}
return true;
}
--- 2127,2137 ----
// Uncommit the bitmap slice:
size_t slice = r->region_number() / _bitmap_regions_per_slice;
size_t off = _bitmap_bytes_per_slice * slice;
size_t len = _bitmap_bytes_per_slice;
! if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
return false;
}
return true;
}
*** 2458,2468 ****
ShenandoahWorkerScope scope(workers(),
ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
"concurrent evacuation");
try_inject_alloc_failure();
! op_evac();
}
void ShenandoahHeap::entry_updaterefs() {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
--- 2331,2341 ----
ShenandoahWorkerScope scope(workers(),
ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
"concurrent evacuation");
try_inject_alloc_failure();
! op_conc_evac();
}
void ShenandoahHeap::entry_updaterefs() {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
*** 2475,2484 ****
--- 2348,2358 ----
"concurrent reference update");
try_inject_alloc_failure();
op_updaterefs();
}
+
void ShenandoahHeap::entry_cleanup() {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
static const char* msg = "Concurrent cleanup";
GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true);
*** 2488,2510 ****
try_inject_alloc_failure();
op_cleanup();
}
! void ShenandoahHeap::entry_cleanup_bitmaps() {
! ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
! static const char* msg = "Concurrent cleanup";
GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true);
EventMark em("%s", msg);
ShenandoahWorkerScope scope(workers(),
! ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
! "concurrent cleanup");
try_inject_alloc_failure();
! op_cleanup_bitmaps();
}
void ShenandoahHeap::entry_preclean() {
if (ShenandoahPreclean && process_references()) {
ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
--- 2362,2384 ----
try_inject_alloc_failure();
op_cleanup();
}
! void ShenandoahHeap::entry_reset() {
! ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
! static const char* msg = "Concurrent reset";
GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true);
EventMark em("%s", msg);
ShenandoahWorkerScope scope(workers(),
! ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
! "concurrent reset");
try_inject_alloc_failure();
! op_reset();
}
void ShenandoahHeap::entry_preclean() {
if (ShenandoahPreclean && process_references()) {
ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
*** 2513,2523 ****
GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true);
EventMark em("%s", msg);
ShenandoahWorkerScope scope(workers(),
ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
! "concurrent preclean");
try_inject_alloc_failure();
op_preclean();
}
}
--- 2387,2398 ----
GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true);
EventMark em("%s", msg);
ShenandoahWorkerScope scope(workers(),
ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
! "concurrent preclean",
! /* check_workers = */ false);
try_inject_alloc_failure();
op_preclean();
}
}
*** 2553,2609 ****
void ShenandoahHeap::leave_evacuation() {
_oom_evac_handler.leave_evacuation();
}
ShenandoahRegionIterator::ShenandoahRegionIterator() :
! _index(0),
! _heap(ShenandoahHeap::heap()) {}
ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
! _index(0),
! _heap(heap) {}
void ShenandoahRegionIterator::reset() {
_index = 0;
}
bool ShenandoahRegionIterator::has_next() const {
return _index < (jint)_heap->num_regions();
}
- void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure& cl) const {
- ShenandoahRegionIterator regions;
- ShenandoahHeapRegion* r = regions.next();
- while (r != NULL) {
- if (cl.heap_region_do(r)) {
- break;
- }
- r = regions.next();
- }
- }
-
char ShenandoahHeap::gc_state() {
return _gc_state.raw_value();
}
const char* ShenandoahHeap::init_mark_event_message() const {
bool update_refs = has_forwarded_objects();
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (update_refs && proc_refs && unload_cls) {
! return "Pause Init Mark (update refs) (process refs) (unload classes)";
} else if (update_refs && proc_refs) {
! return "Pause Init Mark (update refs) (process refs)";
} else if (update_refs && unload_cls) {
return "Pause Init Mark (update refs) (unload classes)";
} else if (proc_refs && unload_cls) {
! return "Pause Init Mark (process refs) (unload classes)";
} else if (update_refs) {
return "Pause Init Mark (update refs)";
} else if (proc_refs) {
! return "Pause Init Mark (process refs)";
} else if (unload_cls) {
return "Pause Init Mark (unload classes)";
} else {
return "Pause Init Mark";
}
--- 2428,2473 ----
void ShenandoahHeap::leave_evacuation() {
_oom_evac_handler.leave_evacuation();
}
ShenandoahRegionIterator::ShenandoahRegionIterator() :
! _heap(ShenandoahHeap::heap()),
! _index(0) {}
ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
! _heap(heap),
! _index(0) {}
void ShenandoahRegionIterator::reset() {
_index = 0;
}
bool ShenandoahRegionIterator::has_next() const {
return _index < (jint)_heap->num_regions();
}
char ShenandoahHeap::gc_state() {
return _gc_state.raw_value();
}
const char* ShenandoahHeap::init_mark_event_message() const {
bool update_refs = has_forwarded_objects();
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (update_refs && proc_refs && unload_cls) {
! return "Pause Init Mark (update refs) (process weakrefs) (unload classes)";
} else if (update_refs && proc_refs) {
! return "Pause Init Mark (update refs) (process weakrefs)";
} else if (update_refs && unload_cls) {
return "Pause Init Mark (update refs) (unload classes)";
} else if (proc_refs && unload_cls) {
! return "Pause Init Mark (process weakrefs) (unload classes)";
} else if (update_refs) {
return "Pause Init Mark (update refs)";
} else if (proc_refs) {
! return "Pause Init Mark (process weakrefs)";
} else if (unload_cls) {
return "Pause Init Mark (unload classes)";
} else {
return "Pause Init Mark";
}
*** 2613,2633 ****
bool update_refs = has_forwarded_objects();
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (update_refs && proc_refs && unload_cls) {
! return "Pause Final Mark (update refs) (process refs) (unload classes)";
} else if (update_refs && proc_refs) {
! return "Pause Final Mark (update refs) (process refs)";
} else if (update_refs && unload_cls) {
return "Pause Final Mark (update refs) (unload classes)";
} else if (proc_refs && unload_cls) {
! return "Pause Final Mark (process refs) (unload classes)";
} else if (update_refs) {
return "Pause Final Mark (update refs)";
} else if (proc_refs) {
! return "Pause Final Mark (process refs)";
} else if (unload_cls) {
return "Pause Final Mark (unload classes)";
} else {
return "Pause Final Mark";
}
--- 2477,2497 ----
bool update_refs = has_forwarded_objects();
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (update_refs && proc_refs && unload_cls) {
! return "Pause Final Mark (update refs) (process weakrefs) (unload classes)";
} else if (update_refs && proc_refs) {
! return "Pause Final Mark (update refs) (process weakrefs)";
} else if (update_refs && unload_cls) {
return "Pause Final Mark (update refs) (unload classes)";
} else if (proc_refs && unload_cls) {
! return "Pause Final Mark (process weakrefs) (unload classes)";
} else if (update_refs) {
return "Pause Final Mark (update refs)";
} else if (proc_refs) {
! return "Pause Final Mark (process weakrefs)";
} else if (unload_cls) {
return "Pause Final Mark (unload classes)";
} else {
return "Pause Final Mark";
}
*** 2637,2657 ****
bool update_refs = has_forwarded_objects();
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (update_refs && proc_refs && unload_cls) {
! return "Concurrent marking (update refs) (process refs) (unload classes)";
} else if (update_refs && proc_refs) {
! return "Concurrent marking (update refs) (process refs)";
} else if (update_refs && unload_cls) {
return "Concurrent marking (update refs) (unload classes)";
} else if (proc_refs && unload_cls) {
! return "Concurrent marking (process refs) (unload classes)";
} else if (update_refs) {
return "Concurrent marking (update refs)";
} else if (proc_refs) {
! return "Concurrent marking (process refs)";
} else if (unload_cls) {
return "Concurrent marking (unload classes)";
} else {
return "Concurrent marking";
}
--- 2501,2521 ----
bool update_refs = has_forwarded_objects();
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (update_refs && proc_refs && unload_cls) {
! return "Concurrent marking (update refs) (process weakrefs) (unload classes)";
} else if (update_refs && proc_refs) {
! return "Concurrent marking (update refs) (process weakrefs)";
} else if (update_refs && unload_cls) {
return "Concurrent marking (update refs) (unload classes)";
} else if (proc_refs && unload_cls) {
! return "Concurrent marking (process weakrefs) (unload classes)";
} else if (update_refs) {
return "Concurrent marking (update refs)";
} else if (proc_refs) {
! return "Concurrent marking (process weakrefs)";
} else if (unload_cls) {
return "Concurrent marking (unload classes)";
} else {
return "Concurrent marking";
}
*** 2673,2682 ****
--- 2537,2568 ----
ShouldNotReachHere();
return "ERROR";
}
}
+ jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) {
+ #ifdef ASSERT
+ assert(worker_id < _max_workers, "sanity");
+ for (uint i = 0; i < num_regions(); i++) {
+ assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
+ }
+ #endif
+ return _liveness_cache[worker_id];
+ }
+
+ void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
+ assert(worker_id < _max_workers, "sanity");
+ jushort* ld = _liveness_cache[worker_id];
+ for (uint i = 0; i < num_regions(); i++) {
+ ShenandoahHeapRegion* r = get_region(i);
+ jushort live = ld[i];
+ if (live > 0) {
+ r->increase_live_data_gc_words(live);
+ ld[i] = 0;
+ }
+ }
+ }
BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
return ShenandoahHeap::heap()->has_forwarded_objects() ? reinterpret_cast<BoolObjectClosure*>(&_fwd_alive_cl)
: reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
}
< prev index next >