--- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp 2014-06-26 09:10:35.884707018 +0200 +++ new/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp 2014-06-26 09:10:35.806704812 +0200 @@ -69,6 +69,11 @@ _start = os::elapsedTime(); } +G1ParScanThreadState::~G1ParScanThreadState() { + retire_alloc_buffers(); + FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base, mtGC); +} + void G1ParScanThreadState::print_termination_stats_hdr(outputStream* const st) { @@ -140,28 +145,15 @@ do { // Drain the overflow stack first, so other threads can steal. while (_refs->pop_overflow(ref)) { - deal_with_reference(ref); + dispatch_reference(ref); } while (_refs->pop_local(ref)) { - deal_with_reference(ref); + dispatch_reference(ref); } } while (!_refs->is_empty()); } -void G1ParScanThreadState::steal_and_trim_queue(RefToScanQueueSet *task_queues) { - StarTask stolen_task; - while (task_queues->steal(queue_num(), hash_seed(), stolen_task)) { - assert(verify_task(stolen_task), "sanity"); - deal_with_reference(stolen_task); - - // We've just processed a reference and we might have made - // available new entries on the queues. So we have to make sure - // we drain the queues as necessary. - trim_queue(); - } -} - oop G1ParScanThreadState::copy_to_survivor_space(oop const old) { size_t word_sz = old->size(); HeapRegion* from_region = _g1h->heap_region_containing_raw(old); @@ -315,97 +307,3 @@ false /* retain */); } } - -template void G1ParScanThreadState::do_oop_evac(T* p, HeapRegion* from) { - assert(!oopDesc::is_null(oopDesc::load_decode_heap_oop(p)), - "Reference should not be NULL here as such are never pushed to the task queue."); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - - // Although we never intentionally push references outside of the collection - // set, due to (benign) races in the claim mechanism during RSet scanning more - // than one thread might claim the same card. So the same card may be - // processed multiple times. So redo this check. - if (_g1h->in_cset_fast_test(obj)) { - oop forwardee; - if (obj->is_forwarded()) { - forwardee = obj->forwardee(); - } else { - forwardee = copy_to_survivor_space(obj); - } - assert(forwardee != NULL, "forwardee should not be NULL"); - oopDesc::encode_store_heap_oop(p, forwardee); - } - - assert(obj != NULL, "Must be"); - update_rs(from, p, queue_num()); -} - -inline void G1ParScanThreadState::do_oop_partial_array(oop* p) { - assert(has_partial_array_mask(p), "invariant"); - oop from_obj = clear_partial_array_mask(p); - - assert(Universe::heap()->is_in_reserved(from_obj), "must be in heap."); - assert(from_obj->is_objArray(), "must be obj array"); - objArrayOop from_obj_array = objArrayOop(from_obj); - // The from-space object contains the real length. - int length = from_obj_array->length(); - - assert(from_obj->is_forwarded(), "must be forwarded"); - oop to_obj = from_obj->forwardee(); - assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); - objArrayOop to_obj_array = objArrayOop(to_obj); - // We keep track of the next start index in the length field of the - // to-space object. - int next_index = to_obj_array->length(); - assert(0 <= next_index && next_index < length, - err_msg("invariant, next index: %d, length: %d", next_index, length)); - - int start = next_index; - int end = length; - int remainder = end - start; - // We'll try not to push a range that's smaller than ParGCArrayScanChunk. - if (remainder > 2 * ParGCArrayScanChunk) { - end = start + ParGCArrayScanChunk; - to_obj_array->set_length(end); - // Push the remainder before we process the range in case another - // worker has run out of things to do and can steal it. - oop* from_obj_p = set_partial_array_mask(from_obj); - push_on_queue(from_obj_p); - } else { - assert(length == end, "sanity"); - // We'll process the final range for this object. Restore the length - // so that the heap remains parsable in case of evacuation failure. - to_obj_array->set_length(end); - } - _scanner.set_region(_g1h->heap_region_containing_raw(to_obj)); - // Process indexes [start,end). It will also process the header - // along with the first chunk (i.e., the chunk with start == 0). - // Note that at this point the length field of to_obj_array is not - // correct given that we are using it to keep track of the next - // start index. oop_iterate_range() (thankfully!) ignores the length - // field and only relies on the start / end parameters. It does - // however return the size of the object which will be incorrect. So - // we have to ignore it even if we wanted to use it. - to_obj_array->oop_iterate_range(&_scanner, start, end); -} - -template inline void G1ParScanThreadState::deal_with_reference(T* ref_to_scan) { - if (!has_partial_array_mask(ref_to_scan)) { - // Note: we can use "raw" versions of "region_containing" because - // "obj_to_scan" is definitely in the heap, and is not in a - // humongous region. - HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan); - do_oop_evac(ref_to_scan, r); - } else { - do_oop_partial_array((oop*)ref_to_scan); - } -} - -inline void G1ParScanThreadState::deal_with_reference(StarTask ref) { - assert(verify_task(ref), "sanity"); - if (ref.is_narrow()) { - deal_with_reference((narrowOop*)ref); - } else { - deal_with_reference((oop*)ref); - } -} --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.hpp 2014-06-26 09:10:36.405721747 +0200 +++ new/src/share/vm/gc_implementation/g1/g1ParScanThreadState.hpp 2014-06-26 09:10:36.328719570 +0200 @@ -98,14 +98,10 @@ } } -public: + public: G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp); - ~G1ParScanThreadState() { - retire_alloc_buffers(); - FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base, mtGC); - } + ~G1ParScanThreadState(); - RefToScanQueue* refs() { return _refs; } ageTable* age_table() { return &_age_table; } G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) { @@ -115,12 +111,6 @@ size_t alloc_buffer_waste() const { return _alloc_buffer_waste; } size_t undo_waste() const { return _undo_waste; } - public: - G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp); - ~G1ParScanThreadState(); - - ageTable* age_table() { return &_age_table; } - #ifdef ASSERT bool queue_is_empty() const { return _refs->is_empty(); } @@ -138,9 +128,9 @@ private: - HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz); - HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz); - void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz); + inline HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz); + inline HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz); + inline void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz); public: @@ -220,18 +210,18 @@ inline void do_oop_partial_array(oop* p); // This method is applied to the fields of the objects that have just been copied. - template void do_oop_evac(T* p, HeapRegion* from); + template inline void do_oop_evac(T* p, HeapRegion* from); template inline void deal_with_reference(T* ref_to_scan); - inline void deal_with_reference(StarTask ref); -public: + inline void dispatch_reference(StarTask ref); + public: oop copy_to_survivor_space(oop const obj); void trim_queue(); - void steal_and_trim_queue(RefToScanQueueSet *task_queues); + inline void steal_and_trim_queue(RefToScanQueueSet *task_queues); }; #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_HPP --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp 2014-06-26 09:10:36.915736166 +0200 +++ new/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp 2014-06-26 09:10:36.837733961 +0200 @@ -43,4 +43,111 @@ } } +template void G1ParScanThreadState::do_oop_evac(T* p, HeapRegion* from) { + assert(!oopDesc::is_null(oopDesc::load_decode_heap_oop(p)), + "Reference should not be NULL here as such are never pushed to the task queue."); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + + // Although we never intentionally push references outside of the collection + // set, due to (benign) races in the claim mechanism during RSet scanning more + // than one thread might claim the same card. So the same card may be + // processed multiple times. So redo this check. + if (_g1h->in_cset_fast_test(obj)) { + oop forwardee; + if (obj->is_forwarded()) { + forwardee = obj->forwardee(); + } else { + forwardee = copy_to_survivor_space(obj); + } + assert(forwardee != NULL, "forwardee should not be NULL"); + oopDesc::encode_store_heap_oop(p, forwardee); + } + + assert(obj != NULL, "Must be"); + update_rs(from, p, queue_num()); +} + +inline void G1ParScanThreadState::do_oop_partial_array(oop* p) { + assert(has_partial_array_mask(p), "invariant"); + oop from_obj = clear_partial_array_mask(p); + + assert(Universe::heap()->is_in_reserved(from_obj), "must be in heap."); + assert(from_obj->is_objArray(), "must be obj array"); + objArrayOop from_obj_array = objArrayOop(from_obj); + // The from-space object contains the real length. + int length = from_obj_array->length(); + + assert(from_obj->is_forwarded(), "must be forwarded"); + oop to_obj = from_obj->forwardee(); + assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); + objArrayOop to_obj_array = objArrayOop(to_obj); + // We keep track of the next start index in the length field of the + // to-space object. + int next_index = to_obj_array->length(); + assert(0 <= next_index && next_index < length, + err_msg("invariant, next index: %d, length: %d", next_index, length)); + + int start = next_index; + int end = length; + int remainder = end - start; + // We'll try not to push a range that's smaller than ParGCArrayScanChunk. + if (remainder > 2 * ParGCArrayScanChunk) { + end = start + ParGCArrayScanChunk; + to_obj_array->set_length(end); + // Push the remainder before we process the range in case another + // worker has run out of things to do and can steal it. + oop* from_obj_p = set_partial_array_mask(from_obj); + push_on_queue(from_obj_p); + } else { + assert(length == end, "sanity"); + // We'll process the final range for this object. Restore the length + // so that the heap remains parsable in case of evacuation failure. + to_obj_array->set_length(end); + } + _scanner.set_region(_g1h->heap_region_containing_raw(to_obj)); + // Process indexes [start,end). It will also process the header + // along with the first chunk (i.e., the chunk with start == 0). + // Note that at this point the length field of to_obj_array is not + // correct given that we are using it to keep track of the next + // start index. oop_iterate_range() (thankfully!) ignores the length + // field and only relies on the start / end parameters. It does + // however return the size of the object which will be incorrect. So + // we have to ignore it even if we wanted to use it. + to_obj_array->oop_iterate_range(&_scanner, start, end); +} + +template inline void G1ParScanThreadState::deal_with_reference(T* ref_to_scan) { + if (!has_partial_array_mask(ref_to_scan)) { + // Note: we can use "raw" versions of "region_containing" because + // "obj_to_scan" is definitely in the heap, and is not in a + // humongous region. + HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan); + do_oop_evac(ref_to_scan, r); + } else { + do_oop_partial_array((oop*)ref_to_scan); + } +} + +inline void G1ParScanThreadState::dispatch_reference(StarTask ref) { + assert(verify_task(ref), "sanity"); + if (ref.is_narrow()) { + deal_with_reference((narrowOop*)ref); + } else { + deal_with_reference((oop*)ref); + } +} + +void G1ParScanThreadState::steal_and_trim_queue(RefToScanQueueSet *task_queues) { + StarTask stolen_task; + while (task_queues->steal(queue_num(), hash_seed(), stolen_task)) { + assert(verify_task(stolen_task), "sanity"); + dispatch_reference(stolen_task); + + // We've just processed a reference and we might have made + // available new entries on the queues. So we have to make sure + // we drain the queues as necessary. + trim_queue(); + } +} + #endif /* SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_INLINE_HPP */