--- old/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp 2020-08-04 04:07:06.240927170 -0400 +++ new/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp 2020-08-04 04:07:05.764911406 -0400 @@ -347,20 +347,57 @@ } } +NOINLINE +HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr, + oop old, + size_t word_sz, + uint age, + uint node_index) { + HeapWord* obj_ptr = NULL; + // Try slow-path allocation unless we're allocating old and old is already full. + if (!(dest_attr->is_old() && _old_gen_is_full)) { + bool plab_refill_failed = false; + obj_ptr = _plab_allocator->allocate_direct_or_new_plab(*dest_attr, + word_sz, + &plab_refill_failed, + node_index); + if (obj_ptr == NULL) { + obj_ptr = allocate_in_next_plab(dest_attr, + word_sz, + plab_refill_failed, + node_index); + } + } + if (obj_ptr != NULL) { + update_numa_stats(node_index); + if (_g1h->_gc_tracer_stw->should_report_promotion_events()) { + // The events are checked individually as part of the actual commit + report_promotion_event(*dest_attr, old, word_sz, age, obj_ptr, node_index); + } + } + return obj_ptr; +} + +NOINLINE +void G1ParScanThreadState::undo_allocation(G1HeapRegionAttr dest_attr, + HeapWord* obj_ptr, + size_t word_sz, + uint node_index) { + _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index); +} + // Private inline function, for direct internal use and providing the // implementation of the public not-inline function. oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const region_attr, oop const old, markWord const old_mark) { + assert(region_attr.is_in_cset(), + "Unexpected region attr type: %s", region_attr.get_type_str()); + const size_t word_sz = old->size(); uint age = 0; G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age); - // The second clause is to prevent premature evacuation failure in case there - // is still space in survivor, but old gen is full. - if (_old_gen_is_full && dest_attr.is_old()) { - return handle_evacuation_failure_par(old, old_mark); - } HeapRegion* const from_region = _g1h->heap_region_containing(old); uint node_index = from_region->node_index(); @@ -369,22 +406,11 @@ // PLAB allocations should succeed most of the time, so we'll // normally check against NULL once and that's it. if (obj_ptr == NULL) { - bool plab_refill_failed = false; - obj_ptr = _plab_allocator->allocate_direct_or_new_plab(dest_attr, word_sz, &plab_refill_failed, node_index); + obj_ptr = allocate_copy_slow(&dest_attr, old, word_sz, age, node_index); if (obj_ptr == NULL) { - assert(region_attr.is_in_cset(), "Unexpected region attr type: %s", region_attr.get_type_str()); - obj_ptr = allocate_in_next_plab(&dest_attr, word_sz, plab_refill_failed, node_index); - if (obj_ptr == NULL) { - // This will either forward-to-self, or detect that someone else has - // installed a forwarding pointer. - return handle_evacuation_failure_par(old, old_mark); - } - } - update_numa_stats(node_index); - - if (_g1h->_gc_tracer_stw->should_report_promotion_events()) { - // The events are checked individually as part of the actual commit - report_promotion_event(dest_attr, old, word_sz, age, obj_ptr, node_index); + // This will either forward-to-self, or detect that someone else has + // installed a forwarding pointer. + return handle_evacuation_failure_par(old, old_mark); } } @@ -396,7 +422,7 @@ if (_g1h->evacuation_should_fail()) { // Doing this after all the allocation attempts also tests the // undo_allocation() method too. - _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index); + undo_allocation(dest_attr, obj_ptr, word_sz, node_index); return handle_evacuation_failure_par(old, old_mark); } #endif // !PRODUCT @@ -409,10 +435,12 @@ if (forward_ptr == NULL) { Copy::aligned_disjoint_words(cast_from_oop(old), obj_ptr, word_sz); - const uint young_index = from_region->young_index_in_cset(); - - assert((from_region->is_young() && young_index > 0) || - (!from_region->is_young() && young_index == 0), "invariant" ); + { + const uint young_index = from_region->young_index_in_cset(); + assert((from_region->is_young() && young_index > 0) || + (!from_region->is_young() && young_index == 0), "invariant" ); + _surviving_young_words[young_index] += word_sz; + } if (dest_attr.is_young()) { if (age < markWord::max_age) { @@ -446,8 +474,6 @@ obj); } - _surviving_young_words[young_index] += word_sz; - if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { // We keep track of the next start index in the length field of // the to-space object. The actual length can be found in the @@ -528,6 +554,7 @@ } } +NOINLINE oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m) { assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));