# HG changeset patch # Parent 3e8e008f9277baecb0ff7196dcda4ac3f8d6a1af diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahEvacLockingBitmap.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacLockingBitmap.cpp Thu Jun 18 21:01:45 2020 +0200 @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahEvacLockingBitmap.hpp" +#include "utilities/bitMap.hpp" + +ShenandoahEvacLockingBitmap::ShenandoahEvacLockingBitmap(MemRegion heap) : + _covered(heap), _shifter(LogMinObjAlignment), _bm(_covered.word_size() >> _shifter, mtGC) { +} diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahEvacLockingBitmap.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacLockingBitmap.hpp Thu Jun 18 21:01:45 2020 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACLOCKINGBITMAP_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACLOCKINGBITMAP_HPP + +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/bitMap.hpp" + +class ShenandoahEvacLockingBitmap : public CHeapObj { +private: + MemRegion _covered; // The heap area covered by this bitmap. + const int _shifter; // Shift amount from heap index to bit index in the bitmap. + CHeapBitMap _bm; // The actual bitmap. + + // Convert from address to bit offset. + inline size_t addr_to_offset(const HeapWord* addr) const; + +public: + ShenandoahEvacLockingBitmap(MemRegion heap); + + inline void acquire(oop obj); + inline void release(oop obj); +}; + +class ShenandoahEvacLocker : public StackObj { +private: + ShenandoahEvacLockingBitmap* _bitmap; + oop const _obj; +public: + inline ShenandoahEvacLocker(ShenandoahEvacLockingBitmap* bitmap, oop obj); + inline ~ShenandoahEvacLocker(); +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACLOCKINGBITMAP_HPP diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahEvacLockingBitmap.inline.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacLockingBitmap.inline.hpp Thu Jun 18 21:01:45 2020 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACLOCKINGBITMAP_INLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACLOCKINGBITMAP_INLINE_HPP + +#include "gc/shenandoah/shenandoahEvacLockingBitmap.hpp" +#include "memory/memRegion.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/bitMap.hpp" + +inline size_t ShenandoahEvacLockingBitmap::addr_to_offset(const HeapWord* addr) const { + return pointer_delta(addr, _covered.start()) >> _shifter; +} + +inline void ShenandoahEvacLockingBitmap::acquire(oop obj) { + size_t offset = addr_to_offset(cast_from_oop(obj)); + int ctr = 0; + int yields = 0; + while (!_bm.par_set_bit(offset)) { + // Spin/yield/sleep-strategy inspired by Thread::SpinAcquire() + ++ctr; + if ((ctr & 0xFFF) == 0 || !os::is_MP()) { + if (yields > 5) { + os::naked_short_sleep(1); + } else { + os::naked_yield(); + ++yields; + } + } else { + SpinPause(); + } + + } +} + +inline void ShenandoahEvacLockingBitmap::release(oop obj) { + size_t offset = addr_to_offset(cast_from_oop(obj)); + assert(_bm.at(offset), ""); + bool success = _bm.par_clear_bit(offset); + assert(success, ""); +} + +inline ShenandoahEvacLocker::ShenandoahEvacLocker(ShenandoahEvacLockingBitmap* bitmap, oop obj): +_bitmap(bitmap), _obj(obj) { + _bitmap->acquire(_obj); +} + +inline ShenandoahEvacLocker::~ShenandoahEvacLocker() { + _bitmap->release(_obj); +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACLOCKINGBITMAP_INLINE_HPP diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp Thu Jun 18 16:08:20 2020 +0200 +++ b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp Thu Jun 18 21:01:45 2020 +0200 @@ -60,7 +60,7 @@ * Returns the new object 'update' upon success, or * the new forwardee that a competing thread installed. */ - static inline oop try_update_forwardee(oop obj, oop update); + static inline void update_forwardee(oop obj, oop update); }; diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp Thu Jun 18 16:08:20 2020 +0200 +++ b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp Thu Jun 18 21:01:45 2020 +0200 @@ -74,19 +74,10 @@ return obj->mark_raw().is_marked(); } -inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) { +inline void ShenandoahForwarding::update_forwardee(oop obj, oop update) { markWord old_mark = obj->mark_raw(); - if (old_mark.is_marked()) { - return oop(old_mark.clear_lock_bits().to_pointer()); - } - markWord new_mark = markWord::encode_pointer_as_mark(update); - markWord prev_mark = obj->cas_set_mark_raw(new_mark, old_mark); - if (prev_mark == old_mark) { - return update; - } else { - return oop(prev_mark.clear_lock_bits().to_pointer()); - } + obj->set_mark_raw(new_mark); } #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Jun 18 16:08:20 2020 +0200 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Jun 18 21:01:45 2020 +0200 @@ -41,6 +41,7 @@ #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" +#include "gc/shenandoah/shenandoahEvacLockingBitmap.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" @@ -259,6 +260,8 @@ _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions); + _evac_locking_bitmap = new ShenandoahEvacLockingBitmap(_heap_region); + if (ShenandoahVerify) { ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size); if (!verify_bitmap.special()) { @@ -495,7 +498,9 @@ _bitmap_region_special(false), _aux_bitmap_region_special(false), _liveness_cache(NULL), - _collection_set(NULL) + _collection_set(NULL), + _evac_locking_bitmap(NULL), + _evac_failed() { _heap = this; @@ -2013,7 +2018,6 @@ collection_set()->clear_current_index(); } - op_stw_evac(); if (cancelled_gc()) { op_degenerated_fail(); diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Thu Jun 18 16:08:20 2020 +0200 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Thu Jun 18 21:01:45 2020 +0200 @@ -41,6 +41,7 @@ class ReferenceProcessor; class ShenandoahCollectorPolicy; class ShenandoahControlThread; +class ShenandoahEvacLockingBitmap; class ShenandoahGCSession; class ShenandoahGCStateResetter; class ShenandoahHeuristics; @@ -678,6 +679,10 @@ private: ShenandoahCollectionSet* _collection_set; + MemRegion _evac_locking_bitmap_region; + ShenandoahEvacLockingBitmap* _evac_locking_bitmap; + ShenandoahSharedFlag _evac_failed; + void evacuate_and_update_roots(); public: diff -r 3e8e008f9277 src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Thu Jun 18 16:08:20 2020 +0200 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp Thu Jun 18 21:01:45 2020 +0200 @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" +#include "gc/shenandoah/shenandoahEvacLockingBitmap.inline.hpp" #include "gc/shenandoah/shenandoahForwarding.inline.hpp" #include "gc/shenandoah/shenandoahWorkGroup.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" @@ -223,6 +224,7 @@ inline void ShenandoahHeap::clear_cancelled_gc() { _cancelled_gc.set(CANCELLABLE); + _evac_failed.unset(); } inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) { @@ -245,66 +247,59 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { - size_t size = p->size(); - - assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); - - bool alloc_from_gclab = true; - HeapWord* copy = NULL; - -#ifdef ASSERT - if (ShenandoahOOMDuringEvacALot && - (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call - copy = NULL; - } else { -#endif - if (UseTLAB) { - copy = allocate_from_gclab(thread, size); - } - if (copy == NULL) { - ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size); - copy = allocate_memory(req); - alloc_from_gclab = false; - } -#ifdef ASSERT - } -#endif - - if (copy == NULL) { - control_thread()->handle_alloc_failure_evac(size); - return ShenandoahBarrierSet::resolve_forwarded(p); + // Fast-path + if (ShenandoahForwarding::is_forwarded(p)) { + return ShenandoahForwarding::get_forwardee(p); } - // Copy the object: - Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); + { + ShenandoahEvacLocker evac_locker(_evac_locking_bitmap, p); + + // Fast-path, double-checked + if (ShenandoahForwarding::is_forwarded(p)) { + return ShenandoahForwarding::get_forwardee(p); + } + + if (_evac_failed.is_set()) { + return p; + } + + assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); + + size_t size = p->size(); + HeapWord* copy = NULL; - // Try to install the new forwarding pointer. - oop copy_val = oop(copy); - oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); - if (result == copy_val) { - // Successfully evacuated. Our copy is now the public one! +#ifdef ASSERT + if (ShenandoahOOMDuringEvacALot && + (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call + copy = NULL; + } else { +#endif + if (UseTLAB) { + copy = allocate_from_gclab(thread, size); + } + if (copy == NULL) { + ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size); + copy = allocate_memory(req); + } +#ifdef ASSERT + } +#endif + + if (copy == NULL) { + control_thread()->handle_alloc_failure_evac(size); + _evac_failed.set(); + return p; + } + + // Copy the object: + Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); + + // Install the new forwarding pointer. + oop copy_val = oop(copy); + ShenandoahForwarding::update_forwardee(p, copy_val); shenandoah_assert_correct(NULL, copy_val); return copy_val; - } else { - // Failed to evacuate. We need to deal with the object that is left behind. Since this - // new allocation is certainly after TAMS, it will be considered live in the next cycle. - // But if it happens to contain references to evacuated regions, those references would - // not get updated for this stale copy during this cycle, and we will crash while scanning - // it the next cycle. - // - // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next - // object will overwrite this stale copy, or the filler object on LAB retirement will - // do this. For non-GCLAB allocations, we have no way to retract the allocation, and - // have to explicitly overwrite the copy with the filler object. With that overwrite, - // we have to keep the fwdptr initialized and pointing to our (stale) copy. - if (alloc_from_gclab) { - ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size); - } else { - fill_with_object(copy, size); - shenandoah_assert_correct(NULL, copy_val); - } - shenandoah_assert_correct(NULL, result); - return result; } }