1 /* 2 * Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 27 28 #include "gc/shared/barrierSet.hpp" 29 #include "gc/shenandoah/shenandoahAsserts.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 31 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" 32 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" 33 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 34 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 35 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" 36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 37 #include "memory/iterator.inline.hpp" 38 #include "oops/oop.inline.hpp" 39 40 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 41 return ShenandoahForwarding::get_forwardee(p); 42 } 43 44 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 45 if (p != NULL) { 46 return resolve_forwarded_not_null(p); 47 } else { 48 return p; 49 } 50 } 51 52 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null_mutator(oop p) { 53 return ShenandoahForwarding::get_forwardee_mutator(p); 54 } 55 56 template <class T> 57 inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) { 58 assert(ShenandoahLoadRefBarrier, "should be enabled"); 59 shenandoah_assert_in_cset(load_addr, obj); 60 61 oop fwd = resolve_forwarded_not_null_mutator(obj); 62 if (obj == fwd) { 63 assert(_heap->is_evacuation_in_progress(), 64 "evac should be in progress"); 65 ShenandoahEvacOOMScope scope; 66 fwd = _heap->evacuate_object(obj, Thread::current()); 67 } 68 69 if (load_addr != NULL && fwd != obj) { 70 // Since we are here and we know the load address, update the reference. 71 ShenandoahHeap::cas_oop(fwd, load_addr, obj); 72 } 73 74 return fwd; 75 } 76 77 inline void ShenandoahBarrierSet::enqueue(oop obj) { 78 assert(_satb_mark_queue_set.is_active(), "only get here when SATB active"); 79 80 // Filter marked objects before hitting the SATB queues. The same predicate would 81 // be used by SATBMQ::filter to eliminate already marked objects downstream, but 82 // filtering here helps to avoid wasteful SATB queueing work to begin with. 83 if (!_heap->requires_marking<false>(obj)) return; 84 85 ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).enqueue_known_active(obj); 86 } 87 88 template <DecoratorSet decorators, typename T> 89 inline void ShenandoahBarrierSet::satb_barrier(T *field) { 90 if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value || 91 HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 92 return; 93 } 94 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { 95 T heap_oop = RawAccess<>::oop_load(field); 96 if (!CompressedOops::is_null(heap_oop)) { 97 enqueue(CompressedOops::decode(heap_oop)); 98 } 99 } 100 } 101 102 inline void ShenandoahBarrierSet::satb_enqueue(oop value) { 103 assert(value != NULL, "checked before"); 104 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { 105 enqueue(value); 106 } 107 } 108 109 inline void ShenandoahBarrierSet::storeval_barrier(oop obj) { 110 if (ShenandoahStoreValEnqueueBarrier && obj != NULL && _heap->is_concurrent_mark_in_progress()) { 111 enqueue(obj); 112 } 113 } 114 115 inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oop value) { 116 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); 117 const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; 118 const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; 119 if (!peek && !on_strong_oop_ref) { 120 satb_enqueue(value); 121 } 122 } 123 124 template <DecoratorSet decorators> 125 inline void ShenandoahBarrierSet::keep_alive_if_weak(oop value) { 126 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); 127 if (!HasDecorator<decorators, ON_STRONG_OOP_REF>::value && 128 !HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 129 satb_enqueue(value); 130 } 131 } 132 133 template <DecoratorSet decorators, typename BarrierSetT> 134 template <typename T> 135 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { 136 oop value = Raw::oop_load_not_in_heap(addr); 137 if (value != NULL) { 138 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 139 value = bs->load_reference_barrier_native(value, addr); 140 if (value != NULL) { 141 bs->keep_alive_if_weak<decorators>(value); 142 } 143 } 144 return value; 145 } 146 147 template <DecoratorSet decorators, typename BarrierSetT> 148 template <typename T> 149 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { 150 oop value = Raw::oop_load_in_heap(addr); 151 if (value != NULL) { 152 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 153 value = bs->load_reference_barrier_not_null(value); 154 bs->keep_alive_if_weak<decorators>(value); 155 } 156 return value; 157 } 158 159 template <DecoratorSet decorators, typename BarrierSetT> 160 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { 161 oop value = Raw::oop_load_in_heap_at(base, offset); 162 if (value != NULL) { 163 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 164 value = bs->load_reference_barrier_not_null(value); 165 bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), 166 value); 167 } 168 return value; 169 } 170 171 template <DecoratorSet decorators, typename BarrierSetT> 172 template <typename T> 173 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) { 174 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); 175 ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); 176 bs->storeval_barrier(value); 177 bs->satb_barrier<decorators>(addr); 178 Raw::oop_store(addr, value); 179 } 180 181 template <DecoratorSet decorators, typename BarrierSetT> 182 template <typename T> 183 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { 184 shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc()); 185 shenandoah_assert_not_forwarded_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); 186 shenandoah_assert_not_in_cset_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); 187 188 oop_store_not_in_heap(addr, value); 189 } 190 191 template <DecoratorSet decorators, typename BarrierSetT> 192 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { 193 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); 194 } 195 196 template <DecoratorSet decorators, typename BarrierSetT> 197 template <typename T> 198 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) { 199 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 200 bs->storeval_barrier(new_value); 201 202 oop res; 203 oop expected = compare_value; 204 do { 205 compare_value = expected; 206 res = Raw::oop_atomic_cmpxchg(addr, compare_value, new_value); 207 expected = res; 208 } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected))); 209 210 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, 211 // because it must be the previous value. 212 if (res != NULL) { 213 res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); 214 bs->satb_enqueue(res); 215 } 216 return res; 217 } 218 219 template <DecoratorSet decorators, typename BarrierSetT> 220 template <typename T> 221 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) { 222 return oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value); 223 } 224 225 template <DecoratorSet decorators, typename BarrierSetT> 226 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) { 227 return oop_atomic_cmpxchg_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), compare_value, new_value); 228 } 229 230 template <DecoratorSet decorators, typename BarrierSetT> 231 template <typename T> 232 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) { 233 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 234 bs->storeval_barrier(new_value); 235 236 oop previous = Raw::oop_atomic_xchg(addr, new_value); 237 238 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, 239 // because it must be the previous value. 240 if (previous != NULL) { 241 previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); 242 bs->satb_enqueue(previous); 243 } 244 return previous; 245 } 246 247 template <DecoratorSet decorators, typename BarrierSetT> 248 template <typename T> 249 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(T* addr, oop new_value) { 250 return oop_atomic_xchg_not_in_heap(addr, new_value); 251 } 252 253 template <DecoratorSet decorators, typename BarrierSetT> 254 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) { 255 return oop_atomic_xchg_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), new_value); 256 } 257 258 // Clone barrier support 259 template <DecoratorSet decorators, typename BarrierSetT> 260 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 261 if (ShenandoahCloneBarrier) { 262 ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src); 263 } 264 Raw::clone(src, dst, size); 265 } 266 267 template <DecoratorSet decorators, typename BarrierSetT> 268 template <typename T> 269 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 270 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 271 size_t length) { 272 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 273 bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw), 274 arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw), 275 length); 276 return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 277 } 278 279 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE> 280 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { 281 assert(HAS_FWD == _heap->has_forwarded_objects(), "Forwarded object status is sane"); 282 283 Thread* thread = Thread::current(); 284 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); 285 ShenandoahMarkingContext* ctx = _heap->marking_context(); 286 const ShenandoahCollectionSet* const cset = _heap->collection_set(); 287 T* end = src + count; 288 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) { 289 T o = RawAccess<>::oop_load(elem_ptr); 290 if (!CompressedOops::is_null(o)) { 291 oop obj = CompressedOops::decode_not_null(o); 292 if (HAS_FWD && cset->is_in(obj)) { 293 oop fwd = resolve_forwarded_not_null(obj); 294 if (EVAC && obj == fwd) { 295 fwd = _heap->evacuate_object(obj, thread); 296 } 297 assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); 298 oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); 299 obj = fwd; 300 } 301 if (ENQUEUE && !ctx->is_marked(obj)) { 302 queue.enqueue_known_active(obj); 303 } 304 } 305 } 306 } 307 308 template <class T> 309 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { 310 if (count == 0) { 311 return; 312 } 313 int gc_state = _heap->gc_state(); 314 if ((gc_state & ShenandoahHeap::MARKING) != 0) { 315 arraycopy_marking(src, dst, count); 316 } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) { 317 arraycopy_evacuation(src, count); 318 } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) { 319 arraycopy_update(src, count); 320 } 321 } 322 323 template <class T> 324 void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) { 325 assert(_heap->is_concurrent_mark_in_progress(), "only during marking"); 326 T* array = ShenandoahSATBBarrier ? dst : src; 327 if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) { 328 arraycopy_work<T, false, false, true>(array, count); 329 } 330 } 331 332 inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) { 333 return ary < _heap->heap_region_containing(ary)->get_update_watermark(); 334 } 335 336 template <class T> 337 void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) { 338 assert(_heap->is_evacuation_in_progress(), "only during evacuation"); 339 if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) { 340 ShenandoahEvacOOMScope oom_evac; 341 arraycopy_work<T, true, true, false>(src, count); 342 } 343 } 344 345 template <class T> 346 void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) { 347 assert(_heap->is_update_refs_in_progress(), "only during update-refs"); 348 if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) { 349 arraycopy_work<T, true, false, false>(src, count); 350 } 351 } 352 353 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP