1 /* 2 * Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 27 28 #include "gc/shared/barrierSet.hpp" 29 #include "gc/shenandoah/shenandoahAsserts.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 31 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" 32 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" 33 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 34 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 35 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" 36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 37 #include "memory/iterator.inline.hpp" 38 #include "oops/oop.inline.hpp" 39 40 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 41 return ShenandoahForwarding::get_forwardee(p); 42 } 43 44 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 45 if (p != NULL) { 46 return resolve_forwarded_not_null(p); 47 } else { 48 return p; 49 } 50 } 51 52 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null_mutator(oop p) { 53 return ShenandoahForwarding::get_forwardee_mutator(p); 54 } 55 56 template <class T> 57 inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) { 58 assert(ShenandoahLoadRefBarrier, "should be enabled"); 59 shenandoah_assert_in_cset(load_addr, obj); 60 61 oop fwd = resolve_forwarded_not_null_mutator(obj); 62 if (obj == fwd) { 63 assert(_heap->is_evacuation_in_progress(), 64 "evac should be in progress"); 65 ShenandoahEvacOOMScope scope; 66 fwd = _heap->evacuate_object_recursively(obj, load_addr, Thread::current()); 67 } 68 69 if (load_addr != NULL && fwd != obj) { 70 // Since we are here and we know the load address, update the reference. 71 ShenandoahHeap::cas_oop(fwd, load_addr, obj); 72 } 73 74 return fwd; 75 } 76 77 inline void ShenandoahBarrierSet::enqueue(oop obj) { 78 assert(_satb_mark_queue_set.is_active(), "only get here when SATB active"); 79 80 // Filter marked objects before hitting the SATB queues. The same predicate would 81 // be used by SATBMQ::filter to eliminate already marked objects downstream, but 82 // filtering here helps to avoid wasteful SATB queueing work to begin with. 83 if (!_heap->requires_marking<false>(obj)) return; 84 85 ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).enqueue_known_active(obj); 86 } 87 88 template <DecoratorSet decorators, typename T> 89 inline void ShenandoahBarrierSet::satb_barrier(T *field) { 90 if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value || 91 HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 92 return; 93 } 94 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { 95 T heap_oop = RawAccess<>::oop_load(field); 96 if (!CompressedOops::is_null(heap_oop)) { 97 enqueue(CompressedOops::decode(heap_oop)); 98 } 99 } 100 } 101 102 inline void ShenandoahBarrierSet::satb_enqueue(oop value) { 103 assert(value != NULL, "checked before"); 104 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { 105 enqueue(value); 106 } 107 } 108 109 inline void ShenandoahBarrierSet::storeval_barrier(oop obj) { 110 if (ShenandoahStoreValEnqueueBarrier && obj != NULL && _heap->is_concurrent_mark_in_progress()) { 111 enqueue(obj); 112 } 113 } 114 115 inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oop value) { 116 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); 117 const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; 118 const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; 119 if (!peek && !on_strong_oop_ref) { 120 satb_enqueue(value); 121 } 122 } 123 124 template <DecoratorSet decorators> 125 inline void ShenandoahBarrierSet::keep_alive_if_weak(oop value) { 126 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); 127 if (!HasDecorator<decorators, ON_STRONG_OOP_REF>::value && 128 !HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 129 satb_enqueue(value); 130 } 131 } 132 133 template <DecoratorSet decorators, typename BarrierSetT> 134 template <typename T> 135 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { 136 oop value = Raw::oop_load_not_in_heap(addr); 137 if (value != NULL) { 138 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 139 value = bs->load_reference_barrier_native(value, addr); 140 if (value != NULL) { 141 bs->keep_alive_if_weak<decorators>(value); 142 } 143 } 144 return value; 145 } 146 147 template <DecoratorSet decorators, typename BarrierSetT> 148 template <typename T> 149 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { 150 oop value = Raw::oop_load_in_heap(addr); 151 if (value != NULL) { 152 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 153 value = bs->load_reference_barrier_not_null(value, addr); 154 bs->keep_alive_if_weak<decorators>(value); 155 } 156 return value; 157 } 158 159 template <DecoratorSet decorators, typename BarrierSetT> 160 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { 161 oop value = Raw::oop_load_in_heap_at(base, offset); 162 if (value != NULL) { 163 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 164 if (UseCompressedOops) { 165 value = bs->load_reference_barrier_not_null(value, (narrowOop*)Raw::field_addr(base, offset)); 166 } else { 167 value = bs->load_reference_barrier_not_null(value, (oop*)Raw::field_addr(base, offset)); 168 } 169 bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), 170 value); 171 } 172 return value; 173 } 174 175 template <DecoratorSet decorators, typename BarrierSetT> 176 template <typename T> 177 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) { 178 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); 179 ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); 180 bs->storeval_barrier(value); 181 bs->satb_barrier<decorators>(addr); 182 Raw::oop_store(addr, value); 183 } 184 185 template <DecoratorSet decorators, typename BarrierSetT> 186 template <typename T> 187 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { 188 shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc()); 189 shenandoah_assert_not_forwarded_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); 190 shenandoah_assert_not_in_cset_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); 191 192 oop_store_not_in_heap(addr, value); 193 } 194 195 template <DecoratorSet decorators, typename BarrierSetT> 196 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { 197 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); 198 } 199 200 template <DecoratorSet decorators, typename BarrierSetT> 201 template <typename T> 202 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) { 203 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 204 bs->storeval_barrier(new_value); 205 206 oop res; 207 oop expected = compare_value; 208 do { 209 compare_value = expected; 210 res = Raw::oop_atomic_cmpxchg(addr, compare_value, new_value); 211 expected = res; 212 } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected))); 213 214 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, 215 // because it must be the previous value. 216 if (res != NULL) { 217 res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res, (oop*)NULL); // TODO: really NULL? 218 bs->satb_enqueue(res); 219 } 220 return res; 221 } 222 223 template <DecoratorSet decorators, typename BarrierSetT> 224 template <typename T> 225 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) { 226 return oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value); 227 } 228 229 template <DecoratorSet decorators, typename BarrierSetT> 230 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) { 231 return oop_atomic_cmpxchg_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), compare_value, new_value); 232 } 233 234 template <DecoratorSet decorators, typename BarrierSetT> 235 template <typename T> 236 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) { 237 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 238 bs->storeval_barrier(new_value); 239 240 oop previous = Raw::oop_atomic_xchg(addr, new_value); 241 242 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, 243 // because it must be the previous value. 244 if (previous != NULL) { 245 previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous, (oop*)NULL); // TODO: really NULL? 246 bs->satb_enqueue(previous); 247 } 248 return previous; 249 } 250 251 template <DecoratorSet decorators, typename BarrierSetT> 252 template <typename T> 253 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(T* addr, oop new_value) { 254 return oop_atomic_xchg_not_in_heap(addr, new_value); 255 } 256 257 template <DecoratorSet decorators, typename BarrierSetT> 258 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) { 259 return oop_atomic_xchg_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), new_value); 260 } 261 262 // Clone barrier support 263 template <DecoratorSet decorators, typename BarrierSetT> 264 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 265 if (ShenandoahCloneBarrier) { 266 ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src); 267 } 268 Raw::clone(src, dst, size); 269 } 270 271 template <DecoratorSet decorators, typename BarrierSetT> 272 template <typename T> 273 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 274 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 275 size_t length) { 276 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 277 bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw), 278 arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw), 279 length); 280 return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 281 } 282 283 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE> 284 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { 285 assert(HAS_FWD == _heap->has_forwarded_objects(), "Forwarded object status is sane"); 286 287 Thread* thread = Thread::current(); 288 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); 289 ShenandoahMarkingContext* ctx = _heap->marking_context(); 290 const ShenandoahCollectionSet* const cset = _heap->collection_set(); 291 T* end = src + count; 292 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) { 293 T o = RawAccess<>::oop_load(elem_ptr); 294 if (!CompressedOops::is_null(o)) { 295 oop obj = CompressedOops::decode_not_null(o); 296 if (HAS_FWD && cset->is_in(obj)) { 297 oop fwd = resolve_forwarded_not_null(obj); 298 if (EVAC && obj == fwd) { 299 fwd = _heap->evacuate_object_recursively(obj, elem_ptr, thread); 300 } 301 assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); 302 oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); 303 obj = fwd; 304 } 305 if (ENQUEUE && !ctx->is_marked(obj)) { 306 queue.enqueue_known_active(obj); 307 } 308 } 309 } 310 } 311 312 template <class T> 313 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { 314 if (count == 0) { 315 return; 316 } 317 int gc_state = _heap->gc_state(); 318 if ((gc_state & ShenandoahHeap::MARKING) != 0) { 319 arraycopy_marking(src, dst, count); 320 } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) { 321 arraycopy_evacuation(src, count); 322 } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) { 323 arraycopy_update(src, count); 324 } 325 } 326 327 template <class T> 328 void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) { 329 assert(_heap->is_concurrent_mark_in_progress(), "only during marking"); 330 T* array = ShenandoahSATBBarrier ? dst : src; 331 if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) { 332 arraycopy_work<T, false, false, true>(array, count); 333 } 334 } 335 336 inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) { 337 return ary < _heap->heap_region_containing(ary)->get_update_watermark(); 338 } 339 340 template <class T> 341 void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) { 342 assert(_heap->is_evacuation_in_progress(), "only during evacuation"); 343 if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) { 344 ShenandoahEvacOOMScope oom_evac; 345 arraycopy_work<T, true, true, false>(src, count); 346 } 347 } 348 349 template <class T> 350 void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) { 351 assert(_heap->is_update_refs_in_progress(), "only during update-refs"); 352 if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) { 353 arraycopy_work<T, true, false, false>(src, count); 354 } 355 } 356 357 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP