1 /* 2 * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 29 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" 30 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 33 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" 34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 35 #include "memory/iterator.inline.hpp" 36 #include "oops/oop.inline.hpp" 37 38 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 39 return ShenandoahForwarding::get_forwardee(p); 40 } 41 42 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 43 if (((HeapWord*) p) != NULL) { 44 return resolve_forwarded_not_null(p); 45 } else { 46 return p; 47 } 48 } 49 50 inline void ShenandoahBarrierSet::enqueue(oop obj) { 51 shenandoah_assert_not_forwarded_if(NULL, obj, _heap->is_concurrent_traversal_in_progress()); 52 assert(_satb_mark_queue_set.is_active(), "only get here when SATB active"); 53 54 // Filter marked objects before hitting the SATB queues. The same predicate would 55 // be used by SATBMQ::filter to eliminate already marked objects downstream, but 56 // filtering here helps to avoid wasteful SATB queueing work to begin with. 57 if (!_heap->requires_marking(obj)) return; 58 59 Thread* thr = Thread::current(); 60 if (thr->is_Java_thread()) { 61 ShenandoahThreadLocalData::satb_mark_queue(thr).enqueue_known_active(obj); 62 } else { 63 MutexLockerEx x(Shared_SATB_Q_lock, Mutex::_no_safepoint_check_flag); 64 _satb_mark_queue_set.shared_satb_queue()->enqueue_known_active(obj); 65 } 66 } 67 68 template <DecoratorSet decorators, typename T> 69 inline void ShenandoahBarrierSet::satb_barrier(T *field) { 70 if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value || 71 HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 72 return; 73 } 74 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { 75 T heap_oop = RawAccess<>::oop_load(field); 76 if (!CompressedOops::is_null(heap_oop)) { 77 enqueue(CompressedOops::decode(heap_oop)); 78 } 79 } 80 } 81 82 inline void ShenandoahBarrierSet::satb_enqueue(oop value) { 83 assert(value != NULL, "checked before"); 84 if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) { 85 enqueue(value); 86 } 87 } 88 89 inline void ShenandoahBarrierSet::storeval_barrier(oop obj) { 90 if (obj != NULL && ShenandoahStoreValEnqueueBarrier && _heap->is_concurrent_traversal_in_progress()) { 91 enqueue(obj); 92 } 93 } 94 95 inline void ShenandoahBarrierSet::keep_alive_barrier(oop value) { 96 assert(value != NULL, "checked before"); 97 if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) { 98 enqueue(value); 99 } 100 } 101 102 inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oop value) { 103 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); 104 const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; 105 const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; 106 if (!peek && !on_strong_oop_ref) { 107 keep_alive_barrier(value); 108 } 109 } 110 111 template <DecoratorSet decorators> 112 inline void ShenandoahBarrierSet::keep_alive_if_weak(oop value) { 113 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); 114 if (!HasDecorator<decorators, ON_STRONG_OOP_REF>::value && 115 !HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 116 keep_alive_barrier(value); 117 } 118 } 119 120 template <DecoratorSet decorators, typename BarrierSetT> 121 template <typename T> 122 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { 123 oop value = Raw::oop_load_not_in_heap(addr); 124 if (value != NULL) { 125 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 126 value = bs->load_reference_barrier_not_null(value); 127 bs->keep_alive_if_weak<decorators>(value); 128 } 129 return value; 130 } 131 132 template <DecoratorSet decorators, typename BarrierSetT> 133 template <typename T> 134 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { 135 oop value = Raw::oop_load_in_heap(addr); 136 if (value != NULL) { 137 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 138 value = bs->load_reference_barrier_not_null(value); 139 bs->keep_alive_if_weak<decorators>(value); 140 } 141 return value; 142 } 143 144 template <DecoratorSet decorators, typename BarrierSetT> 145 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { 146 oop value = Raw::oop_load_in_heap_at(base, offset); 147 if (value != NULL) { 148 ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); 149 value = bs->load_reference_barrier_not_null(value); 150 bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), 151 value); 152 } 153 return value; 154 } 155 156 template <DecoratorSet decorators, typename BarrierSetT> 157 template <typename T> 158 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) { 159 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); 160 ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); 161 bs->storeval_barrier(value); 162 bs->satb_barrier<decorators>(addr); 163 Raw::oop_store(addr, value); 164 } 165 166 template <DecoratorSet decorators, typename BarrierSetT> 167 template <typename T> 168 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { 169 shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc()); 170 shenandoah_assert_not_forwarded_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); 171 shenandoah_assert_not_in_cset_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); 172 173 oop_store_not_in_heap(addr, value); 174 } 175 176 template <DecoratorSet decorators, typename BarrierSetT> 177 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { 178 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); 179 } 180 181 template <DecoratorSet decorators, typename BarrierSetT> 182 template <typename T> 183 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { 184 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 185 bs->storeval_barrier(new_value); 186 187 oop res; 188 oop expected = compare_value; 189 do { 190 compare_value = expected; 191 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 192 expected = res; 193 } while (compare_value != expected && resolve_forwarded(compare_value) == resolve_forwarded(expected)); 194 195 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, 196 // because it must be the previous value. 197 if (res != NULL) { 198 res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); 199 bs->satb_enqueue(res); 200 } 201 return res; 202 } 203 204 template <DecoratorSet decorators, typename BarrierSetT> 205 template <typename T> 206 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { 207 return oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); 208 } 209 210 template <DecoratorSet decorators, typename BarrierSetT> 211 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { 212 return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value); 213 } 214 215 template <DecoratorSet decorators, typename BarrierSetT> 216 template <typename T> 217 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { 218 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 219 bs->storeval_barrier(new_value); 220 221 oop previous = Raw::oop_atomic_xchg(new_value, addr); 222 223 // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, 224 // because it must be the previous value. 225 if (previous != NULL) { 226 previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); 227 bs->satb_enqueue(previous); 228 } 229 return previous; 230 } 231 232 template <DecoratorSet decorators, typename BarrierSetT> 233 template <typename T> 234 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) { 235 return oop_atomic_xchg_in_heap_impl(new_value, addr); 236 } 237 238 template <DecoratorSet decorators, typename BarrierSetT> 239 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { 240 return oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset)); 241 } 242 243 // Clone barrier support 244 template <DecoratorSet decorators, typename BarrierSetT> 245 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 246 if (ShenandoahCloneBarrier) { 247 ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src); 248 } 249 Raw::clone(src, dst, size); 250 } 251 252 template <DecoratorSet decorators, typename BarrierSetT> 253 template <typename T> 254 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 255 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 256 size_t length) { 257 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 258 bs->arraycopy_pre(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw), 259 arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw), 260 length); 261 return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 262 } 263 264 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE> 265 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { 266 Thread* thread = Thread::current(); 267 ShenandoahSATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); 268 ShenandoahMarkingContext* ctx = _heap->marking_context(); 269 const ShenandoahCollectionSet* const cset = _heap->collection_set(); 270 T* end = src + count; 271 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) { 272 T o = RawAccess<>::oop_load(elem_ptr); 273 if (!CompressedOops::is_null(o)) { 274 oop obj = CompressedOops::decode_not_null(o); 275 if (HAS_FWD && cset->is_in((HeapWord *) obj)) { 276 assert(_heap->has_forwarded_objects(), "only get here with forwarded objects"); 277 oop fwd = resolve_forwarded_not_null(obj); 278 if (EVAC && obj == fwd) { 279 fwd = _heap->evacuate_object(obj, thread); 280 } 281 assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); 282 oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); 283 obj = fwd; 284 } 285 if (ENQUEUE && !ctx->is_marked(obj)) { 286 queue.enqueue_known_active(obj); 287 } 288 } 289 } 290 } 291 292 template <class T> 293 void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) { 294 if (_heap->is_concurrent_mark_in_progress()) { 295 if (_heap->has_forwarded_objects()) { 296 arraycopy_work<T, true, false, true>(dst, count); 297 } else { 298 arraycopy_work<T, false, false, true>(dst, count); 299 } 300 } 301 302 arraycopy_update_impl(src, count); 303 } 304 305 void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) { 306 arraycopy_pre_work(src, dst, count); 307 } 308 309 void ShenandoahBarrierSet::arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count) { 310 arraycopy_pre_work(src, dst, count); 311 } 312 313 template <class T> 314 void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) { 315 if (_heap->is_evacuation_in_progress()) { 316 ShenandoahEvacOOMScope oom_evac; 317 arraycopy_work<T, true, true, false>(src, count); 318 } else if (_heap->is_concurrent_traversal_in_progress()){ 319 ShenandoahEvacOOMScope oom_evac; 320 arraycopy_work<T, true, true, true>(src, count); 321 } else if (_heap->has_forwarded_objects()) { 322 arraycopy_work<T, true, false, false>(src, count); 323 } 324 } 325 326 void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count) { 327 arraycopy_update_impl(src, count); 328 } 329 330 void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count) { 331 arraycopy_update_impl(src, count); 332 } 333 334 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP