1 /* 2 * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 25 #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shenandoah/shenandoahAsserts.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 30 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" 31 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" 32 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 33 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 34 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" 35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 36 #include "memory/iterator.inline.hpp" 37 #include "oops/oop.inline.hpp" 38 39 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 40 return ShenandoahForwarding::get_forwardee(p); 41 } 42 43 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 44 if (((HeapWord*) p) != NULL) { 45 return resolve_forwarded_not_null(p); 46 } else { 47 return p; 48 } 49 } 50 51 template <DecoratorSet decorators, typename BarrierSetT> 52 template <typename T> 53 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { 54 oop value = Raw::oop_load_in_heap(addr); 55 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); 56 keep_alive_if_weak(decorators, value); 57 return value; 58 } 59 60 template <DecoratorSet decorators, typename BarrierSetT> 61 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { 62 oop value = Raw::oop_load_in_heap_at(base, offset); 63 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); 64 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value); 65 return value; 66 } 67 68 template <DecoratorSet decorators, typename BarrierSetT> 69 template <typename T> 70 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { 71 oop value = Raw::oop_load_not_in_heap(addr); 72 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_native(value, addr); 73 keep_alive_if_weak(decorators, value); 74 return value; 75 } 76 77 template <DecoratorSet decorators, typename BarrierSetT> 78 template <typename T> 79 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { 80 ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); 81 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; 82 if (keep_alive) { 83 ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); 84 } 85 Raw::oop_store_in_heap(addr, value); 86 } 87 88 template <DecoratorSet decorators, typename BarrierSetT> 89 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { 90 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); 91 } 92 93 template <DecoratorSet decorators, typename BarrierSetT> 94 template <typename T> 95 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) { 96 shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); 97 Raw::oop_store(addr, value); 98 } 99 100 template <DecoratorSet decorators, typename BarrierSetT> 101 template <typename T> 102 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { 103 oop res; 104 oop expected = compare_value; 105 do { 106 compare_value = expected; 107 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 108 expected = res; 109 } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected))); 110 if (res != NULL) { 111 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); 112 } else { 113 return res; 114 } 115 } 116 117 template <DecoratorSet decorators, typename BarrierSetT> 118 template <typename T> 119 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) { 120 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); 121 oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); 122 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; 123 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && 124 (result == compare_value) && 125 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { 126 ShenandoahBarrierSet::barrier_set()->enqueue(result); 127 } 128 return result; 129 } 130 131 template <DecoratorSet decorators, typename BarrierSetT> 132 template <typename T> 133 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { 134 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value); 135 keep_alive_if_weak(decorators, result); 136 return result; 137 } 138 139 template <DecoratorSet decorators, typename BarrierSetT> 140 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { 141 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value); 142 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result); 143 return result; 144 } 145 146 template <DecoratorSet decorators, typename BarrierSetT> 147 template <typename T> 148 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { 149 oop previous = Raw::oop_atomic_xchg(new_value, addr); 150 if (previous != NULL) { 151 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); 152 } else { 153 return previous; 154 } 155 } 156 157 template <DecoratorSet decorators, typename BarrierSetT> 158 template <typename T> 159 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) { 160 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); 161 oop result = oop_atomic_xchg_not_in_heap(new_value, addr); 162 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; 163 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && 164 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { 165 ShenandoahBarrierSet::barrier_set()->enqueue(result); 166 } 167 return result; 168 } 169 170 template <DecoratorSet decorators, typename BarrierSetT> 171 template <typename T> 172 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) { 173 oop result = oop_atomic_xchg_in_heap_impl(new_value, addr); 174 keep_alive_if_weak(addr, result); 175 return result; 176 } 177 178 template <DecoratorSet decorators, typename BarrierSetT> 179 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { 180 oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset)); 181 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result); 182 return result; 183 } 184 185 // Clone barrier support 186 template <DecoratorSet decorators, typename BarrierSetT> 187 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 188 if (ShenandoahCloneBarrier) { 189 ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src); 190 } 191 Raw::clone(src, dst, size); 192 } 193 194 template <DecoratorSet decorators, typename BarrierSetT> 195 template <typename T> 196 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 197 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 198 size_t length) { 199 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 200 bs->arraycopy_pre(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw), 201 arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw), 202 length); 203 return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 204 } 205 206 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE> 207 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { 208 Thread* thread = Thread::current(); 209 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); 210 ShenandoahMarkingContext* ctx = _heap->marking_context(); 211 const ShenandoahCollectionSet* const cset = _heap->collection_set(); 212 T* end = src + count; 213 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) { 214 T o = RawAccess<>::oop_load(elem_ptr); 215 if (!CompressedOops::is_null(o)) { 216 oop obj = CompressedOops::decode_not_null(o); 217 if (HAS_FWD && cset->is_in((HeapWord *) obj)) { 218 assert(_heap->has_forwarded_objects(), "only get here with forwarded objects"); 219 oop fwd = resolve_forwarded_not_null(obj); 220 if (EVAC && obj == fwd) { 221 fwd = _heap->evacuate_object(obj, thread); 222 } 223 assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); 224 oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); 225 obj = fwd; 226 } 227 if (ENQUEUE && !ctx->is_marked(obj)) { 228 queue.enqueue_known_active(obj); 229 } 230 } 231 } 232 } 233 234 template <class T> 235 void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) { 236 if (_heap->is_concurrent_mark_in_progress()) { 237 if (_heap->has_forwarded_objects()) { 238 arraycopy_work<T, true, false, true>(dst, count); 239 } else { 240 arraycopy_work<T, false, false, true>(dst, count); 241 } 242 } 243 244 arraycopy_update_impl(src, count); 245 } 246 247 void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) { 248 arraycopy_pre_work(src, dst, count); 249 } 250 251 void ShenandoahBarrierSet::arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count) { 252 arraycopy_pre_work(src, dst, count); 253 } 254 255 template <class T> 256 void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) { 257 if (_heap->is_evacuation_in_progress()) { 258 ShenandoahEvacOOMScope oom_evac; 259 arraycopy_work<T, true, true, false>(src, count); 260 } else if (_heap->is_concurrent_traversal_in_progress()){ 261 ShenandoahEvacOOMScope oom_evac; 262 arraycopy_work<T, true, true, true>(src, count); 263 } else if (_heap->has_forwarded_objects()) { 264 arraycopy_work<T, true, false, false>(src, count); 265 } 266 } 267 268 void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count) { 269 arraycopy_update_impl(src, count); 270 } 271 272 void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count) { 273 arraycopy_update_impl(src, count); 274 } 275 276 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP