1 /* 2 * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 25 #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 29 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" 30 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 31 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 32 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" 33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 34 35 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 36 return ShenandoahForwarding::get_forwardee(p); 37 } 38 39 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 40 if (((HeapWord*) p) != NULL) { 41 return resolve_forwarded_not_null(p); 42 } else { 43 return p; 44 } 45 } 46 47 template <DecoratorSet decorators, typename BarrierSetT> 48 template <typename T> 49 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) { 50 oop value = Raw::oop_load_in_heap(addr); 51 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); 52 keep_alive_if_weak(decorators, value); 53 return value; 54 } 55 56 template <DecoratorSet decorators, typename BarrierSetT> 57 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { 58 oop value = Raw::oop_load_in_heap_at(base, offset); 59 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); 60 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value); 61 return value; 62 } 63 64 template <DecoratorSet decorators, typename BarrierSetT> 65 template <typename T> 66 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) { 67 oop value = Raw::oop_load_not_in_heap(addr); 68 value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); 69 keep_alive_if_weak(decorators, value); 70 return value; 71 } 72 73 template <DecoratorSet decorators, typename BarrierSetT> 74 template <typename T> 75 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) { 76 ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); 77 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; 78 if (keep_alive) { 79 ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); 80 } 81 Raw::oop_store_in_heap(addr, value); 82 } 83 84 template <DecoratorSet decorators, typename BarrierSetT> 85 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { 86 oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value); 87 } 88 89 template <DecoratorSet decorators, typename BarrierSetT> 90 template <typename T> 91 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { 92 oop res; 93 oop expected = compare_value; 94 do { 95 compare_value = expected; 96 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 97 expected = res; 98 } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected))); 99 if (res != NULL) { 100 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); 101 } else { 102 return res; 103 } 104 } 105 106 template <DecoratorSet decorators, typename BarrierSetT> 107 template <typename T> 108 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) { 109 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); 110 oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); 111 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; 112 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && 113 oopDesc::equals_raw(result, compare_value) && 114 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { 115 ShenandoahBarrierSet::barrier_set()->enqueue(result); 116 } 117 return result; 118 } 119 120 template <DecoratorSet decorators, typename BarrierSetT> 121 template <typename T> 122 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { 123 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value); 124 keep_alive_if_weak(decorators, result); 125 return result; 126 } 127 128 template <DecoratorSet decorators, typename BarrierSetT> 129 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { 130 oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value); 131 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result); 132 return result; 133 } 134 135 template <DecoratorSet decorators, typename BarrierSetT> 136 template <typename T> 137 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { 138 oop previous = Raw::oop_atomic_xchg(new_value, addr); 139 if (previous != NULL) { 140 return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); 141 } else { 142 return previous; 143 } 144 } 145 146 template <DecoratorSet decorators, typename BarrierSetT> 147 template <typename T> 148 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) { 149 ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); 150 oop result = oop_atomic_xchg_not_in_heap(new_value, addr); 151 const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; 152 if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && 153 ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { 154 ShenandoahBarrierSet::barrier_set()->enqueue(result); 155 } 156 return result; 157 } 158 159 template <DecoratorSet decorators, typename BarrierSetT> 160 template <typename T> 161 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) { 162 oop result = oop_atomic_xchg_in_heap_impl(new_value, addr); 163 keep_alive_if_weak(addr, result); 164 return result; 165 } 166 167 template <DecoratorSet decorators, typename BarrierSetT> 168 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { 169 oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset)); 170 keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result); 171 return result; 172 } 173 174 template <typename T> 175 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, 176 bool checkcast, bool satb, bool disjoint, 177 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 178 if (checkcast) { 179 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode); 180 } else { 181 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode); 182 } 183 } 184 185 template <typename T, bool CHECKCAST> 186 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, 187 bool satb, bool disjoint, 188 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 189 if (satb) { 190 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode); 191 } else { 192 return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, disjoint, storeval_mode); 193 } 194 } 195 196 template <typename T, bool CHECKCAST, bool SATB> 197 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint, 198 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 199 switch (storeval_mode) { 200 case NONE: 201 return arraycopy_loop<T, CHECKCAST, SATB, NONE>(src, dst, length, bound, disjoint); 202 case RESOLVE_BARRIER: 203 return arraycopy_loop<T, CHECKCAST, SATB, RESOLVE_BARRIER>(src, dst, length, bound, disjoint); 204 case EVAC_BARRIER: 205 return arraycopy_loop<T, CHECKCAST, SATB, EVAC_BARRIER>(src, dst, length, bound, disjoint); 206 default: 207 ShouldNotReachHere(); 208 return true; // happy compiler 209 } 210 } 211 212 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE> 213 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) { 214 Thread* thread = Thread::current(); 215 ShenandoahMarkingContext* ctx = _heap->marking_context(); 216 ShenandoahEvacOOMScope oom_evac_scope; 217 218 // We need to handle four cases: 219 // 220 // a) src < dst, conjoint, can only copy backward only 221 // [...src...] 222 // [...dst...] 223 // 224 // b) src < dst, disjoint, can only copy forward, because types may mismatch 225 // [...src...] 226 // [...dst...] 227 // 228 // c) src > dst, conjoint, can copy forward only 229 // [...src...] 230 // [...dst...] 231 // 232 // d) src > dst, disjoint, can only copy forward, because types may mismatch 233 // [...src...] 234 // [...dst...] 235 // 236 if (src > dst || disjoint) { 237 // copy forward: 238 T* cur_src = src; 239 T* cur_dst = dst; 240 T* src_end = src + length; 241 for (; cur_src < src_end; cur_src++, cur_dst++) { 242 if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread, ctx)) { 243 return false; 244 } 245 } 246 } else { 247 // copy backward: 248 T* cur_src = src + length - 1; 249 T* cur_dst = dst + length - 1; 250 for (; cur_src >= src; cur_src--, cur_dst--) { 251 if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread, ctx)) { 252 return false; 253 } 254 } 255 } 256 return true; 257 } 258 259 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE> 260 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* const thread, ShenandoahMarkingContext* const ctx) { 261 T o = RawAccess<>::oop_load(cur_src); 262 263 if (SATB) { 264 assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "Shouldn't be here otherwise"); 265 T prev = RawAccess<>::oop_load(cur_dst); 266 if (!CompressedOops::is_null(prev)) { 267 oop prev_obj = CompressedOops::decode_not_null(prev); 268 switch (STOREVAL_MODE) { 269 case NONE: 270 break; 271 case RESOLVE_BARRIER: 272 case EVAC_BARRIER: 273 // The evac-barrier case cannot really happen. It's traversal-only and traversal 274 // doesn't currently use SATB. And even if it did, it would not be fatal to just do the normal resolve here. 275 prev_obj = ShenandoahBarrierSet::resolve_forwarded_not_null(prev_obj); 276 } 277 if (!ctx->is_marked(prev_obj)) { 278 ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(prev_obj); 279 } 280 } 281 } 282 283 if (!CompressedOops::is_null(o)) { 284 oop obj = CompressedOops::decode_not_null(o); 285 286 if (CHECKCAST) { 287 assert(bound != NULL, "need element klass for checkcast"); 288 if (!oopDesc::is_instanceof_or_null(obj, bound)) { 289 return false; 290 } 291 } 292 293 switch (STOREVAL_MODE) { 294 case NONE: 295 break; 296 case RESOLVE_BARRIER: 297 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 298 break; 299 case EVAC_BARRIER: 300 if (_heap->in_collection_set(obj)) { 301 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 302 if (oopDesc::equals_raw(forw, obj)) { 303 forw = _heap->evacuate_object(forw, thread); 304 } 305 obj = forw; 306 } 307 enqueue(obj); 308 break; 309 default: 310 ShouldNotReachHere(); 311 } 312 313 RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj); 314 } else { 315 // Store null. 316 RawAccess<>::oop_store(cur_dst, o); 317 } 318 return true; 319 } 320 321 // Clone barrier support 322 template <DecoratorSet decorators, typename BarrierSetT> 323 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 324 Raw::clone(src, dst, size); 325 ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size)); 326 } 327 328 template <DecoratorSet decorators, typename BarrierSetT> 329 template <typename T> 330 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 331 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 332 size_t length) { 333 ShenandoahHeap* heap = ShenandoahHeap::heap(); 334 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); 335 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value; 336 bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value; 337 ArrayCopyStoreValMode storeval_mode; 338 if (heap->has_forwarded_objects()) { 339 if (heap->is_concurrent_traversal_in_progress()) { 340 storeval_mode = EVAC_BARRIER; 341 } else if (heap->is_update_refs_in_progress()) { 342 storeval_mode = RESOLVE_BARRIER; 343 } else { 344 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress"); 345 storeval_mode = NONE; // E.g. during evac or outside cycle 346 } 347 } else { 348 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress"); 349 storeval_mode = NONE; 350 } 351 352 if (!satb && !checkcast && storeval_mode == NONE) { 353 // Short-circuit to bulk copy. 354 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 355 } 356 357 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); 358 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); 359 360 Klass* bound = objArrayOop(dst_obj)->element_klass(); 361 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); 362 return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, disjoint, storeval_mode); 363 } 364 365 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP