1 /* 2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shenandoah/brooksPointer.inline.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 30 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 33 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 34 return BrooksPointer::forwardee(p); 35 } 36 37 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 38 if (((HeapWord*) p) != NULL) { 39 return resolve_forwarded_not_null(p); 40 } else { 41 return p; 42 } 43 } 44 45 template <DecoratorSet decorators, typename BarrierSetT> 46 template <typename T> 47 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { 48 oop res; 49 oop expected = compare_value; 50 do { 51 compare_value = expected; 52 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 53 expected = res; 54 } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(BarrierSet::barrier_set()->read_barrier(compare_value), BarrierSet::barrier_set()->read_barrier(expected))); 55 if (oopDesc::unsafe_equals(expected, compare_value)) { 56 if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) { 57 ShenandoahBarrierSet::enqueue(compare_value); 58 } 59 if (UseShenandoahMatrix && !CompressedOops::is_null(new_value)) { 60 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix(); 61 matrix->set_connected(addr, new_value); 62 } 63 } 64 return res; 65 } 66 67 template <DecoratorSet decorators, typename BarrierSetT> 68 template <typename T> 69 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) { 70 oop previous = Raw::oop_atomic_xchg(new_value, addr); 71 if (ShenandoahSATBBarrier) { 72 if (!CompressedOops::is_null(previous)) { 73 ShenandoahBarrierSet::enqueue(previous); 74 } 75 } 76 if (UseShenandoahMatrix && !CompressedOops::is_null(new_value)) { 77 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix(); 78 matrix->set_connected(addr, new_value); 79 } 80 return previous; 81 } 82 83 template <DecoratorSet decorators, typename BarrierSetT> 84 template <typename T> 85 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 86 assert(((T*)(void*) src_obj) <= src && ((HeapWord*) src) < (((HeapWord*)(void*) src_obj) + src_obj->size()), "pointer out of object bounds src_obj: %p, src: %p, end: %p, size: %u", (void*) src_obj, src, (((HeapWord*)(void*) src_obj) + src_obj->size()), src_obj->size()); 87 assert(((T*)(void*) dst_obj) <= dst && ((HeapWord*) dst) < ((HeapWord*)(void*) dst_obj) + dst_obj->size(), "pointer out of object bounds dst_obj: %p, dst: %p, end: %p, size: %u", (void*) dst_obj, dst, (((HeapWord*)(void*) dst_obj) + dst_obj->size()), dst_obj->size()); 88 if (!CompressedOops::is_null(src_obj)) { 89 size_t src_offset = pointer_delta((void*) src, (void*) src_obj, sizeof(char)); 90 src_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->read_barrier(src_obj)); 91 src = (T*) (((char*)(void*) src_obj) + src_offset); 92 } 93 if (!CompressedOops::is_null(dst_obj)) { 94 size_t dst_offset = pointer_delta((void*) dst, (void*) dst_obj, sizeof(char)); 95 dst_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_barrier(dst_obj)); 96 dst = (T*) (((char*)(void*) dst_obj) + dst_offset); 97 } 98 return Raw::arraycopy(src_obj, dst_obj, src, dst, length); 99 } 100 101 template <typename T> 102 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, 103 bool checkcast, bool satb, bool matrix, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 104 if (checkcast) { 105 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, matrix, storeval_mode); 106 } else { 107 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, matrix, storeval_mode); 108 } 109 } 110 111 template <typename T, bool CHECKCAST> 112 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, 113 bool satb, bool matrix, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 114 if (satb) { 115 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, matrix, storeval_mode); 116 } else { 117 return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, matrix, storeval_mode); 118 } 119 } 120 121 template <typename T, bool CHECKCAST, bool SATB> 122 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, 123 bool matrix, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 124 if (matrix) { 125 return arraycopy_loop_4<T, CHECKCAST, SATB, true>(src, dst, length, bound, storeval_mode); 126 } else { 127 return arraycopy_loop_4<T, CHECKCAST, SATB, false>(src, dst, length, bound, storeval_mode); 128 } 129 } 130 131 template <typename T, bool CHECKCAST, bool SATB, bool MATRIX> 132 bool ShenandoahBarrierSet::arraycopy_loop_4(T* src, T* dst, size_t length, Klass* bound, 133 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 134 switch (storeval_mode) { 135 case NONE: 136 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, NONE>(src, dst, length, bound); 137 case READ_BARRIER: 138 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, READ_BARRIER>(src, dst, length, bound); 139 case WRITE_BARRIER_MAYBE_ENQUEUE: 140 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, WRITE_BARRIER_MAYBE_ENQUEUE>(src, dst, length, bound); 141 case WRITE_BARRIER_ALWAYS_ENQUEUE: 142 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, WRITE_BARRIER_ALWAYS_ENQUEUE>(src, dst, length, bound); 143 default: 144 ShouldNotReachHere(); 145 return true; // happy compiler 146 } 147 } 148 149 template <typename T, bool CHECKCAST, bool SATB, bool MATRIX, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE> 150 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound) { 151 Thread* thread = Thread::current(); 152 153 ShenandoahEvacOOMScope oom_evac_scope; 154 155 // We need to handle four cases: 156 // 157 // a) src < dst, intersecting, can only copy backward only 158 // [...src...] 159 // [...dst...] 160 // 161 // b) src < dst, non-intersecting, can copy forward/backward 162 // [...src...] 163 // [...dst...] 164 // 165 // c) src > dst, intersecting, can copy forward only 166 // [...src...] 167 // [...dst...] 168 // 169 // d) src > dst, non-intersecting, can copy forward/backward 170 // [...src...] 171 // [...dst...] 172 // 173 if (src > dst) { 174 // copy forward: 175 T* cur_src = src; 176 T* cur_dst = dst; 177 T* src_end = src + length; 178 for (; cur_src < src_end; cur_src++, cur_dst++) { 179 if (!arraycopy_element<T, CHECKCAST, SATB, MATRIX, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) { 180 return false; 181 } 182 } 183 } else { 184 // copy backward: 185 T* cur_src = src + length - 1; 186 T* cur_dst = dst + length - 1; 187 for (; cur_src >= src; cur_src--, cur_dst--) { 188 if (!arraycopy_element<T, CHECKCAST, SATB, MATRIX, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) { 189 return false; 190 } 191 } 192 } 193 return true; 194 } 195 196 template <typename T, bool CHECKCAST, bool SATB, bool MATRIX, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE> 197 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) { 198 T o = RawAccess<>::oop_load(cur_src); 199 200 if (SATB) { 201 T prev = RawAccess<>::oop_load(cur_dst); 202 if (!CompressedOops::is_null(prev)) { 203 oop prev_obj = CompressedOops::decode_not_null(prev); 204 enqueue(prev_obj); 205 } 206 } 207 208 if (!CompressedOops::is_null(o)) { 209 oop obj = CompressedOops::decode_not_null(o); 210 211 if (CHECKCAST) { 212 assert(bound != NULL, "need element klass for checkcast"); 213 if (!oopDesc::is_instanceof_or_null(obj, bound)) { 214 return false; 215 } 216 } 217 218 switch (STOREVAL_MODE) { 219 case NONE: 220 break; 221 case READ_BARRIER: 222 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 223 break; 224 case WRITE_BARRIER_MAYBE_ENQUEUE: 225 if (_heap->in_collection_set(obj)) { 226 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 227 if (oopDesc::unsafe_equals(forw, obj)) { 228 bool evac; 229 forw = _heap->evacuate_object(forw, thread, evac); 230 if (evac) { 231 enqueue(forw); 232 } 233 } 234 obj = forw; 235 } 236 break; 237 case WRITE_BARRIER_ALWAYS_ENQUEUE: 238 if (_heap->in_collection_set(obj)) { 239 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 240 if (oopDesc::unsafe_equals(forw, obj)) { 241 bool evac; 242 forw = _heap->evacuate_object(forw, thread, evac); 243 } 244 obj = forw; 245 } 246 enqueue(obj); 247 break; 248 default: 249 ShouldNotReachHere(); 250 } 251 252 if (MATRIX) { 253 _heap->connection_matrix()->set_connected(cur_dst, obj); 254 } 255 256 RawAccess<OOP_NOT_NULL>::oop_store(cur_dst, obj); 257 } else { 258 // Store null. 259 RawAccess<>::oop_store(cur_dst, o); 260 } 261 return true; 262 } 263 264 // Clone barrier support 265 template <DecoratorSet decorators, typename BarrierSetT> 266 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 267 src = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->read_barrier(src)); 268 dst = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_barrier(dst)); 269 Raw::clone(src, dst, size); 270 ((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_region(MemRegion((HeapWord*) dst, size)); 271 } 272 273 274 template <DecoratorSet decorators, typename BarrierSetT> 275 template <typename T> 276 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 277 assert(((HeapWord*)(void*) src_obj) <= (HeapWord*) src && (HeapWord*) src < (((HeapWord*)(void*) src_obj) + src_obj->size()), "pointer out of object bounds src_obj: %p, src: %p, size: %ul", (void*) src_obj, src, src_obj->size()); 278 assert(((HeapWord*)(void*) dst_obj) <= (HeapWord*) dst && (HeapWord*) dst < (((HeapWord*)(void*) dst_obj) + dst_obj->size()), "pointer out of object bounds dst_obj: "/*POINTER_FORMAT", dst: "POINTER_FORMAT", length: "SIZE_FORMAT, p2i(dst_obj), p2i(dst), length*/); 279 280 ShenandoahHeap* heap = ShenandoahHeap::heap(); 281 282 if (!CompressedOops::is_null(src_obj)) { 283 size_t src_offset = pointer_delta((void*) src, (void*) src_obj, sizeof(T)); 284 src_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->read_barrier(src_obj)); 285 src = ((T*)(void*) src_obj) + src_offset; 286 } 287 if (!CompressedOops::is_null(dst_obj)) { 288 size_t dst_offset = pointer_delta((void*) dst, (void*) dst_obj, sizeof(T)); 289 dst_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_barrier(dst_obj)); 290 dst = ((T*)(void*) dst_obj) + dst_offset; 291 } 292 293 bool satb = (ShenandoahSATBBarrier || ShenandoahConditionalSATBBarrier) && heap->is_concurrent_mark_in_progress(); 294 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value; 295 ArrayCopyStoreValMode storeval_mode; 296 if (heap->has_forwarded_objects()) { 297 if (heap->is_concurrent_partial_in_progress()) { 298 storeval_mode = WRITE_BARRIER_MAYBE_ENQUEUE; 299 } else if (heap->is_concurrent_traversal_in_progress()) { 300 storeval_mode = WRITE_BARRIER_ALWAYS_ENQUEUE; 301 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) { 302 storeval_mode = READ_BARRIER; 303 } else { 304 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress"); 305 storeval_mode = NONE; // E.g. during evac or outside cycle 306 } 307 } else { 308 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress"); 309 storeval_mode = NONE; 310 } 311 312 if (!satb && !checkcast && !UseShenandoahMatrix && storeval_mode == NONE) { 313 // Short-circuit to bulk copy. 314 return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); 315 } 316 317 Klass* bound = objArrayOop(dst_obj)->element_klass(); 318 ShenandoahBarrierSet* bs = (ShenandoahBarrierSet*) BarrierSet::barrier_set(); 319 return bs->arraycopy_loop_1(src, dst, length, bound, checkcast, satb, UseShenandoahMatrix, storeval_mode); 320 } 321 322 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP