1 /* 2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP 26 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shenandoah/brooksPointer.inline.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 30 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 33 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) { 34 return BrooksPointer::forwardee(p); 35 } 36 37 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) { 38 if (((HeapWord*) p) != NULL) { 39 return resolve_forwarded_not_null(p); 40 } else { 41 return p; 42 } 43 } 44 45 template <DecoratorSet decorators, typename BarrierSetT> 46 template <typename T> 47 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { 48 oop res; 49 oop expected = compare_value; 50 do { 51 compare_value = expected; 52 res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 53 expected = res; 54 } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(BarrierSet::barrier_set()->read_barrier(compare_value), BarrierSet::barrier_set()->read_barrier(expected))); 55 if (oopDesc::unsafe_equals(expected, compare_value)) { 56 if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) { 57 ShenandoahBarrierSet::enqueue(compare_value); 58 } 59 if (UseShenandoahMatrix && !CompressedOops::is_null(new_value)) { 60 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix(); 61 matrix->set_connected(addr, new_value); 62 } 63 } 64 return res; 65 } 66 67 template <DecoratorSet decorators, typename BarrierSetT> 68 template <typename T> 69 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) { 70 oop previous = Raw::oop_atomic_xchg(new_value, addr); 71 if (ShenandoahSATBBarrier) { 72 if (!CompressedOops::is_null(previous)) { 73 ShenandoahBarrierSet::enqueue(previous); 74 } 75 } 76 if (UseShenandoahMatrix && !CompressedOops::is_null(new_value)) { 77 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix(); 78 matrix->set_connected(addr, new_value); 79 } 80 return previous; 81 } 82 83 template <DecoratorSet decorators, typename BarrierSetT> 84 template <typename T> 85 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 86 assert(((T*)(void*) src_obj) <= src && ((HeapWord*) src) < (((HeapWord*)(void*) src_obj) + src_obj->size()), "pointer out of object bounds src_obj: %p, src: %p, end: %p, size: %u", (void*) src_obj, src, (((HeapWord*)(void*) src_obj) + src_obj->size()), src_obj->size()); 87 assert(((T*)(void*) dst_obj) <= dst && ((HeapWord*) dst) < ((HeapWord*)(void*) dst_obj) + dst_obj->size(), "pointer out of object bounds dst_obj: %p, dst: %p, end: %p, size: %u", (void*) dst_obj, dst, (((HeapWord*)(void*) dst_obj) + dst_obj->size()), dst_obj->size()); 88 if (!CompressedOops::is_null(src_obj)) { 89 size_t src_offset = pointer_delta((void*) src, (void*) src_obj, sizeof(char)); 90 src_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->read_barrier(src_obj)); 91 src = (T*) (((char*)(void*) src_obj) + src_offset); 92 } 93 if (!CompressedOops::is_null(dst_obj)) { 94 size_t dst_offset = pointer_delta((void*) dst, (void*) dst_obj, sizeof(char)); 95 dst_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_barrier(dst_obj)); 96 dst = (T*) (((char*)(void*) dst_obj) + dst_offset); 97 } 98 return Raw::arraycopy(src_obj, dst_obj, src, dst, length); 99 } 100 101 template <typename T> 102 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, 103 bool checkcast, bool satb, bool matrix, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 104 if (checkcast) { 105 return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, matrix, storeval_mode); 106 } else { 107 return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, matrix, storeval_mode); 108 } 109 } 110 111 template <typename T, bool CHECKCAST> 112 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, 113 bool satb, bool matrix, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 114 if (satb) { 115 return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, matrix, storeval_mode); 116 } else { 117 return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, matrix, storeval_mode); 118 } 119 } 120 121 template <typename T, bool CHECKCAST, bool SATB> 122 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, 123 bool matrix, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 124 if (matrix) { 125 return arraycopy_loop_4<T, CHECKCAST, SATB, true>(src, dst, length, bound, storeval_mode); 126 } else { 127 return arraycopy_loop_4<T, CHECKCAST, SATB, false>(src, dst, length, bound, storeval_mode); 128 } 129 } 130 131 template <typename T, bool CHECKCAST, bool SATB, bool MATRIX> 132 bool ShenandoahBarrierSet::arraycopy_loop_4(T* src, T* dst, size_t length, Klass* bound, 133 ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { 134 switch (storeval_mode) { 135 case NONE: 136 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, NONE>(src, dst, length, bound); 137 case READ_BARRIER: 138 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, READ_BARRIER>(src, dst, length, bound); 139 case WRITE_BARRIER: 140 return arraycopy_loop<T, CHECKCAST, SATB, MATRIX, WRITE_BARRIER>(src, dst, length, bound); 141 default: 142 ShouldNotReachHere(); 143 return true; // happy compiler 144 } 145 } 146 147 template <typename T, bool CHECKCAST, bool SATB, bool MATRIX, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE> 148 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound) { 149 Thread* thread = Thread::current(); 150 151 ShenandoahEvacOOMScope oom_evac_scope; 152 153 // We need to handle four cases: 154 // 155 // a) src < dst, intersecting, can only copy backward only 156 // [...src...] 157 // [...dst...] 158 // 159 // b) src < dst, non-intersecting, can copy forward/backward 160 // [...src...] 161 // [...dst...] 162 // 163 // c) src > dst, intersecting, can copy forward only 164 // [...src...] 165 // [...dst...] 166 // 167 // d) src > dst, non-intersecting, can copy forward/backward 168 // [...src...] 169 // [...dst...] 170 // 171 if (src > dst) { 172 // copy forward: 173 T* cur_src = src; 174 T* cur_dst = dst; 175 T* src_end = src + length; 176 for (; cur_src < src_end; cur_src++, cur_dst++) { 177 if (!arraycopy_element<T, CHECKCAST, SATB, MATRIX, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) { 178 return false; 179 } 180 } 181 } else { 182 // copy backward: 183 T* cur_src = src + length - 1; 184 T* cur_dst = dst + length - 1; 185 for (; cur_src >= src; cur_src--, cur_dst--) { 186 if (!arraycopy_element<T, CHECKCAST, SATB, MATRIX, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) { 187 return false; 188 } 189 } 190 } 191 return true; 192 } 193 194 template <typename T, bool CHECKCAST, bool SATB, bool MATRIX, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE> 195 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) { 196 T o = RawAccess<>::oop_load(cur_src); 197 198 if (SATB) { 199 T prev = RawAccess<>::oop_load(cur_dst); 200 if (!CompressedOops::is_null(prev)) { 201 oop prev_obj = CompressedOops::decode_not_null(prev); 202 enqueue(prev_obj); 203 } 204 } 205 206 if (!CompressedOops::is_null(o)) { 207 oop obj = CompressedOops::decode_not_null(o); 208 209 if (CHECKCAST) { 210 assert(bound != NULL, "need element klass for checkcast"); 211 if (!oopDesc::is_instanceof_or_null(obj, bound)) { 212 return false; 213 } 214 } 215 216 switch (STOREVAL_MODE) { 217 case NONE: 218 break; 219 case READ_BARRIER: 220 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 221 break; 222 case WRITE_BARRIER: 223 if (_heap->in_collection_set(obj)) { 224 oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 225 if (oopDesc::unsafe_equals(forw, obj)) { 226 bool evac; 227 forw = _heap->evacuate_object(forw, thread); 228 } 229 obj = forw; 230 } 231 enqueue(obj); 232 break; 233 default: 234 ShouldNotReachHere(); 235 } 236 237 if (MATRIX) { 238 _heap->connection_matrix()->set_connected(cur_dst, obj); 239 } 240 241 RawAccess<OOP_NOT_NULL>::oop_store(cur_dst, obj); 242 } else { 243 // Store null. 244 RawAccess<>::oop_store(cur_dst, o); 245 } 246 return true; 247 } 248 249 // Clone barrier support 250 template <DecoratorSet decorators, typename BarrierSetT> 251 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 252 src = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->read_barrier(src)); 253 dst = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_barrier(dst)); 254 Raw::clone(src, dst, size); 255 ((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_region(MemRegion((HeapWord*) dst, size)); 256 } 257 258 259 template <DecoratorSet decorators, typename BarrierSetT> 260 template <typename T> 261 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 262 assert(((HeapWord*)(void*) src_obj) <= (HeapWord*) src && (HeapWord*) src < (((HeapWord*)(void*) src_obj) + src_obj->size()), "pointer out of object bounds src_obj: %p, src: %p, size: %ul", (void*) src_obj, src, src_obj->size()); 263 assert(((HeapWord*)(void*) dst_obj) <= (HeapWord*) dst && (HeapWord*) dst < (((HeapWord*)(void*) dst_obj) + dst_obj->size()), "pointer out of object bounds dst_obj: "/*POINTER_FORMAT", dst: "POINTER_FORMAT", length: "SIZE_FORMAT, p2i(dst_obj), p2i(dst), length*/); 264 265 ShenandoahHeap* heap = ShenandoahHeap::heap(); 266 267 if (!CompressedOops::is_null(src_obj)) { 268 size_t src_offset = pointer_delta((void*) src, (void*) src_obj, sizeof(T)); 269 src_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->read_barrier(src_obj)); 270 src = ((T*)(void*) src_obj) + src_offset; 271 } 272 if (!CompressedOops::is_null(dst_obj)) { 273 size_t dst_offset = pointer_delta((void*) dst, (void*) dst_obj, sizeof(T)); 274 dst_obj = arrayOop(((ShenandoahBarrierSet*) BarrierSet::barrier_set())->write_barrier(dst_obj)); 275 dst = ((T*)(void*) dst_obj) + dst_offset; 276 } 277 278 bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); 279 bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value; 280 ArrayCopyStoreValMode storeval_mode; 281 if (heap->has_forwarded_objects()) { 282 if (heap->is_concurrent_traversal_in_progress()) { 283 storeval_mode = WRITE_BARRIER; 284 } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) { 285 storeval_mode = READ_BARRIER; 286 } else { 287 assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress"); 288 storeval_mode = NONE; // E.g. during evac or outside cycle 289 } 290 } else { 291 assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress"); 292 storeval_mode = NONE; 293 } 294 295 if (!satb && !checkcast && !UseShenandoahMatrix && storeval_mode == NONE) { 296 // Short-circuit to bulk copy. 297 return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); 298 } 299 300 Klass* bound = objArrayOop(dst_obj)->element_klass(); 301 ShenandoahBarrierSet* bs = (ShenandoahBarrierSet*) BarrierSet::barrier_set(); 302 return bs->arraycopy_loop_1(src, dst, length, bound, checkcast, satb, UseShenandoahMatrix, storeval_mode); 303 } 304 305 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP