1 /* 2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #ifndef SHARE_GC_Z_ZBARRIER_INLINE_HPP 25 #define SHARE_GC_Z_ZBARRIER_INLINE_HPP 26 27 #include "classfile/javaClasses.hpp" 28 #include "gc/z/zAddress.inline.hpp" 29 #include "gc/z/zBarrier.hpp" 30 #include "gc/z/zOop.inline.hpp" 31 #include "gc/z/zResurrection.inline.hpp" 32 #include "oops/oop.hpp" 33 #include "runtime/atomic.hpp" 34 35 inline void ZBarrier::self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_addr) { 36 if (heal_addr == 0) { 37 // Never heal with null since it interacts badly with reference processing. 38 // A mutator clearing an oop would be similar to calling Reference.clear(), 39 // which would make the reference non-discoverable or silently dropped 40 // by the reference processor. 41 return; 42 } 43 44 for (;;) { 45 if (addr == heal_addr) { 46 // Already healed 47 return; 48 } 49 50 // Heal 51 const uintptr_t prev_addr = Atomic::cmpxchg((volatile uintptr_t*)p, addr, heal_addr); 52 if (prev_addr == addr) { 53 // Success 54 return; 55 } 56 57 if (ZAddress::is_good_or_null(prev_addr)) { 58 // No need to heal 59 return; 60 } 61 62 // The oop location was healed by another barrier, but it is still not 63 // good or null. Re-apply healing to make sure the oop is not left with 64 // weaker (remapped or finalizable) metadata bits than what this barrier 65 // tried to apply. 66 assert(ZAddress::offset(prev_addr) == ZAddress::offset(heal_addr), "Invalid offset"); 67 addr = prev_addr; 68 } 69 } 70 71 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> 72 inline oop ZBarrier::barrier(volatile oop* p, oop o) { 73 uintptr_t addr = ZOop::to_address(o); 74 75 // Fast path 76 if (fast_path(addr)) { 77 return ZOop::from_address(addr); 78 } 79 80 // Slow path 81 const uintptr_t good_addr = slow_path(addr); 82 83 if (p != NULL) { 84 self_heal(p, addr, good_addr); 85 } 86 87 return ZOop::from_address(good_addr); 88 } 89 90 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> 91 inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) { 92 const uintptr_t addr = ZOop::to_address(o); 93 94 // Fast path 95 if (fast_path(addr)) { 96 // Return the good address instead of the weak good address 97 // to ensure that the currently active heap view is used. 98 return ZOop::from_address(ZAddress::good_or_null(addr)); 99 } 100 101 // Slow path 102 const uintptr_t good_addr = slow_path(addr); 103 104 if (p != NULL) { 105 // The slow path returns a good/marked address or null, but we never mark 106 // oops in a weak load barrier so we always heal with the remapped address. 107 self_heal(p, addr, ZAddress::remapped_or_null(good_addr)); 108 } 109 110 return ZOop::from_address(good_addr); 111 } 112 113 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> 114 inline void ZBarrier::root_barrier(oop* p, oop o) { 115 const uintptr_t addr = ZOop::to_address(o); 116 117 // Fast path 118 if (fast_path(addr)) { 119 return; 120 } 121 122 // Slow path 123 const uintptr_t good_addr = slow_path(addr); 124 125 // Non-atomic healing helps speed up root scanning. This is safe to do 126 // since we are always healing roots in a safepoint, or under a lock, 127 // which ensures we are never racing with mutators modifying roots while 128 // we are healing them. It's also safe in case multiple GC threads try 129 // to heal the same root if it is aligned, since they would always heal 130 // the root in the same way and it does not matter in which order it 131 // happens. For misaligned oops, there needs to be mutual exclusion. 132 *p = ZOop::from_address(good_addr); 133 } 134 135 inline bool ZBarrier::is_null_fast_path(uintptr_t addr) { 136 return ZAddress::is_null(addr); 137 } 138 139 inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) { 140 return ZAddress::is_good_or_null(addr); 141 } 142 143 inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) { 144 return ZAddress::is_weak_good_or_null(addr); 145 } 146 147 // 148 // Load barrier 149 // 150 inline oop ZBarrier::load_barrier_on_oop(oop o) { 151 return load_barrier_on_oop_field_preloaded((oop*)NULL, o); 152 } 153 154 inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) { 155 const oop o = *p; 156 return load_barrier_on_oop_field_preloaded(p, o); 157 } 158 159 inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) { 160 return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o); 161 } 162 163 inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) { 164 for (volatile const oop* const end = p + length; p < end; p++) { 165 load_barrier_on_oop_field(p); 166 } 167 } 168 169 // ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents. 170 inline void verify_on_weak(volatile oop* referent_addr) { 171 #ifdef ASSERT 172 if (referent_addr != NULL) { 173 uintptr_t base = (uintptr_t)referent_addr - java_lang_ref_Reference::referent_offset; 174 oop obj = cast_to_oop(base); 175 assert(oopDesc::is_oop(obj), "Verification failed for: ref " PTR_FORMAT " obj: " PTR_FORMAT, (uintptr_t)referent_addr, base); 176 assert(java_lang_ref_Reference::is_referent_field(obj, java_lang_ref_Reference::referent_offset), "Sanity"); 177 } 178 #endif 179 } 180 181 inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) { 182 verify_on_weak(p); 183 184 if (ZResurrection::is_blocked()) { 185 return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o); 186 } 187 188 return load_barrier_on_oop_field_preloaded(p, o); 189 } 190 191 inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) { 192 if (ZResurrection::is_blocked()) { 193 return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o); 194 } 195 196 return load_barrier_on_oop_field_preloaded(p, o); 197 } 198 199 inline void ZBarrier::load_barrier_on_root_oop_field(oop* p) { 200 const oop o = *p; 201 root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o); 202 } 203 204 // 205 // Weak load barrier 206 // 207 inline oop ZBarrier::weak_load_barrier_on_oop_field(volatile oop* p) { 208 assert(!ZResurrection::is_blocked(), "Should not be called during resurrection blocked phase"); 209 const oop o = *p; 210 return weak_load_barrier_on_oop_field_preloaded(p, o); 211 } 212 213 inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) { 214 return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o); 215 } 216 217 inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) { 218 return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o); 219 } 220 221 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) { 222 const oop o = *p; 223 return weak_load_barrier_on_weak_oop_field_preloaded(p, o); 224 } 225 226 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) { 227 verify_on_weak(p); 228 229 if (ZResurrection::is_blocked()) { 230 return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o); 231 } 232 233 return weak_load_barrier_on_oop_field_preloaded(p, o); 234 } 235 236 inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) { 237 return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o); 238 } 239 240 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) { 241 const oop o = *p; 242 return weak_load_barrier_on_phantom_oop_field_preloaded(p, o); 243 } 244 245 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) { 246 if (ZResurrection::is_blocked()) { 247 return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o); 248 } 249 250 return weak_load_barrier_on_oop_field_preloaded(p, o); 251 } 252 253 // 254 // Is alive barrier 255 // 256 inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) { 257 // Check if oop is logically non-null. This operation 258 // is only valid when resurrection is blocked. 259 assert(ZResurrection::is_blocked(), "Invalid phase"); 260 return weak_load_barrier_on_weak_oop(o) != NULL; 261 } 262 263 inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) { 264 // Check if oop is logically non-null. This operation 265 // is only valid when resurrection is blocked. 266 assert(ZResurrection::is_blocked(), "Invalid phase"); 267 return weak_load_barrier_on_phantom_oop(o) != NULL; 268 } 269 270 // 271 // Keep alive barrier 272 // 273 inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) { 274 // This operation is only valid when resurrection is blocked. 275 assert(ZResurrection::is_blocked(), "Invalid phase"); 276 const oop o = *p; 277 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o); 278 } 279 280 inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) { 281 // This operation is only valid when resurrection is blocked. 282 assert(ZResurrection::is_blocked(), "Invalid phase"); 283 const oop o = *p; 284 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o); 285 } 286 287 inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) { 288 // This operation is only valid when resurrection is blocked. 289 assert(ZResurrection::is_blocked(), "Invalid phase"); 290 const oop o = *p; 291 root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o); 292 } 293 294 // 295 // Mark barrier 296 // 297 inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) { 298 // The fast path only checks for null since the GC worker 299 // threads doing marking wants to mark through good oops. 300 const oop o = *p; 301 302 if (finalizable) { 303 barrier<is_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o); 304 } else { 305 barrier<is_null_fast_path, mark_barrier_on_oop_slow_path>(p, o); 306 } 307 } 308 309 inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) { 310 for (volatile const oop* const end = p + length; p < end; p++) { 311 mark_barrier_on_oop_field(p, finalizable); 312 } 313 } 314 315 inline void ZBarrier::mark_barrier_on_root_oop_field(oop* p) { 316 const oop o = *p; 317 root_barrier<is_good_or_null_fast_path, mark_barrier_on_root_oop_slow_path>(p, o); 318 } 319 320 inline void ZBarrier::mark_barrier_on_invisible_root_oop_field(oop* p) { 321 const oop o = *p; 322 root_barrier<is_good_or_null_fast_path, mark_barrier_on_invisible_root_oop_slow_path>(p, o); 323 } 324 325 // 326 // Relocate barrier 327 // 328 inline void ZBarrier::relocate_barrier_on_root_oop_field(oop* p) { 329 const oop o = *p; 330 root_barrier<is_good_or_null_fast_path, relocate_barrier_on_root_oop_slow_path>(p, o); 331 } 332 333 #endif // SHARE_GC_Z_ZBARRIER_INLINE_HPP