1 /* 2 * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #ifndef SHARE_GC_Z_ZBARRIER_INLINE_HPP 25 #define SHARE_GC_Z_ZBARRIER_INLINE_HPP 26 27 #include "gc/z/zAddress.inline.hpp" 28 #include "gc/z/zBarrier.hpp" 29 #include "gc/z/zOop.inline.hpp" 30 #include "gc/z/zResurrection.inline.hpp" 31 #include "runtime/atomic.hpp" 32 33 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> 34 inline oop ZBarrier::barrier(volatile oop* p, oop o) { 35 uintptr_t addr = ZOop::to_address(o); 36 37 retry: 38 // Fast path 39 if (fast_path(addr)) { 40 return ZOop::from_address(addr); 41 } 42 43 // Slow path 44 const uintptr_t good_addr = slow_path(addr); 45 46 // Self heal, but only if the address was actually updated by the slow path, 47 // which might not be the case, e.g. when marking through an already good oop. 48 if (p != NULL && good_addr != addr) { 49 const uintptr_t prev_addr = Atomic::cmpxchg(good_addr, (volatile uintptr_t*)p, addr); 50 if (prev_addr != addr) { 51 // Some other thread overwrote the oop. If this oop was updated by a 52 // weak barrier the new oop might not be good, in which case we need 53 // to re-apply this barrier. 54 addr = prev_addr; 55 goto retry; 56 } 57 } 58 59 return ZOop::from_address(good_addr); 60 } 61 62 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> 63 inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) { 64 const uintptr_t addr = ZOop::to_address(o); 65 66 // Fast path 67 if (fast_path(addr)) { 68 // Return the good address instead of the weak good address 69 // to ensure that the currently active heap view is used. 70 return ZOop::from_address(ZAddress::good_or_null(addr)); 71 } 72 73 // Slow path 74 uintptr_t good_addr = slow_path(addr); 75 76 // Self heal unless the address returned from the slow path is null, 77 // in which case resurrection was blocked and we must let the reference 78 // processor clear the oop. Mutators are not allowed to clear oops in 79 // these cases, since that would be similar to calling Reference.clear(), 80 // which would make the reference non-discoverable or silently dropped 81 // by the reference processor. 82 if (p != NULL && good_addr != 0) { 83 // The slow path returns a good/marked address, but we never mark oops 84 // in a weak load barrier so we always self heal with the remapped address. 85 const uintptr_t weak_good_addr = ZAddress::remapped(good_addr); 86 const uintptr_t prev_addr = Atomic::cmpxchg(weak_good_addr, (volatile uintptr_t*)p, addr); 87 if (prev_addr != addr) { 88 // Some other thread overwrote the oop. The new 89 // oop is guaranteed to be weak good or null. 90 assert(ZAddress::is_weak_good_or_null(prev_addr), "Bad weak overwrite"); 91 92 // Return the good address instead of the weak good address 93 // to ensure that the currently active heap view is used. 94 good_addr = ZAddress::good_or_null(prev_addr); 95 } 96 } 97 98 return ZOop::from_address(good_addr); 99 } 100 101 template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> 102 inline void ZBarrier::root_barrier(oop* p, oop o) { 103 const uintptr_t addr = ZOop::to_address(o); 104 105 // Fast path 106 if (fast_path(addr)) { 107 return; 108 } 109 110 // Slow path 111 const uintptr_t good_addr = slow_path(addr); 112 113 // Non-atomic healing helps speed up root scanning. This is safe to do 114 // since we are always healing roots in a safepoint, or under a lock, 115 // which ensures we are never racing with mutators modifying roots while 116 // we are healing them. It's also safe in case multiple GC threads try 117 // to heal the same root if it is aligned, since they would always heal 118 // the root in the same way and it does not matter in which order it 119 // happens. For misaligned oops, there needs to be mutual exclusion. 120 *p = ZOop::from_address(good_addr); 121 } 122 123 inline bool ZBarrier::is_null_fast_path(uintptr_t addr) { 124 return ZAddress::is_null(addr); 125 } 126 127 inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) { 128 return ZAddress::is_good_or_null(addr); 129 } 130 131 inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) { 132 return ZAddress::is_weak_good_or_null(addr); 133 } 134 135 inline bool ZBarrier::is_resurrection_blocked(volatile oop* p, oop* o) { 136 const bool is_blocked = ZResurrection::is_blocked(); 137 138 // Reload oop after checking the resurrection blocked state. This is 139 // done to prevent a race where we first load an oop, which is logically 140 // null but not yet cleared, then this oop is cleared by the reference 141 // processor and resurrection is unblocked. At this point the mutator 142 // would see the unblocked state and pass this invalid oop through the 143 // normal barrier path, which would incorrectly try to mark this oop. 144 if (p != NULL) { 145 // First assign to reloaded_o to avoid compiler warning about 146 // implicit dereference of volatile oop. 147 const oop reloaded_o = *p; 148 *o = reloaded_o; 149 } 150 151 return is_blocked; 152 } 153 154 // 155 // Load barrier 156 // 157 inline oop ZBarrier::load_barrier_on_oop(oop o) { 158 return load_barrier_on_oop_field_preloaded((oop*)NULL, o); 159 } 160 161 inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) { 162 const oop o = *p; 163 return load_barrier_on_oop_field_preloaded(p, o); 164 } 165 166 inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) { 167 return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o); 168 } 169 170 inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) { 171 for (volatile const oop* const end = p + length; p < end; p++) { 172 load_barrier_on_oop_field(p); 173 } 174 } 175 176 inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) { 177 if (is_resurrection_blocked(p, &o)) { 178 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o); 179 } 180 181 return load_barrier_on_oop_field_preloaded(p, o); 182 } 183 184 inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) { 185 if (is_resurrection_blocked(p, &o)) { 186 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o); 187 } 188 189 return load_barrier_on_oop_field_preloaded(p, o); 190 } 191 192 inline void ZBarrier::load_barrier_on_root_oop_field(oop* p) { 193 const oop o = *p; 194 root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o); 195 } 196 197 // 198 // Weak load barrier 199 // 200 inline oop ZBarrier::weak_load_barrier_on_oop_field(volatile oop* p) { 201 assert(!ZResurrection::is_blocked(), "Should not be called during resurrection blocked phase"); 202 const oop o = *p; 203 return weak_load_barrier_on_oop_field_preloaded(p, o); 204 } 205 206 inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) { 207 return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o); 208 } 209 210 inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) { 211 return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o); 212 } 213 214 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) { 215 const oop o = *p; 216 return weak_load_barrier_on_weak_oop_field_preloaded(p, o); 217 } 218 219 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) { 220 if (is_resurrection_blocked(p, &o)) { 221 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o); 222 } 223 224 return weak_load_barrier_on_oop_field_preloaded(p, o); 225 } 226 227 inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) { 228 return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o); 229 } 230 231 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) { 232 const oop o = *p; 233 return weak_load_barrier_on_phantom_oop_field_preloaded(p, o); 234 } 235 236 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) { 237 if (is_resurrection_blocked(p, &o)) { 238 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o); 239 } 240 241 return weak_load_barrier_on_oop_field_preloaded(p, o); 242 } 243 244 // 245 // Is alive barrier 246 // 247 inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) { 248 // Check if oop is logically non-null. This operation 249 // is only valid when resurrection is blocked. 250 assert(ZResurrection::is_blocked(), "Invalid phase"); 251 return weak_load_barrier_on_weak_oop(o) != NULL; 252 } 253 254 inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) { 255 // Check if oop is logically non-null. This operation 256 // is only valid when resurrection is blocked. 257 assert(ZResurrection::is_blocked(), "Invalid phase"); 258 return weak_load_barrier_on_phantom_oop(o) != NULL; 259 } 260 261 // 262 // Keep alive barrier 263 // 264 inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) { 265 // This operation is only valid when resurrection is blocked. 266 assert(ZResurrection::is_blocked(), "Invalid phase"); 267 const oop o = *p; 268 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o); 269 } 270 271 inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) { 272 // This operation is only valid when resurrection is blocked. 273 assert(ZResurrection::is_blocked(), "Invalid phase"); 274 const oop o = *p; 275 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o); 276 } 277 278 inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) { 279 // This operation is only valid when resurrection is blocked. 280 assert(ZResurrection::is_blocked(), "Invalid phase"); 281 const oop o = *p; 282 root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o); 283 } 284 285 // 286 // Mark barrier 287 // 288 inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) { 289 // The fast path only checks for null since the GC worker 290 // threads doing marking wants to mark through good oops. 291 const oop o = *p; 292 293 if (finalizable) { 294 barrier<is_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o); 295 } else { 296 barrier<is_null_fast_path, mark_barrier_on_oop_slow_path>(p, o); 297 } 298 } 299 300 inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) { 301 for (volatile const oop* const end = p + length; p < end; p++) { 302 mark_barrier_on_oop_field(p, finalizable); 303 } 304 } 305 306 inline void ZBarrier::mark_barrier_on_root_oop_field(oop* p) { 307 const oop o = *p; 308 root_barrier<is_good_or_null_fast_path, mark_barrier_on_root_oop_slow_path>(p, o); 309 } 310 311 // 312 // Relocate barrier 313 // 314 inline void ZBarrier::relocate_barrier_on_root_oop_field(oop* p) { 315 const oop o = *p; 316 root_barrier<is_good_or_null_fast_path, relocate_barrier_on_root_oop_slow_path>(p, o); 317 } 318 319 #endif // SHARE_GC_Z_ZBARRIER_INLINE_HPP