211 inline bool ZBarrier::is_marked_or_null_fast_path(uintptr_t addr) {
 212   return ZAddress::is_marked_or_null(addr);
 213 }
 214 
 215 inline bool ZBarrier::during_mark() {
 216   return ZGlobalPhase == ZPhaseMark;
 217 }
 218 
 219 inline bool ZBarrier::during_relocate() {
 220   return ZGlobalPhase == ZPhaseRelocate;
 221 }
 222 
 223 //
 224 // Load barrier
 225 //
 226 inline oop ZBarrier::load_barrier_on_oop(oop o) {
 227   return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
 228 }
 229 
 230 inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) {
 231   const oop o = *p;
 232   return load_barrier_on_oop_field_preloaded(p, o);
 233 }
 234 
 235 inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
 236   return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
 237 }
 238 
 239 inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
 240   for (volatile const oop* const end = p + length; p < end; p++) {
 241     load_barrier_on_oop_field(p);
 242   }
 243 }
 244 
 245 // ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
 246 inline void verify_on_weak(volatile oop* referent_addr) {
 247 #ifdef ASSERT
 248   if (referent_addr != NULL) {
 249     uintptr_t base = (uintptr_t)referent_addr - java_lang_ref_Reference::referent_offset;
 250     oop obj = cast_to_oop(base);
 251     assert(oopDesc::is_oop(obj), "Verification failed for: ref " PTR_FORMAT " obj: " PTR_FORMAT, (uintptr_t)referent_addr, base);

 265 }
 266 
 267 inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
 268   if (ZResurrection::is_blocked()) {
 269     return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
 270   }
 271 
 272   return load_barrier_on_oop_field_preloaded(p, o);
 273 }
 274 
 275 inline void ZBarrier::load_barrier_on_root_oop_field(oop* p) {
 276   const oop o = *p;
 277   root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
 278 }
 279 
 280 //
 281 // Weak load barrier
 282 //
 283 inline oop ZBarrier::weak_load_barrier_on_oop_field(volatile oop* p) {
 284   assert(!ZResurrection::is_blocked(), "Should not be called during resurrection blocked phase");
 285   const oop o = *p;
 286   return weak_load_barrier_on_oop_field_preloaded(p, o);
 287 }
 288 
 289 inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
 290   return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
 291 }
 292 
 293 inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) {
 294   return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
 295 }
 296 
 297 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) {
 298   const oop o = *p;
 299   return weak_load_barrier_on_weak_oop_field_preloaded(p, o);
 300 }
 301 
 302 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
 303   verify_on_weak(p);
 304 
 305   if (ZResurrection::is_blocked()) {
 306     return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
 307   }
 308 
 309   return weak_load_barrier_on_oop_field_preloaded(p, o);
 310 }
 311 
 312 inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) {
 313   return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
 314 }
 315 
 316 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) {
 317   const oop o = *p;
 318   return weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
 319 }
 320 
 321 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
 322   if (ZResurrection::is_blocked()) {
 323     return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
 324   }
 325 
 326   return weak_load_barrier_on_oop_field_preloaded(p, o);
 327 }
 328 
 329 //
 330 // Is alive barrier
 331 //
 332 inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) {
 333   // Check if oop is logically non-null. This operation
 334   // is only valid when resurrection is blocked.
 335   assert(ZResurrection::is_blocked(), "Invalid phase");
 336   return weak_load_barrier_on_weak_oop(o) != NULL;
 337 }
 338 
 339 inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) {
 340   // Check if oop is logically non-null. This operation
 341   // is only valid when resurrection is blocked.
 342   assert(ZResurrection::is_blocked(), "Invalid phase");
 343   return weak_load_barrier_on_phantom_oop(o) != NULL;
 344 }
 345 
 346 //
 347 // Keep alive barrier
 348 //
 349 inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
 350   // This operation is only valid when resurrection is blocked.
 351   assert(ZResurrection::is_blocked(), "Invalid phase");
 352   const oop o = *p;
 353   barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
 354 }
 355 
 356 inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
 357   // This operation is only valid when resurrection is blocked.
 358   assert(ZResurrection::is_blocked(), "Invalid phase");
 359   const oop o = *p;
 360   barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
 361 }
 362 
 363 inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) {
 364   // This operation is only valid when resurrection is blocked.
 365   assert(ZResurrection::is_blocked(), "Invalid phase");
 366   const oop o = *p;
 367   root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
 368 }
 369 
 370 inline void ZBarrier::keep_alive_barrier_on_oop(oop o) {
 371   const uintptr_t addr = ZOop::to_address(o);
 372   assert(ZAddress::is_good(addr), "Invalid address");
 373 
 374   if (during_mark()) {
 375     mark_barrier_on_oop_slow_path(addr);
 376   }
 377 }
 378 
 379 //
 380 // Mark barrier
 381 //
 382 inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
 383   const oop o = *p;
 384 
 385   if (finalizable) {
 386     barrier<is_marked_or_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
 387   } else {
 388     const uintptr_t addr = ZOop::to_address(o);
 389     if (ZAddress::is_good(addr)) {
 390       // Mark through good oop
 391       mark_barrier_on_oop_slow_path(addr);
 392     } else {
 393       // Mark through bad oop
 394       barrier<is_good_or_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
 395     }
 396   }
 397 }
 398 
 399 inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
 400   for (volatile const oop* const end = p + length; p < end; p++) {
 401     mark_barrier_on_oop_field(p, finalizable);
 402   }
 403 }

 211 inline bool ZBarrier::is_marked_or_null_fast_path(uintptr_t addr) {
 212   return ZAddress::is_marked_or_null(addr);
 213 }
 214 
 215 inline bool ZBarrier::during_mark() {
 216   return ZGlobalPhase == ZPhaseMark;
 217 }
 218 
 219 inline bool ZBarrier::during_relocate() {
 220   return ZGlobalPhase == ZPhaseRelocate;
 221 }
 222 
 223 //
 224 // Load barrier
 225 //
 226 inline oop ZBarrier::load_barrier_on_oop(oop o) {
 227   return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
 228 }
 229 
 230 inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) {
 231   const oop o = Atomic::load(p);
 232   return load_barrier_on_oop_field_preloaded(p, o);
 233 }
 234 
 235 inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
 236   return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
 237 }
 238 
 239 inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
 240   for (volatile const oop* const end = p + length; p < end; p++) {
 241     load_barrier_on_oop_field(p);
 242   }
 243 }
 244 
 245 // ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
 246 inline void verify_on_weak(volatile oop* referent_addr) {
 247 #ifdef ASSERT
 248   if (referent_addr != NULL) {
 249     uintptr_t base = (uintptr_t)referent_addr - java_lang_ref_Reference::referent_offset;
 250     oop obj = cast_to_oop(base);
 251     assert(oopDesc::is_oop(obj), "Verification failed for: ref " PTR_FORMAT " obj: " PTR_FORMAT, (uintptr_t)referent_addr, base);

 265 }
 266 
 267 inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
 268   if (ZResurrection::is_blocked()) {
 269     return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
 270   }
 271 
 272   return load_barrier_on_oop_field_preloaded(p, o);
 273 }
 274 
 275 inline void ZBarrier::load_barrier_on_root_oop_field(oop* p) {
 276   const oop o = *p;
 277   root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
 278 }
 279 
 280 //
 281 // Weak load barrier
 282 //
 283 inline oop ZBarrier::weak_load_barrier_on_oop_field(volatile oop* p) {
 284   assert(!ZResurrection::is_blocked(), "Should not be called during resurrection blocked phase");
 285   const oop o = Atomic::load(p);
 286   return weak_load_barrier_on_oop_field_preloaded(p, o);
 287 }
 288 
 289 inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
 290   return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
 291 }
 292 
 293 inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) {
 294   return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
 295 }
 296 
 297 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) {
 298   const oop o = Atomic::load(p);
 299   return weak_load_barrier_on_weak_oop_field_preloaded(p, o);
 300 }
 301 
 302 inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
 303   verify_on_weak(p);
 304 
 305   if (ZResurrection::is_blocked()) {
 306     return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
 307   }
 308 
 309   return weak_load_barrier_on_oop_field_preloaded(p, o);
 310 }
 311 
 312 inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) {
 313   return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
 314 }
 315 
 316 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) {
 317   const oop o = Atomic::load(p);
 318   return weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
 319 }
 320 
 321 inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
 322   if (ZResurrection::is_blocked()) {
 323     return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
 324   }
 325 
 326   return weak_load_barrier_on_oop_field_preloaded(p, o);
 327 }
 328 
 329 //
 330 // Is alive barrier
 331 //
 332 inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) {
 333   // Check if oop is logically non-null. This operation
 334   // is only valid when resurrection is blocked.
 335   assert(ZResurrection::is_blocked(), "Invalid phase");
 336   return weak_load_barrier_on_weak_oop(o) != NULL;
 337 }
 338 
 339 inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) {
 340   // Check if oop is logically non-null. This operation
 341   // is only valid when resurrection is blocked.
 342   assert(ZResurrection::is_blocked(), "Invalid phase");
 343   return weak_load_barrier_on_phantom_oop(o) != NULL;
 344 }
 345 
 346 //
 347 // Keep alive barrier
 348 //
 349 inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
 350   // This operation is only valid when resurrection is blocked.
 351   assert(ZResurrection::is_blocked(), "Invalid phase");
 352   const oop o = Atomic::load(p);
 353   barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
 354 }
 355 
 356 inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
 357   // This operation is only valid when resurrection is blocked.
 358   assert(ZResurrection::is_blocked(), "Invalid phase");
 359   const oop o = Atomic::load(p);
 360   barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
 361 }
 362 
 363 inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) {
 364   // This operation is only valid when resurrection is blocked.
 365   assert(ZResurrection::is_blocked(), "Invalid phase");
 366   const oop o = *p;
 367   root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
 368 }
 369 
 370 inline void ZBarrier::keep_alive_barrier_on_oop(oop o) {
 371   const uintptr_t addr = ZOop::to_address(o);
 372   assert(ZAddress::is_good(addr), "Invalid address");
 373 
 374   if (during_mark()) {
 375     mark_barrier_on_oop_slow_path(addr);
 376   }
 377 }
 378 
 379 //
 380 // Mark barrier
 381 //
 382 inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
 383   const oop o = Atomic::load(p);
 384 
 385   if (finalizable) {
 386     barrier<is_marked_or_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
 387   } else {
 388     const uintptr_t addr = ZOop::to_address(o);
 389     if (ZAddress::is_good(addr)) {
 390       // Mark through good oop
 391       mark_barrier_on_oop_slow_path(addr);
 392     } else {
 393       // Mark through bad oop
 394       barrier<is_good_or_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
 395     }
 396   }
 397 }
 398 
 399 inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
 400   for (volatile const oop* const end = p + length; p < end; p++) {
 401     mark_barrier_on_oop_field(p, finalizable);
 402   }
 403 }