# HG changeset patch
# User zgu
# Date 1572617485 14400
#      Fri Nov 01 10:11:25 2019 -0400
# Node ID b93aecccb3d358770e240ccedf8dffba0b3dfab1
# Parent  75099fcf7962f769cda4c932a46a7ae3421cd2ea
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet

diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
@@ -346,20 +346,33 @@
   }
 }
 
+
+//
+// Arguments:
+//
+// Inputs:
+//   src:        oop location to load from, might be clobbered
+//   tmp1:       unused
+//   tmp_thread: unused
+//
+// Output:
+//   dst:        oop loaded from src location
+//
+// Kill:
+//   rscratch1 (scratch reg)
+//
+// Alias:
+//   dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
+//
 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                             Register dst, Address src, Register tmp1, Register tmp_thread) {
-  bool on_oop = is_reference_type(type);
-  bool not_in_heap = (decorators & IN_NATIVE) != 0;
-  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
-  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
-  bool on_reference = on_weak || on_phantom;
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-  bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
 
   Register result_dst = dst;
+  bool need_load_reference_barrier = ShenandoahBarrierSet::need_load_reference_barrier(decorators, type);
 
-  if (on_oop) {
-    // We want to preserve src
+  // Only preserve src address if we need load reference barrier
+  if (need_load_reference_barrier) {
+    // Use rscratch1 as temporary output register to avoid clobbering src
     if (dst == src.base() || dst == src.index()) {
       dst = rscratch1;
     }
@@ -367,29 +380,32 @@
   }
 
   BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
-  if (on_oop) {
-    if (not_in_heap && !is_traversal_mode) {
-      load_reference_barrier_native(masm, dst, src);
-    } else {
-      load_reference_barrier(masm, dst, src);
-    }
+  if (!need_load_reference_barrier) {
+    return;
+  }
 
-    if (dst != result_dst) {
-      __ mov(result_dst, dst);
-      dst = result_dst;
-    }
+  if (ShenandoahBarrierSet::use_native_load_reference_barrier(decorators, type)) {
+    load_reference_barrier_native(masm, dst, src);
+  } else {
+    load_reference_barrier(masm, dst, src);
+  }
 
-    if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
-      __ enter();
-      satb_write_barrier_pre(masm /* masm */,
-                             noreg /* obj */,
-                             dst /* pre_val */,
-                             rthread /* thread */,
-                             tmp1 /* tmp */,
-                             true /* tosca_live */,
-                             true /* expand_call */);
-      __ leave();
-    }
+  // Move loaded oop to final destination
+  if (dst != result_dst) {
+    __ mov(result_dst, dst);
+    dst = result_dst;
+  }
+
+  if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
+    __ enter();
+    satb_write_barrier_pre(masm /* masm */,
+                           noreg /* obj */,
+                           dst /* pre_val */,
+                           rthread /* thread */,
+                           tmp1 /* tmp */,
+                           true /* tosca_live */,
+                           true /* expand_call */);
+    __ leave();
   }
 }
 
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -445,21 +445,29 @@
   }
 }
 
+//
+// Arguments:
+//
+// Inputs:
+//   src:        oop location, might be clobbered
+//   tmp1:       scratch register, might not be valid.
+//   tmp_thread: unused
+//
+// Output:
+//   dst:        oop loaded from src location
+//
+// Kill:
+//   tmp1 (if it is valid)
+//
 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
              Register dst, Address src, Register tmp1, Register tmp_thread) {
-  bool on_oop = is_reference_type(type);
-  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
-  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
-  bool not_in_heap = (decorators & IN_NATIVE) != 0;
-  bool on_reference = on_weak || on_phantom;
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-  bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
-
   Register result_dst = dst;
   bool use_tmp1_for_dst = false;
+  bool need_load_reference_barrier = ShenandoahBarrierSet::need_load_reference_barrier(decorators, type);
 
-  if (on_oop) {
-    // We want to preserve src
+  // Only preserve src address if we need load reference barrier
+  if (need_load_reference_barrier) {
+    // Use tmp1 or rdi as temporary output register to avoid clobbering src
     if (dst == src.base() || dst == src.index()) {
       // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
       if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
@@ -475,37 +483,38 @@
 
   BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 
-  if (on_oop) {
-    if (not_in_heap && !is_traversal_mode) {
-      load_reference_barrier_native(masm, dst, src);
-    } else {
-      load_reference_barrier(masm, dst, src);
+  if (!need_load_reference_barrier) return;
+
+  if (ShenandoahBarrierSet::use_native_load_reference_barrier(decorators, type)) {
+    load_reference_barrier_native(masm, dst, src);
+  } else {
+    load_reference_barrier(masm, dst, src);
+  }
+
+  // Move loaded oop to final destination
+  if (dst != result_dst) {
+    __ movptr(result_dst, dst);
+
+    if (!use_tmp1_for_dst) {
+      __ pop(dst);
     }
 
-    if (dst != result_dst) {
-      __ movptr(result_dst, dst);
-
-      if (!use_tmp1_for_dst) {
-        __ pop(dst);
-      }
-
-      dst = result_dst;
-    }
+    dst = result_dst;
+  }
 
-    if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
-      const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
-      assert_different_registers(dst, tmp1, tmp_thread);
-      NOT_LP64(__ get_thread(thread));
-      // Generate the SATB pre-barrier code to log the value of
-      // the referent field in an SATB buffer.
-      shenandoah_write_barrier_pre(masm /* masm */,
-                                   noreg /* obj */,
-                                   dst /* pre_val */,
-                                   thread /* thread */,
-                                   tmp1 /* tmp */,
-                                   true /* tosca_live */,
-                                   true /* expand_call */);
-    }
+  if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
+    const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
+    assert_different_registers(dst, tmp1, tmp_thread);
+    NOT_LP64(__ get_thread(thread));
+    // Generate the SATB pre-barrier code to log the value of
+    // the referent field in an SATB buffer.
+    shenandoah_write_barrier_pre(masm /* masm */,
+                                 noreg /* obj */,
+                                 dst /* pre_val */,
+                                 thread /* thread */,
+                                 tmp1 /* tmp */,
+                                 true /* tosca_live */,
+                                 true /* expand_call */);
   }
 }
 
diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
@@ -203,18 +203,20 @@
 }
 
 void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
-  if (!access.is_oop()) {
+  DecoratorSet decorators = access.decorators();
+  assert((decorators & AS_RAW) == 0, "Unexpected decorator");
+  assert((decorators & AS_NO_KEEPALIVE) == 0, "Unexpected decorator");
+  BasicType type = access.type();
+  bool need_load_reference_barrier = ShenandoahBarrierSet::need_load_reference_barrier(decorators, type);
+
+  if (!need_load_reference_barrier) {
     BarrierSetC1::load_at_resolved(access, result);
     return;
   }
 
   LIRGenerator* gen = access.gen();
 
-  DecoratorSet decorators = access.decorators();
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-
-  if ((decorators & IN_NATIVE) != 0 && !is_traversal_mode) {
-    assert(access.is_oop(), "IN_NATIVE access only for oop values");
+  if (ShenandoahBarrierSet::use_native_load_reference_barrier(decorators, type)) {
     BarrierSetC1::load_at_resolved(access, result);
     LIR_OprList* args = new LIR_OprList();
     LIR_Opr addr = access.resolved_addr();
@@ -229,34 +231,25 @@
                                             objectType, NULL);
     __ move(call_result, result);
   } else {
-    if (ShenandoahLoadRefBarrier) {
       LIR_Opr tmp = gen->new_register(T_OBJECT);
       BarrierSetC1::load_at_resolved(access, tmp);
       tmp = load_reference_barrier(access.gen(), tmp, access.resolved_addr());
       __ move(tmp, result);
-    } else {
-      BarrierSetC1::load_at_resolved(access, result);
-    }
   }
 
-  if (ShenandoahKeepAliveBarrier) {
-    bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
-    bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+  if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
     bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
-    bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
 
-    if ((is_weak || is_phantom || is_anonymous) && keep_alive) {
-      // Register the value in the referent field with the pre-barrier
-      LabelObj *Lcont_anonymous;
-      if (is_anonymous) {
-        Lcont_anonymous = new LabelObj();
-        generate_referent_check(access, Lcont_anonymous);
-      }
-      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */,
-                  result /* pre_val */);
-      if (is_anonymous) {
-        __ branch_destination(Lcont_anonymous->label());
-      }
+    // Register the value in the referent field with the pre-barrier
+    LabelObj *Lcont_anonymous;
+    if (is_anonymous) {
+      Lcont_anonymous = new LabelObj();
+      generate_referent_check(access, Lcont_anonymous);
+    }
+    pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */,
+                result /* pre_val */);
+    if (is_anonymous) {
+      __ branch_destination(Lcont_anonymous->label());
     }
   }
 }
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
@@ -535,6 +535,10 @@
 
 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
   DecoratorSet decorators = access.decorators();
+  BasicType type = access.type();
+
+  assert((decorators & AS_RAW) == 0, "Unexpected decorator");
+  assert((decorators & AS_NO_KEEPALIVE) == 0, "Unexpected decorator");
 
   Node* adr = access.addr().node();
   Node* obj = access.base();
@@ -545,36 +549,32 @@
   bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
   bool is_unordered = (decorators & MO_UNORDERED) != 0;
   bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
-  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
-  bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
-  bool in_native = (decorators & IN_NATIVE) != 0;
 
   Node* top = Compile::current()->top();
 
   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
-
-  if (access.is_oop()) {
-    if (ShenandoahLoadRefBarrier) {
-      load = new ShenandoahLoadReferenceBarrierNode(NULL, load, in_native && !is_traversal_mode);
-      if (access.is_parse_access()) {
-        load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
-      } else {
-        load = static_cast<C2OptAccess &>(access).gvn().transform(load);
-      }
-    }
+  if (!ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
+    return load;
   }
 
+  load = new ShenandoahLoadReferenceBarrierNode(NULL, load, ShenandoahBarrierSet::use_native_load_reference_barrier(decorators, type));
+  if (access.is_parse_access()) {
+    load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
+  } else {
+    load = static_cast<C2OptAccess &>(access).gvn().transform(load);
+  }
+
+  if (!ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
+    return load;
+  }
   // If we are reading the value of the referent field of a Reference
   // object (either by using Unsafe directly or through reflection)
   // then, if SATB is enabled, we need to record the referent in an
   // SATB log buffer using the pre-barrier mechanism.
   // Also we need to add memory barrier to prevent commoning reads
   // from this field across safepoint since GC can change its value.
-  bool need_read_barrier = ShenandoahKeepAliveBarrier &&
-    (on_weak_ref || (unknown && offset != top && obj != top));
-
-  if (!access.is_oop() || !need_read_barrier) {
+  if (unknown && (offset == top || obj == top)) {
     return load;
   }
 
@@ -582,7 +582,7 @@
   C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
   GraphKit* kit = parse_access.kit();
 
-  if (on_weak_ref && keep_alive) {
+  if (on_weak_ref) {
     // Use the pre-barrier to record the value in the referent field
     satb_write_barrier_pre(kit, false /* do_load */,
                            NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -27,6 +27,7 @@
 #include "gc/shenandoah/shenandoahBarrierSetClone.inline.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
@@ -71,6 +72,36 @@
   return true;
 }
 
+bool ShenandoahBarrierSet::need_load_reference_barrier(DecoratorSet decorators, BasicType type) {
+  if (!ShenandoahLoadRefBarrier) return false;
+  // Only needed for references
+  if (!is_reference_type(type)) return false;
+  return true;
+}
+
+bool ShenandoahBarrierSet::use_native_load_reference_barrier(DecoratorSet decorators, BasicType type) {
+  assert(need_load_reference_barrier(decorators, type), "Why ask?");
+  assert(is_reference_type(type), "Why we here?");
+  // Native load reference barrier is only needed for concurrent root processing
+  if (!ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
+    return false;
+  }
+
+  return (decorators & IN_NATIVE) != 0;
+}
+
+bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators,BasicType type) {
+  if (!ShenandoahKeepAliveBarrier) return false;
+  // Only needed for references
+  if (!is_reference_type(type)) return false;
+
+  bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
+  bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+  bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
+  bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
+  return (on_weak_ref || unknown) && (keep_alive || is_traversal_mode);
+}
+
 template <class T>
 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) {
   shenandoah_assert_not_in_cset_loc_except(field, _heap->cancelled_gc());
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -57,6 +57,10 @@
     return barrier_set()->_satb_mark_queue_set;
   }
 
+  static bool need_load_reference_barrier(DecoratorSet decorators, BasicType type);
+  static bool use_native_load_reference_barrier(DecoratorSet decorators, BasicType type);
+  static bool need_keep_alive_barrier(DecoratorSet decorators, BasicType type);
+
   void print_on(outputStream* st) const;
 
   bool is_a(BarrierSet::Name bsn);