# HG changeset patch
# Parent b9787f124d93c1f71194b2485d2996f92fee80d5
* * *
changeset:   51378:aa251c268e06
tag:         tip
user:        roland
summary:     fix
* * *
changeset:   51452:8f0eb8cbd541
tag:         tip
user:        roland
summary:     fix

diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -418,11 +418,13 @@
 
 #ifdef _LP64
   if (ShenandoahStoreValEnqueueBarrier) {
-    Label is_null;
-    __ testptr(dst, dst);
-    __ jcc(Assembler::zero, is_null);
-    write_barrier_impl(masm, dst);
-    __ bind(is_null);
+    if (!ShenandoahLoadValBarrier) {
+      Label is_null;
+      __ testptr(dst, dst);
+      __ jcc(Assembler::zero, is_null);
+      write_barrier_impl(masm, dst);
+      __ bind(is_null);
+    }
 
     // The set of registers to be saved+restored is the same as in the write-barrier above.
     // Those are the commonly used registers in the interpreter.
@@ -437,7 +439,7 @@
     //__ pop_callee_saved_registers();
     __ popa();
   }
-  if (ShenandoahStoreValReadBarrier) {
+  if (ShenandoahStoreValReadBarrier && !ShenandoahLoadValBarrier) {
     read_barrier_impl(masm, dst);
   }
 #else
@@ -452,10 +454,17 @@
   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
   bool on_reference = on_weak || on_phantom;
-  if (in_heap) {
+  if (in_heap && !ShenandoahLoadValBarrier) {
     read_barrier_not_null(masm, src.base());
   }
   BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  if (ShenandoahLoadValBarrier && on_oop /*&& in_heap*/) {
+    Label is_null;
+    __ testptr(dst, dst);
+    __ jcc(Assembler::zero, is_null);
+    write_barrier(masm, dst);
+    __ bind(is_null);
+  }
   if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
     const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
     NOT_LP64(__ get_thread(thread));
@@ -477,7 +486,7 @@
 
   bool in_heap = (decorators & IN_HEAP) != 0;
   bool as_normal = (decorators & AS_NORMAL) != 0;
-  if (in_heap) {
+  if (in_heap && !ShenandoahLoadValBarrier) {
     write_barrier(masm, dst.base());
   }
   if (type == T_OBJECT || type == T_ARRAY) {
@@ -556,7 +565,7 @@
 
 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
   __ cmpptr(op1, op2);
-  if (ShenandoahAcmpBarrier) {
+  if (ShenandoahAcmpBarrier && !ShenandoahLoadValBarrier) {
     Label done;
     __ jccb(Assembler::equal, done);
     read_barrier(masm, op1);
@@ -568,7 +577,7 @@
 
 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) {
   __ cmpptr(src1, src2);
-  if (ShenandoahAcmpBarrier) {
+  if (ShenandoahAcmpBarrier && !ShenandoahLoadValBarrier) {
     Label done;
     __ jccb(Assembler::equal, done);
     __ movptr(rscratch2, src2);
@@ -629,24 +638,28 @@
 }
 
 void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
-  bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
-  if (oop_not_null) {
-    read_barrier_not_null(masm, obj);
-  } else {
-    read_barrier(masm, obj);
+  if (!ShenandoahLoadValBarrier) {
+    bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
+    if (oop_not_null) {
+      read_barrier_not_null(masm, obj);
+    } else {
+      read_barrier(masm, obj);
+    }
   }
 }
 
 void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
-  bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
-  if (oop_not_null) {
-    write_barrier(masm, obj);
-  } else {
-    Label done;
-    __ testptr(obj, obj);
-    __ jcc(Assembler::zero, done);
-    write_barrier(masm, obj);
-    __ bind(done);
+  if (!ShenandoahLoadValBarrier) {
+    bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
+    if (oop_not_null) {
+      write_barrier(masm, obj);
+    } else {
+      Label done;
+      __ testptr(obj, obj);
+      __ jcc(Assembler::zero, done);
+      write_barrier(masm, obj);
+      __ bind(done);
+    }
   }
 }
 
@@ -715,7 +728,9 @@
   if (UseCompressedOops) {
     __ decode_heap_oop(tmp1);
   }
-  __ resolve_for_read(0, tmp1);
+  if (!ShenandoahLoadValBarrier) {
+    __ resolve_for_read(0, tmp1);
+  }
 
   if (UseCompressedOops) {
     __ movl(tmp2, oldval);
diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
@@ -264,10 +264,13 @@
   LIRGenerator* gen = access.gen();
   if (UseShenandoahGC) {
     if (ShenandoahStoreValEnqueueBarrier) {
-      obj = write_barrier_impl(access, obj, info, need_null_check);
+      if (!ShenandoahLoadValBarrier) {
+        obj = write_barrier_impl(access, obj, info, need_null_check);
+      }
+      obj = ensure_in_register(access, obj);
       pre_barrier(access, LIR_OprFact::illegalOpr, obj);
     }
-    if (ShenandoahStoreValReadBarrier) {
+    if (ShenandoahStoreValReadBarrier && !ShenandoahLoadValBarrier) {
       obj = read_barrier_impl(access, obj, info, true /*need_null_check*/);
     }
   }
@@ -275,7 +278,9 @@
 }
 
 void ShenandoahBarrierSetC1::store_at(LIRAccess& access, LIR_Opr value) {
-  access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  if (!ShenandoahLoadValBarrier) {
+    access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  }
   LIR_Opr resolved = resolve_address(access, false);
   access.set_resolved_addr(resolved);
   if (access.is_oop()) {
@@ -293,11 +298,23 @@
 
 void ShenandoahBarrierSetC1::load_at(LIRAccess& access, LIR_Opr result) {
   LIRItem& base_item = access.base().item();
-  access.set_base(read_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  if (!ShenandoahLoadValBarrier) {
+    access.set_base(read_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  }
   LIR_Opr resolved = resolve_address(access, false);
   access.set_resolved_addr(resolved);
-  BarrierSetC1::load_at_resolved(access, result);
-  access.set_base(base_item);
+
+  if (ShenandoahLoadValBarrier && access.is_oop()) {
+    LIRGenerator* gen = access.gen();
+    LIR_Opr result_tmp = gen->new_register(T_OBJECT);
+    BarrierSetC1::load_at_resolved(access, result_tmp);
+    access.set_base(base_item);
+    result_tmp = write_barrier(access, result_tmp, access.access_emit_info(), true /*access.needs_null_check()*/);
+    gen->lir()->move(result_tmp, result);
+  } else {
+    BarrierSetC1::load_at_resolved(access, result);
+    access.set_base(base_item);
+  }
 
   if (ShenandoahKeepAliveBarrier) {
     DecoratorSet decorators = access.decorators();
@@ -323,7 +340,9 @@
 
 LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
   access.load_address();
-  access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  if (!ShenandoahLoadValBarrier) {
+    access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  }
   LIR_Opr resolved = resolve_address(access, true);
   access.set_resolved_addr(resolved);
   if (access.is_oop()) {
@@ -340,7 +359,9 @@
 }
 
 LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at(LIRAccess& access, LIRItem& value) {
-  access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  if (!ShenandoahLoadValBarrier) {
+    access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  }
   LIR_Opr resolved = resolve_address(access, true);
   access.set_resolved_addr(resolved);
   if (access.is_oop()) {
@@ -350,6 +371,9 @@
     }
   }
   LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value);
+  if (ShenandoahLoadValBarrier && access.is_oop()) {
+    result = write_barrier(access, result, access.access_emit_info(), true /*access.needs_null_check()*/);
+  }
   if (access.is_oop() && UseShenandoahMatrix) {
     post_barrier(access, access.resolved_addr(), value.result());
   }
@@ -358,18 +382,29 @@
 
 LIR_Opr ShenandoahBarrierSetC1::atomic_add_at(LIRAccess& access, LIRItem& value) {
   access.load_address();
-  access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  if (!ShenandoahLoadValBarrier) {
+    access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check()));
+  }
   LIR_Opr resolved = resolve_address(access, true);
   access.set_resolved_addr(resolved);
   return BarrierSetC1::atomic_add_at_resolved(access, value);
 }
 
 LIR_Opr ShenandoahBarrierSetC1::resolve_for_read(LIRAccess& access) {
-  return read_barrier(access, access.base().opr(), access.access_emit_info(), access.needs_null_check());
+  if (!ShenandoahLoadValBarrier) {
+    return read_barrier(access, access.base().opr(), access.access_emit_info(), access.needs_null_check());
+  } else {
+    return access.base().opr();
+  }
 }
 
 LIR_Opr ShenandoahBarrierSetC1::resolve_for_write(LIRAccess& access) {
-  return write_barrier(access, access.base().opr(), access.access_emit_info(), access.needs_null_check());
+  if (!ShenandoahLoadValBarrier) {
+    return write_barrier(access, access.base().opr(), access.access_emit_info(), access.needs_null_check());
+  } else {
+    return access.base().opr();
+  }
+
 }
 
 class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
@@ -69,16 +69,19 @@
 
 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
   if (ShenandoahStoreValEnqueueBarrier) {
-    obj = shenandoah_write_barrier(kit, obj);
+    if (!ShenandoahLoadValBarrier) {
+      obj = shenandoah_write_barrier(kit, obj);
+    }
     obj = shenandoah_enqueue_barrier(kit, obj);
   }
-  if (ShenandoahStoreValReadBarrier) {
+  if (ShenandoahStoreValReadBarrier && !ShenandoahLoadValBarrier) {
     obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
   }
   return obj;
 }
 
 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_acmp(GraphKit* kit, Node* obj) {
+  assert(!ShenandoahLoadValBarrier, "Should not get here with load-value barriers");
   return shenandoah_read_barrier_impl(kit, obj, true, true, false);
 }
 
@@ -680,6 +683,10 @@
   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top();
   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 
+  if (ShenandoahLoadValBarrier && access.is_oop() && on_heap) {
+    load = shenandoah_write_barrier(kit, load);
+  }
+
   // If we are reading the value of the referent field of a Reference
   // object (either by using Unsafe directly or through reflection)
   // then, if SATB is enabled, we need to record the referent in an
@@ -712,6 +719,8 @@
 
 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
                                                    Node* val, const Type* value_type) const {
+  DecoratorSet decorators = access.decorators();
+  bool on_heap = (decorators & IN_HEAP) != 0;
   GraphKit* kit = access.kit();
   if (access.is_oop()) {
     val = shenandoah_storeval_barrier(kit, val);
@@ -720,7 +729,11 @@
                                  expected_val /* pre_val */, T_OBJECT);
 
   }
-  return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, val, value_type);
+  Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, val, value_type);
+  if (on_heap && access.is_oop() && ShenandoahLoadValBarrier) {
+    result = shenandoah_write_barrier(kit, result);
+  }
+  return result;
 }
 
 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val,
@@ -748,6 +761,8 @@
 }
 
 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* val, const Type* value_type) const {
+  DecoratorSet decorators = access.decorators();
+  bool on_heap = (decorators & IN_HEAP) != 0;
   GraphKit* kit = access.kit();
   if (access.is_oop()) {
     val = shenandoah_storeval_barrier(kit, val);
@@ -757,6 +772,9 @@
     shenandoah_write_barrier_pre(kit, false /* do_load */,
                                  NULL, NULL, max_juint, NULL, NULL,
                                  result /* pre_val */, T_OBJECT);
+    if (on_heap && ShenandoahLoadValBarrier) {
+      result = shenandoah_write_barrier(kit, result);
+    }
   }
   return result;
 }
@@ -777,11 +795,19 @@
 }
 
 Node* ShenandoahBarrierSetC2::resolve_for_read(GraphKit* kit, Node* n) const {
-  return shenandoah_read_barrier(kit, n);
+  if (!ShenandoahLoadValBarrier) {
+    return shenandoah_read_barrier(kit, n);
+  } else {
+    return n;
+  }
 }
 
 Node* ShenandoahBarrierSetC2::resolve_for_write(GraphKit* kit, Node* n) const {
-  return shenandoah_write_barrier(kit, n);
+  if (!ShenandoahLoadValBarrier) {
+    return shenandoah_write_barrier(kit, n);
+  } else {
+    return n;
+  }
 }
 
 // Support for GC barriers emitted during parsing
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -193,7 +193,7 @@
  */
 void ShenandoahBarrierNode::do_cmpp_if(GraphKit& kit, Node*& taken_branch, Node*& untaken_branch, Node*& taken_memory, Node*& untaken_memory) {
   assert(taken_memory == NULL && untaken_memory == NULL, "unexpected memory inputs");
-  if (!UseShenandoahGC || !ShenandoahAcmpBarrier || ShenandoahVerifyOptoBarriers) {
+  if (!UseShenandoahGC || !ShenandoahAcmpBarrier || ShenandoahLoadValBarrier || ShenandoahVerifyOptoBarriers) {
     return;
   }
   if (taken_branch->is_top() || untaken_branch->is_top()) {
@@ -568,7 +568,7 @@
 
 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) {
   if (UseShenandoahGC) {
-    if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) {
+    //if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier) ) {
       bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
       C->clear_major_progress();
       PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
@@ -582,7 +582,7 @@
         }
         C->clear_major_progress();
       }
-    }
+      //}
   }
   return true;
 }
@@ -2603,7 +2603,7 @@
   phase->register_new_node(heap_stable_cmp, ctrl);
   Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
   phase->register_new_node(heap_stable_test, ctrl);
-  IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+  IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.9), COUNT_UNKNOWN);
   phase->register_control(heap_stable_iff, loop, ctrl);
 
   heap_stable = new IfFalseNode(heap_stable_iff);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
+#include "gc/shared/stringdedup/stringDedupThread.hpp"
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
@@ -267,7 +268,7 @@
 
 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) {
   bool eq = oopDesc::unsafe_equals(obj1, obj2);
-  if (! eq && ShenandoahAcmpBarrier) {
+  if (! eq && ShenandoahAcmpBarrier && !ShenandoahLoadValBarrier) {
     OrderAccess::loadload();
     obj1 = resolve_forwarded(obj1);
     obj2 = resolve_forwarded(obj2);
@@ -284,8 +285,14 @@
     if (evac_in_progress &&
         _heap->in_collection_set(obj) &&
         oopDesc::unsafe_equals(obj, fwd)) {
-      ShenandoahEvacOOMScope oom_evac_scope;
-      return _heap->evacuate_object(obj, Thread::current());
+      Thread* thread = Thread::current();
+      if (thread->is_Worker_thread()) {
+        assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must have done OOM protocol");
+        return _heap->evacuate_object(obj, thread);
+      } else {
+        ShenandoahEvacOOMScope oom_evac_scope;
+        return _heap->evacuate_object(obj, thread);
+      }
     } else {
       return fwd;
     }
@@ -305,11 +312,13 @@
 oop ShenandoahBarrierSet::storeval_barrier(oop obj) {
   if (ShenandoahStoreValEnqueueBarrier) {
     if (!CompressedOops::is_null(obj)) {
-      obj = write_barrier(obj);
+      if (!ShenandoahLoadValBarrier) {
+        obj = write_barrier(obj);
+      }
       enqueue(obj);
     }
   }
-  if (ShenandoahStoreValReadBarrier) {
+  if (ShenandoahStoreValReadBarrier && !ShenandoahLoadValBarrier) {
     obj = resolve_forwarded(obj);
   }
   return obj;
@@ -342,6 +351,12 @@
   }
 }
 
+#ifdef ASSERT
+void ShenandoahBarrierSet::verify_safe_addr(void* p) {
+  shenandoah_assert_not_in_cset_except(p, NULL, ShenandoahHeap::heap()->cancelled_gc());
+}
+#endif
+
 void ShenandoahBarrierSet::on_thread_create(Thread* thread) {
   // Create thread local data
   ShenandoahThreadLocalData::create(thread);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -102,6 +102,8 @@
 
   bool obj_equals(oop obj1, oop obj2);
 
+  void verify_safe_addr(void* p) PRODUCT_RETURN;
+
   void enqueue(oop obj);
 
 private:
@@ -161,19 +163,23 @@
 
     template <typename T>
     static T load_in_heap_at(oop base, ptrdiff_t offset) {
-      base = ShenandoahBarrierSet::resolve_forwarded(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::resolve_forwarded(base);
+      }
       return Raw::template load_at<T>(base, offset);
     }
 
     template <typename T>
     static void store_in_heap(T* addr, T value) {
-      ShouldNotReachHere();
+      ShenandoahBarrierSet::barrier_set()->verify_safe_addr(addr);
       Raw::store(addr, value);
     }
 
     template <typename T>
     static void store_in_heap_at(oop base, ptrdiff_t offset, T value) {
-      base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      }
       Raw::store_at(base, offset, value);
     }
 
@@ -185,7 +191,9 @@
 
     template <typename T>
     static T atomic_cmpxchg_in_heap_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
-      base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      }
       return Raw::atomic_cmpxchg_at(new_value, base, offset, compare_value);
     }
 
@@ -197,7 +205,9 @@
 
     template <typename T>
     static T atomic_xchg_in_heap_at(T new_value, oop base, ptrdiff_t offset) {
-      base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      }
       return Raw::atomic_xchg_at(new_value, base, offset);
     }
 
@@ -210,19 +220,9 @@
     // IN_HEAP is set (e.g. when using the HeapAccess API), it is
     // an oop_* overload, and the barrier strength is AS_NORMAL.
     template <typename T>
-    static oop oop_load_in_heap(T* addr) {
-      // ShouldNotReachHere();
-      oop value = Raw::template oop_load<oop>(addr);
-      keep_alive_if_weak(decorators, value);
-      return value;
-    }
+    static oop oop_load_in_heap(T* addr);
 
-    static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) {
-      base = ShenandoahBarrierSet::resolve_forwarded(base);
-      oop value = Raw::template oop_load_at<oop>(base, offset);
-      keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
-      return value;
-    }
+    static oop oop_load_in_heap_at(oop base, ptrdiff_t offset);
 
     template <typename T>
     static void oop_store_in_heap(T* addr, oop value) {
@@ -231,9 +231,10 @@
     }
 
     static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
-      base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      }
       value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(value);
-
       oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
     }
 
@@ -241,16 +242,21 @@
     static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value);
 
     static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
-      base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      }
       new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
-      return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
+      oop oldval = oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
+      return oldval;
     }
 
     template <typename T>
     static oop oop_atomic_xchg_in_heap(oop new_value, T* addr);
 
     static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
-      base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      if (!ShenandoahLoadValBarrier) {
+        base = ShenandoahBarrierSet::barrier_set()->write_barrier(base);
+      }
       new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
       return oop_atomic_xchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
     }
@@ -268,15 +274,25 @@
     static oop oop_load_not_in_heap(T* addr) {
       oop value = Raw::oop_load_not_in_heap(addr);
       keep_alive_if_weak(decorators, value);
+      if (ShenandoahLoadValBarrier) {
+        value = ShenandoahBarrierSet::barrier_set()->write_barrier(value);
+      }
       return value;
     }
 
     static oop resolve(oop obj) {
-      return ShenandoahBarrierSet::barrier_set()->write_barrier(obj);
+      if (!ShenandoahLoadValBarrier) {
+        obj = ShenandoahBarrierSet::barrier_set()->write_barrier(obj);
+      }
+      return obj;
     }
 
     static bool equals(oop o1, oop o2) {
-      return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2);
+      if (!ShenandoahLoadValBarrier) {
+        return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2);
+      } else {
+        return Raw::equals(o1, o2);
+      }
     }
 
   };
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
@@ -29,6 +29,7 @@
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 
 bool ShenandoahBarrierSet::need_update_refs_barrier() {
   return UseShenandoahMatrix ||
@@ -51,6 +52,31 @@
 
 template <DecoratorSet decorators, typename BarrierSetT>
 template <typename T>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
+  // ShouldNotReachHere();
+  oop value = Raw::template oop_load<oop>(addr);
+  keep_alive_if_weak(decorators, value);
+  if (ShenandoahLoadValBarrier) {
+    value = ShenandoahBarrierSet::barrier_set()->write_barrier(value);
+  }
+  return value;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
+  if (!ShenandoahLoadValBarrier) {
+    base = ShenandoahBarrierSet::resolve_forwarded(base);
+  }
+  oop value = Raw::template oop_load_at<oop>(base, offset);
+  keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
+  if (ShenandoahLoadValBarrier) {
+    value = ShenandoahBarrierSet::barrier_set()->write_barrier(value);
+  }
+  return value;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
   oop res;
   oop expected = compare_value;
@@ -68,6 +94,9 @@
       matrix->set_connected(addr, new_value);
     }
   }
+  if (ShenandoahLoadValBarrier) {
+    res = ShenandoahBarrierSet::barrier_set()->write_barrier(res);
+  }
   return res;
 }
 
@@ -84,6 +113,9 @@
     ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
     matrix->set_connected(addr, new_value);
   }
+  if (ShenandoahLoadValBarrier) {
+    previous = ShenandoahBarrierSet::barrier_set()->write_barrier(previous);
+  }
   return previous;
 }
 
@@ -92,11 +124,13 @@
 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
                                                                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
                                                                                      size_t length) {
-  if (!CompressedOops::is_null(src_obj)) {
-    src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
-  }
-  if (!CompressedOops::is_null(dst_obj)) {
-    dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
+  if (!ShenandoahLoadValBarrier) {
+    if (!CompressedOops::is_null(src_obj)) {
+      src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
+    }
+    if (!CompressedOops::is_null(dst_obj)) {
+      dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
+    }
   }
   Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
 }
@@ -251,8 +285,10 @@
 // Clone barrier support
 template <DecoratorSet decorators, typename BarrierSetT>
 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
-  src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));
-  dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));
+  if (!ShenandoahLoadValBarrier) {
+    src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));
+    dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));
+  }
   Raw::clone(src, dst, size);
   ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
 }
@@ -264,13 +300,14 @@
                                                                                          arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
                                                                                          size_t length) {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
-  if (!CompressedOops::is_null(src_obj)) {
-    src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
+  if (!ShenandoahLoadValBarrier) {
+    if (!CompressedOops::is_null(src_obj)) {
+      src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));
+    }
+    if (!CompressedOops::is_null(dst_obj)) {
+      dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
+    }
   }
-  if (!CompressedOops::is_null(dst_obj)) {
-    dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));
-  }
-
   bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
   bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
   ArrayCopyStoreValMode storeval_mode;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -1274,6 +1274,27 @@
   }
 };
 
+class ShenandoahEvacuateUpdateAllRootsTask : public AbstractGangTask {
+  ShenandoahRootProcessor* _rp;
+public:
+
+  ShenandoahEvacuateUpdateAllRootsTask(ShenandoahRootProcessor* rp) :
+    AbstractGangTask("Shenandoah evacuate and update all roots"),
+    _rp(rp)
+  {
+    // Nothing else to do.
+  }
+
+  void work(uint worker_id) {
+    ShenandoahEvacOOMScope oom_evac_scope;
+    ShenandoahEvacuateUpdateRootsClosure cl;
+    CLDToOopClosure cldCl(&cl);
+    // MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
+    CodeBlobToOopClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
+    _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id);
+  }
+};
+
 class ShenandoahFixRootsTask : public AbstractGangTask {
   ShenandoahRootEvacuator* _rp;
 public:
@@ -1295,6 +1316,26 @@
   }
 };
 
+class ShenandoahFixAllRootsTask : public AbstractGangTask {
+  ShenandoahRootProcessor* _rp;
+public:
+
+  ShenandoahFixAllRootsTask(ShenandoahRootProcessor* rp) :
+    AbstractGangTask("Shenandoah update all roots"),
+    _rp(rp)
+  {
+    // Nothing else to do.
+  }
+
+  void work(uint worker_id) {
+    ShenandoahEvacOOMScope oom_evac_scope;
+    ShenandoahUpdateRefsClosure cl;
+    CLDToOopClosure cldCl(&cl);
+    MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
+    _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id);
+  }
+};
+
 void ShenandoahHeap::evacuate_and_update_roots() {
 
 #if defined(COMPILER2) || INCLUDE_JVMCI
@@ -1302,7 +1343,11 @@
 #endif
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
 
-  {
+  if (ShenandoahLoadValBarrier) {
+    ShenandoahRootProcessor rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
+    ShenandoahEvacuateUpdateAllRootsTask roots_task(&rp);
+    workers()->run_task(&roots_task);
+  } else {
     ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
     workers()->run_task(&roots_task);
@@ -1329,9 +1374,15 @@
 #if defined(COMPILER2) || INCLUDE_JVMCI
     DerivedPointerTable::clear();
 #endif
-    ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
-    ShenandoahFixRootsTask update_roots_task(&rp);
-    workers()->run_task(&update_roots_task);
+    if (ShenandoahLoadValBarrier) {
+      ShenandoahRootProcessor rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
+      ShenandoahFixAllRootsTask update_roots_task(&rp);
+      workers()->run_task(&update_roots_task);
+    } else {
+      ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
+      ShenandoahFixRootsTask update_roots_task(&rp);
+      workers()->run_task(&update_roots_task);
+    }
 #if defined(COMPILER2) || INCLUDE_JVMCI
     DerivedPointerTable::update_pointers();
 #endif
@@ -2513,7 +2564,9 @@
   }
   assert(!cancelled_gc(), "Should have been done right before");
 
-  concurrentMark()->update_roots(ShenandoahPhaseTimings::final_update_refs_roots);
+  if (!ShenandoahLoadValBarrier) {
+    concurrentMark()->update_roots(ShenandoahPhaseTimings::final_update_refs_roots);
+  }
 
   // Allocations might have happened before we STWed here, record peak:
   heuristics()->record_peak_occupancy();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
@@ -193,6 +193,11 @@
       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
       if (unload_classes) {
         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &code_cl, NULL, worker_id);
+        if (ShenandoahLoadValBarrier) {
+          ShenandoahAllCodeRootsIterator iter = ShenandoahCodeRoots::iterator();
+          // TODO: Don't need to mark all that stuff, only pre-evac to to-space
+          iter.possibly_parallel_blobs_do(&code_cl);
+        }
       } else {
         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, NULL, worker_id);
       }
@@ -268,6 +273,11 @@
       if (unload_classes) {
         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
+        if (ShenandoahLoadValBarrier) {
+          // TODO: Don't need to mark all that stuff, only pre-evac to to-space
+          ShenandoahAllCodeRootsIterator iter = ShenandoahCodeRoots::iterator();
+          iter.possibly_parallel_blobs_do(&code_cl);
+        }
       } else {
         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
       }
diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
@@ -321,6 +321,9 @@
   diagnostic(bool, ShenandoahCloneBarrier, true,                            \
           "Turn on/off clone barriers in Shenandoah")                       \
                                                                             \
+  diagnostic(bool, ShenandoahLoadValBarrier, true,                          \
+          "Turn on/off load-value-style barriers")                          \
+                                                                            \
   diagnostic(bool, UseShenandoahMatrix, false,                              \
           "Turn on/off Shenandoah connection matrix collection")            \
                                                                             \
diff --git a/src/hotspot/share/oops/accessBackend.hpp b/src/hotspot/share/oops/accessBackend.hpp
--- a/src/hotspot/share/oops/accessBackend.hpp
+++ b/src/hotspot/share/oops/accessBackend.hpp
@@ -410,7 +410,13 @@
 
   static oop resolve(oop obj) { return obj; }
 
-  static bool equals(oop o1, oop o2) { return o1 == o2; }
+  static bool equals(oop o1, oop o2) {
+#ifndef CHECK_UNHANDLED_OOPS
+    return o1 == o2;
+#else
+    return o1.obj() == o2.obj();
+#endif
+  }
 };
 
 // Below is the implementation of the first 4 steps of the template pipeline:
diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp
--- a/src/hotspot/share/opto/callGenerator.cpp
+++ b/src/hotspot/share/opto/callGenerator.cpp
@@ -839,7 +839,7 @@
       // Get MethodHandle receiver:
       Node* receiver = kit.argument(0);
 #if INCLUDE_SHENANDOAHGC
-      assert(!(ShenandoahBarrierNode::skip_through_barrier(receiver)->is_Con() && !receiver->is_Con()), "barrier prevents optimization");
+      receiver = ShenandoahBarrierNode::skip_through_barrier(receiver);
 #endif
       if (receiver->Opcode() == Op_ConP) {
         input_not_const = false;
@@ -874,7 +874,7 @@
       // Get MemberName argument:
       Node* member_name = kit.argument(callee->arg_size() - 1);
 #if INCLUDE_SHENANDOAHGC
-      assert(!(ShenandoahBarrierNode::skip_through_barrier(member_name)->is_Con() && !member_name->is_Con()), "barrier prevents optimization");
+      member_name = ShenandoahBarrierNode::skip_through_barrier(member_name);
 #endif
       if (member_name->Opcode() == Op_ConP) {
         input_not_const = false;
diff --git a/src/hotspot/share/opto/replacednodes.cpp b/src/hotspot/share/opto/replacednodes.cpp
--- a/src/hotspot/share/opto/replacednodes.cpp
+++ b/src/hotspot/share/opto/replacednodes.cpp
@@ -151,7 +151,7 @@
         if (use->outcnt() == 0) {
           continue;
         }
-        if (n->is_CFG() || (n->in(0) != NULL && !n->in(0)->is_top())) {
+        if (n->is_CFG() || (n->in(0) != NULL && !n->in(0)->is_top()) && n->Opcode() != Op_ShenandoahWBMemProj) {
           int depth = 0;
           Node *m = n;
           if (!n->is_CFG()) {
diff --git a/src/hotspot/share/opto/stringopts.cpp b/src/hotspot/share/opto/stringopts.cpp
--- a/src/hotspot/share/opto/stringopts.cpp
+++ b/src/hotspot/share/opto/stringopts.cpp
@@ -603,6 +603,8 @@
 
   // Collect the types needed to talk about the various slices of memory
   byte_adr_idx = C->get_alias_index(TypeAryPtr::BYTES);
+  const Type* ary_t = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE, TypeInt::POS), ciTypeArrayKlass::make(T_BYTE), true, 0);
+  byte_adr_idx_wb = C->get_alias_index(ShenandoahBarrierNode::brooks_pointer_type(ary_t));
 
   // For each locally allocated StringBuffer see if the usages can be
   // collapsed into a single String construction.
@@ -1245,7 +1247,7 @@
 }
 
 // Simplified version of Integer.getChars
-void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, int merge_index) {
+void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, Node* final_mem_wb, int merge_index) {
   // if (i < 0) {
   //     sign = '-';
   //     i = -i;
@@ -1336,11 +1338,13 @@
 
   final_merge->init_req(merge_index + 2, __ IfFalse(iff));
   final_mem->init_req(merge_index + 2, kit.memory(byte_adr_idx));
+  final_mem_wb->init_req(merge_index + 2, kit.memory(byte_adr_idx_wb));
 
   kit.set_control(__ IfTrue(iff));
   if (kit.stopped()) {
     final_merge->init_req(merge_index + 1, C->top());
     final_mem->init_req(merge_index + 1, C->top());
+    final_mem_wb->init_req(merge_index + 1, C->top());
   } else {
     Node* index = __ SubI(charPos, __ intcon((bt == T_BYTE) ? 1 : 2));
     st = __ store_to_memory(kit.control(), kit.array_element_address(dst_array, index, T_BYTE),
@@ -1348,6 +1352,7 @@
 
     final_merge->init_req(merge_index + 1, kit.control());
     final_mem->init_req(merge_index + 1, st);
+    final_mem_wb->init_req(merge_index + 1, kit.memory(byte_adr_idx_wb));
   }
 }
 
@@ -1366,12 +1371,16 @@
 
   Node* final_mem = PhiNode::make(final_merge, kit.memory(byte_adr_idx), Type::MEMORY, TypeAryPtr::BYTES);
   kit.gvn().set_type(final_mem, Type::MEMORY);
+  const Type* ary_t = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE, TypeInt::POS), ciTypeArrayKlass::make(T_BYTE), true, 0);
+  Node* final_mem_wb = PhiNode::make(final_merge, kit.memory(byte_adr_idx_wb), Type::MEMORY, ShenandoahBarrierNode::brooks_pointer_type(ary_t));
+  kit.gvn().set_type(final_mem_wb, Type::MEMORY);
 
   // need to handle arg == Integer.MIN_VALUE specially because negating doesn't make it positive
   IfNode* iff = kit.create_and_map_if(kit.control(), __ Bool(__ CmpI(arg, __ intcon(0x80000000)), BoolTest::ne),
                                       PROB_FAIR, COUNT_UNKNOWN);
 
   Node* old_mem = kit.memory(byte_adr_idx);
+  Node* old_mem_wb = kit.memory(byte_adr_idx_wb);
 
   kit.set_control(__ IfFalse(iff));
   if (kit.stopped()) {
@@ -1382,23 +1391,26 @@
                 dst_array, dst_coder, start);
     final_merge->init_req(3, kit.control());
     final_mem->init_req(3, kit.memory(byte_adr_idx));
+    final_mem_wb->init_req(3, kit.memory(byte_adr_idx_wb));
   }
 
   kit.set_control(__ IfTrue(iff));
   kit.set_memory(old_mem, byte_adr_idx);
+  kit.set_memory(old_mem_wb, byte_adr_idx_wb);
 
   if (!dcon) {
     // Check encoding of destination
     iff = kit.create_and_map_if(kit.control(), __ Bool(__ CmpI(dst_coder, __ intcon(0)), BoolTest::eq),
                                 PROB_FAIR, COUNT_UNKNOWN);
     old_mem = kit.memory(byte_adr_idx);
+    old_mem_wb = kit.memory(byte_adr_idx_wb);
   }
   if (!dcon || dbyte) {
     // Destination is Latin1,
     if (!dcon) {
       kit.set_control(__ IfTrue(iff));
     }
-    getChars(kit, arg, dst_array, T_BYTE, end, final_merge, final_mem);
+    getChars(kit, arg, dst_array, T_BYTE, end, final_merge, final_mem, final_mem_wb);
   }
   if (!dcon || !dbyte) {
     // Destination is UTF16
@@ -1406,17 +1418,20 @@
     if (!dcon) {
       kit.set_control(__ IfFalse(iff));
       kit.set_memory(old_mem, byte_adr_idx);
+      kit.set_memory(old_mem_wb, byte_adr_idx_wb);
       merge_index = 3; // Account for Latin1 case
     }
-    getChars(kit, arg, dst_array, T_CHAR, end, final_merge, final_mem, merge_index);
+    getChars(kit, arg, dst_array, T_CHAR, end, final_merge, final_mem, final_mem_wb, merge_index);
   }
 
   // Final merge point for Latin1 and UTF16 case
   kit.set_control(final_merge);
   kit.set_memory(final_mem, byte_adr_idx);
+  kit.set_memory(final_mem_wb, byte_adr_idx_wb);
 
   C->record_for_igvn(final_merge);
   C->record_for_igvn(final_mem);
+  C->record_for_igvn(final_mem_wb);
   return end;
 }
 
@@ -1749,6 +1764,7 @@
   // Create a region for the overflow checks to merge into.
   int args = MAX2(sc->num_arguments(), 1);
   RegionNode* overflow = new RegionNode(args);
+  PhiNode* overflow_phi = new PhiNode(overflow, Type::MEMORY, TypePtr::BOTTOM);
   kit.gvn().set_type(overflow, Type::CONTROL);
 
   // Create a hook node to hold onto the individual sizes since they
@@ -1790,6 +1806,9 @@
           Node* p = __ Bool(__ CmpP(arg, kit.null()), BoolTest::ne);
           IfNode* iff = kit.create_and_map_if(kit.control(), p, PROB_MIN, COUNT_UNKNOWN);
           overflow->add_req(__ IfFalse(iff));
+          Node* mem = __ reset_memory();
+          overflow_phi->add_req(mem);
+          __ set_all_memory(mem);
           Node* notnull = __ IfTrue(iff);
           kit.set_control(notnull); // set control for the cast_not_null
           arg = kit.cast_not_null(arg, false);
@@ -1910,6 +1929,9 @@
                                           PROB_MIN, COUNT_UNKNOWN);
       kit.set_control(__ IfFalse(iff));
       overflow->set_req(argi, __ IfTrue(iff));
+      Node* mem = __ reset_memory();
+      overflow_phi->set_req(argi, mem);
+      __ set_all_memory(mem);
     }
   }
 
@@ -1917,6 +1939,7 @@
     // Hook
     PreserveJVMState pjvms(&kit);
     kit.set_control(overflow);
+    kit.set_all_memory(__ gvn().transform(overflow_phi));
     C->record_for_igvn(overflow);
     kit.uncommon_trap(Deoptimization::Reason_intrinsic,
                       Deoptimization::Action_make_not_entrant);
diff --git a/src/hotspot/share/opto/stringopts.hpp b/src/hotspot/share/opto/stringopts.hpp
--- a/src/hotspot/share/opto/stringopts.hpp
+++ b/src/hotspot/share/opto/stringopts.hpp
@@ -42,6 +42,7 @@
 
   // Memory slices needed for code gen
   int byte_adr_idx;
+  int byte_adr_idx_wb;
 
   // Integer.sizeTable - used for int to String conversion
   ciField* size_table_field;
@@ -66,7 +67,7 @@
   Node* int_stringSize(GraphKit& kit, Node* value);
 
   // Simplified version of Integer.getChars
-  void getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, int merge_index = 0);
+  void getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicType bt, Node* end, Node* final_merge, Node* final_mem, Node* final_mem_wb, int merge_index = 0);
 
   // Copy the characters representing arg into dst_array starting at start
   Node* int_getChars(GraphKit& kit, Node* arg, Node* dst_array, Node* dst_coder, Node* start, Node* size);