# HG changeset patch
# User rkennke
# Date 1444164613 -7200
#      Tue Oct 06 22:50:13 2015 +0200
# Node ID c055621bfd32b457fb231adeb7e3742d84c78d4c
# Parent  90b308169cb2e201e2d02c9d8cbec5e3a7d941e5
[mq]: diff-shenandoah.patch

diff --git a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
+++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
@@ -32,6 +32,7 @@
 #include "c1/c1_ValueStack.hpp"
 #include "ci/ciArrayKlass.hpp"
 #include "ci/ciInstance.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/cardTableModRefBS.hpp"
 #include "gc/shared/collectedHeap.hpp"
@@ -1466,6 +1467,55 @@
   }
 }
 
+void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) {
+  Label done;
+  Register obj = op->in_opr()->as_register();
+  Register res = op->result_opr()->as_register();
+  Register tmp1 = op->tmp1_opr()->as_register();
+  Register tmp2 = op->tmp2_opr()->as_register();
+  assert_different_registers(res, tmp1, tmp2);
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+
+  // Check for null.
+  if (op->need_null_check()) {
+    __ testptr(res, res);
+    __ jcc(Assembler::zero, done);
+  }
+
+  // Check for evacuation-in-progress
+  Address evacuation_in_progress = Address(r15_thread, in_bytes(JavaThread::evacuation_in_progress_offset()));
+  __ cmpb(evacuation_in_progress, 0);
+
+  // The read-barrier.
+  __ movptr(res, Address(res, -8));
+
+  __ jcc(Assembler::equal, done);
+
+  // Check for object in collection set.
+  __ movptr(tmp1, res);
+  __ shrptr(tmp1, ShenandoahHeapRegion::RegionSizeShift);
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testb(tmp2, 0x1);
+  __ jcc(Assembler::zero, done);
+
+  if (res != rax) {
+    __ xchgptr(res, rax); // Move obj into rax and save rax into obj.
+  }
+
+  __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_write_barrier_slow_id)));
+
+  if (res != rax) {
+    __ xchgptr(rax, res); // Swap back obj with rax.
+  }
+
+  __ bind(done);
+
+}
+
 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
   LIR_Opr src  = op->in_opr();
   LIR_Opr dest = op->result_opr();
@@ -1962,10 +2012,43 @@
       } else
 #endif
       {
-        if (os::is_MP()) {
-          __ lock();
+        if (UseShenandoahGC) {
+          Label done;
+          Label retry;
+
+          __ bind(retry);
+
+          // Save original cmp-value into tmp1, before following cas destroys it.
+          __ movptr(op->tmp1()->as_register(), op->cmp_value()->as_register());
+
+          if (os::is_MP()) {
+            __ lock();
+          }
+          __ cmpxchgptr(newval, Address(addr, 0));
+
+          // If the cmpxchg succeeded, then we're done.
+          __ jcc(Assembler::equal, done);
+
+          // Resolve the original cmp value.
+          oopDesc::bs()->interpreter_read_barrier(masm(), op->tmp1()->as_register());
+          // Resolve the old value at address. We get the old value in cmp/rax
+          // when the comparison in cmpxchg failed.
+          __ movptr(op->tmp2()->as_register(), cmpval);
+          oopDesc::bs()->interpreter_read_barrier(masm(), op->tmp2()->as_register());
+
+          // We're done if the expected/cmp value is not the same as old. It's a valid
+          // cmpxchg failure then. Otherwise we need special treatment for Shenandoah
+          // to prevent false positives.
+          __ cmpptr(op->tmp1()->as_register(), op->tmp2()->as_register());
+          __ jcc(Assembler::equal, retry);
+
+          __ bind(done);
+        } else {
+          if (os::is_MP()) {
+            __ lock();
+          }
+          __ cmpxchgptr(newval, Address(addr, 0));
         }
-        __ cmpxchgptr(newval, Address(addr, 0));
       }
     } else {
       assert(op->code() == lir_cas_int, "lir_cas_int expected");
diff --git a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
+++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
@@ -300,15 +300,26 @@
     null_check_info = new CodeEmitInfo(range_check_info);
   }
 
+  LIR_Opr ary = array.result();
+  ary = shenandoah_write_barrier(ary, null_check_info, x->needs_null_check());
+  LIR_Opr val = value.result();
+  if (obj_store && UseShenandoahGC) {
+    if (! val->is_register()) {
+      assert(val->is_constant(), "expect constant");
+    } else {
+      val = shenandoah_read_barrier(val, NULL, true);
+    }
+  }
+
   // emit array address setup early so it schedules better
-  LIR_Address* array_addr = emit_array_address(array.result(), index.result(), x->elt_type(), obj_store);
+  LIR_Address* array_addr = emit_array_address(ary, index.result(), x->elt_type(), obj_store);
 
   if (GenerateRangeChecks && needs_range_check) {
     if (use_length) {
       __ cmp(lir_cond_belowEqual, length.result(), index.result());
       __ branch(lir_cond_belowEqual, T_INT, new RangeCheckStub(range_check_info, index.result()));
     } else {
-      array_range_check(array.result(), index.result(), null_check_info, range_check_info);
+      array_range_check(ary, index.result(), null_check_info, range_check_info);
       // range_check also does the null check
       null_check_info = NULL;
     }
@@ -320,18 +331,18 @@
     LIR_Opr tmp3 = new_register(objectType);
 
     CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
-    __ store_check(value.result(), array.result(), tmp1, tmp2, tmp3, store_check_info, x->profiled_method(), x->profiled_bci());
+    __ store_check(val, ary, tmp1, tmp2, tmp3, store_check_info, x->profiled_method(), x->profiled_bci());
   }
 
   if (obj_store) {
     // Needs GC write barriers.
     pre_barrier(LIR_OprFact::address(array_addr), LIR_OprFact::illegalOpr /* pre_val */,
                 true /* do_load */, false /* patch */, NULL);
-    __ move(value.result(), array_addr, null_check_info);
+    __ move(val, array_addr, null_check_info);
     // Seems to be a precise
     post_barrier(LIR_OprFact::address(array_addr), value.result());
   } else {
-    __ move(value.result(), array_addr, null_check_info);
+    __ move(val, array_addr, null_check_info);
   }
 }
 
@@ -358,7 +369,9 @@
   // this CodeEmitInfo must not have the xhandlers because here the
   // object is already locked (xhandlers expect object to be unlocked)
   CodeEmitInfo* info = state_for(x, x->state(), true);
-  monitor_enter(obj.result(), lock, syncTempOpr(), scratch,
+  LIR_Opr obj_opr = obj.result();
+  obj_opr = shenandoah_write_barrier(obj_opr, state_for(x), x->needs_null_check());
+  monitor_enter(obj_opr, lock, syncTempOpr(), scratch,
                         x->monitor_no(), info_for_exception, info);
 }
 
@@ -750,27 +763,31 @@
 
   LIR_Opr addr = new_pointer_register();
   LIR_Address* a;
+
+  LIR_Opr obj_op = obj.result();
+  obj_op = shenandoah_write_barrier(obj_op, NULL, false);
+
   if(offset.result()->is_constant()) {
 #ifdef _LP64
     jlong c = offset.result()->as_jlong();
     if ((jlong)((jint)c) == c) {
-      a = new LIR_Address(obj.result(),
+      a = new LIR_Address(obj_op,
                           (jint)c,
                           as_BasicType(type));
     } else {
       LIR_Opr tmp = new_register(T_LONG);
       __ move(offset.result(), tmp);
-      a = new LIR_Address(obj.result(),
+      a = new LIR_Address(obj_op,
                           tmp,
                           as_BasicType(type));
     }
 #else
-    a = new LIR_Address(obj.result(),
+    a = new LIR_Address(obj_op,
                         offset.result()->as_jint(),
                         as_BasicType(type));
 #endif
   } else {
-    a = new LIR_Address(obj.result(),
+    a = new LIR_Address(obj_op,
                         offset.result(),
                         LIR_Address::times_1,
                         0,
@@ -785,12 +802,17 @@
   }
 
   LIR_Opr ill = LIR_OprFact::illegalOpr;  // for convenience
-  if (type == objectType)
-    __ cas_obj(addr, cmp.result(), val.result(), ill, ill);
+
+  LIR_Opr val_op = val.result();
+
+  if (type == objectType) {
+    val_op = shenandoah_read_barrier(val_op, NULL, true);
+    __ cas_obj(addr, cmp.result(), val_op, new_register(T_OBJECT), new_register(T_OBJECT));
+  }
   else if (type == intType)
-    __ cas_int(addr, cmp.result(), val.result(), ill, ill);
+    __ cas_int(addr, cmp.result(), val_op, ill, ill);
   else if (type == longType)
-    __ cas_long(addr, cmp.result(), val.result(), ill, ill);
+    __ cas_long(addr, cmp.result(), val_op, ill, ill);
   else {
     ShouldNotReachHere();
   }
@@ -801,7 +823,7 @@
            result, as_BasicType(type));
   if (type == objectType) {   // Write-barrier needed for Object fields.
     // Seems to be precise
-    post_barrier(addr, val.result());
+    post_barrier(addr, val_op);
   }
 }
 
@@ -893,14 +915,19 @@
   LIRItem dst_pos(x->argument_at(3), this);
   LIRItem length(x->argument_at(4), this);
 
+  LIR_Opr dst_op = dst.result();
+  dst_op = shenandoah_write_barrier(dst_op, info, x->arg_needs_null_check(2));
+  LIR_Opr src_op = src.result();
+  src_op = shenandoah_read_barrier(src_op, info, x->arg_needs_null_check(0));
+
   // operands for arraycopy must use fixed registers, otherwise
   // LinearScan will fail allocation (because arraycopy always needs a
   // call)
 
 #ifndef _LP64
-  src.load_item_force     (FrameMap::rcx_oop_opr);
+  src_op = force_opr_to(src_op, FrameMap::rcx_oop_opr);
   src_pos.load_item_force (FrameMap::rdx_opr);
-  dst.load_item_force     (FrameMap::rax_oop_opr);
+  dst_op = force_opr_to(dst_op, FrameMap::rax_oop_opr);
   dst_pos.load_item_force (FrameMap::rbx_opr);
   length.load_item_force  (FrameMap::rdi_opr);
   LIR_Opr tmp =           (FrameMap::rsi_opr);
@@ -914,9 +941,9 @@
   // of the C convention we can process the java args trivially into C
   // args without worry of overwriting during the xfer
 
-  src.load_item_force     (FrameMap::as_oop_opr(j_rarg0));
+  src_op = force_opr_to(src_op, FrameMap::as_oop_opr(j_rarg0));
   src_pos.load_item_force (FrameMap::as_opr(j_rarg1));
-  dst.load_item_force     (FrameMap::as_oop_opr(j_rarg2));
+  dst_op = force_opr_to(dst_op, FrameMap::as_oop_opr(j_rarg2));
   dst_pos.load_item_force (FrameMap::as_opr(j_rarg3));
   length.load_item_force  (FrameMap::as_opr(j_rarg4));
 
@@ -929,7 +956,7 @@
   ciArrayKlass* expected_type;
   arraycopy_helper(x, &flags, &expected_type);
 
-  __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
+  __ arraycopy(src_op, src_pos.result(), dst_op, dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
 }
 
 void LIRGenerator::do_update_CRC32(Intrinsic* x) {
@@ -980,6 +1007,10 @@
       }
 #endif
 
+      if (is_updateBytes) {
+        base_op = shenandoah_read_barrier(base_op, NULL, false);
+      }
+
       LIR_Address* a = new LIR_Address(base_op,
                                        index,
                                        LIR_Address::times_1,
@@ -1312,6 +1343,10 @@
 
   LIR_Opr left = xin->result();
   LIR_Opr right = yin->result();
+  if (tag == objectTag && UseShenandoahGC && x->y()->type() != objectNull) { // Don't need to resolve for ifnull.
+    left = shenandoah_write_barrier(left, NULL, true);
+    right = shenandoah_read_barrier(right, NULL, true);
+  }
   __ cmp(lir_cond(cond), left, right);
   // Generate branch profiling. Profiling code doesn't kill flags.
   profile_branch(x, cond);
@@ -1391,6 +1426,7 @@
 
 void LIRGenerator::get_Object_unsafe(LIR_Opr dst, LIR_Opr src, LIR_Opr offset,
                                      BasicType type, bool is_volatile) {
+  src = shenandoah_read_barrier(src, NULL, false);
   if (is_volatile && type == T_LONG) {
     LIR_Address* addr = new LIR_Address(src, offset, T_DOUBLE);
     LIR_Opr tmp = new_register(T_DOUBLE);
@@ -1408,6 +1444,7 @@
 
 void LIRGenerator::put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data,
                                      BasicType type, bool is_volatile) {
+  src = shenandoah_write_barrier(src, NULL, false);
   if (is_volatile && type == T_LONG) {
     LIR_Address* addr = new LIR_Address(src, offset, T_DOUBLE);
     LIR_Opr tmp = new_register(T_DOUBLE);
@@ -1423,6 +1460,7 @@
       // Do the pre-write barrier, if any.
       pre_barrier(LIR_OprFact::address(addr), LIR_OprFact::illegalOpr /* pre_val */,
                   true /* do_load */, false /* patch */, NULL);
+      data = shenandoah_read_barrier(data, NULL, true);
       __ move(data, addr);
       assert(src->is_register(), "must be register");
       // Seems to be a precise address
@@ -1449,22 +1487,29 @@
   LIR_Opr offset = off.result();
 
   assert (type == T_INT || (!x->is_add() && is_obj) LP64_ONLY( || type == T_LONG ), "unexpected type");
+
+  LIR_Opr src_op = src.result();
+  src_op = shenandoah_write_barrier(src_op, NULL, false);
+  if (is_obj) {
+    data = shenandoah_read_barrier(data, NULL, true);
+  }
+
   LIR_Address* addr;
   if (offset->is_constant()) {
 #ifdef _LP64
     jlong c = offset->as_jlong();
     if ((jlong)((jint)c) == c) {
-      addr = new LIR_Address(src.result(), (jint)c, type);
+      addr = new LIR_Address(src_op, (jint)c, type);
     } else {
       LIR_Opr tmp = new_register(T_LONG);
       __ move(offset, tmp);
-      addr = new LIR_Address(src.result(), tmp, type);
+      addr = new LIR_Address(src_op, tmp, type);
     }
 #else
-    addr = new LIR_Address(src.result(), offset->as_jint(), type);
+    addr = new LIR_Address(src_op, offset->as_jint(), type);
 #endif
   } else {
-    addr = new LIR_Address(src.result(), offset, type);
+    addr = new LIR_Address(src_op, offset, type);
   }
 
   // Because we want a 2-arg form of xchg and xadd
diff --git a/src/cpu/x86/vm/c1_Runtime1_x86.cpp b/src/cpu/x86/vm/c1_Runtime1_x86.cpp
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp
@@ -39,6 +39,9 @@
 #include "utilities/macros.hpp"
 #include "vmreg_x86.inline.hpp"
 #if INCLUDE_ALL_GCS
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
 #endif
 
@@ -1657,13 +1660,24 @@
       break;
 
 #if INCLUDE_ALL_GCS
+    case shenandoah_write_barrier_slow_id:
+      {
+        StubFrame f(sasm, "shenandoah_write_barrier", dont_gc_arguments);
+
+        save_live_registers(sasm, 1);
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1), r15_thread, rax);
+        restore_live_registers_except_rax(sasm);
+        __ verify_oop(rax);
+
+      }
+      break;
     case g1_pre_barrier_slow_id:
       {
         StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
         // arg0 : previous value of memory
 
         BarrierSet* bs = Universe::heap()->barrier_set();
-        if (bs->kind() != BarrierSet::G1SATBCTLogging) {
+        if (bs->kind() != BarrierSet::G1SATBCTLogging && bs->kind() != BarrierSet::ShenandoahBarrierSet) {
           __ movptr(rax, (int)id);
           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
           __ should_not_reach_here();
@@ -1741,8 +1755,15 @@
         // arg0: store_address
         Address store_addr(rbp, 2*BytesPerWord);
 
+        BarrierSet* bs = Universe::heap()->barrier_set();
+        if (bs->kind() == BarrierSet::ShenandoahBarrierSet) {
+          __ movptr(rax, (int)id);
+          __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
+          __ should_not_reach_here();
+          break;
+        }
         CardTableModRefBS* ct =
-          barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
+          barrier_set_cast<CardTableModRefBS>(bs);
         assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
 
         Label done;
diff --git a/src/cpu/x86/vm/interp_masm_x86.cpp b/src/cpu/x86/vm/interp_masm_x86.cpp
--- a/src/cpu/x86/vm/interp_masm_x86.cpp
+++ b/src/cpu/x86/vm/interp_masm_x86.cpp
@@ -1070,6 +1070,9 @@
     // Load object pointer into obj_reg
     movptr(obj_reg, Address(lock_reg, obj_offset));
 
+    // Need to preemptively evacuate obj because we CAS the mark word later.
+    oopDesc::bs()->interpreter_write_barrier(this, obj_reg);
+
     if (UseBiasedLocking) {
       biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, false, done, &slow_case);
     }
@@ -1165,6 +1168,9 @@
     // Load oop into obj_reg(%c_rarg3)
     movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
 
+    // Need to preemptively evacuate obj because we CAS the mark word later.
+    oopDesc::bs()->interpreter_write_barrier(this, obj_reg);
+
     // Free entry
     movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
 
diff --git a/src/cpu/x86/vm/jniFastGetField_x86_64.cpp b/src/cpu/x86/vm/jniFastGetField_x86_64.cpp
--- a/src/cpu/x86/vm/jniFastGetField_x86_64.cpp
+++ b/src/cpu/x86/vm/jniFastGetField_x86_64.cpp
@@ -81,6 +81,7 @@
                                                 // robj is data dependent on rcounter.
   }
   __ movptr(robj, Address(robj, 0));             // *obj
+  oopDesc::bs()->interpreter_read_barrier(masm, robj);
   __ mov   (roffset, c_rarg2);
   __ shrptr(roffset, 2);                         // offset
 
@@ -179,6 +180,7 @@
                                                 // robj is data dependent on rcounter.
   }
   __ movptr(robj, Address(robj, 0));             // *obj
+  oopDesc::bs()->interpreter_read_barrier(masm, robj);
   __ mov   (roffset, c_rarg2);
   __ shrptr(roffset, 2);                         // offset
 
diff --git a/src/cpu/x86/vm/macroAssembler_x86.cpp b/src/cpu/x86/vm/macroAssembler_x86.cpp
--- a/src/cpu/x86/vm/macroAssembler_x86.cpp
+++ b/src/cpu/x86/vm/macroAssembler_x86.cpp
@@ -3729,6 +3729,11 @@
     // provoke OS NULL exception if reg = NULL by
     // accessing M[reg] w/o changing any (non-CC) registers
     // NOTE: cmpl is plenty here to provoke a segv
+
+    if (ShenandoahVerifyReadsToFromSpace) {
+      oopDesc::bs()->interpreter_read_barrier(this, reg);
+    }
+
     cmpptr(rax, Address(reg, 0));
     // Note: should probably use testl(rax, Address(reg, 0));
     //       may be shorter code (however, this version of
@@ -4227,6 +4232,13 @@
   assert(thread == r15_thread, "must be");
 #endif // _LP64
 
+  if (UseShenandoahGC) {
+    // No need for this in Shenandoah.
+    return;
+  }
+
+  assert(UseG1GC, "expect G1 GC");
+
   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
                                        PtrQueue::byte_offset_of_index()));
   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
@@ -4411,10 +4423,15 @@
 
   NOT_LP64(get_thread(thread));
 
+  uint oop_extra_words = Universe::heap()->oop_extra_words();
+
   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
   if (var_size_in_bytes == noreg) {
-    lea(end, Address(obj, con_size_in_bytes));
+    lea(end, Address(obj, con_size_in_bytes + oop_extra_words * HeapWordSize));
   } else {
+    if (oop_extra_words > 0) {
+      addq(var_size_in_bytes, oop_extra_words * HeapWordSize);
+    }
     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
   }
   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
@@ -4423,6 +4440,8 @@
   // update the tlab top pointer
   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
 
+  Universe::heap()->compile_prepare_oop(this, obj);
+
   // recover var_size_in_bytes if necessary
   if (var_size_in_bytes == end) {
     subptr(var_size_in_bytes, obj);
@@ -5690,6 +5709,9 @@
 
 
 void MacroAssembler::load_klass(Register dst, Register src) {
+  if (ShenandoahVerifyReadsToFromSpace) {
+    oopDesc::bs()->interpreter_read_barrier(this, src);
+  }
 #ifdef _LP64
   if (UseCompressedClassPointers) {
     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
diff --git a/src/cpu/x86/vm/methodHandles_x86.cpp b/src/cpu/x86/vm/methodHandles_x86.cpp
--- a/src/cpu/x86/vm/methodHandles_x86.cpp
+++ b/src/cpu/x86/vm/methodHandles_x86.cpp
@@ -165,8 +165,10 @@
   //NOT_PRODUCT({ FlagSetting fs(TraceMethodHandles, true); trace_method_handle(_masm, "LZMH"); });
 
   // Load the invoker, as MH -> MH.form -> LF.vmentry
+  oopDesc::bs()->interpreter_read_barrier(_masm, recv);
   __ verify_oop(recv);
   __ load_heap_oop(method_temp, Address(recv, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes())));
+  oopDesc::bs()->interpreter_read_barrier(_masm, method_temp);
   __ verify_oop(method_temp);
   __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes())));
   __ verify_oop(method_temp);
@@ -180,8 +182,10 @@
                         Address(temp2, ConstMethod::size_of_parameters_offset()),
                         sizeof(u2), /*is_signed*/ false);
     // assert(sizeof(u2) == sizeof(Method::_size_of_parameters), "");
+    __ movptr(temp2, __ argument_address(temp2, -1));
+    oopDesc::bs()->interpreter_read_barrier(_masm, temp2);
     Label L;
-    __ cmpptr(recv, __ argument_address(temp2, -1));
+    __ cmpptr(recv, temp2);
     __ jcc(Assembler::equal, L);
     __ movptr(rax, __ argument_address(temp2, -1));
     __ STOP("receiver not on stack");
@@ -376,6 +380,7 @@
     //  rsi/r13 - interpreter linkage (if interpreted)
     //  rcx, rdx, rsi, rdi, r8 - compiler arguments (if compiled)
 
+    oopDesc::bs()->interpreter_read_barrier(_masm, member_reg);
     Label L_incompatible_class_change_error;
     switch (iid) {
     case vmIntrinsics::_linkToSpecial:
diff --git a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp
--- a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp
+++ b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp
@@ -2446,7 +2446,7 @@
 
     // Load the oop from the handle
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
-
+    oopDesc::bs()->interpreter_read_barrier(masm, obj_reg);
     if (UseBiasedLocking) {
       __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, lock_done, &slow_path_lock);
     }
@@ -2615,6 +2615,7 @@
 
     // Get locked oop from the handle we passed to jni
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
+    oopDesc::bs()->interpreter_read_barrier(masm, obj_reg);
 
     Label done;
 
diff --git a/src/cpu/x86/vm/shenandoahBarrierSet_x86.cpp b/src/cpu/x86/vm/shenandoahBarrierSet_x86.cpp
new file mode 100644
--- /dev/null
+++ b/src/cpu/x86/vm/shenandoahBarrierSet_x86.cpp
@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+
+#include "asm/macroAssembler.hpp"
+#include "interpreter/interpreter.hpp"
+
+#define __ masm->
+
+#ifndef CC_INTERP
+void ShenandoahBarrierSet::compile_resolve_oop_runtime(MacroAssembler* masm, Register dst) {
+
+  __ push(rscratch1);
+
+  if (dst != rax) {
+    __ push(rax);
+  }
+  if (dst != rbx) {
+    __ push(rbx);
+  }
+  if (dst != rcx) {
+    __ push(rcx);
+  }
+  if (dst != rdx) {
+    __ push(rdx);
+  }
+  if (dst != rdi) {
+    __ push(rdi);
+  }
+  if (dst != rsi) {
+    __ push(rsi);
+  }
+  if (dst != rbp) {
+    __ push(rbp);
+  }
+  if (dst != r8) {
+    __ push(r8);
+  }
+  if (dst != r9) {
+    __ push(r9);
+  }
+  if (dst != r11) {
+    __ push(r11);
+  }
+  if (dst != r12) {
+    __ push(r12);
+  }
+  if (dst != r13) {
+    __ push(r13);
+  }
+  if (dst != r14) {
+    __ push(r14);
+  }
+  if (dst != r15) {
+    __ push(r15);
+  }
+
+  __ subptr(rsp, 128);
+  __ movdbl(Address(rsp, 0), xmm0);
+  __ movdbl(Address(rsp, 8), xmm1);
+  __ movdbl(Address(rsp, 16), xmm2);
+  __ movdbl(Address(rsp, 24), xmm3);
+  __ movdbl(Address(rsp, 32), xmm4);
+  __ movdbl(Address(rsp, 40), xmm5);
+  __ movdbl(Address(rsp, 48), xmm6);
+  __ movdbl(Address(rsp, 56), xmm7);
+  __ movdbl(Address(rsp, 64), xmm8);
+  __ movdbl(Address(rsp, 72), xmm9);
+  __ movdbl(Address(rsp, 80), xmm10);
+  __ movdbl(Address(rsp, 88), xmm11);
+  __ movdbl(Address(rsp, 96), xmm12);
+  __ movdbl(Address(rsp, 104), xmm13);
+  __ movdbl(Address(rsp, 112), xmm14);
+  __ movdbl(Address(rsp, 120), xmm15);
+
+  __ mov(c_rarg1, dst);
+  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_oop_static), c_rarg1);
+  __ mov(rscratch1, rax);
+
+  __ movdbl(xmm0, Address(rsp, 0));
+  __ movdbl(xmm1, Address(rsp, 8));
+  __ movdbl(xmm2, Address(rsp, 16));
+  __ movdbl(xmm3, Address(rsp, 24));
+  __ movdbl(xmm4, Address(rsp, 32));
+  __ movdbl(xmm5, Address(rsp, 40));
+  __ movdbl(xmm6, Address(rsp, 48));
+  __ movdbl(xmm7, Address(rsp, 56));
+  __ movdbl(xmm8, Address(rsp, 64));
+  __ movdbl(xmm9, Address(rsp, 72));
+  __ movdbl(xmm10, Address(rsp, 80));
+  __ movdbl(xmm11, Address(rsp, 88));
+  __ movdbl(xmm12, Address(rsp, 96));
+  __ movdbl(xmm13, Address(rsp, 104));
+  __ movdbl(xmm14, Address(rsp, 112));
+  __ movdbl(xmm15, Address(rsp, 120));
+  __ addptr(rsp, 128);
+
+  if (dst != r15) {
+    __ pop(r15);
+  }
+  if (dst != r14) {
+    __ pop(r14);
+  }
+  if (dst != r13) {
+    __ pop(r13);
+  }
+  if (dst != r12) {
+    __ pop(r12);
+  }
+  if (dst != r11) {
+    __ pop(r11);
+  }
+  if (dst != r9) {
+    __ pop(r9);
+  }
+  if (dst != r8) {
+    __ pop(r8);
+  }
+  if (dst != rbp) {
+    __ pop(rbp);
+  }
+  if (dst != rsi) {
+    __ pop(rsi);
+  }
+  if (dst != rdi) {
+    __ pop(rdi);
+  }
+  if (dst != rdx) {
+    __ pop(rdx);
+  }
+  if (dst != rcx) {
+    __ pop(rcx);
+  }
+  if (dst != rbx) {
+    __ pop(rbx);
+  }
+  if (dst != rax) {
+    __ pop(rax);
+  }
+
+  __ mov(dst, rscratch1);
+
+  __ pop(rscratch1);
+}
+
+// TODO: The following should really live in an X86 specific subclass.
+void ShenandoahBarrierSet::interpreter_read_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+
+    Label is_null;
+    __ testptr(dst, dst);
+    __ jcc(Assembler::zero, is_null);
+    interpreter_read_barrier_not_null(masm, dst);
+    __ bind(is_null);
+  }
+}
+
+void ShenandoahBarrierSet::interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    if (ShenandoahVerifyReadsToFromSpace) {
+      compile_resolve_oop_runtime(masm, dst);
+      return;
+    }
+    __ movptr(dst, Address(dst, -8));
+  }
+}
+
+void ShenandoahBarrierSet::interpreter_write_barrier(MacroAssembler* masm, Register dst) {
+
+  if (! ShenandoahWriteBarrier) {
+    assert(! ShenandoahConcurrentEvacuation, "Can only do this without concurrent evacuation");
+    return interpreter_read_barrier(masm, dst);
+  }
+
+  assert(dst != rscratch1, "different regs");
+  //assert(dst != rscratch2, "Need rscratch2");
+
+  Label done;
+
+  Address evacuation_in_progress = Address(r15_thread, in_bytes(JavaThread::evacuation_in_progress_offset()));
+
+  __ cmpb(evacuation_in_progress, 0);
+
+  // Now check if evacuation is in progress.
+  interpreter_read_barrier_not_null(masm, dst);
+
+  __ jcc(Assembler::equal, done);
+  __ push(rscratch1);
+  __ push(rscratch2);
+
+  __ movptr(rscratch1, dst);
+  __ shrptr(rscratch1, ShenandoahHeapRegion::RegionSizeShift);
+  __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1));
+  __ testb(rscratch2, 0x1);
+
+  __ pop(rscratch2);
+  __ pop(rscratch1);
+
+  __ jcc(Assembler::zero, done);
+
+  __ push(rscratch1);
+
+  // Save possibly live regs.
+  if (dst != rax) {
+    __ push(rax);
+  }
+  if (dst != rbx) {
+    __ push(rbx);
+  }
+  if (dst != rcx) {
+    __ push(rcx);
+  }
+  if (dst != rdx) {
+    __ push(rdx);
+  }
+  if (dst != c_rarg1) {
+    __ push(c_rarg1);
+  }
+
+  __ subptr(rsp, 2 * wordSize);
+  __ movdbl(Address(rsp, 0), xmm0);
+
+  // Call into runtime
+  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_interp), dst);
+  __ mov(rscratch1, rax);
+
+  // Restore possibly live regs.
+  __ movdbl(xmm0, Address(rsp, 0));
+  __ addptr(rsp, 2 * Interpreter::stackElementSize);
+
+  if (dst != c_rarg1) {
+    __ pop(c_rarg1);
+  }
+  if (dst != rdx) {
+    __ pop(rdx);
+  }
+  if (dst != rcx) {
+    __ pop(rcx);
+  }
+  if (dst != rbx) {
+    __ pop(rbx);
+  }
+  if (dst != rax) {
+    __ pop(rax);
+  }
+
+  // Move result into dst reg.
+  __ mov(dst, rscratch1);
+
+  __ pop(rscratch1);
+
+  __ bind(done);
+}
+
+void ShenandoahHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) {
+  __ incrementq(obj, BrooksPointer::BROOKS_POINTER_OBJ_SIZE * HeapWordSize);
+  __ movptr(Address(obj, -1 * HeapWordSize), obj);
+}
+#endif
+
diff --git a/src/cpu/x86/vm/stubGenerator_x86_64.cpp b/src/cpu/x86/vm/stubGenerator_x86_64.cpp
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp
+++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp
@@ -804,6 +804,92 @@
     return start;
   }
 
+  address generate_shenandoah_wb() {
+    StubCodeMark mark(this, "StubRoutines", "shenandoah_wb");
+    address start = __ pc();
+
+    Label done;
+
+    __ push(rbx);
+    // Check for object beeing in the collection set.
+    // TODO: Can we use only 1 register here?
+    __ movptr(rdi, rax);
+    __ shrptr(rdi, ShenandoahHeapRegion::RegionSizeShift);
+    __ movptr(rbx, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+    __ movbool(rbx, Address(rbx, rdi, Address::times_1));
+    __ testbool(rbx);
+    __ jcc(Assembler::zero, done);
+
+    __ push(rcx);
+    __ push(rdx);
+    __ push(rdi);
+    __ push(rsi);
+    __ push(r8);
+    __ push(r9);
+    __ push(r10);
+    __ push(r11);
+    __ push(r12);
+    __ push(r13);
+    __ push(r14);
+    __ push(r15);
+    __ subptr(rsp, 128);
+    __ movdbl(Address(rsp, 0), xmm0);
+    __ movdbl(Address(rsp, 8), xmm1);
+    __ movdbl(Address(rsp, 16), xmm2);
+    __ movdbl(Address(rsp, 24), xmm3);
+    __ movdbl(Address(rsp, 32), xmm4);
+    __ movdbl(Address(rsp, 40), xmm5);
+    __ movdbl(Address(rsp, 48), xmm6);
+    __ movdbl(Address(rsp, 56), xmm7);
+    __ movdbl(Address(rsp, 64), xmm8);
+    __ movdbl(Address(rsp, 72), xmm9);
+    __ movdbl(Address(rsp, 80), xmm10);
+    __ movdbl(Address(rsp, 88), xmm11);
+    __ movdbl(Address(rsp, 96), xmm12);
+    __ movdbl(Address(rsp, 104), xmm13);
+    __ movdbl(Address(rsp, 112), xmm14);
+    __ movdbl(Address(rsp, 120), xmm15);
+    __ movptr(rdi, rax);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2), rdi);
+    __ movdbl(xmm0, Address(rsp, 0));
+    __ movdbl(xmm1, Address(rsp, 8));
+    __ movdbl(xmm2, Address(rsp, 16));
+    __ movdbl(xmm3, Address(rsp, 24));
+    __ movdbl(xmm4, Address(rsp, 32));
+    __ movdbl(xmm5, Address(rsp, 40));
+    __ movdbl(xmm6, Address(rsp, 48));
+    __ movdbl(xmm7, Address(rsp, 56));
+    __ movdbl(xmm8, Address(rsp, 64));
+    __ movdbl(xmm9, Address(rsp, 72));
+    __ movdbl(xmm10, Address(rsp, 80));
+    __ movdbl(xmm11, Address(rsp, 88));
+    __ movdbl(xmm12, Address(rsp, 96));
+    __ movdbl(xmm13, Address(rsp, 104));
+    __ movdbl(xmm14, Address(rsp, 112));
+    __ movdbl(xmm15, Address(rsp, 120));
+    __ addptr(rsp, 128);
+    __ pop(r15);
+    __ pop(r14);
+    __ pop(r13);
+    __ pop(r12);
+    __ pop(r11);
+    __ pop(r10);
+    __ pop(r9);
+    __ pop(r8);
+    __ pop(rsi);
+    __ pop(rdi);
+    __ pop(rdx);
+    __ pop(rcx);
+
+    __ bind(done);
+
+    __ pop(rbx);
+
+    __ ret(0);
+
+    return start;
+  }
+
   address generate_f2i_fixup() {
     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
     Address inout(rsp, 5 * wordSize); // return address + 4 saves
@@ -1234,6 +1320,7 @@
     BarrierSet* bs = Universe::heap()->barrier_set();
     switch (bs->kind()) {
       case BarrierSet::G1SATBCTLogging:
+      case BarrierSet::ShenandoahBarrierSet:
         // With G1, don't generate the call if we statically know that the target in uninitialized
         if (!dest_uninitialized) {
            __ pusha();                      // push registers
@@ -1278,6 +1365,7 @@
     BarrierSet* bs = Universe::heap()->barrier_set();
     switch (bs->kind()) {
       case BarrierSet::G1SATBCTLogging:
+      case BarrierSet::ShenandoahBarrierSet:
         {
           __ pusha();             // push registers (overkill)
           if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
@@ -4275,6 +4363,9 @@
                                                 throw_NullPointerException_at_call));
 
     // entry points that are platform specific
+    if (UseShenandoahGC) {
+      StubRoutines::x86::_shenandoah_wb = generate_shenandoah_wb();
+    }
     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
diff --git a/src/cpu/x86/vm/stubRoutines_x86_64.cpp b/src/cpu/x86/vm/stubRoutines_x86_64.cpp
--- a/src/cpu/x86/vm/stubRoutines_x86_64.cpp
+++ b/src/cpu/x86/vm/stubRoutines_x86_64.cpp
@@ -34,6 +34,7 @@
 address StubRoutines::x86::_get_previous_fp_entry = NULL;
 address StubRoutines::x86::_get_previous_sp_entry = NULL;
 
+address StubRoutines::x86::_shenandoah_wb = NULL;
 address StubRoutines::x86::_f2i_fixup = NULL;
 address StubRoutines::x86::_f2l_fixup = NULL;
 address StubRoutines::x86::_d2i_fixup = NULL;
diff --git a/src/cpu/x86/vm/stubRoutines_x86_64.hpp b/src/cpu/x86/vm/stubRoutines_x86_64.hpp
--- a/src/cpu/x86/vm/stubRoutines_x86_64.hpp
+++ b/src/cpu/x86/vm/stubRoutines_x86_64.hpp
@@ -43,6 +43,7 @@
   static address _get_previous_fp_entry;
   static address _get_previous_sp_entry;
 
+  static address _shenandoah_wb;
   static address _f2i_fixup;
   static address _f2l_fixup;
   static address _d2i_fixup;
@@ -65,6 +66,11 @@
     return _get_previous_sp_entry;
   }
 
+  static address shenandoah_wb()
+  {
+    return _shenandoah_wb;
+  }
+
   static address f2i_fixup()
   {
     return _f2i_fixup;
diff --git a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
--- a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
+++ b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
@@ -545,6 +545,7 @@
 #endif // ASSERT
 
     __ bind(done);
+    oopDesc::bs()->interpreter_read_barrier(_masm, rax);
   }
 
   // add space for monitor & lock
@@ -637,7 +638,7 @@
   const int referent_offset = java_lang_ref_Reference::referent_offset;
   guarantee(referent_offset > 0, "referent offset not initialized");
 
-  if (UseG1GC) {
+  if (UseG1GC || UseShenandoahGC) {
     Label slow_path;
     // rbx: method
 
@@ -648,6 +649,8 @@
     __ testptr(rax, rax);
     __ jcc(Assembler::zero, slow_path);
 
+    oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax);
+
     // rax: local 0
     // rbx: method (but can be used as scratch now)
     // rdx: scratch
@@ -778,6 +781,7 @@
       __ movl(crc,   Address(rsp, 5*wordSize)); // Initial CRC
     } else {
       __ movptr(buf, Address(rsp, 3*wordSize)); // byte[] array
+      oopDesc::bs()->interpreter_read_barrier_not_null(_masm, buf);
       __ addptr(buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
       __ movl2ptr(off, Address(rsp, 2*wordSize)); // offset
       __ addq(buf, off); // + offset
diff --git a/src/cpu/x86/vm/templateTable_x86.cpp b/src/cpu/x86/vm/templateTable_x86.cpp
--- a/src/cpu/x86/vm/templateTable_x86.cpp
+++ b/src/cpu/x86/vm/templateTable_x86.cpp
@@ -157,6 +157,7 @@
   switch (barrier) {
 #if INCLUDE_ALL_GCS
     case BarrierSet::G1SATBCTLogging:
+    case BarrierSet::ShenandoahBarrierSet:
       {
         // flatten object address if needed
         // We do it regardless of precise because we need the registers
@@ -189,6 +190,9 @@
             new_val = rbx;
             __ movptr(new_val, val);
           }
+          // For Shenandoah, make sure we only store refs into to-space.
+          oopDesc::bs()->interpreter_read_barrier(_masm, val);
+
           __ store_heap_oop(Address(rdx, 0), val);
           __ g1_write_barrier_post(rdx /* store_adr */,
                                    new_val /* new_val */,
@@ -684,6 +688,11 @@
 void TemplateTable::index_check_without_pop(Register array, Register index) {
   // destroys rbx
   // check array
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    oopDesc::bs()->interpreter_read_barrier(_masm, array);
+  }
+
   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
   // sign extend index for use by indexed load
   __ movl2ptr(index, index);
@@ -704,6 +713,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ movl(rax, Address(rdx, rax,
                        Address::times_4,
                        arrayOopDesc::base_offset_in_bytes(T_INT)));
@@ -716,6 +726,7 @@
   index_check(rdx, rax); // kills rbx
   NOT_LP64(__ mov(rbx, rax));
   // rbx,: index
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ movptr(rax, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize));
   NOT_LP64(__ movl(rdx, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize)));
 }
@@ -727,6 +738,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_float(Address(rdx, rax,
                         Address::times_4,
                         arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
@@ -737,6 +749,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_double(Address(rdx, rax,
                          Address::times_8,
                          arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
@@ -747,6 +760,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_heap_oop(rax, Address(rdx, rax,
                                 UseCompressedOops ? Address::times_4 : Address::times_ptr,
                                 arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
@@ -757,6 +771,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_signed_byte(rax, Address(rdx, rax, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 }
 
@@ -765,6 +780,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_unsigned_short(rax, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 }
 
@@ -778,6 +794,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_unsigned_short(rax,
                          Address(rdx, rax,
                                  Address::times_2,
@@ -790,6 +807,7 @@
   // rax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
   __ load_signed_short(rax, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 }
 
@@ -984,6 +1002,7 @@
   // rbx: index
   // rdx: array
   index_check(rdx, rbx); // prefer index in rbx
+  oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
   __ movl(Address(rdx, rbx,
                   Address::times_4,
                   arrayOopDesc::base_offset_in_bytes(T_INT)),
@@ -998,6 +1017,7 @@
   // rdx: high(value)
   index_check(rcx, rbx);  // prefer index in rbx,
   // rbx,: index
+  oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
   __ movptr(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize), rax);
   NOT_LP64(__ movl(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize), rdx));
 }
@@ -1010,6 +1030,7 @@
   // rbx:  index
   // rdx:  array
   index_check(rdx, rbx); // prefer index in rbx
+  oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
   __ store_float(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 }
 
@@ -1020,6 +1041,7 @@
   // rbx:  index
   // rdx:  array
   index_check(rdx, rbx); // prefer index in rbx
+  oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
   __ store_double(Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 }
 
@@ -1036,6 +1058,7 @@
                           arrayOopDesc::base_offset_in_bytes(T_OBJECT));
 
   index_check_without_pop(rdx, rcx);     // kills rbx
+  oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
   __ testptr(rax, rax);
   __ jcc(Assembler::zero, is_null);
 
@@ -1084,6 +1107,7 @@
   // rbx: index
   // rdx: array
   index_check(rdx, rbx); // prefer index in rbx
+  oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
   __ movb(Address(rdx, rbx,
                   Address::times_1,
                   arrayOopDesc::base_offset_in_bytes(T_BYTE)),
@@ -1097,6 +1121,7 @@
   // rbx: index
   // rdx: array
   index_check(rdx, rbx);  // prefer index in rbx
+  oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
   __ movw(Address(rdx, rbx,
                   Address::times_2,
                   arrayOopDesc::base_offset_in_bytes(T_CHAR)),
@@ -2311,6 +2336,17 @@
   // assume branch is more often taken than not (loops use backward branches)
   Label not_taken;
   __ pop_ptr(rdx);
+  if (UseShenandoahGC) {
+    // For Shenandoah, if the objects are not equal, we try again after
+    // resolving both objects through a read barrier, to make sure we're
+    // not comparing from-space and to-space copies of the same object.
+    Label eq;
+    __ cmpptr(rdx, rax);
+    __ jcc(Assembler::equal, eq);
+    oopDesc::bs()->interpreter_read_barrier(_masm, rax);
+    oopDesc::bs()->interpreter_read_barrier(_masm, rdx);
+    __ bind(eq);
+  }
   __ cmpptr(rdx, rax);
   __ jcc(j_not(cc), not_taken);
   branch(false, false);
@@ -2751,6 +2787,7 @@
   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
 
   if (!is_static) pop_and_check_object(obj);
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, obj);
 
   const Address field(obj, off, Address::times_1, 0*wordSize);
   NOT_LP64(const Address hi(obj, off, Address::times_1, 1*wordSize));
@@ -3020,6 +3057,7 @@
   {
     __ pop(btos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ movb(field, rax);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_bputfield, bc, rbx, true, byte_no);
@@ -3035,6 +3073,7 @@
   {
     __ pop(atos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     // Store into the field
     do_oop_store(_masm, field, rax, _bs->kind(), false);
     if (!is_static && rc == may_rewrite) {
@@ -3051,6 +3090,7 @@
   {
     __ pop(itos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ movl(field, rax);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_iputfield, bc, rbx, true, byte_no);
@@ -3066,6 +3106,7 @@
   {
     __ pop(ctos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ movw(field, rax);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_cputfield, bc, rbx, true, byte_no);
@@ -3081,6 +3122,7 @@
   {
     __ pop(stos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ movw(field, rax);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_sputfield, bc, rbx, true, byte_no);
@@ -3097,6 +3139,7 @@
   {
     __ pop(ltos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ movq(field, rax);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_lputfield, bc, rbx, true, byte_no);
@@ -3143,6 +3186,7 @@
   {
     __ pop(ftos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ store_float(field);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_fputfield, bc, rbx, true, byte_no);
@@ -3160,6 +3204,7 @@
   {
     __ pop(dtos);
     if (!is_static) pop_and_check_object(obj);
+    oopDesc::bs()->interpreter_write_barrier(_masm, obj);
     __ store_double(field);
     if (!is_static && rc == may_rewrite) {
       patch_bytecode(Bytecodes::_fast_dputfield, bc, rbx, true, byte_no);
@@ -3286,9 +3331,11 @@
   // access field
   switch (bytecode()) {
   case Bytecodes::_fast_aputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
     do_oop_store(_masm, field, rax, _bs->kind(), false);
     break;
   case Bytecodes::_fast_lputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
 #ifdef _LP64
   __ movq(field, rax);
 #else
@@ -3296,20 +3343,25 @@
 #endif
     break;
   case Bytecodes::_fast_iputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
     __ movl(field, rax);
     break;
   case Bytecodes::_fast_bputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
     __ movb(field, rax);
     break;
   case Bytecodes::_fast_sputfield:
     // fall through
   case Bytecodes::_fast_cputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
     __ movw(field, rax);
     break;
   case Bytecodes::_fast_fputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
     __ store_float(field);
     break;
   case Bytecodes::_fast_dputfield:
+    oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
     __ store_double(field);
     break;
   default:
@@ -3367,6 +3419,7 @@
   // rax: object
   __ verify_oop(rax);
   __ null_check(rax);
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax);
   Address field(rax, rbx, Address::times_1);
 
   // access field
@@ -3428,6 +3481,7 @@
   // next instruction)
   __ increment(rbcp);
   __ null_check(rax);
+  oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax);
   const Address field = Address(rax, rbx, Address::times_1, 0*wordSize);
   switch (state) {
   case itos:
@@ -3830,11 +3884,17 @@
 #endif // _LP64
 
   if (UseTLAB) {
+    uint oop_extra_words = Universe::heap()->oop_extra_words();
+    if (oop_extra_words > 0) {
+      __ addq(rdx, oop_extra_words * HeapWordSize);
+    }
+
     __ movptr(rax, Address(thread, in_bytes(JavaThread::tlab_top_offset())));
     __ lea(rbx, Address(rax, rdx, Address::times_1));
     __ cmpptr(rbx, Address(thread, in_bytes(JavaThread::tlab_end_offset())));
     __ jcc(Assembler::above, allow_shared_alloc ? allocate_shared : slow_case);
     __ movptr(Address(thread, in_bytes(JavaThread::tlab_top_offset())), rbx);
+    Universe::heap()->compile_prepare_oop(_masm);
     if (ZeroTLAB) {
       // the fields have been already cleared
       __ jmp(initialize_header);
@@ -3975,6 +4035,9 @@
 
 void TemplateTable::arraylength() {
   transition(atos, itos);
+  if (ShenandoahVerifyReadsToFromSpace) {
+    oopDesc::bs()->interpreter_read_barrier(_masm, rax);
+  }
   __ null_check(rax, arrayOopDesc::length_offset_in_bytes());
   __ movl(rax, Address(rax, arrayOopDesc::length_offset_in_bytes()));
 }
@@ -4075,7 +4138,11 @@
   __ pop_ptr(rdx); // restore receiver
   __ verify_oop(rdx);
   __ load_klass(rdx, rdx);
-  __ jmpb(resolved);
+  if (ShenandoahVerifyReadsToFromSpace) {
+    __ jmp(resolved);
+  } else {
+    __ jmpb(resolved);
+  }
 
   // Get superklass in rax and subklass in rdx
   __ bind(quicked);
@@ -4171,6 +4238,11 @@
   // check for NULL object
   __ null_check(rax);
 
+  // We need to preemptively evacuate the object, because we later compare
+  // it to objects in the BasicObjectLock list, and we might get false negatives
+  // if another thread evacuates the object in the meantime. See acmp.
+  oopDesc::bs()->interpreter_write_barrier(_masm, rax);
+
   const Address monitor_block_top(
         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
   const Address monitor_block_bot(
@@ -4193,7 +4265,11 @@
                                         // starting with top-most entry
     __ lea(rbot, monitor_block_bot);    // points to word before bottom
                                         // of monitor block
-    __ jmpb(entry);
+    if (UseShenandoahGC && ShenandoahVerifyReadsToFromSpace) {
+      __ jmp(entry);
+    } else {
+      __ jmpb(entry);
+    }
 
     __ bind(loop);
     // check if current entry is used
@@ -4201,7 +4277,9 @@
     // if not used then remember entry in rmon
     __ cmovptr(Assembler::equal, rmon, rtop);   // cmov => cmovptr
     // check if current entry is for same object
-    __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+    __ movptr(rscratch1, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+    oopDesc::bs()->interpreter_read_barrier(_masm, rscratch1);
+    __ cmpptr(rax, rscratch1);
     // if same object then stop searching
     __ jccb(Assembler::equal, exit);
     // otherwise advance to next entry
@@ -4268,6 +4346,11 @@
   // check for NULL object
   __ null_check(rax);
 
+  // We need to preemptively evacuate the object, because we later compare
+  // it to objects in the BasicObjectLock list, and we might get false negatives
+  // if another thread evacuates the object in the meantime. See acmp.
+  oopDesc::bs()->interpreter_write_barrier(_masm, rax);
+
   const Address monitor_block_top(
         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
   const Address monitor_block_bot(
@@ -4286,11 +4369,17 @@
                                         // starting with top-most entry
     __ lea(rbot, monitor_block_bot);    // points to word before bottom
                                         // of monitor block
-    __ jmpb(entry);
+    if (UseShenandoahGC && ShenandoahVerifyReadsToFromSpace) {
+      __ jmp(entry);
+    } else {
+      __ jmpb(entry);
+    }
 
     __ bind(loop);
     // check if current entry is for same object
-    __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+    __ movptr(rscratch1, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+    oopDesc::bs()->interpreter_read_barrier(_masm, rscratch1);
+    __ cmpptr(rax, rscratch1);
     // if same object then stop searching
     __ jcc(Assembler::equal, found);
     // otherwise advance to next entry
diff --git a/src/cpu/x86/vm/vtableStubs_x86_64.cpp b/src/cpu/x86/vm/vtableStubs_x86_64.cpp
--- a/src/cpu/x86/vm/vtableStubs_x86_64.cpp
+++ b/src/cpu/x86/vm/vtableStubs_x86_64.cpp
@@ -222,11 +222,13 @@
 int VtableStub::pd_code_size_limit(bool is_vtable_stub) {
   if (is_vtable_stub) {
     // Vtable stub size
-    return (DebugVtables ? 512 : 24) + (CountCompiledCalls ? 13 : 0) +
+    return (ShenandoahVerifyReadsToFromSpace ? 512 : 0) +
+           (DebugVtables ? 512 : 24) + (CountCompiledCalls ? 13 : 0) +
            (UseCompressedClassPointers ?  MacroAssembler::instr_size_for_decode_klass_not_null() : 0);
   } else {
     // Itable stub size
-    return (DebugVtables ? 512 : 74) + (CountCompiledCalls ? 13 : 0) +
+    return (ShenandoahVerifyReadsToFromSpace ? 512 : 0) +
+           (DebugVtables ? 512 : 74) + (CountCompiledCalls ? 13 : 0) +
            (UseCompressedClassPointers ?  MacroAssembler::instr_size_for_decode_klass_not_null() : 0);
   }
   // In order to tune these parameters, run the JVM with VM options
diff --git a/src/cpu/x86/vm/x86_64.ad b/src/cpu/x86/vm/x86_64.ad
--- a/src/cpu/x86/vm/x86_64.ad
+++ b/src/cpu/x86/vm/x86_64.ad
@@ -6403,6 +6403,41 @@
   ins_pipe(ialu_reg_reg); // XXX
 %}
 
+instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{
+  match(Set dst (ShenandoahReadBarrier src));
+  effect(DEF dst, USE src);
+  ins_cost(125); // XXX
+  format %{ "shenandoah_rb $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ movptr(d, Address(s, -8));
+  %}
+  ins_pipe(ialu_reg_mem);
+%}
+
+instruct shenandoahWB(rax_RegP dst, rdi_RegP src, rFlagsReg cr) %{
+  match(Set dst (ShenandoahWriteBarrier src));
+  effect(DEF dst, USE_KILL src, KILL cr);
+  ins_cost(300); // XXX
+  format %{ "shenandoah_wb $dst,$src" %}
+  ins_encode %{
+    Label done;
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    assert(s == rdi, "need rdi");
+    assert(d == rax, "result in rax");
+    Address evacuation_in_progress = Address(r15_thread, in_bytes(JavaThread::evacuation_in_progress_offset()));
+    __ movptr(d, Address(s, -8));
+    __ cmpb(evacuation_in_progress, 0);
+    __ movptr(d, Address(s, -8));
+    __ jcc(Assembler::equal, done);
+    __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::shenandoah_wb())));
+    __ bind(done);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // Convert oop pointer into compressed form
 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
diff --git a/src/os_cpu/linux_x86/vm/os_linux_x86.cpp b/src/os_cpu/linux_x86/vm/os_linux_x86.cpp
--- a/src/os_cpu/linux_x86/vm/os_linux_x86.cpp
+++ b/src/os_cpu/linux_x86/vm/os_linux_x86.cpp
@@ -435,6 +435,22 @@
     }
   }
 
+    if ((sig == SIGSEGV) && UseShenandoahGC
+        && (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace)) {
+      if (Universe::heap()->is_in(info->si_addr)) {
+        tty->print("OK, we got the SIGSEGV,on address %p now what?\n", (address) info->si_addr);
+        ucontext_t* uc = (ucontext_t*) ucVoid;
+        address pc = (address) os::Linux::ucontext_get_pc(uc);
+        os::print_context(tty, ucVoid);
+        Universe::heap()->print();
+        if (ShenandoahVerifyReadsToFromSpace) {
+          assert(false, "Illegal read to From Space");
+        } else {
+          assert(false, "Illegal write to From Space");
+        }
+      }
+    }
+
 #ifndef AMD64
   // Execution protection violation
   //
diff --git a/src/share/vm/adlc/formssel.cpp b/src/share/vm/adlc/formssel.cpp
--- a/src/share/vm/adlc/formssel.cpp
+++ b/src/share/vm/adlc/formssel.cpp
@@ -781,7 +781,9 @@
         !strcmp(_matrule->_rChild->_opType,"CreateEx")     ||  // type of exception
         !strcmp(_matrule->_rChild->_opType,"CheckCastPP")  ||
         !strcmp(_matrule->_rChild->_opType,"GetAndSetP")   ||
-        !strcmp(_matrule->_rChild->_opType,"GetAndSetN")) )  return true;
+        !strcmp(_matrule->_rChild->_opType,"GetAndSetN")   ||
+        !strcmp(_matrule->_rChild->_opType,"ShenandoahReadBarrier") ||
+        !strcmp(_matrule->_rChild->_opType,"ShenandoahWriteBarrier")) )  return true;
   else if ( is_ideal_load() == Form::idealP )                return true;
   else if ( is_ideal_store() != Form::none  )                return true;
 
@@ -3487,6 +3489,7 @@
     "ClearArray",
     "GetAndAddI", "GetAndSetI", "GetAndSetP",
     "GetAndAddL", "GetAndSetL", "GetAndSetN",
+    "ShenandoahReadBarrier", "ShenandoahWriteBarrier",
   };
   int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*);
   if( strcmp(_opType,"PrefetchAllocation")==0 )
diff --git a/src/share/vm/adlc/main.cpp b/src/share/vm/adlc/main.cpp
--- a/src/share/vm/adlc/main.cpp
+++ b/src/share/vm/adlc/main.cpp
@@ -237,6 +237,7 @@
   AD.addInclude(AD._HPP_file, "opto/regalloc.hpp");
   AD.addInclude(AD._HPP_file, "opto/subnode.hpp");
   AD.addInclude(AD._HPP_file, "opto/vectornode.hpp");
+  AD.addInclude(AD._HPP_file, "gc/shenandoah/shenandoahBarrierSet.hpp");
   AD.addInclude(AD._CPP_CLONE_file, "precompiled.hpp");
   AD.addInclude(AD._CPP_CLONE_file, "adfiles", get_basename(AD._HPP_file._name));
   AD.addInclude(AD._CPP_EXPAND_file, "precompiled.hpp");
diff --git a/src/share/vm/asm/assembler.cpp b/src/share/vm/asm/assembler.cpp
--- a/src/share/vm/asm/assembler.cpp
+++ b/src/share/vm/asm/assembler.cpp
@@ -312,5 +312,5 @@
     }
   }
 #endif
-  return offset < 0 || os::vm_page_size() <= offset;
+  return (offset < 0 && ((!UseShenandoahGC) || offset != -8)) || os::vm_page_size() <= offset;
 }
diff --git a/src/share/vm/c1/c1_LIR.cpp b/src/share/vm/c1/c1_LIR.cpp
--- a/src/share/vm/c1/c1_LIR.cpp
+++ b/src/share/vm/c1/c1_LIR.cpp
@@ -1010,6 +1010,15 @@
       do_temp(opProfileType->_tmp);
       break;
     }
+    case lir_shenandoah_wb: {
+      assert(op->as_OpShenandoahWriteBarrier() != NULL, "must be");
+      LIR_OpShenandoahWriteBarrier* opShenandoahWB = (LIR_OpShenandoahWriteBarrier*) op;
+      do_input(opShenandoahWB->_opr);
+      do_output(opShenandoahWB->_result);
+      do_temp(opShenandoahWB->_tmp1);
+      do_temp(opShenandoahWB->_tmp2);
+      break;
+    }
   default:
     ShouldNotReachHere();
   }
@@ -1108,6 +1117,10 @@
   }
 }
 
+void LIR_OpShenandoahWriteBarrier::emit_code(LIR_Assembler* masm) {
+  masm->emit_opShenandoahWriteBarrier(this);
+}
+
 void LIR_OpConvert::emit_code(LIR_Assembler* masm) {
   masm->emit_opConvert(this);
   if (stub() != NULL) {
@@ -1816,6 +1829,7 @@
      case lir_profile_call:          s = "profile_call";  break;
      // LIR_OpProfileType
      case lir_profile_type:          s = "profile_type";  break;
+  case lir_shenandoah_wb:            s = "shenandoah_wb"; break;
      // LIR_OpAssert
 #ifdef ASSERT
      case lir_assert:                s = "assert";        break;
@@ -1826,6 +1840,13 @@
   return s;
 }
 
+void LIR_OpShenandoahWriteBarrier::print_instr(outputStream* out) const {
+  out->print("[obj: "); in_opr()->print(out); out->print("]");
+  out->print("[res: "); result_opr()->print(out); out->print("]");
+  out->print("[tmp1: "); tmp1_opr()->print(out); out->print("]");
+  out->print("[tmp2: "); tmp2_opr()->print(out); out->print("]");
+}
+
 // LIR_OpJavaCall
 void LIR_OpJavaCall::print_instr(outputStream* out) const {
   out->print("call: ");
diff --git a/src/share/vm/c1/c1_LIR.hpp b/src/share/vm/c1/c1_LIR.hpp
--- a/src/share/vm/c1/c1_LIR.hpp
+++ b/src/share/vm/c1/c1_LIR.hpp
@@ -876,6 +876,7 @@
 class      LIR_OpConvert;
 class      LIR_OpAllocObj;
 class      LIR_OpRoundFP;
+class      LIR_OpShenandoahWriteBarrier;
 class    LIR_Op2;
 class    LIR_OpDelay;
 class    LIR_Op3;
@@ -940,6 +941,7 @@
       , lir_pack64
       , lir_unpack64
       , lir_unwind
+      , lir_shenandoah_wb
   , end_op1
   , begin_op2
       , lir_cmp
@@ -1154,6 +1156,7 @@
   virtual LIR_OpCompareAndSwap* as_OpCompareAndSwap() { return NULL; }
   virtual LIR_OpProfileCall* as_OpProfileCall() { return NULL; }
   virtual LIR_OpProfileType* as_OpProfileType() { return NULL; }
+  virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return NULL; }
 #ifdef ASSERT
   virtual LIR_OpAssert* as_OpAssert() { return NULL; }
 #endif
@@ -1468,6 +1471,25 @@
   virtual void print_instr(outputStream* out) const PRODUCT_RETURN;
 };
 
+class LIR_OpShenandoahWriteBarrier : public LIR_Op1 {
+ friend class LIR_OpVisitState;
+
+private:
+  bool _need_null_check;
+  LIR_Opr _tmp1;
+  LIR_Opr _tmp2;
+
+public:
+  LIR_OpShenandoahWriteBarrier(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, CodeEmitInfo* info, bool need_null_check) : LIR_Op1(lir_shenandoah_wb, obj, result, T_OBJECT, lir_patch_none, info), _tmp1(tmp1), _tmp2(tmp2), _need_null_check(need_null_check) {
+  }
+  LIR_Opr tmp1_opr() const { return _tmp1; }
+  LIR_Opr tmp2_opr() const { return _tmp2; }
+  bool need_null_check() const { return _need_null_check; }
+  virtual void emit_code(LIR_Assembler* masm);
+  virtual LIR_OpShenandoahWriteBarrier* as_OpShenandoahWriteBarrier() { return this; }
+  virtual void print_instr(outputStream* out) const PRODUCT_RETURN;
+
+};
 
 class ConversionStub;
 
@@ -2149,6 +2171,8 @@
 #endif
   void convert(Bytecodes::Code code, LIR_Opr left, LIR_Opr dst, ConversionStub* stub = NULL/*, bool is_32bit = false*/) { append(new LIR_OpConvert(code, left, dst, stub)); }
 
+  void shenandoah_wb(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, CodeEmitInfo* info, bool need_null_check) { append(new LIR_OpShenandoahWriteBarrier(obj, result, tmp1, tmp2, info, need_null_check)); }
+
   void logical_and (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_and,  left, right, dst)); }
   void logical_or  (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_or,   left, right, dst)); }
   void logical_xor (LIR_Opr left, LIR_Opr right, LIR_Opr dst) { append(new LIR_Op2(lir_logic_xor,  left, right, dst)); }
diff --git a/src/share/vm/c1/c1_LIRAssembler.hpp b/src/share/vm/c1/c1_LIRAssembler.hpp
--- a/src/share/vm/c1/c1_LIRAssembler.hpp
+++ b/src/share/vm/c1/c1_LIRAssembler.hpp
@@ -196,6 +196,7 @@
   void emit_opLabel(LIR_OpLabel* op);
   void emit_arraycopy(LIR_OpArrayCopy* op);
   void emit_updatecrc32(LIR_OpUpdateCRC32* op);
+  void emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op);
   void emit_opConvert(LIR_OpConvert* op);
   void emit_alloc_obj(LIR_OpAllocObj* op);
   void emit_alloc_array(LIR_OpAllocArray* op);
diff --git a/src/share/vm/c1/c1_LIRGenerator.cpp b/src/share/vm/c1/c1_LIRGenerator.cpp
--- a/src/share/vm/c1/c1_LIRGenerator.cpp
+++ b/src/share/vm/c1/c1_LIRGenerator.cpp
@@ -247,14 +247,22 @@
 void LIRItem::load_item_force(LIR_Opr reg) {
   LIR_Opr r = result();
   if (r != reg) {
+    _result = _gen->force_opr_to(r, reg);
+  }
+}
+
+LIR_Opr LIRGenerator::force_opr_to(LIR_Opr op, LIR_Opr reg) {
+  if (op != reg) {
 #if !defined(ARM) && !defined(E500V2)
-    if (r->type() != reg->type()) {
+    if (op->type() != reg->type()) {
       // moves between different types need an intervening spill slot
-      r = _gen->force_to_spill(r, reg->type());
+      op = force_to_spill(op, reg->type());
     }
 #endif
-    __ move(r, reg);
-    _result = reg;
+    __ move(op, reg);
+    return reg;
+  } else {
+    return op;
   }
 }
 
@@ -1422,6 +1430,7 @@
   switch (_bs->kind()) {
 #if INCLUDE_ALL_GCS
     case BarrierSet::G1SATBCTLogging:
+    case BarrierSet::ShenandoahBarrierSet:
       G1SATBCardTableModRef_pre_barrier(addr_opr, pre_val, do_load, patch, info);
       break;
 #endif // INCLUDE_ALL_GCS
@@ -1444,6 +1453,8 @@
     case BarrierSet::G1SATBCTLogging:
       G1SATBCardTableModRef_post_barrier(addr,  new_val);
       break;
+    case BarrierSet::ShenandoahBarrierSet:
+      break;
 #endif // INCLUDE_ALL_GCS
     case BarrierSet::CardTableForRS:
     case BarrierSet::CardTableExtension:
@@ -1714,11 +1725,23 @@
   }
 #endif
 
+  LIR_Opr obj = object.result();
+
   if (x->needs_null_check() &&
       (needs_patching ||
        MacroAssembler::needs_explicit_null_check(x->offset()))) {
     // emit an explicit null check because the offset is too large
-    __ null_check(object.result(), new CodeEmitInfo(info));
+    __ null_check(obj, new CodeEmitInfo(info));
+  }
+
+  obj = shenandoah_write_barrier(obj, info, x->needs_null_check());
+  LIR_Opr val = value.result();
+  if (is_oop && UseShenandoahGC) {
+    if (! val->is_register()) {
+      assert(val->is_constant(), "expect constant");
+    } else {
+      val = shenandoah_read_barrier(val, NULL, true);
+    }
   }
 
   LIR_Address* address;
@@ -1727,9 +1750,9 @@
     // generate_address to try to be smart about emitting the -1.
     // Otherwise the patching code won't know how to find the
     // instruction to patch.
-    address = new LIR_Address(object.result(), PATCHED_ADDR, field_type);
+    address = new LIR_Address(obj, PATCHED_ADDR, field_type);
   } else {
-    address = generate_address(object.result(), x->offset(), field_type);
+    address = generate_address(obj, x->offset(), field_type);
   }
 
   if (is_volatile && os::is_MP()) {
@@ -1747,15 +1770,15 @@
 
   bool needs_atomic_access = is_volatile || AlwaysAtomicAccesses;
   if (needs_atomic_access && !needs_patching) {
-    volatile_field_store(value.result(), address, info);
+    volatile_field_store(val, address, info);
   } else {
     LIR_PatchCode patch_code = needs_patching ? lir_patch_normal : lir_patch_none;
-    __ store(value.result(), address, info, patch_code);
+    __ store(val, address, info, patch_code);
   }
 
   if (is_oop) {
     // Store to object so mark the card of the header
-    post_barrier(object.result(), value.result());
+    post_barrier(obj, val);
   }
 
   if (is_volatile && os::is_MP()) {
@@ -1793,12 +1816,12 @@
   }
 #endif
 
+  LIR_Opr obj = object.result();
   bool stress_deopt = StressLoopInvariantCodeMotion && info && info->deoptimize_on_exception();
   if (x->needs_null_check() &&
       (needs_patching ||
        MacroAssembler::needs_explicit_null_check(x->offset()) ||
        stress_deopt)) {
-    LIR_Opr obj = object.result();
     if (stress_deopt) {
       obj = new_register(T_OBJECT);
       __ move(LIR_OprFact::oopConst(NULL), obj);
@@ -1807,6 +1830,7 @@
     __ null_check(obj, new CodeEmitInfo(info));
   }
 
+  obj = shenandoah_read_barrier(obj, info, x->needs_null_check() && x->explicit_null_check() != NULL);
   LIR_Opr reg = rlock_result(x, field_type);
   LIR_Address* address;
   if (needs_patching) {
@@ -1814,9 +1838,9 @@
     // generate_address to try to be smart about emitting the -1.
     // Otherwise the patching code won't know how to find the
     // instruction to patch.
-    address = new LIR_Address(object.result(), PATCHED_ADDR, field_type);
+    address = new LIR_Address(obj, PATCHED_ADDR, field_type);
   } else {
-    address = generate_address(object.result(), x->offset(), field_type);
+    address = generate_address(obj, x->offset(), field_type);
   }
 
   bool needs_atomic_access = is_volatile || AlwaysAtomicAccesses;
@@ -1832,6 +1856,39 @@
   }
 }
 
+LIR_Opr LIRGenerator::shenandoah_read_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+  if (UseShenandoahGC) {
+
+    LabelObj* done = new LabelObj();
+    LIR_Opr result = new_register(T_OBJECT);
+    __ move(obj, result);
+    if (need_null_check) {
+      __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL));
+      __ branch(lir_cond_equal, T_LONG, done->label());
+    }
+    LIR_Address* brooks_ptr_address = generate_address(result, -8, T_ADDRESS);
+    __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none);
+
+    __ branch_destination(done->label());
+    return result;
+  } else {
+    return obj;
+  }
+}
+
+LIR_Opr LIRGenerator::shenandoah_write_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
+  if (UseShenandoahGC) {
+
+    LIR_Opr result = new_register(T_OBJECT);
+    LIR_Opr tmp1 = new_register(T_INT);
+    LIR_Opr tmp2 = new_register(T_INT);
+    __ shenandoah_wb(obj, result, tmp1, tmp2, info ? new CodeEmitInfo(info) : NULL, need_null_check);
+    return result;
+
+  } else {
+    return obj;
+  }
+}
 
 //------------------------java.nio.Buffer.checkIndex------------------------
 
@@ -1928,8 +1985,11 @@
     }
   }
 
+  LIR_Opr ary = array.result();
+  ary = shenandoah_read_barrier(ary, null_check_info, null_check_info != NULL);
+
   // emit array address setup early so it schedules better
-  LIR_Address* array_addr = emit_array_address(array.result(), index.result(), x->elt_type(), false);
+  LIR_Address* array_addr = emit_array_address(ary, index.result(), x->elt_type(), false);
 
   if (GenerateRangeChecks && needs_range_check) {
     if (StressLoopInvariantCodeMotion && range_check_info->deoptimize_on_exception()) {
@@ -1940,7 +2000,7 @@
       __ cmp(lir_cond_belowEqual, length.result(), index.result());
       __ branch(lir_cond_belowEqual, T_INT, new RangeCheckStub(range_check_info, index.result()));
     } else {
-      array_range_check(array.result(), index.result(), null_check_info, range_check_info);
+      array_range_check(ary, index.result(), null_check_info, range_check_info);
       // The range check performs the null check, so clear it out for the load
       null_check_info = NULL;
     }
@@ -2253,7 +2313,7 @@
   //   }
   // }
 
-  if (UseG1GC && type == T_OBJECT) {
+  if ((UseShenandoahGC || UseG1GC) && type == T_OBJECT) {
     bool gen_pre_barrier = true;     // Assume we need to generate pre_barrier.
     bool gen_offset_check = true;    // Assume we need to generate the offset guard.
     bool gen_source_check = true;    // Assume we need to check the src object for null.
@@ -2795,6 +2855,7 @@
       __ load_stack_address_monitor(0, lock);
 
       CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, x->check_flag(Instruction::DeoptimizeOnException));
+      obj = shenandoah_write_barrier(obj, info, false);
       CodeStub* slow_path = new MonitorEnterStub(obj, lock, info);
 
       // receiver is guaranteed non-NULL so don't need CodeEmitInfo
@@ -3015,9 +3076,9 @@
 
 // Code for  :  x->x() {x->cond()} x->y() ? x->tval() : x->fval()
 void LIRGenerator::do_IfOp(IfOp* x) {
+  ValueTag xtag = x->x()->type()->tag();
 #ifdef ASSERT
   {
-    ValueTag xtag = x->x()->type()->tag();
     ValueTag ttag = x->tval()->type()->tag();
     assert(xtag == intTag || xtag == objectTag, "cannot handle others");
     assert(ttag == addressTag || ttag == intTag || ttag == objectTag || ttag == longTag, "cannot handle others");
@@ -3040,7 +3101,14 @@
   f_val.dont_load_item();
   LIR_Opr reg = rlock_result(x);
 
-  __ cmp(lir_cond(x->cond()), left.result(), right.result());
+  LIR_Opr left_opr = left.result();
+  LIR_Opr right_opr = right.result();
+  if (xtag == objectTag && UseShenandoahGC && x->y()->type() != objectNull) { // Don't need to resolve for ifnull.
+    left_opr = shenandoah_write_barrier(left_opr, NULL, true);
+    right_opr = shenandoah_read_barrier(right_opr, NULL, true);
+  }
+
+  __ cmp(lir_cond(x->cond()), left_opr, right_opr);
   __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
 }
 
diff --git a/src/share/vm/c1/c1_LIRGenerator.hpp b/src/share/vm/c1/c1_LIRGenerator.hpp
--- a/src/share/vm/c1/c1_LIRGenerator.hpp
+++ b/src/share/vm/c1/c1_LIRGenerator.hpp
@@ -226,6 +226,8 @@
   LIR_Opr round_item(LIR_Opr opr);
   LIR_Opr force_to_spill(LIR_Opr value, BasicType t);
 
+  LIR_Opr force_opr_to(LIR_Opr op, LIR_Opr reg);
+
   PhiResolverState& resolver_state() { return _resolver_state; }
 
   void  move_to_phi(PhiResolver* resolver, Value cur_val, Value sux_val);
@@ -265,6 +267,9 @@
   void pre_barrier(LIR_Opr addr_opr, LIR_Opr pre_val, bool do_load, bool patch, CodeEmitInfo* info);
   void post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
 
+  LIR_Opr shenandoah_read_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+  LIR_Opr shenandoah_write_barrier(LIR_Opr obj, CodeEmitInfo* info, bool need_null_check);
+
   // specific implementations
   // pre barriers
 
diff --git a/src/share/vm/c1/c1_Runtime1.cpp b/src/share/vm/c1/c1_Runtime1.cpp
--- a/src/share/vm/c1/c1_Runtime1.cpp
+++ b/src/share/vm/c1/c1_Runtime1.cpp
@@ -60,6 +60,7 @@
 #include "runtime/vm_version.hpp"
 #include "utilities/copy.hpp"
 #include "utilities/events.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
 
 
 // Implementation of StubAssembler
@@ -204,6 +205,7 @@
     switch (id) {
       // These stubs don't need to have an oopmap
     case dtrace_object_alloc_id:
+    case shenandoah_write_barrier_slow_id:
     case g1_pre_barrier_slow_id:
     case g1_post_barrier_slow_id:
     case slow_subtype_check_id:
@@ -317,6 +319,7 @@
   FUNCTION_CASE(entry, TRACE_TIME_METHOD);
 #endif
   FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32());
+  FUNCTION_CASE(entry, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1);
 
 #undef FUNCTION_CASE
 
@@ -671,7 +674,7 @@
   if (PrintBiasedLockingStatistics) {
     Atomic::inc(BiasedLocking::slow_path_entry_count_addr());
   }
-  Handle h_obj(thread, obj);
+Handle h_obj(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(obj));
   assert(h_obj()->is_oop(), "must be NULL or an object");
   if (UseBiasedLocking) {
     // Retry fast entry if bias is revoked to avoid unnecessary inflation
@@ -696,7 +699,7 @@
   // monitorexit is non-blocking (leaf routine) => no exceptions can be thrown
   EXCEPTION_MARK;
 
-  oop obj = lock->obj();
+  oop obj = oopDesc::bs()->resolve_and_maybe_copy_oop(lock->obj());
   assert(obj->is_oop(), "must be NULL or an object");
   if (UseFastLocking) {
     // When using fast locking, the compiled code has already tried the fast case
@@ -1406,6 +1409,10 @@
   if ((unsigned int) arrayOop(dst)->length() < (unsigned int)dst_pos + (unsigned int)length) return ac_failed;
 
   if (length == 0) return ac_ok;
+
+  oopDesc::bs()->resolve_oop(src);
+  oopDesc::bs()->resolve_and_maybe_copy_oop(dst);
+
   if (src->is_typeArray()) {
     Klass* klass_oop = src->klass();
     if (klass_oop != dst->klass()) return ac_failed;
diff --git a/src/share/vm/c1/c1_Runtime1.hpp b/src/share/vm/c1/c1_Runtime1.hpp
--- a/src/share/vm/c1/c1_Runtime1.hpp
+++ b/src/share/vm/c1/c1_Runtime1.hpp
@@ -68,6 +68,7 @@
   stub(load_klass_patching)          \
   stub(load_mirror_patching)         \
   stub(load_appendix_patching)       \
+  stub(shenandoah_write_barrier_slow)          \
   stub(g1_pre_barrier_slow)          \
   stub(g1_post_barrier_slow)         \
   stub(fpu2long_stub)                \
diff --git a/src/share/vm/ci/ciInstanceKlass.cpp b/src/share/vm/ci/ciInstanceKlass.cpp
--- a/src/share/vm/ci/ciInstanceKlass.cpp
+++ b/src/share/vm/ci/ciInstanceKlass.cpp
@@ -179,12 +179,12 @@
 //
 ciInstanceKlass* ciInstanceKlass::get_canonical_holder(int offset) {
   #ifdef ASSERT
-  if (!(offset >= 0 && offset < layout_helper())) {
+  if (!(offset >= 0 && offset < layout_helper() || (offset == -8 && UseShenandoahGC))) {
     tty->print("*** get_canonical_holder(%d) on ", offset);
     this->print();
     tty->print_cr(" ***");
   };
-  assert(offset >= 0 && offset < layout_helper(), "offset must be tame");
+  assert(offset >= 0 && offset < layout_helper() || (offset == -8 && UseShenandoahGC), "offset must be tame");
   #endif
 
   if (offset < instanceOopDesc::base_offset_in_bytes()) {
diff --git a/src/share/vm/ci/ciObject.cpp b/src/share/vm/ci/ciObject.cpp
--- a/src/share/vm/ci/ciObject.cpp
+++ b/src/share/vm/ci/ciObject.cpp
@@ -52,9 +52,9 @@
 ciObject::ciObject(oop o) {
   ASSERT_IN_VM;
   if (ciObjectFactory::is_initialized()) {
-    _handle = JNIHandles::make_local(o);
+    _handle = JNIHandles::make_local(oopDesc::bs()->resolve_and_maybe_copy_oop(o));
   } else {
-    _handle = JNIHandles::make_global(o);
+    _handle = JNIHandles::make_global(oopDesc::bs()->resolve_and_maybe_copy_oop(o));
   }
   _klass = NULL;
   init_flags_from(o);
diff --git a/src/share/vm/ci/ciObject.hpp b/src/share/vm/ci/ciObject.hpp
--- a/src/share/vm/ci/ciObject.hpp
+++ b/src/share/vm/ci/ciObject.hpp
@@ -27,6 +27,7 @@
 
 #include "ci/ciBaseObject.hpp"
 #include "ci/ciClassList.hpp"
+#include "gc/shared/barrierSet.hpp"
 #include "memory/allocation.hpp"
 #include "runtime/handles.hpp"
 #include "runtime/jniHandles.hpp"
@@ -69,7 +70,9 @@
   // Get the VM oop that this object holds.
   oop get_oop() const {
     assert(_handle != NULL, "null oop");
-    return JNIHandles::resolve_non_null(_handle);
+    oop obj = JNIHandles::resolve_non_null(_handle);
+    assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
+    return obj;
   }
 
   void init_flags_from(oop x);
diff --git a/src/share/vm/ci/ciObjectFactory.cpp b/src/share/vm/ci/ciObjectFactory.cpp
--- a/src/share/vm/ci/ciObjectFactory.cpp
+++ b/src/share/vm/ci/ciObjectFactory.cpp
@@ -242,6 +242,10 @@
 
   assert(Universe::heap()->is_in_reserved(key), "must be");
 
+  // In Shenandoah we need to make sure that nobody forwards the key elsewhere
+  // under our hood.
+  key = oopDesc::bs()->resolve_and_maybe_copy_oop(key);
+
   NonPermObject* &bucket = find_non_perm(key);
   if (bucket != NULL) {
     return bucket->object();
diff --git a/src/share/vm/ci/ciObjectFactory.hpp b/src/share/vm/ci/ciObjectFactory.hpp
--- a/src/share/vm/ci/ciObjectFactory.hpp
+++ b/src/share/vm/ci/ciObjectFactory.hpp
@@ -27,6 +27,7 @@
 
 #include "ci/ciClassList.hpp"
 #include "ci/ciObject.hpp"
+#include "gc/shared/barrierSet.hpp"
 #include "utilities/growableArray.hpp"
 
 // ciObjectFactory
@@ -78,6 +79,11 @@
   void ensure_metadata_alive(ciMetadata* m);
 
   static bool is_equal(NonPermObject* p, oop key) {
+    // Shenandoah: We already force-forwarded the key earlier.
+    // The oop in the object should always point to to-space.
+    assert(key == oopDesc::bs()->resolve_oop(key), "should be forwarded");
+    assert(p->object()->get_oop() == oopDesc::bs()->resolve_oop(p->object()->get_oop()),
+           "should be forwarded");
     return p->object()->get_oop() == key;
   }
 
diff --git a/src/share/vm/classfile/classLoaderData.cpp b/src/share/vm/classfile/classLoaderData.cpp
--- a/src/share/vm/classfile/classLoaderData.cpp
+++ b/src/share/vm/classfile/classLoaderData.cpp
@@ -242,7 +242,7 @@
   // Have to lock and put the new dependency on the end of the dependency
   // array so the card mark for CMS sees that this dependency is new.
   // Can probably do this lock free with some effort.
-  ObjectLocker ol(Handle(THREAD, _list_head), THREAD);
+  ObjectLocker ol(Handle(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(_list_head)), THREAD);
 
   oop loader_or_mirror = new_dependency->obj_at(0);
 
diff --git a/src/share/vm/classfile/dictionary.cpp b/src/share/vm/classfile/dictionary.cpp
--- a/src/share/vm/classfile/dictionary.cpp
+++ b/src/share/vm/classfile/dictionary.cpp
@@ -80,13 +80,14 @@
 
 bool DictionaryEntry::contains_protection_domain(oop protection_domain) const {
 #ifdef ASSERT
-  if (protection_domain == InstanceKlass::cast(klass())->protection_domain()) {
+  protection_domain = oopDesc::bs()->resolve_and_maybe_copy_oop(protection_domain);
+  if (protection_domain == oopDesc::bs()->resolve_and_maybe_copy_oop(InstanceKlass::cast(klass())->protection_domain())) {
     // Ensure this doesn't show up in the pd_set (invariant)
     bool in_pd_set = false;
     for (ProtectionDomainEntry* current = _pd_set;
                                 current != NULL;
                                 current = current->next()) {
-      if (current->protection_domain() == protection_domain) {
+      if (oopDesc::bs()->resolve_and_maybe_copy_oop(current->protection_domain()) == protection_domain) {
         in_pd_set = true;
         break;
       }
@@ -98,7 +99,7 @@
   }
 #endif /* ASSERT */
 
-  if (protection_domain == InstanceKlass::cast(klass())->protection_domain()) {
+  if (protection_domain == oopDesc::bs()->resolve_and_maybe_copy_oop(InstanceKlass::cast(klass())->protection_domain())) {
     // Succeeds trivially
     return true;
   }
@@ -106,7 +107,7 @@
   for (ProtectionDomainEntry* current = _pd_set;
                               current != NULL;
                               current = current->next()) {
-    if (current->protection_domain() == protection_domain) return true;
+    if (oopDesc::bs()->resolve_and_maybe_copy_oop(current->protection_domain()) == protection_domain) return true;
   }
   return false;
 }
diff --git a/src/share/vm/classfile/javaClasses.cpp b/src/share/vm/classfile/javaClasses.cpp
--- a/src/share/vm/classfile/javaClasses.cpp
+++ b/src/share/vm/classfile/javaClasses.cpp
@@ -212,7 +212,9 @@
   int length = UTF8::unicode_length(utf8_str);
   Handle h_obj = basic_create(length, CHECK_NH);
   if (length > 0) {
-    UTF8::convert_to_unicode(utf8_str, value(h_obj())->char_at_addr(0), length);
+    typeArrayOop buffer = value(h_obj());
+    buffer = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(buffer));
+    UTF8::convert_to_unicode(utf8_str, buffer->char_at_addr(0), length);
   }
   return h_obj;
 }
@@ -226,7 +228,9 @@
   int length = UTF8::unicode_length((char*)symbol->bytes(), symbol->utf8_length());
   Handle h_obj = basic_create(length, CHECK_NH);
   if (length > 0) {
-    UTF8::convert_to_unicode((char*)symbol->bytes(), value(h_obj())->char_at_addr(0), length);
+    typeArrayOop buffer = value(h_obj());
+    buffer = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(buffer));
+    UTF8::convert_to_unicode((char*)symbol->bytes(), buffer->char_at_addr(0), length);
   }
   return h_obj;
 }
@@ -347,6 +351,7 @@
 
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   return java_lang_String::hash_code(value->char_at_addr(offset), length);
 }
 
@@ -355,6 +360,7 @@
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
 
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* base = (length == 0) ? NULL : value->char_at_addr(offset);
   if (base == NULL) return NULL;
 
@@ -375,6 +381,7 @@
 
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   return StringTable::hash_string(value->char_at_addr(offset), length);
 }
 
@@ -383,6 +390,7 @@
   typeArrayOop value  = java_lang_String::value(obj);
   int          offset = java_lang_String::offset(obj);
   int          length = java_lang_String::length(obj);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* base = (length == 0) ? NULL : value->char_at_addr(offset);
   Symbol* sym = SymbolTable::lookup_unicode(base, length, THREAD);
   return sym;
@@ -392,6 +400,7 @@
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* base = (length == 0) ? NULL : value->char_at_addr(offset);
   return SymbolTable::probe_unicode(base, length);
 }
@@ -401,6 +410,7 @@
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* position = (length == 0) ? NULL : value->char_at_addr(offset);
   return UNICODE::utf8_length(position, length);
 }
@@ -409,6 +419,7 @@
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* position = (length == 0) ? NULL : value->char_at_addr(offset);
   return UNICODE::as_utf8(position, length);
 }
@@ -417,6 +428,7 @@
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* position = (length == 0) ? NULL : value->char_at_addr(offset);
   return UNICODE::as_utf8(position, length, buf, buflen);
 }
@@ -426,6 +438,7 @@
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
   assert(start + len <= length, "just checking");
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* position = value->char_at_addr(offset + start);
   return UNICODE::as_utf8(position, len);
 }
@@ -435,6 +448,7 @@
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
   assert(start + len <= length, "just checking");
+  value = typeArrayOop(oopDesc::bs()->resolve_oop(value));
   jchar* position = value->char_at_addr(offset + start);
   return UNICODE::as_utf8(position, len, buf, buflen);
 }
@@ -443,6 +457,9 @@
   assert(java_string->klass() == SystemDictionary::String_klass(),
          "must be java_string");
   typeArrayOop value  = java_lang_String::value(java_string);
+  if (ShenandoahVerifyReadsToFromSpace) {
+    value = (typeArrayOop) oopDesc::bs()->resolve_oop(value);
+  }
   int          offset = java_lang_String::offset(java_string);
   int          length = java_lang_String::length(java_string);
   if (length != len) {
@@ -863,9 +880,9 @@
     // Note: create_basic_type_mirror above initializes ak to a non-null value.
     type = ArrayKlass::cast(ak)->element_type();
   } else {
-    assert(java_class == Universe::void_mirror(), "only valid non-array primitive");
+    assert(oopDesc::bs()->resolve_and_maybe_copy_oop(java_class) == oopDesc::bs()->resolve_and_maybe_copy_oop(Universe::void_mirror()), "only valid non-array primitive");
   }
-  assert(Universe::java_mirror(type) == java_class, "must be consistent");
+  assert(oopDesc::bs()->resolve_and_maybe_copy_oop(Universe::java_mirror(type)) == oopDesc::bs()->resolve_and_maybe_copy_oop(java_class), "must be consistent");
   return type;
 }
 
@@ -3087,6 +3104,7 @@
 
 ClassLoaderData** java_lang_ClassLoader::loader_data_addr(oop loader) {
     assert(loader != NULL && loader->is_oop(), "loader must be oop");
+    loader = oopDesc::bs()->resolve_and_maybe_copy_oop(loader);
     return (ClassLoaderData**) loader->address_field_addr(_loader_data_offset);
 }
 
@@ -3107,6 +3125,9 @@
 }
 
 oop java_lang_ClassLoader::parent(oop loader) {
+  if (ShenandoahVerifyReadsToFromSpace) {
+    loader = oopDesc::bs()->resolve_oop(loader);
+  }
   assert(is_instance(loader), "loader must be oop");
   return loader->obj_field(parent_offset);
 }
diff --git a/src/share/vm/classfile/javaClasses.hpp b/src/share/vm/classfile/javaClasses.hpp
--- a/src/share/vm/classfile/javaClasses.hpp
+++ b/src/share/vm/classfile/javaClasses.hpp
@@ -157,7 +157,11 @@
     if (count_offset > 0) {
       return java_string->int_field(count_offset);
     } else {
-      return ((typeArrayOop)java_string->obj_field(value_offset))->length();
+      oop value = java_string->obj_field(value_offset);
+      if (ShenandoahVerifyReadsToFromSpace) {
+        value = oopDesc::bs()->resolve_oop(value);
+      }
+      return ((typeArrayOop) value)->length();
     }
   }
   static int utf8_length(oop java_string);
diff --git a/src/share/vm/classfile/systemDictionary.cpp b/src/share/vm/classfile/systemDictionary.cpp
--- a/src/share/vm/classfile/systemDictionary.cpp
+++ b/src/share/vm/classfile/systemDictionary.cpp
@@ -498,7 +498,7 @@
   bool calledholdinglock
       = ObjectSynchronizer::current_thread_holds_lock((JavaThread*)THREAD, lockObject);
   assert(calledholdinglock,"must hold lock for notify");
-  assert((!(lockObject() == _system_loader_lock_obj) && !is_parallelCapable(lockObject)), "unexpected double_lock_wait");
+  assert((!(oopDesc::bs()->resolve_and_maybe_copy_oop(lockObject()) == oopDesc::bs()->resolve_and_maybe_copy_oop(_system_loader_lock_obj)) && !is_parallelCapable(lockObject)), "unexpected double_lock_wait");
   ObjectSynchronizer::notifyall(lockObject, THREAD);
   intptr_t recursions =  ObjectSynchronizer::complete_exit(lockObject, THREAD);
   SystemDictionary_lock->wait();
@@ -1531,9 +1531,9 @@
 Handle SystemDictionary::compute_loader_lock_object(Handle class_loader, TRAPS) {
   // If class_loader is NULL we synchronize on _system_loader_lock_obj
   if (class_loader.is_null()) {
-    return Handle(THREAD, _system_loader_lock_obj);
+    return Handle(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(_system_loader_lock_obj));
   } else {
-    return class_loader;
+    return Handle(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(class_loader()));
   }
 }
 
@@ -1552,7 +1552,7 @@
       == ObjectSynchronizer::owner_other) {
     // contention will likely happen, so increment the corresponding
     // contention counter.
-    if (loader_lock() == _system_loader_lock_obj) {
+    if (oopDesc::bs()->resolve_and_maybe_copy_oop(loader_lock()) == oopDesc::bs()->resolve_and_maybe_copy_oop(_system_loader_lock_obj)) {
       ClassLoader::sync_systemLoaderLockContentionRate()->inc();
     } else {
       ClassLoader::sync_nonSystemLoaderLockContentionRate()->inc();
diff --git a/src/share/vm/code/nmethod.cpp b/src/share/vm/code/nmethod.cpp
--- a/src/share/vm/code/nmethod.cpp
+++ b/src/share/vm/code/nmethod.cpp
@@ -900,7 +900,9 @@
       handle == (jobject) Universe::non_oop_word()) {
     (*dest) = (oop) handle;
   } else {
-    (*dest) = JNIHandles::resolve_non_null(handle);
+    oop obj = JNIHandles::resolve_non_null(handle);
+    assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
+    (*dest) = obj;
   }
 }
 
diff --git a/src/share/vm/gc/g1/concurrentMark.cpp b/src/share/vm/gc/g1/concurrentMark.cpp
--- a/src/share/vm/gc/g1/concurrentMark.cpp
+++ b/src/share/vm/gc/g1/concurrentMark.cpp
@@ -58,81 +58,9 @@
 #include "runtime/prefetch.inline.hpp"
 #include "services/memTracker.hpp"
 
-// Concurrent marking bit map wrapper
-
-CMBitMapRO::CMBitMapRO(int shifter) :
-  _bm(),
-  _shifter(shifter) {
-  _bmStartWord = 0;
-  _bmWordSize = 0;
-}
-
-HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
-                                               const HeapWord* limit) const {
-  // First we must round addr *up* to a possible object boundary.
-  addr = (HeapWord*)align_size_up((intptr_t)addr,
-                                  HeapWordSize << _shifter);
-  size_t addrOffset = heapWordToOffset(addr);
-  if (limit == NULL) {
-    limit = _bmStartWord + _bmWordSize;
-  }
-  size_t limitOffset = heapWordToOffset(limit);
-  size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
-  HeapWord* nextAddr = offsetToHeapWord(nextOffset);
-  assert(nextAddr >= addr, "get_next_one postcondition");
-  assert(nextAddr == limit || isMarked(nextAddr),
-         "get_next_one postcondition");
-  return nextAddr;
-}
-
-HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr,
-                                                 const HeapWord* limit) const {
-  size_t addrOffset = heapWordToOffset(addr);
-  if (limit == NULL) {
-    limit = _bmStartWord + _bmWordSize;
-  }
-  size_t limitOffset = heapWordToOffset(limit);
-  size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
-  HeapWord* nextAddr = offsetToHeapWord(nextOffset);
-  assert(nextAddr >= addr, "get_next_one postcondition");
-  assert(nextAddr == limit || !isMarked(nextAddr),
-         "get_next_one postcondition");
-  return nextAddr;
-}
-
-int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
-  assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
-  return (int) (diff >> _shifter);
-}
-
-#ifndef PRODUCT
-bool CMBitMapRO::covers(MemRegion heap_rs) const {
-  // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
-  assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize,
-         "size inconsistency");
-  return _bmStartWord == (HeapWord*)(heap_rs.start()) &&
-         _bmWordSize  == heap_rs.word_size();
-}
-#endif
-
-void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const {
-  _bm.print_on_error(st, prefix);
-}
-
-size_t CMBitMap::compute_size(size_t heap_size) {
-  return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
-}
-
-size_t CMBitMap::mark_distance() {
-  return MinObjAlignmentInBytes * BitsPerByte;
-}
-
-void CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
-  _bmStartWord = heap.start();
-  _bmWordSize = heap.word_size();
-
-  _bm.set_map((BitMap::bm_word_t*) storage->reserved().start());
-  _bm.set_size(_bmWordSize >> _shifter);
+void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
+
+  CMBitMap::initialize(heap, storage->reserved());
 
   storage->set_mapping_changed_listener(&_listener);
 }
@@ -200,7 +128,7 @@
   }
 };
 
-void CMBitMap::clearAll() {
+void G1CMBitMap::clearAll() {
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
   ClearBitmapHRClosure cl(NULL, this, false /* may_yield */);
   uint n_workers = g1h->workers()->active_workers();
@@ -210,39 +138,6 @@
   return;
 }
 
-void CMBitMap::markRange(MemRegion mr) {
-  mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
-  assert(!mr.is_empty(), "unexpected empty region");
-  assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
-          ((HeapWord *) mr.end())),
-         "markRange memory region end is not card aligned");
-  // convert address range into offset range
-  _bm.at_put_range(heapWordToOffset(mr.start()),
-                   heapWordToOffset(mr.end()), true);
-}
-
-void CMBitMap::clearRange(MemRegion mr) {
-  mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
-  assert(!mr.is_empty(), "unexpected empty region");
-  // convert address range into offset range
-  _bm.at_put_range(heapWordToOffset(mr.start()),
-                   heapWordToOffset(mr.end()), false);
-}
-
-MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
-                                            HeapWord* end_addr) {
-  HeapWord* start = getNextMarkedWordAddress(addr);
-  start = MIN2(start, end_addr);
-  HeapWord* end   = getNextUnmarkedWordAddress(start);
-  end = MIN2(end, end_addr);
-  assert(start <= end, "Consistency check");
-  MemRegion mr(start, end);
-  if (!mr.is_empty()) {
-    clearRange(mr);
-  }
-  return mr;
-}
-
 CMMarkStack::CMMarkStack(ConcurrentMark* cm) :
   _base(NULL), _cm(cm)
 #ifdef ASSERT
diff --git a/src/share/vm/gc/g1/concurrentMark.hpp b/src/share/vm/gc/g1/concurrentMark.hpp
--- a/src/share/vm/gc/g1/concurrentMark.hpp
+++ b/src/share/vm/gc/g1/concurrentMark.hpp
@@ -28,6 +28,7 @@
 #include "classfile/javaClasses.hpp"
 #include "gc/g1/g1RegionToSpaceMapper.hpp"
 #include "gc/g1/heapRegionSet.hpp"
+#include "gc/shared/cmBitMap.inline.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/taskqueue.hpp"
 
@@ -52,74 +53,6 @@
   bool do_object_b(oop obj);
 };
 
-// A generic CM bit map.  This is essentially a wrapper around the BitMap
-// class, with one bit per (1<<_shifter) HeapWords.
-
-class CMBitMapRO VALUE_OBJ_CLASS_SPEC {
- protected:
-  HeapWord* _bmStartWord;      // base address of range covered by map
-  size_t    _bmWordSize;       // map size (in #HeapWords covered)
-  const int _shifter;          // map to char or bit
-  BitMap    _bm;               // the bit map itself
-
- public:
-  // constructor
-  CMBitMapRO(int shifter);
-
-  enum { do_yield = true };
-
-  // inquiries
-  HeapWord* startWord()   const { return _bmStartWord; }
-  size_t    sizeInWords() const { return _bmWordSize;  }
-  // the following is one past the last word in space
-  HeapWord* endWord()     const { return _bmStartWord + _bmWordSize; }
-
-  // read marks
-
-  bool isMarked(HeapWord* addr) const {
-    assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
-           "outside underlying space?");
-    return _bm.at(heapWordToOffset(addr));
-  }
-
-  // iteration
-  inline bool iterate(BitMapClosure* cl, MemRegion mr);
-  inline bool iterate(BitMapClosure* cl);
-
-  // Return the address corresponding to the next marked bit at or after
-  // "addr", and before "limit", if "limit" is non-NULL.  If there is no
-  // such bit, returns "limit" if that is non-NULL, or else "endWord()".
-  HeapWord* getNextMarkedWordAddress(const HeapWord* addr,
-                                     const HeapWord* limit = NULL) const;
-  // Return the address corresponding to the next unmarked bit at or after
-  // "addr", and before "limit", if "limit" is non-NULL.  If there is no
-  // such bit, returns "limit" if that is non-NULL, or else "endWord()".
-  HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr,
-                                       const HeapWord* limit = NULL) const;
-
-  // conversion utilities
-  HeapWord* offsetToHeapWord(size_t offset) const {
-    return _bmStartWord + (offset << _shifter);
-  }
-  size_t heapWordToOffset(const HeapWord* addr) const {
-    return pointer_delta(addr, _bmStartWord) >> _shifter;
-  }
-  int heapWordDiffToOffsetDiff(size_t diff) const;
-
-  // The argument addr should be the start address of a valid object
-  HeapWord* nextObject(HeapWord* addr) {
-    oop obj = (oop) addr;
-    HeapWord* res =  addr + obj->size();
-    assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity");
-    return res;
-  }
-
-  void print_on_error(outputStream* st, const char* prefix) const;
-
-  // debugging
-  NOT_PRODUCT(bool covers(MemRegion rs) const;)
-};
-
 class CMBitMapMappingChangedListener : public G1MappingChangedListener {
  private:
   CMBitMap* _bm;
@@ -131,11 +64,12 @@
   virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled);
 };
 
-class CMBitMap : public CMBitMapRO {
+class G1CMBitMap : public CMBitMap {
  private:
   CMBitMapMappingChangedListener _listener;
 
  public:
+  G1CMBitMap() : CMBitMap(), _listener() { _listener.set_bitmap(this); }
   static size_t compute_size(size_t heap_size);
   // Returns the amount of bytes on the heap between two marks in the bitmap.
   static size_t mark_distance();
@@ -145,29 +79,9 @@
     return mark_distance();
   }
 
-  CMBitMap() : CMBitMapRO(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); }
-
   // Initializes the underlying BitMap to cover the given area.
   void initialize(MemRegion heap, G1RegionToSpaceMapper* storage);
 
-  // Write marks.
-  inline void mark(HeapWord* addr);
-  inline void clear(HeapWord* addr);
-  inline bool parMark(HeapWord* addr);
-  inline bool parClear(HeapWord* addr);
-
-  void markRange(MemRegion mr);
-  void clearRange(MemRegion mr);
-
-  // Starting at the bit corresponding to "addr" (inclusive), find the next
-  // "1" bit, if any.  This bit starts some run of consecutive "1"'s; find
-  // the end of this run (stopping at "end_addr").  Return the MemRegion
-  // covering from the start of the region corresponding to the first bit
-  // of the run to the end of the region corresponding to the last bit of
-  // the run.  If there is no "1" bit at or after "addr", return an empty
-  // MemRegion.
-  MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr);
-
   // Clear the whole mark bitmap.
   void clearAll();
 };
@@ -378,8 +292,8 @@
   FreeRegionList        _cleanup_list;
 
   // Concurrent marking support structures
-  CMBitMap                _markBitMap1;
-  CMBitMap                _markBitMap2;
+  G1CMBitMap                _markBitMap1;
+  G1CMBitMap                _markBitMap2;
   CMBitMapRO*             _prevMarkBitMap; // Completed mark bitmap
   CMBitMap*               _nextMarkBitMap; // Under-construction mark bitmap
 
diff --git a/src/share/vm/gc/g1/concurrentMark.inline.hpp b/src/share/vm/gc/g1/concurrentMark.inline.hpp
--- a/src/share/vm/gc/g1/concurrentMark.inline.hpp
+++ b/src/share/vm/gc/g1/concurrentMark.inline.hpp
@@ -166,63 +166,6 @@
   return false;
 }
 
-inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
-  HeapWord* start_addr = MAX2(startWord(), mr.start());
-  HeapWord* end_addr = MIN2(endWord(), mr.end());
-
-  if (end_addr > start_addr) {
-    // Right-open interval [start-offset, end-offset).
-    BitMap::idx_t start_offset = heapWordToOffset(start_addr);
-    BitMap::idx_t end_offset = heapWordToOffset(end_addr);
-
-    start_offset = _bm.get_next_one_offset(start_offset, end_offset);
-    while (start_offset < end_offset) {
-      if (!cl->do_bit(start_offset)) {
-        return false;
-      }
-      HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr);
-      BitMap::idx_t next_offset = heapWordToOffset(next_addr);
-      start_offset = _bm.get_next_one_offset(next_offset, end_offset);
-    }
-  }
-  return true;
-}
-
-inline bool CMBitMapRO::iterate(BitMapClosure* cl) {
-  MemRegion mr(startWord(), sizeInWords());
-  return iterate(cl, mr);
-}
-
-#define check_mark(addr)                                                       \
-  assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize),      \
-         "outside underlying space?");                                         \
-  assert(G1CollectedHeap::heap()->is_in_exact(addr),                           \
-         err_msg("Trying to access not available bitmap " PTR_FORMAT           \
-                 " corresponding to " PTR_FORMAT " (%u)",                      \
-                 p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr)));
-
-inline void CMBitMap::mark(HeapWord* addr) {
-  check_mark(addr);
-  _bm.set_bit(heapWordToOffset(addr));
-}
-
-inline void CMBitMap::clear(HeapWord* addr) {
-  check_mark(addr);
-  _bm.clear_bit(heapWordToOffset(addr));
-}
-
-inline bool CMBitMap::parMark(HeapWord* addr) {
-  check_mark(addr);
-  return _bm.par_set_bit(heapWordToOffset(addr));
-}
-
-inline bool CMBitMap::parClear(HeapWord* addr) {
-  check_mark(addr);
-  return _bm.par_clear_bit(heapWordToOffset(addr));
-}
-
-#undef check_mark
-
 template<typename Fn>
 inline void CMMarkStack::iterate(Fn fn) {
   assert(_saved_index == _index,
diff --git a/src/share/vm/gc/g1/satbQueue.cpp b/src/share/vm/gc/g1/satbQueue.cpp
--- a/src/share/vm/gc/g1/satbQueue.cpp
+++ b/src/share/vm/gc/g1/satbQueue.cpp
@@ -74,30 +74,13 @@
 // processing must be somewhat circumspect and not assume entries
 // in an unfiltered buffer refer to valid objects.
 
-inline bool requires_marking(const void* entry, G1CollectedHeap* heap) {
-  // Includes rejection of NULL pointers.
-  assert(heap->is_in_reserved(entry),
-         err_msg("Non-heap pointer in SATB buffer: " PTR_FORMAT, p2i(entry)));
-
-  HeapRegion* region = heap->heap_region_containing_raw(entry);
-  assert(region != NULL, err_msg("No region for " PTR_FORMAT, p2i(entry)));
-  if (entry >= region->next_top_at_mark_start()) {
-    return false;
-  }
-
-  assert(((oop)entry)->is_oop(true /* ignore mark word */),
-         err_msg("Invalid oop in SATB buffer: " PTR_FORMAT, p2i(entry)));
-
-  return true;
-}
-
 // This method removes entries from a SATB buffer that will not be
 // useful to the concurrent marking threads.  Entries are retained if
 // they require marking and are not already marked. Retained entries
 // are compacted toward the top of the buffer.
 
 void ObjPtrQueue::filter() {
-  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+  CollectedHeap* heap = Universe::heap();
   void** buf = _buf;
   size_t sz = _sz;
 
@@ -124,7 +107,8 @@
     // far, we'll just end up copying it to the same place.
     *p = NULL;
 
-    if (requires_marking(entry, g1h) && !g1h->isMarkedNext((oop)entry)) {
+    bool retain = heap->is_obj_ill(oop(entry));
+    if (retain) {
       assert(new_index > 0, "we should not have already filled up the buffer");
       new_index -= oopSize;
       assert(new_index >= i,
diff --git a/src/share/vm/gc/serial/genMarkSweep.cpp b/src/share/vm/gc/serial/genMarkSweep.cpp
--- a/src/share/vm/gc/serial/genMarkSweep.cpp
+++ b/src/share/vm/gc/serial/genMarkSweep.cpp
@@ -173,7 +173,7 @@
 
 
 void GenMarkSweep::deallocate_stacks() {
-  if (!UseG1GC) {
+  if (!UseG1GC  && !UseShenandoahGC) {
     GenCollectedHeap* gch = GenCollectedHeap::heap();
     gch->release_scratch();
   }
diff --git a/src/share/vm/gc/serial/genMarkSweep.hpp b/src/share/vm/gc/serial/genMarkSweep.hpp
--- a/src/share/vm/gc/serial/genMarkSweep.hpp
+++ b/src/share/vm/gc/serial/genMarkSweep.hpp
@@ -29,6 +29,7 @@
 
 class GenMarkSweep : public MarkSweep {
   friend class VM_MarkSweep;
+  friend class ShenandoahMarkCompact;
   friend class G1MarkSweep;
  public:
   static void invoke_at_safepoint(ReferenceProcessor* rp, bool clear_all_softrefs);
diff --git a/src/share/vm/gc/serial/markSweep.cpp b/src/share/vm/gc/serial/markSweep.cpp
--- a/src/share/vm/gc/serial/markSweep.cpp
+++ b/src/share/vm/gc/serial/markSweep.cpp
@@ -85,6 +85,9 @@
   T heap_oop = oopDesc::load_heap_oop(p);
   if (!oopDesc::is_null(heap_oop)) {
     oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+    if (UseShenandoahGC) {
+      obj = ShenandoahBarrierSet::resolve_and_update_oop_static(p, obj);
+    }
     if (!obj->mark()->is_marked() &&
         !is_archive_object(obj)) {
       mark_object(obj);
@@ -182,6 +185,9 @@
   T heap_oop = oopDesc::load_heap_oop(p);
   if (!oopDesc::is_null(heap_oop)) {
     oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+    if (UseShenandoahGC) {
+      obj = ShenandoahBarrierSet::resolve_and_update_oop_static(p, obj);
+    }
     if (!obj->mark()->is_marked() &&
         !is_archive_object(obj)) {
       mark_object(obj);
diff --git a/src/share/vm/gc/shared/barrierSet.hpp b/src/share/vm/gc/shared/barrierSet.hpp
--- a/src/share/vm/gc/shared/barrierSet.hpp
+++ b/src/share/vm/gc/shared/barrierSet.hpp
@@ -27,11 +27,14 @@
 
 #include "memory/memRegion.hpp"
 #include "oops/oopsHierarchy.hpp"
+#include "asm/register.hpp"
 #include "utilities/fakeRttiSupport.hpp"
 
 // This class provides the interface between a barrier implementation and
 // the rest of the system.
 
+class MacroAssembler;
+
 class BarrierSet: public CHeapObj<mtGC> {
   friend class VMStructs;
 public:
@@ -52,7 +55,8 @@
     CardTableForRS,             // CardTableModRefBSForCTRS
     CardTableExtension,         // CardTableExtension
     G1SATBCT,                   // G1SATBCardTableModRefBS
-    G1SATBCTLogging             // G1SATBCardTableLoggingModRefBS
+    G1SATBCTLogging,            // G1SATBCardTableLoggingModRefBS
+    ShenandoahBarrierSet
   };
 
 protected:
@@ -213,6 +217,27 @@
 
   // Print a description of the memory for the barrier set
   virtual void print_on(outputStream* st) const = 0;
+
+  virtual oop resolve_oop(oop src) {
+    return src;
+  }
+  virtual oop resolve_and_maybe_copy_oop(oop src) {
+    return src;
+  }
+
+#ifndef CC_INTERP
+  virtual void interpreter_read_barrier(MacroAssembler* masm, Register dst) {
+    // Default implementation does nothing.
+  }
+
+  virtual void interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst) {
+    // Default implementation does nothing.
+  }
+
+  virtual void interpreter_write_barrier(MacroAssembler* masm, Register dst) {
+    // Default implementation does nothing.
+  }
+#endif
 };
 
 template<typename T>
diff --git a/src/share/vm/gc/shared/cmBitMap.cpp b/src/share/vm/gc/shared/cmBitMap.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shared/cmBitMap.cpp
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+// Concurrent marking bit map wrapper
+
+#include "gc/shared/cmBitMap.inline.hpp"
+#include "utilities/bitMap.inline.hpp"
+
+CMBitMapRO::CMBitMapRO(int shifter) :
+  _bm(),
+  _shifter(shifter) {
+  _bmStartWord = 0;
+  _bmWordSize = 0;
+}
+
+HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
+                                               const HeapWord* limit) const {
+  // First we must round addr *up* to a possible object boundary.
+  addr = (HeapWord*)align_size_up((intptr_t)addr,
+                                  HeapWordSize << _shifter);
+  size_t addrOffset = heapWordToOffset(addr);
+  if (limit == NULL) {
+    limit = _bmStartWord + _bmWordSize;
+  }
+  size_t limitOffset = heapWordToOffset(limit);
+  size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
+  HeapWord* nextAddr = offsetToHeapWord(nextOffset);
+  assert(nextAddr >= addr, "get_next_one postcondition");
+  assert(nextAddr == limit || isMarked(nextAddr),
+         "get_next_one postcondition");
+  return nextAddr;
+}
+
+HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr,
+                                                 const HeapWord* limit) const {
+  size_t addrOffset = heapWordToOffset(addr);
+  if (limit == NULL) {
+    limit = _bmStartWord + _bmWordSize;
+  }
+  size_t limitOffset = heapWordToOffset(limit);
+  size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
+  HeapWord* nextAddr = offsetToHeapWord(nextOffset);
+  assert(nextAddr >= addr, "get_next_one postcondition");
+  assert(nextAddr == limit || !isMarked(nextAddr),
+         "get_next_one postcondition");
+  return nextAddr;
+}
+
+int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
+  assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
+  return (int) (diff >> _shifter);
+}
+
+#ifndef PRODUCT
+bool CMBitMapRO::covers(MemRegion heap_rs) const {
+  // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
+  assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize,
+         "size inconsistency");
+  return _bmStartWord == (HeapWord*)(heap_rs.start()) &&
+         _bmWordSize  == heap_rs.word_size();
+}
+#endif
+
+void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const {
+  _bm.print_on_error(st, prefix);
+}
+
+size_t CMBitMap::compute_size(size_t heap_size) {
+  return heap_size / mark_distance();
+}
+
+size_t CMBitMap::mark_distance() {
+  return MinObjAlignmentInBytes * BitsPerByte;
+}
+
+void CMBitMap::initialize(MemRegion heap, MemRegion bitmap) {
+  _bmStartWord = heap.start();
+  _bmWordSize = heap.word_size();
+
+  _bm.set_map((BitMap::bm_word_t*) bitmap.start());
+  _bm.set_size(_bmWordSize >> _shifter);
+}
+
+void CMBitMap::clearAll() {
+  _bm.clear();
+}
+
+void CMBitMap::markRange(MemRegion mr) {
+  mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
+  assert(!mr.is_empty(), "unexpected empty region");
+  assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
+          ((HeapWord *) mr.end())),
+         "markRange memory region end is not card aligned");
+  // convert address range into offset range
+  _bm.at_put_range(heapWordToOffset(mr.start()),
+                   heapWordToOffset(mr.end()), true);
+}
+
+void CMBitMap::parMarkRange(MemRegion mr) {
+  mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
+  assert(!mr.is_empty(), "unexpected empty region");
+  assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
+          ((HeapWord *) mr.end())),
+         "markRange memory region end is not card aligned");
+  // convert address range into offset range
+  _bm.par_at_put_range(heapWordToOffset(mr.start()),
+                   heapWordToOffset(mr.end()), true);
+}
+
+void CMBitMap::clearRange(MemRegion mr) {
+  mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
+  assert(!mr.is_empty(), "unexpected empty region");
+  // convert address range into offset range
+  _bm.at_put_range(heapWordToOffset(mr.start()),
+                   heapWordToOffset(mr.end()), false);
+}
+
+MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
+                                            HeapWord* end_addr) {
+  HeapWord* start = getNextMarkedWordAddress(addr);
+  start = MIN2(start, end_addr);
+  HeapWord* end   = getNextUnmarkedWordAddress(start);
+  end = MIN2(end, end_addr);
+  assert(start <= end, "Consistency check");
+  MemRegion mr(start, end);
+  if (!mr.is_empty()) {
+    clearRange(mr);
+  }
+  return mr;
+}
diff --git a/src/share/vm/gc/shared/cmBitMap.hpp b/src/share/vm/gc/shared/cmBitMap.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shared/cmBitMap.hpp
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHARED_CMBITMAP_HPP
+#define SHARE_VM_GC_SHARED_CMBITMAP_HPP
+
+#include "memory/memRegion.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/bitMap.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// A generic CM bit map.  This is essentially a wrapper around the BitMap
+// class, with one bit per (1<<_shifter) HeapWords.
+
+class CMBitMapRO VALUE_OBJ_CLASS_SPEC {
+ protected:
+  HeapWord* _bmStartWord;      // base address of range covered by map
+  size_t    _bmWordSize;       // map size (in #HeapWords covered)
+  const int _shifter;          // map to char or bit
+  BitMap    _bm;               // the bit map itself
+
+ public:
+  // constructor
+  CMBitMapRO(int shifter);
+
+  enum { do_yield = true };
+
+  // inquiries
+  HeapWord* startWord()   const { return _bmStartWord; }
+  size_t    sizeInWords() const { return _bmWordSize;  }
+  // the following is one past the last word in space
+  HeapWord* endWord()     const { return _bmStartWord + _bmWordSize; }
+
+  // read marks
+
+  bool isMarked(HeapWord* addr) const {
+    assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
+           "outside underlying space?");
+    return _bm.at(heapWordToOffset(addr));
+  }
+
+  // iteration
+  inline bool iterate(BitMapClosure* cl, MemRegion mr);
+  inline bool iterate(BitMapClosure* cl);
+
+  // Return the address corresponding to the next marked bit at or after
+  // "addr", and before "limit", if "limit" is non-NULL.  If there is no
+  // such bit, returns "limit" if that is non-NULL, or else "endWord()".
+  HeapWord* getNextMarkedWordAddress(const HeapWord* addr,
+                                     const HeapWord* limit = NULL) const;
+  // Return the address corresponding to the next unmarked bit at or after
+  // "addr", and before "limit", if "limit" is non-NULL.  If there is no
+  // such bit, returns "limit" if that is non-NULL, or else "endWord()".
+  HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr,
+                                       const HeapWord* limit = NULL) const;
+
+  // conversion utilities
+  HeapWord* offsetToHeapWord(size_t offset) const {
+    return _bmStartWord + (offset << _shifter);
+  }
+  size_t heapWordToOffset(const HeapWord* addr) const {
+    return pointer_delta(addr, _bmStartWord) >> _shifter;
+  }
+  int heapWordDiffToOffsetDiff(size_t diff) const;
+
+  // The argument addr should be the start address of a valid object
+  HeapWord* nextObject(HeapWord* addr) {
+    oop obj = (oop) addr;
+    HeapWord* res =  addr + obj->size();
+    assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity");
+    return res;
+  }
+
+  void print_on_error(outputStream* st, const char* prefix) const;
+
+  // debugging
+  NOT_PRODUCT(bool covers(MemRegion rs) const;)
+};
+
+class CMBitMap : public CMBitMapRO {
+
+ public:
+  static size_t compute_size(size_t heap_size);
+  // Returns the amount of bytes on the heap between two marks in the bitmap.
+  static size_t mark_distance();
+  // Returns how many bytes (or bits) of the heap a single byte (or bit) of the
+  // mark bitmap corresponds to. This is the same as the mark distance above.  static size_t heap_map_factor() {
+  static size_t heap_map_factor() {
+    return mark_distance();
+  }
+
+  CMBitMap() : CMBitMapRO(LogMinObjAlignment) {}
+
+  // Initializes the underlying BitMap to cover the given area.
+  void initialize(MemRegion heap, MemRegion bitmap);
+
+  // Write marks.
+  inline void mark(HeapWord* addr);
+  inline void clear(HeapWord* addr);
+  inline bool parMark(HeapWord* addr);
+  inline bool parClear(HeapWord* addr);
+
+  void markRange(MemRegion mr);
+  void parMarkRange(MemRegion mr);
+  void clearRange(MemRegion mr);
+
+  // Starting at the bit corresponding to "addr" (inclusive), find the next
+  // "1" bit, if any.  This bit starts some run of consecutive "1"'s; find
+  // the end of this run (stopping at "end_addr").  Return the MemRegion
+  // covering from the start of the region corresponding to the first bit
+  // of the run to the end of the region corresponding to the last bit of
+  // the run.  If there is no "1" bit at or after "addr", return an empty
+  // MemRegion.
+  MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr);
+
+  // Clear the whole mark bitmap.
+  virtual void clearAll();
+};
+
+#endif // SHARE_VM_GC_SHARED_CMBITMAP_HPP
diff --git a/src/share/vm/gc/shared/cmBitMap.inline.hpp b/src/share/vm/gc/shared/cmBitMap.inline.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shared/cmBitMap.inline.hpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHARED_CMBITMAP_INLINE_HPP
+#define SHARE_VM_GC_SHARED_CMBITMAP_INLINE_HPP
+
+#include "gc/shared/cmBitMap.hpp"
+#include "utilities/bitMap.inline.hpp"
+
+inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
+  HeapWord* start_addr = MAX2(startWord(), mr.start());
+  HeapWord* end_addr = MIN2(endWord(), mr.end());
+
+  if (end_addr > start_addr) {
+    // Right-open interval [start-offset, end-offset).
+    BitMap::idx_t start_offset = heapWordToOffset(start_addr);
+    BitMap::idx_t end_offset = heapWordToOffset(end_addr);
+
+    start_offset = _bm.get_next_one_offset(start_offset, end_offset);
+    while (start_offset < end_offset) {
+      if (!cl->do_bit(start_offset)) {
+        return false;
+      }
+      HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr);
+      BitMap::idx_t next_offset = heapWordToOffset(next_addr);
+      start_offset = _bm.get_next_one_offset(next_offset, end_offset);
+    }
+  }
+  return true;
+}
+
+inline bool CMBitMapRO::iterate(BitMapClosure* cl) {
+  MemRegion mr(startWord(), sizeInWords());
+  return iterate(cl, mr);
+}
+
+#define check_mark(addr)                                                       \
+  assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize),      \
+         "outside underlying space?");                                         \
+  /* assert(G1CollectedHeap::heap()->is_in_exact(addr),                  \
+         err_msg("Trying to access not available bitmap "PTR_FORMAT            \
+                 " corresponding to "PTR_FORMAT" (%u)",                        \
+                 p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr))); */
+
+inline void CMBitMap::mark(HeapWord* addr) {
+  check_mark(addr);
+  _bm.set_bit(heapWordToOffset(addr));
+}
+
+inline void CMBitMap::clear(HeapWord* addr) {
+  check_mark(addr);
+  _bm.clear_bit(heapWordToOffset(addr));
+}
+
+inline bool CMBitMap::parMark(HeapWord* addr) {
+  check_mark(addr);
+  return _bm.par_set_bit(heapWordToOffset(addr));
+}
+
+inline bool CMBitMap::parClear(HeapWord* addr) {
+  check_mark(addr);
+  return _bm.par_clear_bit(heapWordToOffset(addr));
+}
+
+#undef check_mark
+
+#endif // SHARE_VM_GC_SHARED_CMBITMAP_INLINE_HPP
diff --git a/src/share/vm/gc/shared/collectedHeap.cpp b/src/share/vm/gc/shared/collectedHeap.cpp
--- a/src/share/vm/gc/shared/collectedHeap.cpp
+++ b/src/share/vm/gc/shared/collectedHeap.cpp
@@ -319,7 +319,7 @@
 #endif // ASSERT
   }
   thread->tlab().fill(obj, obj + size, new_tlab_size);
-  return obj;
+  return Universe::heap()->tlab_post_allocation_setup(obj);
 }
 
 void CollectedHeap::flush_deferred_store_barrier(JavaThread* thread) {
@@ -631,3 +631,26 @@
       err_msg("after_heap: " PTR_FORMAT " is unexpectedly in the heap", p2i(after_heap)));
 }
 #endif
+
+HeapWord* CollectedHeap::tlab_post_allocation_setup(HeapWord* obj) {
+  return obj;
+}
+
+uint CollectedHeap::oop_extra_words() {
+  // Default implementation doesn't need extra space for oops.
+  return 0;
+}
+
+void CollectedHeap::shutdown() {
+  // Default implementation does nothing.
+}
+
+void CollectedHeap::accumulate_statistics_all_gclabs() {
+  // Default implementation does nothing.
+}
+
+#ifndef CC_INTERP
+void CollectedHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) {
+  // Default implementation does nothing.
+}
+#endif
diff --git a/src/share/vm/gc/shared/collectedHeap.hpp b/src/share/vm/gc/shared/collectedHeap.hpp
--- a/src/share/vm/gc/shared/collectedHeap.hpp
+++ b/src/share/vm/gc/shared/collectedHeap.hpp
@@ -77,6 +77,7 @@
 // CollectedHeap
 //   GenCollectedHeap
 //   G1CollectedHeap
+//   ShenandoahHeap
 //   ParallelScavengeHeap
 //
 class CollectedHeap : public CHeapObj<mtInternal> {
@@ -185,7 +186,8 @@
   enum Name {
     GenCollectedHeap,
     ParallelScavengeHeap,
-    G1CollectedHeap
+    G1CollectedHeap,
+    ShenandoahHeap
   };
 
   static inline size_t filler_array_max_size() {
@@ -194,6 +196,8 @@
 
   virtual Name kind() const = 0;
 
+  virtual HeapWord* tlab_post_allocation_setup(HeapWord* obj);
+
   /**
    * Returns JNI error code JNI_ENOMEM if memory could not be allocated,
    * and JNI_OK on success.
@@ -298,6 +302,12 @@
   inline static void post_allocation_install_obj_klass(KlassHandle klass,
                                                        oop obj);
 
+  virtual uint oop_extra_words();
+
+#ifndef CC_INTERP
+  virtual void compile_prepare_oop(MacroAssembler* masm, Register obj = rax);
+#endif
+
   // Raw memory allocation facilities
   // The obj and array allocate methods are covers for these methods.
   // mem_allocate() should never be
@@ -570,6 +580,12 @@
   // Heap verification
   virtual void verify(bool silent, VerifyOption option) = 0;
 
+  // Shut down all GC workers and other GC related threads.
+  virtual void shutdown();
+
+  // Accumulate additional statistics from GCLABs.
+  virtual void accumulate_statistics_all_gclabs();
+
   // Non product verification and debugging.
 #ifndef PRODUCT
   // Support for PromotionFailureALot.  Return true if it's time to cause a
@@ -603,6 +619,10 @@
     return false;
   }
 
+  virtual bool is_obj_ill(const oop obj) const {
+    return true;
+  }
+
   /////////////// Unit tests ///////////////
 
   NOT_PRODUCT(static void test_is_in();)
diff --git a/src/share/vm/gc/shared/collectedHeap.inline.hpp b/src/share/vm/gc/shared/collectedHeap.inline.hpp
--- a/src/share/vm/gc/shared/collectedHeap.inline.hpp
+++ b/src/share/vm/gc/shared/collectedHeap.inline.hpp
@@ -180,8 +180,10 @@
 HeapWord* CollectedHeap::allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size) {
   assert(UseTLAB, "should use UseTLAB");
 
+  size += Universe::heap()->oop_extra_words();
   HeapWord* obj = thread->tlab().allocate(size);
   if (obj != NULL) {
+    obj = Universe::heap()->tlab_post_allocation_setup(obj);
     return obj;
   }
   // Otherwise...
diff --git a/src/share/vm/gc/shared/collectorPolicy.hpp b/src/share/vm/gc/shared/collectorPolicy.hpp
--- a/src/share/vm/gc/shared/collectorPolicy.hpp
+++ b/src/share/vm/gc/shared/collectorPolicy.hpp
@@ -51,6 +51,7 @@
 #if INCLUDE_ALL_GCS
 class ConcurrentMarkSweepPolicy;
 class G1CollectorPolicy;
+class ShenandoahCollectorPolicy;
 #endif // INCLUDE_ALL_GCS
 
 class GCPolicyCounters;
@@ -130,6 +131,7 @@
 #if INCLUDE_ALL_GCS
   virtual ConcurrentMarkSweepPolicy*    as_concurrent_mark_sweep_policy() { return NULL; }
   virtual G1CollectorPolicy*            as_g1_policy()                    { return NULL; }
+  virtual ShenandoahCollectorPolicy*    as_pgc_policy()                    { return NULL; }
 #endif // INCLUDE_ALL_GCS
   // Note that these are not virtual.
   bool is_generation_policy()            { return as_generation_policy() != NULL; }
@@ -137,9 +139,11 @@
 #if INCLUDE_ALL_GCS
   bool is_concurrent_mark_sweep_policy() { return as_concurrent_mark_sweep_policy() != NULL; }
   bool is_g1_policy()                    { return as_g1_policy() != NULL; }
+  bool is_pgc_policy()                   { return as_pgc_policy() != NULL; }
 #else  // INCLUDE_ALL_GCS
   bool is_concurrent_mark_sweep_policy() { return false; }
   bool is_g1_policy()                    { return false; }
+  bool is_pgc_policy()                    { return false; }
 #endif // INCLUDE_ALL_GCS
 
 
diff --git a/src/share/vm/gc/shared/gcCause.hpp b/src/share/vm/gc/shared/gcCause.hpp
--- a/src/share/vm/gc/shared/gcCause.hpp
+++ b/src/share/vm/gc/shared/gcCause.hpp
@@ -73,6 +73,8 @@
     _g1_inc_collection_pause,
     _g1_humongous_allocation,
 
+    _shenandoah_init_mark,
+
     _last_ditch_collection,
 
     _dcmd_gc_run,
diff --git a/src/share/vm/gc/shared/gcName.hpp b/src/share/vm/gc/shared/gcName.hpp
--- a/src/share/vm/gc/shared/gcName.hpp
+++ b/src/share/vm/gc/shared/gcName.hpp
@@ -37,6 +37,7 @@
   G1New,
   ConcurrentMarkSweep,
   G1Old,
+  Shenandoah,
   GCNameEndSentinel
 };
 
diff --git a/src/share/vm/gc/shared/gcTrace.hpp b/src/share/vm/gc/shared/gcTrace.hpp
--- a/src/share/vm/gc/shared/gcTrace.hpp
+++ b/src/share/vm/gc/shared/gcTrace.hpp
@@ -206,6 +206,11 @@
   void send_concurrent_mode_failure_event();
 };
 
+class ShenandoahTracer : public GCTracer {
+public:
+  ShenandoahTracer() : GCTracer(Shenandoah) {}
+};
+
 class ParallelOldTracer : public OldGCTracer {
   ParallelOldGCInfo _parallel_old_gc_info;
 
diff --git a/src/share/vm/gc/shared/referenceProcessor.cpp b/src/share/vm/gc/shared/referenceProcessor.cpp
--- a/src/share/vm/gc/shared/referenceProcessor.cpp
+++ b/src/share/vm/gc/shared/referenceProcessor.cpp
@@ -446,6 +446,7 @@
   _next = discovered;
   _referent_addr = java_lang_ref_Reference::referent_addr(_ref);
   _referent = java_lang_ref_Reference::referent(_ref);
+  assert(_referent == oopDesc::bs()->resolve_oop(_referent), "expect forwarded referent");
   assert(Universe::heap()->is_in_reserved_or_null(_referent),
          "Wrong oop found in java.lang.Reference object");
   assert(allow_null_referent ?
@@ -647,6 +648,7 @@
   oop next = refs_list.head();
   while (next != obj) {
     obj = next;
+    assert(obj == oopDesc::bs()->resolve_oop(obj), "expect forwarded obj");
     next = java_lang_ref_Reference::discovered(obj);
     java_lang_ref_Reference::set_discovered_raw(obj, NULL);
   }
@@ -1094,7 +1096,7 @@
       // Check assumption that an object is not potentially
       // discovered twice except by concurrent collectors that potentially
       // trace the same Reference object twice.
-      assert(UseConcMarkSweepGC || UseG1GC,
+      assert(UseConcMarkSweepGC || UseG1GC || UseShenandoahGC,
              "Only possible with a concurrent marking collector");
       return true;
     }
diff --git a/src/share/vm/gc/shared/space.cpp b/src/share/vm/gc/shared/space.cpp
--- a/src/share/vm/gc/shared/space.cpp
+++ b/src/share/vm/gc/shared/space.cpp
@@ -392,7 +392,7 @@
 
   // store the forwarding pointer into the mark word
   if ((HeapWord*)q != compact_top) {
-    q->forward_to(oop(compact_top));
+    q->forward_to(compact_oop(compact_top));
     assert(q->is_gc_marked(), "encoding the pointer should preserve the mark");
   } else {
     // if the object isn't moving we can just set the mark to the default
@@ -631,8 +631,11 @@
 
 // This version requires locking.
 inline HeapWord* ContiguousSpace::allocate_impl(size_t size) {
+  // Shenandoah is currently partitioning by region so this assertion
+  // is too strong.  If we move to a smaller granularity we will
+  // need to revisit this.
   assert(Heap_lock->owned_by_self() ||
-         (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()),
+         (SafepointSynchronize::is_at_safepoint() && (Thread::current()->is_VM_thread() || UseShenandoahGC)),
          "not locked");
   HeapWord* obj = top();
   if (pointer_delta(end(), obj) >= size) {
diff --git a/src/share/vm/gc/shared/space.hpp b/src/share/vm/gc/shared/space.hpp
--- a/src/share/vm/gc/shared/space.hpp
+++ b/src/share/vm/gc/shared/space.hpp
@@ -367,6 +367,10 @@
     return oop(addr)->size();
   }
 
+  inline oop make_oop(HeapWord* addr) const {
+    return oop(addr);
+  }
+
 public:
   CompactibleSpace() :
    _compaction_top(NULL), _next_compaction_space(NULL) {}
@@ -442,6 +446,10 @@
   virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp,
                     HeapWord* compact_top);
 
+  virtual oop compact_oop(HeapWord* addr) const {
+    return oop(addr);
+  }
+
   // Return a size with adjustments as required of the space.
   virtual size_t adjust_object_size_v(size_t size) const { return size; }
 
@@ -584,7 +592,7 @@
 
   // Iteration
   void oop_iterate(ExtendedOopClosure* cl);
-  void object_iterate(ObjectClosure* blk);
+  virtual void object_iterate(ObjectClosure* blk);
   // For contiguous spaces this method will iterate safely over objects
   // in the space (i.e., between bottom and top) when at a safepoint.
   void safe_object_iterate(ObjectClosure* blk);
diff --git a/src/share/vm/gc/shared/space.inline.hpp b/src/share/vm/gc/shared/space.inline.hpp
--- a/src/share/vm/gc/shared/space.inline.hpp
+++ b/src/share/vm/gc/shared/space.inline.hpp
@@ -121,14 +121,18 @@
 
   while (q < t) {
     assert(!space->scanned_block_is_obj(q) ||
-           oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() ||
-           oop(q)->mark()->has_bias_pattern(),
+           space->make_oop(q)->mark()->is_marked() ||
+           oopDesc::bs()->resolve_oop(space->make_oop(q))->mark()->is_marked() ||
+           space->make_oop(q)->mark()->is_unlocked() ||
+           oopDesc::bs()->resolve_oop(space->make_oop(q))->mark()->is_unlocked() ||
+           space->make_oop(q)->mark()->has_bias_pattern() ||
+           oopDesc::bs()->resolve_oop(space->make_oop(q))->mark()->has_bias_pattern(),
            "these are the only valid states during a mark sweep");
-    if (space->scanned_block_is_obj(q) && oop(q)->is_gc_marked()) {
+    if (space->scanned_block_is_obj(q) && space->make_oop(q)->is_gc_marked()) {
       // prefetch beyond q
       Prefetch::write(q, interval);
       size_t size = space->scanned_block_size(q);
-      compact_top = cp->space->forward(oop(q), size, cp, compact_top);
+      compact_top = cp->space->forward(space->make_oop(q), size, cp, compact_top);
       q += size;
       end_of_live = q;
     } else {
@@ -138,14 +142,14 @@
         // prefetch beyond end
         Prefetch::write(end, interval);
         end += space->scanned_block_size(end);
-      } while (end < t && (!space->scanned_block_is_obj(end) || !oop(end)->is_gc_marked()));
+      } while (end < t && (!space->scanned_block_is_obj(end) || !space->make_oop(end)->is_gc_marked()));
 
       // see if we might want to pretend this object is alive so that
       // we don't have to compact quite as often.
       if (allowed_deadspace > 0 && q == compact_top) {
         size_t sz = pointer_delta(end, q);
         if (space->insert_deadspace(allowed_deadspace, q, sz)) {
-          compact_top = cp->space->forward(oop(q), sz, cp, compact_top);
+          compact_top = cp->space->forward(space->make_oop(q), sz, cp, compact_top);
           q = end;
           end_of_live = end;
           continue;
@@ -161,7 +165,7 @@
 
       // record the current LiveRange object.
       // liveRange->start() is overlaid on the mark word.
-      liveRange = (LiveRange*)q;
+      liveRange = (LiveRange*) (HeapWord*) space->make_oop(q);
       liveRange->set_start(end);
       liveRange->set_end(end);
 
@@ -199,7 +203,7 @@
 
   assert(space->_first_dead <= space->_end_of_live, "Stands to reason, no?");
 
-  if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) {
+  if (q < t && space->_first_dead > q && !space->make_oop(q)->is_gc_marked()) {
     // we have a chunk of the space which hasn't moved and we've
     // reinitialized the mark word during the previous pass, so we can't
     // use is_gc_marked for the traversal.
@@ -214,7 +218,7 @@
       assert(space->block_is_obj(q), "should be at block boundaries, and should be looking at objs");
 
       // point all the oops to the new location
-      size_t size = MarkSweep::adjust_pointers(oop(q));
+      size_t size = MarkSweep::adjust_pointers(space->make_oop(q));
       size = space->adjust_obj_size(size);
 
       q += size;
@@ -235,10 +239,10 @@
   while (q < t) {
     // prefetch beyond q
     Prefetch::write(q, interval);
-    if (oop(q)->is_gc_marked()) {
+    if (space->make_oop(q)->is_gc_marked()) {
       // q is alive
       // point all the oops to the new location
-      size_t size = MarkSweep::adjust_pointers(oop(q));
+      size_t size = MarkSweep::adjust_pointers(space->make_oop(q));
       size = space->adjust_obj_size(size);
       debug_only(prev_q = q);
       q += size;
@@ -246,7 +250,7 @@
       // q is not a live object, so its mark should point at the next
       // live object
       debug_only(prev_q = q);
-      q = (HeapWord*) oop(q)->mark()->decode_pointer();
+      q = (HeapWord*) space->make_oop(q)->mark()->decode_pointer();
       assert(q > prev_q, "we should be moving forward through memory");
     }
   }
@@ -263,7 +267,7 @@
   HeapWord* const t = space->_end_of_live;
   debug_only(HeapWord* prev_q = NULL);
 
-  if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) {
+  if (q < t && space->_first_dead > q && !space->make_oop(q)->is_gc_marked()) {
     #ifdef ASSERT // Debug only
       // we have a chunk of the space which hasn't moved and we've reinitialized
       // the mark word during the previous pass, so we can't use is_gc_marked for
@@ -272,7 +276,7 @@
 
       while (q < end) {
         size_t size = space->obj_size(q);
-        assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)");
+        assert(!space->make_oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)");
         prev_q = q;
         q += size;
       }
@@ -289,10 +293,10 @@
   const intx scan_interval = PrefetchScanIntervalInBytes;
   const intx copy_interval = PrefetchCopyIntervalInBytes;
   while (q < t) {
-    if (!oop(q)->is_gc_marked()) {
+    if (!space->make_oop(q)->is_gc_marked()) {
       // mark is pointer to next marked oop
       debug_only(prev_q = q);
-      q = (HeapWord*) oop(q)->mark()->decode_pointer();
+      q = (HeapWord*) space->make_oop(q)->mark()->decode_pointer();
       assert(q > prev_q, "we should be moving forward through memory");
     } else {
       // prefetch beyond q
@@ -300,14 +304,14 @@
 
       // size and destination
       size_t size = space->obj_size(q);
-      HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee();
+      HeapWord* compaction_top = (HeapWord*)space->make_oop(q)->forwardee();
 
       // prefetch beyond compaction_top
       Prefetch::write(compaction_top, copy_interval);
 
       // copy object and reinit its mark
       assert(q != compaction_top, "everything in this pass should be moving");
-      Copy::aligned_conjoint_words(q, compaction_top, size);
+      Copy::aligned_conjoint_words((HeapWord*) space->make_oop(q), compaction_top, size);
       oop(compaction_top)->init_mark();
       assert(oop(compaction_top)->klass() != NULL, "should have a class");
 
diff --git a/src/share/vm/gc/shared/taskqueue.cpp b/src/share/vm/gc/shared/taskqueue.cpp
--- a/src/share/vm/gc/shared/taskqueue.cpp
+++ b/src/share/vm/gc/shared/taskqueue.cpp
@@ -107,6 +107,17 @@
          err_msg("overflow_max_len=" SIZE_FORMAT " overflow=" SIZE_FORMAT,
                  get(overflow_max_len), get(overflow)));
 }
+
+void TaskQueueStats::verify_only_pushes() const
+{
+  assert((get(pop) == 0),
+         err_msg("pops=" SIZE_FORMAT ,
+                 get(pop)));
+  assert((get(steal) == 0),
+         err_msg("steals=" SIZE_FORMAT ,
+                 get(steal)));
+}
+
 #endif // ASSERT
 #endif // TASKQUEUE_STATS
 
diff --git a/src/share/vm/gc/shared/taskqueue.hpp b/src/share/vm/gc/shared/taskqueue.hpp
--- a/src/share/vm/gc/shared/taskqueue.hpp
+++ b/src/share/vm/gc/shared/taskqueue.hpp
@@ -79,6 +79,7 @@
   void print(outputStream* const stream = tty, unsigned int width = 10) const;
 
   DEBUG_ONLY(void verify() const;)
+  DEBUG_ONLY(void verify_only_pushes() const;)
 
 private:
   size_t                    _stats[last_stat_id];
diff --git a/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp b/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp
--- a/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp
+++ b/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp
@@ -51,6 +51,8 @@
     thread->tlab().initialize_statistics();
   }
 
+  Universe::heap()->accumulate_statistics_all_gclabs();
+
   // Publish new stats if some allocation occurred.
   if (global_stats()->allocation() != 0) {
     global_stats()->publish();
@@ -66,7 +68,7 @@
   size_t used     = Universe::heap()->tlab_used(thread);
 
   _gc_waste += (unsigned)remaining();
-  size_t total_allocated = thread->allocated_bytes();
+  size_t total_allocated = _gclab ? thread->allocated_bytes_gclab() : thread->allocated_bytes();
   size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc;
   _allocated_before_last_gc = total_allocated;
 
@@ -113,10 +115,15 @@
     invariants();
 
     if (retire) {
-      myThread()->incr_allocated_bytes(used_bytes());
+      if (_gclab) {
+        myThread()->incr_allocated_bytes_gclab(used_bytes());
+      } else {
+        myThread()->incr_allocated_bytes(used_bytes());
+      }
     }
 
-    CollectedHeap::fill_with_object(top(), hard_end(), retire);
+    HeapWord* obj = Universe::heap()->tlab_post_allocation_setup(top());
+    CollectedHeap::fill_with_object(obj, hard_end(), retire);
 
     if (retire || ZeroTLAB) {  // "Reset" the TLAB
       set_start(NULL);
@@ -191,7 +198,8 @@
   invariants();
 }
 
-void ThreadLocalAllocBuffer::initialize() {
+void ThreadLocalAllocBuffer::initialize(bool gclab) {
+  _gclab = gclab;
   initialize(NULL,                    // start
              NULL,                    // top
              NULL);                   // end
@@ -224,7 +232,8 @@
   // During jvm startup, the main (primordial) thread is initialized
   // before the heap is initialized.  So reinitialize it now.
   guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread");
-  Thread::current()->tlab().initialize();
+  Thread::current()->tlab().initialize(false);
+  Thread::current()->gclab().initialize(true);
 
   if (PrintTLAB && Verbose) {
     gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n",
@@ -256,12 +265,12 @@
   double waste_percent = alloc == 0 ? 0.0 :
                       100.0 * waste / alloc;
   size_t tlab_used  = Universe::heap()->tlab_used(thrd);
-  gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]"
+  gclog_or_tty->print("TLAB: %s %s thread: " INTPTR_FORMAT " [id: %2d]"
                       " desired_size: " SIZE_FORMAT "KB"
                       " slow allocs: %d  refill waste: " SIZE_FORMAT "B"
                       " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB"
                       " slow: %dB fast: %dB\n",
-                      tag, p2i(thrd), thrd->osthread()->thread_id(),
+                      tag, _gclab ? "gclab" : "tlab ", p2i(thrd), thrd->osthread()->thread_id(),
                       _desired_size / (K / HeapWordSize),
                       _slow_allocations, _refill_waste_limit * HeapWordSize,
                       _allocation_fraction.average(),
@@ -285,9 +294,27 @@
 }
 
 Thread* ThreadLocalAllocBuffer::myThread() {
-  return (Thread*)(((char *)this) +
-                   in_bytes(start_offset()) -
-                   in_bytes(Thread::tlab_start_offset()));
+  ByteSize gclab_offset = Thread::gclab_start_offset();
+  ByteSize tlab_offset = Thread::tlab_start_offset();
+  ByteSize offs = _gclab ? gclab_offset : tlab_offset;
+  Thread* thread = (Thread*)(((char *)this) +
+                   in_bytes(start_offset()) - in_bytes(offs));
+#ifdef ASSERT
+  assert(this == (_gclab ? &thread->gclab() : &thread->tlab()), "must be");
+#endif
+  return thread;
+}
+
+size_t ThreadLocalAllocBuffer::end_reserve() {
+  int reserve_size = typeArrayOopDesc::header_size(T_INT) + Universe::heap()->oop_extra_words();
+  return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch());
+}
+
+void ThreadLocalAllocBuffer::rollback(size_t size) {
+  HeapWord* old_top = top();
+  if (old_top != NULL) { // Pathological case: we accept that we can't rollback.
+    set_top(old_top - size);
+  }
 }
 
 
diff --git a/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp b/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp
--- a/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp
+++ b/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp
@@ -59,8 +59,7 @@
 
   AdaptiveWeightedAverage _allocation_fraction;  // fraction of eden allocated in tlabs
 
-  void accumulate_statistics();
-  void initialize_statistics();
+  bool _gclab;
 
   void set_start(HeapWord* start)                { _start = start; }
   void set_end(HeapWord* end)                    { _end = end; }
@@ -79,9 +78,6 @@
   // Make parsable and release it.
   void reset();
 
-  // Resize based on amount of allocation, etc.
-  void resize();
-
   void invariants() const { assert(top() >= start() && top() <= end(), "invalid tlab"); }
 
   void initialize(HeapWord* start, HeapWord* top, HeapWord* end);
@@ -125,11 +121,17 @@
   // Allocate size HeapWords. The memory is NOT initialized to zero.
   inline HeapWord* allocate(size_t size);
 
+  // Resize based on amount of allocation, etc.
+  void resize();
+
+  void accumulate_statistics();
+  void initialize_statistics();
+
+  // Rolls back a single allocation of the given size.
+  void rollback(size_t size);
+
   // Reserve space at the end of TLAB
-  static size_t end_reserve() {
-    int reserve_size = typeArrayOopDesc::header_size(T_INT);
-    return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch());
-  }
+  static size_t end_reserve();
   static size_t alignment_reserve()              { return align_object_size(end_reserve()); }
   static size_t alignment_reserve_in_bytes()     { return alignment_reserve() * HeapWordSize; }
 
@@ -157,7 +159,7 @@
   static void resize_all_tlabs();
 
   void fill(HeapWord* start, HeapWord* top, size_t new_size);
-  void initialize();
+  void initialize(bool gclab);
 
   static size_t refill_waste_limit_increment()   { return TLABWasteIncrement; }
 
diff --git a/src/share/vm/gc/shenandoah/brooksPointer.cpp b/src/share/vm/gc/shenandoah/brooksPointer.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/brooksPointer.cpp
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "memory/universe.hpp"
+#include "gc/shared/collectedHeap.hpp"
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+
+BrooksPointer::BrooksPointer(HeapWord** hw) : _heap_word(hw) {}
+
+BrooksPointer BrooksPointer::get(oop obj) {
+  HeapWord* hw_obj = (HeapWord*) obj;
+  HeapWord* brooks_ptr = hw_obj - 1;
+  // We know that the value in that memory location is a pointer to another
+  // heapword/oop.
+  return BrooksPointer((HeapWord**) brooks_ptr);
+}
+
+void BrooksPointer::set_forwardee(oop forwardee) {
+  assert(ShenandoahHeap::heap()->is_in(forwardee), "forwardee must be valid oop in the heap");
+  *_heap_word = (HeapWord*) forwardee;
+#ifdef ASSERT
+  if (ShenandoahTraceBrooksPointers) {
+    tty->print_cr("setting_forwardee to "PTR_FORMAT" = "PTR_FORMAT, p2i((HeapWord*) forwardee), p2i(*_heap_word));
+  }
+#endif
+}
+
+HeapWord* BrooksPointer::cas_forwardee(HeapWord* old, HeapWord* forwardee) {
+  assert(ShenandoahHeap::heap()->is_in(forwardee), "forwardee must point to a heap address");
+
+
+
+  HeapWord* o = old;
+  HeapWord* n = forwardee;
+  HeapWord* result;
+
+#ifdef ASSERT
+  if (ShenandoahTraceBrooksPointers) {
+    tty->print_cr("Attempting to CAS "PTR_FORMAT" value "PTR_FORMAT" from "PTR_FORMAT" to "PTR_FORMAT, p2i(_heap_word), p2i(*_heap_word), p2i(o), p2i(n));
+  }
+#endif
+
+#ifdef ASSERT
+  if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) {
+    ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+    ShenandoahHeapRegion* hr = sh->heap_region_containing(old);
+
+    {
+      hr->memProtectionOff();
+      result =  (HeapWord*) (HeapWord*) Atomic::cmpxchg_ptr(n, _heap_word, o);
+      hr->memProtectionOn();
+    }
+  } else {
+    result =  (HeapWord*) (HeapWord*) Atomic::cmpxchg_ptr(n, _heap_word, o);
+  }
+#else
+  result =  (HeapWord*) (HeapWord*) Atomic::cmpxchg_ptr(n, _heap_word, o);
+#endif
+
+#ifdef ASSERT
+  if (ShenandoahTraceBrooksPointers) {
+    tty->print_cr("Result of CAS from "PTR_FORMAT" to "PTR_FORMAT" was "PTR_FORMAT" read value was "PTR_FORMAT, p2i(o), p2i(n), p2i(result), p2i(*_heap_word));
+  }
+#endif
+
+  return result;
+}
+
+bool BrooksPointer::check_forwardee_is_in_heap(oop forwardee) {
+   return Universe::heap()->is_in(forwardee);
+}
diff --git a/src/share/vm/gc/shenandoah/brooksPointer.hpp b/src/share/vm/gc/shenandoah/brooksPointer.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/brooksPointer.hpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_HPP
+#define SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_HPP
+
+#include "oops/oop.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+
+class BrooksPointer {
+
+public:
+  static const uint BROOKS_POINTER_OBJ_SIZE = 1;
+
+private:
+
+  HeapWord** _heap_word;
+
+  BrooksPointer(HeapWord** heap_word);
+
+public:
+
+  bool check_forwardee_is_in_heap(oop forwardee);
+
+  inline oop get_forwardee_raw() {
+    return oop(*_heap_word);
+  }
+
+  inline oop get_forwardee() {
+    oop forwardee;
+
+#ifdef ASSERT
+    if (ShenandoahVerifyReadsToFromSpace) {
+      ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+      ShenandoahHeapRegion* hr = sh->heap_region_containing(_heap_word);
+
+      {
+        hr->memProtectionOff();
+        forwardee = (oop) (*_heap_word);
+        hr->memProtectionOn();
+      }
+    } else {
+      forwardee = get_forwardee_raw();
+    }
+#else
+    forwardee = get_forwardee_raw();
+#endif
+
+    assert(check_forwardee_is_in_heap(forwardee), "forwardee must be in heap");
+    assert(forwardee->is_oop(), "forwardee must be valid oop");
+    return forwardee;
+  }
+
+  void set_forwardee(oop forwardee);
+  HeapWord* cas_forwardee(HeapWord* old, HeapWord* forwardee);
+
+  static BrooksPointer get(oop obj);
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahBarrierSet.cpp b/src/share/vm/gc/shenandoah/shenandoahBarrierSet.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -0,0 +1,352 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1SATBCardTableModRefBS.hpp"
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "memory/universe.hpp"
+#include "utilities/array.hpp"
+
+class UpdateRefsForOopClosure: public ExtendedOopClosure {
+
+private:
+  ShenandoahHeap* _heap;
+public:
+  UpdateRefsForOopClosure() {
+    _heap = ShenandoahHeap::heap();
+  }
+
+  void do_oop(oop* p)       {
+    _heap->maybe_update_oop_ref(p);
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+
+};
+
+ShenandoahBarrierSet::ShenandoahBarrierSet() :
+  BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet))
+{
+}
+
+void ShenandoahBarrierSet::print_on(outputStream* st) const {
+  st->print("ShenandoahBarrierSet");
+}
+
+bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) {
+  return bsn == BarrierSet::ShenandoahBarrierSet;
+}
+
+bool ShenandoahBarrierSet::has_read_prim_array_opt() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_read_prim_barrier() {
+  return false;
+}
+
+bool ShenandoahBarrierSet::has_read_ref_array_opt() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_read_ref_barrier() {
+  return false;
+}
+
+bool ShenandoahBarrierSet::has_read_region_opt() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_write_prim_array_opt() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_write_prim_barrier() {
+  return false;
+}
+
+bool ShenandoahBarrierSet::has_write_ref_array_opt() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_write_ref_barrier() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_write_ref_pre_barrier() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::has_write_region_opt() {
+  return true;
+}
+
+bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) {
+  return true;
+}
+
+void ShenandoahBarrierSet::read_prim_array(MemRegion mr) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){
+  Unimplemented();
+}
+
+bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) {
+  return false;
+}
+
+void ShenandoahBarrierSet::read_ref_array(MemRegion mr) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSet::read_ref_field(void* v) {
+  //    tty->print_cr("read_ref_field: v = "PTR_FORMAT, v);
+  // return *v;
+}
+
+bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) {
+  Unimplemented();
+  return false;
+}
+
+void ShenandoahBarrierSet::read_region(MemRegion mr) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSet::write_prim_array(MemRegion mr) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) {
+  Unimplemented();
+}
+
+bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) {
+  Unimplemented();
+  return false;
+}
+
+bool ShenandoahBarrierSet::need_update_refs_barrier() {
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  return heap->is_update_references_in_progress() || (heap->concurrent_mark_in_progress() && heap->need_update_refs());
+}
+
+void ShenandoahBarrierSet::write_ref_array_work(MemRegion mr) {
+  if (! need_update_refs_barrier()) return;
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  for (HeapWord* word = mr.start(); word < mr.end(); word++) {
+    oop* oop_ptr = (oop*) word;
+    heap->maybe_update_oop_ref(oop_ptr);
+  }
+}
+
+template <class T>
+void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) {
+
+#ifdef ASSERT
+    ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+    if (sh->is_in(dst) &&
+        sh->heap_region_containing((HeapWord*) dst)->is_in_collection_set() &&
+        ! sh->cancelled_concgc()) {
+      tty->print_cr("dst = "PTR_FORMAT, p2i(dst));
+      sh->heap_region_containing((HeapWord*) dst)->print();
+      assert(false, "We should have fixed this earlier");
+    }
+#endif
+
+  if (! JavaThread::satb_mark_queue_set().is_active()) return;
+  // tty->print_cr("write_ref_array_pre_work: "PTR_FORMAT", "INT32_FORMAT, dst, count);
+  T* elem_ptr = dst;
+  for (int i = 0; i < count; i++, elem_ptr++) {
+    T heap_oop = oopDesc::load_heap_oop(elem_ptr);
+    if (!oopDesc::is_null(heap_oop)) {
+      G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
+    }
+    // tty->print_cr("write_ref_array_pre_work: oop: "PTR_FORMAT, heap_oop);
+  }
+}
+
+void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) {
+  if (! dest_uninitialized) {
+    write_ref_array_pre_work(dst, count);
+  }
+}
+
+void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) {
+  if (! dest_uninitialized) {
+    write_ref_array_pre_work(dst, count);
+  }
+}
+
+template <class T>
+void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop newVal) {
+  T heap_oop = oopDesc::load_heap_oop(field);
+
+#ifdef ASSERT
+    ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+    if (sh->is_in(field) &&
+        sh->heap_region_containing((HeapWord*)field)->is_in_collection_set() &&
+        ! sh->cancelled_concgc()) {
+      tty->print_cr("field = "PTR_FORMAT, p2i(field));
+      sh->heap_region_containing((HeapWord*)field)->print();
+      assert(false, "We should have fixed this earlier");
+    }
+#endif
+
+  if (!oopDesc::is_null(heap_oop)) {
+    G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop));
+    // tty->print_cr("write_ref_field_pre_static: v = "PTR_FORMAT" o = "PTR_FORMAT" old: "PTR_FORMAT, field, newVal, heap_oop);
+  }
+}
+
+template <class T>
+inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) {
+  write_ref_field_pre_static(field, newVal);
+}
+
+// These are the more general virtual versions.
+void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) {
+  write_ref_field_pre_static(field, new_val);
+}
+
+void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) {
+  write_ref_field_pre_static(field, new_val);
+}
+
+void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) {
+  guarantee(false, "Not needed");
+}
+
+void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) {
+  if (! need_update_refs_barrier()) return;
+  assert (! UseCompressedOops, "compressed oops not supported yet");
+  ShenandoahHeap::heap()->maybe_update_oop_ref((oop*) v);
+  // tty->print_cr("write_ref_field_work: v = "PTR_FORMAT" o = "PTR_FORMAT, v, o);
+}
+
+void ShenandoahBarrierSet::write_region_work(MemRegion mr) {
+
+  if (! need_update_refs_barrier()) return;
+
+  // This is called for cloning an object (see jvm.cpp) after the clone
+  // has been made. We are not interested in any 'previous value' because
+  // it would be NULL in any case. But we *are* interested in any oop*
+  // that potentially need to be updated.
+
+  // tty->print_cr("write_region_work: "PTR_FORMAT", "PTR_FORMAT, mr.start(), mr.end());
+  oop obj = oop(mr.start());
+  assert(obj->is_oop(), "must be an oop");
+  UpdateRefsForOopClosure cl;
+  obj->oop_iterate(&cl);
+}
+
+oop ShenandoahBarrierSet::resolve_oop(oop src) {
+  return ShenandoahBarrierSet::resolve_oop_static(src);
+}
+
+oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work(oop src) {
+  ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+  assert(src != NULL, "only evacuated non NULL oops");
+
+  if (sh->in_cset_fast_test((HeapWord*) src)) {
+    return resolve_and_maybe_copy_oop_work2(src);
+  } else {
+    return src;
+  }
+}
+
+oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work2(oop src) {
+  ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+  if (! sh->is_evacuation_in_progress()) {
+    // We may get here through a barrier that just took a safepoint that
+    // turned off evacuation. In this case, return right away.
+    return ShenandoahBarrierSet::resolve_oop_static(src);
+  }
+  assert(src != NULL, "only evacuated non NULL oops");
+  assert(sh->heap_region_containing(src)->is_in_collection_set(), "only evacuate objects in collection set");
+  assert(! sh->heap_region_containing(src)->is_humongous(), "never evacuate humongous objects");
+  // TODO: Consider passing thread from caller.
+  oop dst = sh->evacuate_object(src, Thread::current());
+#ifdef ASSERT
+    if (ShenandoahTraceEvacuations) {
+      tty->print_cr("src = "PTR_FORMAT" dst = "PTR_FORMAT" src = "PTR_FORMAT" src-2 = "PTR_FORMAT,
+                 p2i((HeapWord*) src), p2i((HeapWord*) dst), p2i((HeapWord*) src), p2i(((HeapWord*) src) - 2));
+    }
+#endif
+  assert(sh->is_in(dst), "result should be in the heap");
+  return dst;
+}
+
+oop ShenandoahBarrierSet::resolve_and_maybe_copy_oopHelper(oop src) {
+    if (src != NULL) {
+      ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+      oop tmp = resolve_oop_static(src);
+      if (! sh->is_evacuation_in_progress()) {
+        return tmp;
+      }
+      return resolve_and_maybe_copy_oop_work(src);
+    } else {
+      return NULL;
+    }
+}
+
+JRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2(oopDesc* src))
+  oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src));
+  // tty->print_cr("called C2 write barrier with: %p result: %p copy: %d", (oopDesc*) src, (oopDesc*) result, src != result);
+  return (oopDesc*) result;
+JRT_END
+
+IRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_interp(oopDesc* src))
+  oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src));
+  // tty->print_cr("called interpreter write barrier with: %p result: %p", src, result);
+  return (oopDesc*) result;
+IRT_END
+
+JRT_LEAF(oopDesc*, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src))
+  oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->resolve_and_maybe_copy_oop_work2(oop(src));
+  // tty->print_cr("called static write barrier (2) with: "PTR_FORMAT" result: "PTR_FORMAT, p2i(src), p2i((oopDesc*)(result)));
+  return (oopDesc*) result;
+JRT_END
+
+oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop(oop src) {
+    ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+    oop result;
+    if (src != NULL && sh->is_in(src)) {
+      result = resolve_and_maybe_copy_oopHelper(src);
+      assert(sh->is_in(result), "result should be in the heap");
+    } else {
+      result = src;
+    }
+    assert(result == NULL || (sh->is_in(result) && result->is_oop()), "resolved oop must be NULL, or a valid oop in the heap");
+    return result;
+  }
diff --git a/src/share/vm/gc/shenandoah/shenandoahBarrierSet.hpp b/src/share/vm/gc/shenandoah/shenandoahBarrierSet.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP
+
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shared/barrierSet.hpp"
+
+class ShenandoahBarrierSet: public BarrierSet {
+private:
+
+  static inline oop get_shenandoah_forwardee_helper(oop p) {
+    assert(UseShenandoahGC, "must only be called when Shenandoah is used.");
+    assert(Universe::heap()->is_in(p), "We shouldn't be calling this on objects not in the heap");
+    oop forwardee;
+#ifdef ASSERT
+    if (ShenandoahVerifyReadsToFromSpace) {
+      ShenandoahHeap* heap = (ShenandoahHeap *) Universe::heap();
+      ShenandoahHeapRegion* region = heap->heap_region_containing(p);
+      {
+        region->memProtectionOff();
+        forwardee = oop( *((HeapWord**) ((HeapWord*) p) - 1));
+        region->memProtectionOn();
+      }
+    } else {
+      forwardee = oop( *((HeapWord**) ((HeapWord*) p) - 1));
+    }
+#else
+    forwardee = oop( *((HeapWord**) ((HeapWord*) p) - 1));
+#endif
+    return forwardee;
+  }
+
+public:
+
+  ShenandoahBarrierSet();
+
+  void print_on(outputStream* st) const;
+
+  bool is_a(BarrierSet::Name bsn);
+
+  bool has_read_prim_array_opt();
+  bool has_read_prim_barrier();
+  bool has_read_ref_array_opt();
+  bool has_read_ref_barrier();
+  bool has_read_region_opt();
+  bool has_write_prim_array_opt();
+  bool has_write_prim_barrier();
+  bool has_write_ref_array_opt();
+  bool has_write_ref_barrier();
+  bool has_write_ref_pre_barrier();
+  bool has_write_region_opt();
+  bool is_aligned(HeapWord* hw);
+  void read_prim_array(MemRegion mr);
+  void read_prim_field(HeapWord* hw, size_t s);
+  bool read_prim_needs_barrier(HeapWord* hw, size_t s);
+  void read_ref_array(MemRegion mr);
+
+  void read_ref_field(void* v);
+
+  bool read_ref_needs_barrier(void* v);
+  void read_region(MemRegion mr);
+  void resize_covered_region(MemRegion mr);
+  void write_prim_array(MemRegion mr);
+  void write_prim_field(HeapWord* hw, size_t s , juint x, juint y);
+  bool write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y);
+  void write_ref_array_work(MemRegion mr);
+
+  template <class T> void
+  write_ref_array_pre_work(T* dst, int count);
+
+  void write_ref_array_pre(oop* dst, int count, bool dest_uninitialized);
+
+  void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized);
+
+
+  template <class T> static void write_ref_field_pre_static(T* field, oop newVal);
+
+  // We export this to make it available in cases where the static
+  // type of the barrier set is known.  Note that it is non-virtual.
+  template <class T> inline void inline_write_ref_field_pre(T* field, oop newVal);
+
+  // These are the more general virtual versions.
+  void write_ref_field_pre_work(oop* field, oop new_val);
+  void write_ref_field_pre_work(narrowOop* field, oop new_val);
+  void write_ref_field_pre_work(void* field, oop new_val);
+
+  void write_ref_field_work(void* v, oop o, bool release = false);
+  void write_region_work(MemRegion mr);
+
+  virtual oop resolve_oop(oop src);
+
+  template <class T>
+  static inline oop resolve_and_update_oop_static(T p, oop obj) {
+    oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
+    if (forw != obj) {
+      obj = forw;
+      oopDesc::encode_store_heap_oop_not_null(p, obj);
+    }
+    return obj;
+  }
+
+  static inline oop resolve_oop_static_not_null(oop p) {
+    assert(p != NULL, "Must be NULL checked");
+
+    oop result = get_shenandoah_forwardee_helper(p);
+
+    if (result != NULL) {
+#ifdef ASSERT
+      if (result != p) {
+        oop second_forwarding = get_shenandoah_forwardee_helper(result);
+
+        // We should never be forwarded more than once.
+        if (result != second_forwarding) {
+          ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+          tty->print("first reference "PTR_FORMAT" is in heap region:\n", p2i((HeapWord*) p));
+          sh->heap_region_containing(p)->print();
+          tty->print("first_forwarding "PTR_FORMAT" is in heap region:\n", p2i((HeapWord*) result));
+          sh->heap_region_containing(result)->print();
+          tty->print("final reference "PTR_FORMAT" is in heap region:\n", p2i((HeapWord*) second_forwarding));
+          sh->heap_region_containing(second_forwarding)->print();
+          assert(get_shenandoah_forwardee_helper(result) == result, "Only one fowarding per customer");
+        }
+      }
+#endif
+      if (! ShenandoahVerifyReadsToFromSpace) {
+        // is_oop() would trigger a SEGFAULT when we're checking from-space-access.
+        assert(ShenandoahHeap::heap()->is_in(result) && result->is_oop(), "resolved oop must be a valid oop in the heap");
+      }
+    }
+    return result;
+  }
+
+  static inline oop resolve_oop_static(oop p) {
+    if (((HeapWord*) p) != NULL) {
+      return resolve_oop_static_not_null(p);
+    } else {
+      return p;
+    }
+  }
+
+  static inline oop resolve_oop_static_no_check(oop p) {
+    if (((HeapWord*) p) != NULL) {
+      return get_shenandoah_forwardee_helper(p);
+    } else {
+      return p;
+    }
+  }
+
+
+  oop resolve_and_maybe_copy_oopHelper(oop src);
+  oop resolve_and_maybe_copy_oop_work(oop src);
+  oop resolve_and_maybe_copy_oop_work2(oop src);
+  virtual oop resolve_and_maybe_copy_oop(oop src);
+
+  static oopDesc* resolve_and_maybe_copy_oop_c2(oopDesc* src);
+  static oopDesc* resolve_and_maybe_copy_oop_interp(oopDesc* src);
+  static oopDesc* resolve_and_maybe_copy_oop_c1(JavaThread* thread, oopDesc* src);
+
+private:
+  bool need_update_refs_barrier();
+
+#ifndef CC_INTERP
+public:
+  virtual void interpreter_read_barrier(MacroAssembler* masm, Register dst);
+  virtual void interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst);
+  void interpreter_write_barrier(MacroAssembler* masm, Register dst);
+
+private:
+  void compile_resolve_oop_runtime(MacroAssembler* masm, Register dst);
+
+#endif
+};
+
+class ShenandoahMarkCompactBarrierSet : public ShenandoahBarrierSet {
+
+   oop resolve_oop(oop src) {
+     return src;
+   }
+   oop maybe_resolve_oop(oop src) {
+     return src;
+   }
+};
+
+#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.cpp
@@ -0,0 +1,779 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+
+class ShenandoahHeuristics : public CHeapObj<mtGC> {
+
+  NumberSeq _allocation_rate_bytes;
+  NumberSeq _reclamation_rate_bytes;
+
+  size_t _bytes_allocated_since_CM;
+  size_t _bytes_reclaimed_this_cycle;
+
+protected:
+  size_t _bytes_allocated_start_CM;
+  size_t _bytes_allocated_during_CM;
+
+public:
+
+  ShenandoahHeuristics();
+
+  void record_bytes_allocated(size_t bytes);
+  void record_bytes_reclaimed(size_t bytes);
+  void record_bytes_start_CM(size_t bytes);
+  void record_bytes_end_CM(size_t bytes);
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const=0;
+  virtual bool update_refs_early();
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set) =0;
+  void print_tracing_info();
+};
+
+ShenandoahHeuristics::ShenandoahHeuristics() :
+  _bytes_allocated_since_CM(0),
+  _bytes_reclaimed_this_cycle(0),
+  _bytes_allocated_start_CM(0),
+  _bytes_allocated_during_CM(0)
+{
+  if (PrintGCDetails)
+    tty->print_cr("initializing heuristics");
+}
+
+void ShenandoahCollectorPolicy::record_phase_start(TimingPhase phase) {
+  _timing_data[phase]._start = os::elapsedTime();
+
+  if (PrintGCTimeStamps) {
+    if (phase == init_mark)
+      _tracer->report_gc_start(GCCause::_shenandoah_init_mark, _conc_timer->gc_start());
+    else if (phase == full_gc)
+      _tracer->report_gc_start(GCCause::_last_ditch_collection, _stw_timer->gc_start());
+
+    gclog_or_tty->gclog_stamp(_tracer->gc_id());
+    gclog_or_tty->print("[GC %s start", _phase_names[phase]);
+    ShenandoahHeap* heap = (ShenandoahHeap*) Universe::heap();
+
+    gclog_or_tty->print(" total = " SIZE_FORMAT " K, used = " SIZE_FORMAT " K free = " SIZE_FORMAT " K", heap->capacity()/ K, heap->used() /K,
+                        ((heap->capacity() - heap->used())/K) );
+
+    if (heap->calculateUsed() != heap->used()) {
+      gclog_or_tty->print("calc used = " SIZE_FORMAT " K heap used = " SIZE_FORMAT " K",
+                            heap->calculateUsed() / K, heap->used() / K);
+    }
+    //    assert(heap->calculateUsed() == heap->used(), "Just checking");
+    gclog_or_tty->print_cr("]");
+  }
+}
+
+void ShenandoahCollectorPolicy::record_phase_end(TimingPhase phase) {
+  double end = os::elapsedTime();
+  double elapsed = end - _timing_data[phase]._start;
+  _timing_data[phase]._ms.add(elapsed * 1000);
+
+  if (ShenandoahGCVerbose && PrintGCDetails) {
+    tty->print_cr("PolicyPrint: %s "SIZE_FORMAT" took %lf ms", _phase_names[phase],
+                  _timing_data[phase]._count++, elapsed * 1000);
+  }
+  if (PrintGCTimeStamps) {
+    ShenandoahHeap* heap = (ShenandoahHeap*) Universe::heap();
+    gclog_or_tty->gclog_stamp(_tracer->gc_id());
+
+    gclog_or_tty->print("[GC %s end, %lf secs", _phase_names[phase], elapsed );
+    gclog_or_tty->print(" total = " SIZE_FORMAT " K, used = " SIZE_FORMAT " K free = " SIZE_FORMAT " K", heap->capacity()/ K, heap->used() /K,
+                        ((heap->capacity() - heap->used())/K) );
+
+    if (heap->calculateUsed() != heap->used()) {
+      gclog_or_tty->print("calc used = " SIZE_FORMAT " K heap used = " SIZE_FORMAT " K",
+                            heap->calculateUsed() / K, heap->used() / K);
+    }
+    //    assert(heap->calculateUsed() == heap->used(), "Stashed heap used must be equal to calculated heap used");
+    gclog_or_tty->print_cr("]");
+
+    if (phase == recycle_regions) {
+      _tracer->report_gc_end(_conc_timer->gc_end(), _conc_timer->time_partitions());
+    } else if (phase == full_gc) {
+      _tracer->report_gc_end(_stw_timer->gc_end(), _stw_timer->time_partitions());
+    } else if (phase == conc_mark || phase == conc_evac || phase == conc_uprefs || phase == prepare_evac) {
+      if (_conc_gc_aborted) {
+        _tracer->report_gc_end(_conc_timer->gc_end(), _conc_timer->time_partitions());
+        clear_conc_gc_aborted();
+      }
+    } else if (phase == final_evac) {
+      ShenandoahHeap* heap = ShenandoahHeap::heap();
+      this->record_bytes_end_CM(heap->_bytesAllocSinceCM);
+    }
+  }
+}
+
+void ShenandoahCollectorPolicy::report_concgc_cancelled() {
+  if (PrintGCTimeStamps)  {
+    gclog_or_tty->print("Concurrent GC Cancelled\n");
+    set_conc_gc_aborted();
+    //    _tracer->report_gc_end(_conc_timer->gc_end(), _conc_timer->time_partitions());
+  }
+}
+
+bool ShenandoahHeuristics::update_refs_early() {
+  return ShenandoahUpdateRefsEarly;
+}
+
+void ShenandoahHeuristics::record_bytes_allocated(size_t bytes) {
+  _bytes_allocated_since_CM = bytes;
+  _bytes_allocated_start_CM = bytes;
+  _allocation_rate_bytes.add(bytes);
+}
+
+void ShenandoahHeuristics::record_bytes_reclaimed(size_t bytes) {
+  _bytes_reclaimed_this_cycle = bytes;
+  _reclamation_rate_bytes.add(bytes);
+}
+
+void ShenandoahHeuristics::record_bytes_start_CM(size_t bytes) {
+  _bytes_allocated_start_CM = bytes;
+}
+
+void ShenandoahHeuristics::record_bytes_end_CM(size_t bytes) {
+  _bytes_allocated_during_CM = (bytes > _bytes_allocated_start_CM) ? (bytes - _bytes_allocated_start_CM)
+                                                                   : bytes;
+}
+
+class AggressiveHeuristics : public ShenandoahHeuristics {
+public:
+  AggressiveHeuristics() : ShenandoahHeuristics(){
+  if (PrintGCDetails)
+    tty->print_cr("Initializing aggressive heuristics");
+  }
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const {
+    return true;
+  }
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set) {
+    region_set->set_garbage_threshold(8);
+    region_set->choose_collection_and_free_sets(collection_set, free_set);
+  }
+};
+
+class HalfwayHeuristics : public ShenandoahHeuristics {
+public:
+  HalfwayHeuristics() : ShenandoahHeuristics() {
+  if (PrintGCDetails)
+    tty->print_cr("Initializing halfway heuristics");
+  }
+
+  bool should_start_concurrent_mark(size_t used, size_t capacity) const {
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    size_t threshold_bytes_allocated = heap->capacity() / 4;
+    if (used * 2 > capacity && heap->_bytesAllocSinceCM > threshold_bytes_allocated)
+      return true;
+    else
+      return false;
+  }
+  void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                       ShenandoahHeapRegionSet* collection_set,
+                                       ShenandoahHeapRegionSet* free_set) {
+    region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2);
+    region_set->choose_collection_and_free_sets(collection_set, free_set);
+  }
+};
+
+// GC as little as possible
+class LazyHeuristics : public ShenandoahHeuristics {
+public:
+  LazyHeuristics() : ShenandoahHeuristics() {
+    if (PrintGCDetails) {
+      tty->print_cr("Initializing lazy heuristics");
+    }
+  }
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const {
+    size_t targetStartMarking = (capacity / 5) * 4;
+    if (used > targetStartMarking) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set) {
+    region_set->choose_collection_and_free_sets(collection_set, free_set);
+  }
+};
+
+// These are the heuristics in place when we made this class
+class StatusQuoHeuristics : public ShenandoahHeuristics {
+public:
+  StatusQuoHeuristics() : ShenandoahHeuristics() {
+    if (PrintGCDetails) {
+      tty->print_cr("Initializing status quo heuristics");
+    }
+  }
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const {
+    size_t targetStartMarking = capacity / 16;
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    size_t threshold_bytes_allocated = heap->capacity() / 4;
+
+    if (used > targetStartMarking
+        && heap->_bytesAllocSinceCM > threshold_bytes_allocated) {
+      // Need to check that an appropriate number of regions have
+      // been allocated since last concurrent mark too.
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set) {
+    region_set->choose_collection_and_free_sets(collection_set, free_set);
+  }
+};
+
+static uintx clamp(uintx value, uintx min, uintx max) {
+  value = MAX2(value, min);
+  value = MIN2(value, max);
+  return value;
+}
+
+static double get_percent(uintx value) {
+  double _percent = static_cast<double>(clamp(value, 0, 100));
+  return _percent / 100.;
+}
+
+class DynamicHeuristics : public ShenandoahHeuristics {
+private:
+  double _free_threshold_factor;
+  double _garbage_threshold_factor;
+  double _allocation_threshold_factor;
+
+  uintx _free_threshold;
+  uintx _garbage_threshold;
+  uintx _allocation_threshold;
+
+public:
+  DynamicHeuristics() : ShenandoahHeuristics() {
+    if (PrintGCDetails) {
+      tty->print_cr("Initializing dynamic heuristics");
+    }
+
+    _free_threshold = 0;
+    _garbage_threshold = 0;
+    _allocation_threshold = 0;
+
+    _free_threshold_factor = 0.;
+    _garbage_threshold_factor = 0.;
+    _allocation_threshold_factor = 0.;
+  }
+
+  virtual ~DynamicHeuristics() {}
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const {
+
+    bool shouldStartConcurrentMark = false;
+
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    size_t available = heap->free_regions()->available();
+    uintx factor = heap->need_update_refs() ? ShenandoahFreeThreshold : ShenandoahInitialFreeThreshold;
+    size_t targetStartMarking = (capacity * factor) / 100;
+
+    size_t threshold_bytes_allocated = heap->capacity() * _allocation_threshold_factor;
+    if (available < targetStartMarking &&
+        heap->_bytesAllocSinceCM > threshold_bytes_allocated)
+    {
+      // Need to check that an appropriate number of regions have
+      // been allocated since last concurrent mark too.
+      shouldStartConcurrentMark = true;
+    }
+
+    if (shouldStartConcurrentMark && ShenandoahTracePhases) {
+      tty->print_cr("Start GC at available: "SIZE_FORMAT", factor: "UINTX_FORMAT", update-refs: %s", available, factor, BOOL_TO_STR(heap->need_update_refs()));
+    }
+    return shouldStartConcurrentMark;
+  }
+
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set)
+  {
+    region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes * _garbage_threshold_factor);
+    region_set->choose_collection_and_free_sets(collection_set, free_set);
+  }
+
+  void set_free_threshold(uintx free_threshold) {
+    this->_free_threshold_factor = get_percent(free_threshold);
+    this->_free_threshold = free_threshold;
+  }
+
+  void set_garbage_threshold(uintx garbage_threshold) {
+    this->_garbage_threshold_factor = get_percent(garbage_threshold);
+    this->_garbage_threshold = garbage_threshold;
+  }
+
+  void set_allocation_threshold(uintx allocationThreshold) {
+    this->_allocation_threshold_factor = get_percent(allocationThreshold);
+    this->_allocation_threshold = allocationThreshold;
+  }
+
+  uintx get_allocation_threshold() {
+    return this->_allocation_threshold;
+  }
+
+  uintx get_garbage_threshold() {
+    return this->_garbage_threshold;
+  }
+
+  uintx get_free_threshold() {
+    return this->_free_threshold;
+  }
+};
+
+
+class AdaptiveHeuristics : public ShenandoahHeuristics {
+private:
+  size_t _max_live_data;
+  double _used_threshold_factor;
+  double _garbage_threshold_factor;
+  double _allocation_threshold_factor;
+
+  uintx _used_threshold;
+  uintx _garbage_threshold;
+  uintx _allocation_threshold;
+
+public:
+  AdaptiveHeuristics() : ShenandoahHeuristics() {
+    if (PrintGCDetails) {
+      tty->print_cr("Initializing dynamic heuristics");
+    }
+
+    _max_live_data = 0;
+
+    _used_threshold = 0;
+    _garbage_threshold = 0;
+    _allocation_threshold = 0;
+
+    _used_threshold_factor = 0.;
+    _garbage_threshold_factor = 0.1;
+    _allocation_threshold_factor = 0.;
+  }
+
+  virtual ~AdaptiveHeuristics() {}
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const {
+
+    ShenandoahHeap* _heap = ShenandoahHeap::heap();
+    bool shouldStartConcurrentMark = false;
+
+    size_t max_live_data = _max_live_data;
+    if (max_live_data == 0) {
+      max_live_data = capacity * 0.2; // Very generous initial value.
+    } else {
+      max_live_data *= 1.3; // Add some wiggle room.
+    }
+    size_t max_cycle_allocated = _heap->_max_allocated_gc;
+    if (max_cycle_allocated == 0) {
+      max_cycle_allocated = capacity * 0.3; // Very generous.
+    } else {
+      max_cycle_allocated *= 1.3; // Add 20% wiggle room. Should be enough.
+    }
+    size_t threshold = _heap->capacity() - max_cycle_allocated - max_live_data;
+    if (used > threshold)
+    {
+      shouldStartConcurrentMark = true;
+    }
+
+    return shouldStartConcurrentMark;
+  }
+
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set)
+  {
+    size_t bytes_alloc = ShenandoahHeap::heap()->_bytesAllocSinceCM;
+    size_t min_garbage =  bytes_alloc/* * 1.1*/;
+    region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes * _garbage_threshold_factor);
+    region_set->choose_collection_and_free_sets_min_garbage(collection_set, free_set, min_garbage);
+    /*
+    tty->print_cr("garbage to be collected: "SIZE_FORMAT, collection_set->garbage());
+    tty->print_cr("objects to be evacuated: "SIZE_FORMAT, collection_set->live_data());
+    */
+    _max_live_data = MAX2(_max_live_data, collection_set->live_data());
+  }
+
+  void set_used_threshold(uintx used_threshold) {
+    this->_used_threshold_factor = get_percent(used_threshold);
+    this->_used_threshold = used_threshold;
+  }
+
+  void set_garbage_threshold(uintx garbage_threshold) {
+    this->_garbage_threshold_factor = get_percent(garbage_threshold);
+    this->_garbage_threshold = garbage_threshold;
+  }
+
+  void set_allocation_threshold(uintx allocationThreshold) {
+    this->_allocation_threshold_factor = get_percent(allocationThreshold);
+    this->_allocation_threshold = allocationThreshold;
+  }
+
+  uintx get_allocation_threshold() {
+    return this->_allocation_threshold;
+  }
+
+  uintx get_garbage_threshold() {
+    return this->_garbage_threshold;
+  }
+
+  uintx get_used_threshold() {
+    return this->_used_threshold;
+  }
+};
+
+class NewAdaptiveHeuristics : public ShenandoahHeuristics {
+private:
+  size_t _max_live_data;
+  double _target_heap_occupancy_factor;
+  double _allocation_threshold_factor;
+  size_t _last_bytesAllocSinceCM;
+
+  uintx _target_heap_occupancy;
+  uintx _allocation_threshold;
+
+public:
+  NewAdaptiveHeuristics() : ShenandoahHeuristics()
+  {
+    if (PrintGCDetails) {
+      tty->print_cr("Initializing newadaptive heuristics");
+    }
+    _max_live_data = 0;
+    _allocation_threshold = 0;
+    _target_heap_occupancy_factor = 0.;
+    _allocation_threshold_factor = 0.;
+    _last_bytesAllocSinceCM = 0;
+  }
+
+  virtual ~NewAdaptiveHeuristics() {}
+
+  virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const
+  {
+      if (this->_bytes_allocated_during_CM > 0) {
+          // Not the first concurrent mark.
+          // _bytes_allocated_during_CM
+          ShenandoahHeap *heap = ShenandoahHeap::heap();
+          size_t threshold_bytes_allocated = heap->capacity() / 4;
+          size_t targetStartMarking = (size_t) capacity * this->_target_heap_occupancy_factor;
+          return (used > targetStartMarking) && (this->_bytes_allocated_during_CM > threshold_bytes_allocated);
+      } else {
+          // First concurrent mark.
+          size_t targetStartMarking = capacity / 2;
+          ShenandoahHeap *heap = ShenandoahHeap::heap();
+          size_t threshold_bytes_allocated = heap->capacity() / 4;
+
+          // Need to check that an appropriate number of regions have
+          // been allocated since last concurrent mark too.
+          return (used > targetStartMarking) && (heap->_bytesAllocSinceCM > threshold_bytes_allocated);
+      }
+  }
+
+  virtual void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                               ShenandoahHeapRegionSet* collection_set,
+                                               ShenandoahHeapRegionSet* free_set)
+  {
+    ShenandoahHeap *_heap = ShenandoahHeap::heap();
+    this->_last_bytesAllocSinceCM = ShenandoahHeap::heap()->_bytesAllocSinceCM;
+    if (this->_last_bytesAllocSinceCM > 0) {
+      size_t min_garbage = this->_last_bytesAllocSinceCM;
+      region_set->choose_collection_and_free_sets_min_garbage(collection_set, free_set, min_garbage);
+    } else {
+      region_set->set_garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2);
+      region_set->choose_collection_and_free_sets(collection_set, free_set);
+    }
+    this->_max_live_data = MAX2(this->_max_live_data, collection_set->live_data());
+  }
+
+  void set_target_heap_occupancy(uintx target_heap_occupancy) {
+    this->_target_heap_occupancy_factor = get_percent(target_heap_occupancy);
+    this->_target_heap_occupancy = target_heap_occupancy;
+  }
+
+  void set_allocation_threshold(uintx allocationThreshold) {
+    this->_allocation_threshold_factor = get_percent(allocationThreshold);
+    this->_allocation_threshold = allocationThreshold;
+  }
+
+  uintx get_allocation_threshold() {
+    return this->_allocation_threshold;
+  }
+
+  uintx get_target_heap_occupancy() {
+    return this->_target_heap_occupancy;
+  }
+};
+
+
+static DynamicHeuristics *configureDynamicHeuristics() {
+  DynamicHeuristics *heuristics = new DynamicHeuristics();
+
+  heuristics->set_garbage_threshold(ShenandoahGarbageThreshold);
+  heuristics->set_allocation_threshold(ShenandoahAllocationThreshold);
+  heuristics->set_free_threshold(ShenandoahFreeThreshold);
+  if (ShenandoahLogConfig) {
+    tty->print_cr("Shenandoah dynamic heuristics thresholds: allocation "SIZE_FORMAT", used "SIZE_FORMAT", garbage "SIZE_FORMAT,
+                  heuristics->get_allocation_threshold(),
+                  heuristics->get_free_threshold(),
+                  heuristics->get_garbage_threshold());
+  }
+  return heuristics;
+}
+
+
+static NewAdaptiveHeuristics* configureNewAdaptiveHeuristics() {
+  NewAdaptiveHeuristics* heuristics = new NewAdaptiveHeuristics();
+
+  heuristics->set_target_heap_occupancy(ShenandoahTargetHeapOccupancy);
+  if (ShenandoahLogConfig) {
+    tty->print_cr( "Shenandoah newadaptive heuristics target heap occupancy: "SIZE_FORMAT,
+                   heuristics->get_target_heap_occupancy() );
+  }
+  return heuristics;
+}
+
+
+ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() {
+
+  ShenandoahHeapRegion::setup_heap_region_size(initial_heap_byte_size(), initial_heap_byte_size());
+
+  initialize_all();
+
+  _tracer = new (ResourceObj::C_HEAP, mtGC) ShenandoahTracer();
+  _stw_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer();
+  _conc_timer = new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer();
+  _user_requested_gcs = 0;
+  _allocation_failure_gcs = 0;
+  _conc_gc_aborted = false;
+
+  _phase_names[init_mark] = "InitMark";
+  _phase_names[final_mark] = "FinalMark";
+  _phase_names[rescan_roots] = "RescanRoots";
+  _phase_names[drain_satb] = "DrainSATB";
+  _phase_names[drain_queues] = "DrainQueues";
+  _phase_names[weakrefs] = "WeakRefs";
+  _phase_names[prepare_evac] = "PrepareEvac";
+  _phase_names[init_evac] = "InitEvac";
+  _phase_names[final_evac] = "FinalEvacuation";
+  _phase_names[final_uprefs] = "FinalUpdateRefs";
+
+  _phase_names[update_roots] = "UpdateRoots";
+  _phase_names[recycle_regions] = "RecycleRegions";
+  _phase_names[reset_bitmaps] = "ResetBitmaps";
+  _phase_names[resize_tlabs] = "ResizeTLABs";
+
+  _phase_names[full_gc] = "FullGC";
+  _phase_names[conc_mark] = "ConcurrentMark";
+  _phase_names[conc_evac] = "ConcurrentEvacuation";
+  _phase_names[conc_uprefs] = "ConcurrentUpdateReferences";
+
+  if (ShenandoahGCHeuristics != NULL) {
+    if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: aggressive");
+      }
+      _heuristics = new AggressiveHeuristics();
+    } else if (strcmp(ShenandoahGCHeuristics, "statusquo") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: statusquo");
+      }
+      _heuristics = new StatusQuoHeuristics();
+    } else if (strcmp(ShenandoahGCHeuristics, "halfway") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: halfway");
+      }
+      _heuristics = new HalfwayHeuristics();
+    } else if (strcmp(ShenandoahGCHeuristics, "lazy") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: lazy");
+      }
+      _heuristics = new LazyHeuristics();
+    } else if (strcmp(ShenandoahGCHeuristics, "dynamic") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: dynamic");
+      }
+      _heuristics = configureDynamicHeuristics();
+    } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: adaptive");
+      }
+      _heuristics = new AdaptiveHeuristics();
+    } else if (strcmp(ShenandoahGCHeuristics, "newadaptive") == 0) {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: newadaptive");
+      }
+      _heuristics = configureNewAdaptiveHeuristics();
+    } else {
+      fatal("Unknown -XX:ShenandoahGCHeuristics option");
+    }
+  } else {
+      if (ShenandoahLogConfig) {
+        tty->print_cr("Shenandoah heuristics: statusquo (default)");
+      }
+    _heuristics = new StatusQuoHeuristics();
+  }
+
+}
+
+ShenandoahCollectorPolicy* ShenandoahCollectorPolicy::as_pgc_policy() {
+  return this;
+}
+
+BarrierSet::Name ShenandoahCollectorPolicy::barrier_set_name() {
+  return BarrierSet::ShenandoahBarrierSet;
+}
+
+HeapWord* ShenandoahCollectorPolicy::mem_allocate_work(size_t size,
+                                                       bool is_tlab,
+                                                       bool* gc_overhead_limit_was_exceeded) {
+  guarantee(false, "Not using this policy feature yet.");
+  return NULL;
+}
+
+HeapWord* ShenandoahCollectorPolicy::satisfy_failed_allocation(size_t size, bool is_tlab) {
+  guarantee(false, "Not using this policy feature yet.");
+  return NULL;
+}
+
+void ShenandoahCollectorPolicy::initialize_alignments() {
+
+  // This is expected by our algorithm for ShenandoahHeap::heap_region_containing().
+  _space_alignment = ShenandoahHeapRegion::RegionSizeBytes;
+  _heap_alignment = ShenandoahHeapRegion::RegionSizeBytes;
+}
+
+void ShenandoahCollectorPolicy::post_heap_initialize() {
+  // Nothing to do here (yet).
+}
+
+void ShenandoahCollectorPolicy::record_bytes_allocated(size_t bytes) {
+  _heuristics->record_bytes_allocated(bytes);
+}
+
+void ShenandoahCollectorPolicy::record_bytes_start_CM(size_t bytes) {
+  _heuristics->record_bytes_start_CM(bytes);
+}
+
+void ShenandoahCollectorPolicy::record_bytes_end_CM(size_t bytes) {
+  _heuristics->record_bytes_end_CM(bytes);
+}
+
+void ShenandoahCollectorPolicy::record_bytes_reclaimed(size_t bytes) {
+  _heuristics->record_bytes_reclaimed(bytes);
+}
+
+void ShenandoahCollectorPolicy::record_user_requested_gc() {
+  _user_requested_gcs++;
+}
+
+void ShenandoahCollectorPolicy::record_allocation_failure_gc() {
+  _allocation_failure_gcs++;
+}
+
+bool ShenandoahCollectorPolicy::should_start_concurrent_mark(size_t used,
+                                                             size_t capacity) {
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  return _heuristics->should_start_concurrent_mark(used, capacity);
+}
+
+bool ShenandoahCollectorPolicy::update_refs_early() {
+  return _heuristics->update_refs_early();
+}
+
+void ShenandoahCollectorPolicy::choose_collection_and_free_sets(
+                             ShenandoahHeapRegionSet* region_set,
+                             ShenandoahHeapRegionSet* collection_set,
+                             ShenandoahHeapRegionSet* free_set) {
+  _heuristics->choose_collection_and_free_sets(region_set, collection_set, free_set);
+}
+
+void ShenandoahCollectorPolicy::print_tracing_info() {
+  print_summary_sd("Initial Mark Pauses", 0, &(_timing_data[init_mark]._ms));
+  print_summary_sd("Final Mark Pauses", 0, &(_timing_data[final_mark]._ms));
+
+  print_summary_sd("Rescan Roots", 2, &(_timing_data[rescan_roots]._ms));
+  print_summary_sd("Drain SATB", 2, &(_timing_data[drain_satb]._ms));
+  print_summary_sd("Drain Queues", 2, &(_timing_data[drain_queues]._ms));
+  if (ShenandoahProcessReferences) {
+    print_summary_sd("Weak References", 2, &(_timing_data[weakrefs]._ms));
+  }
+  print_summary_sd("Prepare Evacuation", 2, &(_timing_data[prepare_evac]._ms));
+  print_summary_sd("Initial Evacuation", 2, &(_timing_data[init_evac]._ms));
+
+  print_summary_sd("Final Evacuation Pauses", 0, &(_timing_data[final_evac]._ms));
+  print_summary_sd("Final Update Refs Pauses", 0, &(_timing_data[final_uprefs]._ms));
+  print_summary_sd("Update roots", 2, &(_timing_data[update_roots]._ms));
+  print_summary_sd("Recycle regions", 2, &(_timing_data[recycle_regions]._ms));
+  print_summary_sd("Reset bitmaps", 2, &(_timing_data[reset_bitmaps]._ms));
+  print_summary_sd("Resize TLABs", 2, &(_timing_data[resize_tlabs]._ms));
+  gclog_or_tty->print_cr(" ");
+  print_summary_sd("Concurrent Marking Times", 0, &(_timing_data[conc_mark]._ms));
+  print_summary_sd("Concurrent Evacuation Times", 0, &(_timing_data[conc_evac]._ms));
+  print_summary_sd("Concurrent Update References Times", 0, &(_timing_data[conc_uprefs]._ms));
+  print_summary_sd("Full GC Times", 0, &(_timing_data[full_gc]._ms));
+
+  gclog_or_tty->print_cr("User requested GCs: "SIZE_FORMAT, _user_requested_gcs);
+  gclog_or_tty->print_cr("Allocation failure GCs: "SIZE_FORMAT, _allocation_failure_gcs);
+
+  gclog_or_tty->print_cr(" ");
+  double total_sum = _timing_data[init_mark]._ms.sum() +
+    _timing_data[final_mark]._ms.sum() +
+    _timing_data[final_evac]._ms.sum() +
+    _timing_data[final_uprefs]._ms.sum();
+  double total_avg = (_timing_data[init_mark]._ms.avg() +
+                      _timing_data[final_mark]._ms.avg() +
+                      _timing_data[final_evac]._ms.avg() +
+                      _timing_data[final_uprefs]._ms.avg()) / 4.0;
+  double total_max = MAX2(
+                          MAX2(
+                               MAX2(_timing_data[init_mark]._ms.maximum(),
+                                    _timing_data[final_mark]._ms.maximum()),
+                               _timing_data[final_evac]._ms.maximum()),
+                          _timing_data[final_uprefs]._ms.maximum());
+
+  gclog_or_tty->print_cr("%-27s = %8.2lf s, avg = %8.2lf ms, max = %8.2lf ms",
+                         "Total", total_sum / 1000.0, total_avg, total_max);
+
+}
+
+void ShenandoahCollectorPolicy::print_summary_sd(const char* str, uint indent, const NumberSeq* seq)  {
+  double sum = seq->sum();
+  for (uint i = 0; i < indent; i++) gclog_or_tty->print(" ");
+  gclog_or_tty->print_cr("%-27s = %8.2lf s (avg = %8.2lf ms)",
+                         str, sum / 1000.0, seq->avg());
+  for (uint i = 0; i < indent; i++) gclog_or_tty->print(" ");
+  gclog_or_tty->print_cr("%s = "INT32_FORMAT_W(5)", std dev = %8.2lf ms, max = %8.2lf ms)",
+                         "(num", seq->num(), seq->sd(), seq->maximum());
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahCollectorPolicy.hpp
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAH_COLLECTOR_POLICY_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAH_COLLECTOR_POLICY_HPP
+
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
+#include "gc/shared/gcTrace.hpp"
+#include "gc/shared/gcTimer.hpp"
+#include "gc/shared/collectorPolicy.hpp"
+#include "runtime/arguments.hpp"
+#include "utilities/numberSeq.hpp"
+
+
+class ShenandoahHeap;
+class ShenandoahHeuristics;
+
+class ShenandoahCollectorPolicy: public CollectorPolicy {
+
+public:
+  enum TimingPhase {
+    init_mark,
+    final_mark,
+    rescan_roots,
+    drain_satb,
+    drain_queues,
+    weakrefs,
+    prepare_evac,
+    init_evac,
+
+    final_evac,
+    final_uprefs,
+    update_roots,
+    recycle_regions,
+    reset_bitmaps,
+    resize_tlabs,
+    full_gc,
+    conc_mark,
+    conc_evac,
+    conc_uprefs,
+
+    _num_phases
+  };
+
+private:
+  struct TimingData {
+    NumberSeq _ms;
+    double _start;
+    size_t _count;
+  };
+
+private:
+  TimingData _timing_data[_num_phases];
+  const char* _phase_names[_num_phases];
+
+  size_t _user_requested_gcs;
+  size_t _allocation_failure_gcs;
+
+  ShenandoahHeap* _pgc;
+  ShenandoahHeuristics* _heuristics;
+  ShenandoahTracer* _tracer;
+  STWGCTimer* _stw_timer;
+  ConcurrentGCTimer* _conc_timer;
+
+  bool _conc_gc_aborted;
+
+public:
+  ShenandoahCollectorPolicy();
+
+  virtual ShenandoahCollectorPolicy* as_pgc_policy();
+
+  BarrierSet::Name barrier_set_name();
+
+  HeapWord* mem_allocate_work(size_t size,
+                              bool is_tlab,
+                              bool* gc_overhead_limit_was_exceeded);
+
+  HeapWord* satisfy_failed_allocation(size_t size, bool is_tlab);
+
+  void initialize_alignments();
+
+  void post_heap_initialize();
+
+  void record_phase_start(TimingPhase phase);
+  void record_phase_end(TimingPhase phase);
+  void report_concgc_cancelled();
+
+  void record_user_requested_gc();
+  void record_allocation_failure_gc();
+
+  void record_bytes_allocated(size_t bytes);
+  void record_bytes_reclaimed(size_t bytes);
+  void record_bytes_start_CM(size_t bytes);
+  void record_bytes_end_CM(size_t bytes);
+  bool should_start_concurrent_mark(size_t used, size_t capacity);
+  void choose_collection_and_free_sets(ShenandoahHeapRegionSet* region_set,
+                                       ShenandoahHeapRegionSet* collection_set,
+                                       ShenandoahHeapRegionSet* free_set);
+
+  bool update_refs_early();
+
+  void print_tracing_info();
+
+  GCTimer* conc_timer(){return _conc_timer;}
+  GCTimer* stw_timer() {return _stw_timer;}
+  ShenandoahTracer* tracer() {return _tracer;}
+
+  void set_conc_gc_aborted() { _conc_gc_aborted = true;}
+  void clear_conc_gc_aborted() {_conc_gc_aborted = false;}
+
+private:
+  void print_summary_sd(const char* str, uint indent, const NumberSeq* seq);
+};
+
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAH_COLLECTOR_POLICY_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.cpp
@@ -0,0 +1,737 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "classfile/stringTable.hpp"
+#include "gc/shared/gcTimer.hpp"
+#include "gc/shared/isGCActiveMark.hpp"
+#include "gc/shared/strongRootsScope.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahRootProcessor.hpp"
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shared/referenceProcessor.hpp"
+#include "code/codeCache.hpp"
+#include "classfile/symbolTable.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "memory/iterator.inline.hpp"
+#include "oops/oop.inline.hpp"
+#include "gc/shared/taskqueue.inline.hpp"
+
+// Mark the object and add it to the queue to be scanned
+ShenandoahMarkObjsClosure::ShenandoahMarkObjsClosure(SCMObjToScanQueue* q, bool update_refs) :
+  _heap((ShenandoahHeap*)(Universe::heap())),
+  _mark_refs(ShenandoahMarkRefsClosure(q, update_refs)),
+  _live_data(NEW_C_HEAP_ARRAY(size_t, _heap->max_regions(), mtGC))
+{
+  Copy::zero_to_bytes(_live_data, _heap->max_regions() * sizeof(size_t));
+}
+
+ShenandoahMarkObjsClosure::~ShenandoahMarkObjsClosure() {
+  // Merge liveness data back into actual regions.
+
+  // We need to lock the heap here, to avoid race with growing of heap.
+  MutexLockerEx ml(ShenandoahHeap_lock, true);
+  ShenandoahHeapRegion** regions = _heap->heap_regions();
+  for (uint i = 0; i < _heap->num_regions(); i++) {
+    regions[i]->increase_live_data(_live_data[i]);
+  }
+  FREE_C_HEAP_ARRAY(size_t, _live_data);
+}
+
+ShenandoahMarkRefsClosure::ShenandoahMarkRefsClosure(SCMObjToScanQueue* q, bool update_refs) :
+  MetadataAwareOopClosure(((ShenandoahHeap *) Universe::heap())->ref_processor()),
+  _queue(q),
+  _heap((ShenandoahHeap*) Universe::heap()),
+  _scm(_heap->concurrentMark()),
+  _update_refs(update_refs)
+{
+}
+
+void ShenandoahMarkRefsClosure::do_oop(narrowOop* p) {
+  Unimplemented();
+}
+
+
+// Walks over all the objects in the generation updating any
+// references to from space.
+
+class CLDMarkAliveClosure : public CLDClosure {
+private:
+  CLDClosure* _cl;
+public:
+  CLDMarkAliveClosure(CLDClosure* cl) : _cl(cl) {
+  }
+  void do_cld(ClassLoaderData* cld) {
+    ShenandoahIsAliveClosure is_alive;
+    if (cld->is_alive(&is_alive)) {
+      _cl->do_cld(cld);
+    }
+  }
+};
+
+class ShenandoahMarkRootsTask : public AbstractGangTask {
+private:
+  ShenandoahRootProcessor* _rp;
+  bool _update_refs;
+public:
+  ShenandoahMarkRootsTask(ShenandoahRootProcessor* rp, bool update_refs) :
+    AbstractGangTask("Shenandoah update roots task"), _update_refs(update_refs),
+    _rp(rp) {
+  }
+
+  void work(uint worker_id) {
+    // tty->print_cr("start mark roots worker: "INT32_FORMAT, worker_id);
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    SCMObjToScanQueue* q = heap->concurrentMark()->get_queue(worker_id);
+    ShenandoahMarkRefsClosure cl(q, _update_refs);
+
+    CodeBlobToOopClosure blobsCl(&cl, true);
+    CLDToOopClosure cldCl(&cl);
+
+    ResourceMark m;
+    if (ShenandoahProcessReferences && ClassUnloadingWithConcurrentMark) {
+      _rp->process_strong_roots(&cl, &cldCl, &blobsCl);
+    } else {
+      _rp->process_all_roots(&cl, &cldCl, &blobsCl);
+    }
+    // tty->print_cr("finish mark roots worker: "INT32_FORMAT, worker_id);
+  }
+};
+
+class SCMConcurrentMarkingTask : public AbstractGangTask {
+private:
+  ShenandoahConcurrentMark* _cm;
+  ParallelTaskTerminator* _terminator;
+  int _seed;
+  bool _update_refs;
+
+public:
+  SCMConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator, bool update_refs) :
+    AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator), _update_refs(update_refs), _seed(17) {
+  }
+
+
+  void work(uint worker_id) {
+
+    SCMObjToScanQueue* q = _cm->get_queue(worker_id);
+    ShenandoahMarkObjsClosure cl(q, _update_refs);
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    while (true) {
+      if (heap->cancelled_concgc() ||
+          (!_cm->try_queue(q, &cl) &&
+           !_cm->try_draining_an_satb_buffer(worker_id) &&
+           !_cm->try_to_steal(worker_id, &cl, &_seed))
+          ) {
+        if (_terminator->offer_termination()) break;
+      }
+    }
+    if (ShenandoahTracePhases && heap->cancelled_concgc()) {
+      tty->print_cr("Cancelled concurrent marking");
+    }
+  }
+};
+
+void ShenandoahConcurrentMark::prepare_unmarked_root_objs() {
+
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  bool update_refs = heap->need_update_refs();
+
+  if (update_refs) {
+    COMPILER2_PRESENT(DerivedPointerTable::clear());
+  }
+
+  prepare_unmarked_root_objs_no_derived_ptrs(update_refs);
+
+  if (update_refs) {
+    COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+  }
+
+}
+
+void ShenandoahConcurrentMark::prepare_unmarked_root_objs_no_derived_ptrs(bool update_refs) {
+  assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
+
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  if (ShenandoahParallelRootScan) {
+
+    ClassLoaderDataGraph::clear_claimed_marks();
+    heap->conc_workers()->set_active_workers(_max_conc_worker_id);
+    ShenandoahRootProcessor root_proc(heap, _max_conc_worker_id);
+    TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
+    ShenandoahMarkRootsTask mark_roots(&root_proc, update_refs);
+    heap->conc_workers()->run_task(&mark_roots);
+
+    // Mark through any class loaders that have been found alive.
+    ShenandoahMarkRefsClosure cl(get_queue(0), update_refs);
+    CLDToOopClosure cldCl(&cl);
+    CLDMarkAliveClosure cld_keep_alive(&cldCl);
+    ClassLoaderDataGraph::roots_cld_do(NULL, &cld_keep_alive);
+
+  } else {
+    ShenandoahMarkRefsClosure cl(get_queue(0), update_refs);
+    heap->roots_iterate(&cl);
+  }
+
+  if (!(ShenandoahProcessReferences && ClassUnloadingWithConcurrentMark)) {
+    ShenandoahMarkRefsClosure cl(get_queue(0), update_refs);
+    heap->weak_roots_iterate(&cl);
+  }
+
+  // tty->print_cr("all root marker threads done");
+}
+
+
+void ShenandoahConcurrentMark::initialize() {
+  _max_conc_worker_id = MAX2((uint) ConcGCThreads, 1U);
+  _task_queues = new SCMObjToScanQueueSet((int) _max_conc_worker_id);
+
+  for (uint i = 0; i < _max_conc_worker_id; ++i) {
+    SCMObjToScanQueue* task_queue = new SCMObjToScanQueue();
+    task_queue->initialize();
+    _task_queues->register_queue(i, task_queue);
+  }
+  JavaThread::satb_mark_queue_set().set_buffer_size(1014 /* G1SATBBufferSize */);
+}
+
+void ShenandoahConcurrentMark::mark_from_roots() {
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("STOPPING THE WORLD: before marking");
+    tty->print_cr("Starting markFromRoots");
+  }
+
+  ShenandoahHeap* sh = (ShenandoahHeap *) Universe::heap();
+
+  bool update_refs = sh->need_update_refs();
+
+  sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::conc_mark);
+  ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues);
+
+  if (ShenandoahProcessReferences) {
+    ReferenceProcessor* rp = sh->ref_processor();
+    // enable ("weak") refs discovery
+    rp->enable_discovery(true /*verify_no_refs*/);
+    rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
+  }
+
+  SCMConcurrentMarkingTask markingTask = SCMConcurrentMarkingTask(this, &terminator, update_refs);
+  sh->conc_workers()->set_active_workers(_max_conc_worker_id);
+  sh->conc_workers()->run_task(&markingTask);
+
+  if (ShenandoahGCVerbose) {
+    tty->print("total workers = %u active workers = %u\n",
+               sh->conc_workers()->total_workers(),
+               sh->conc_workers()->active_workers());
+    TASKQUEUE_STATS_ONLY(print_taskqueue_stats());
+    TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
+  }
+
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("Finishing markFromRoots");
+    tty->print_cr("RESUMING THE WORLD: after marking");
+  }
+
+  sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::conc_mark);
+}
+
+class FinishDrainSATBBuffersTask : public AbstractGangTask {
+private:
+  ShenandoahConcurrentMark* _cm;
+  ParallelTaskTerminator* _terminator;
+public:
+  FinishDrainSATBBuffersTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator) :
+    AbstractGangTask("Finish draining SATB buffers"), _cm(cm), _terminator(terminator) {
+  }
+
+  void work(uint worker_id) {
+    _cm->drain_satb_buffers(worker_id, true);
+  }
+};
+
+class ShenandoahUpdateAliveRefs : public OopClosure {
+private:
+  ShenandoahHeap* _heap;
+public:
+  ShenandoahUpdateAliveRefs() : _heap(ShenandoahHeap::heap()) {
+  }
+  virtual void do_oop(oop* p) {
+    _heap->maybe_update_oop_ref(p);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+};
+
+void ShenandoahConcurrentMark::finish_mark_from_roots() {
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("Starting finishMarkFromRoots");
+  }
+
+  IsGCActiveMark is_active;
+
+  ShenandoahHeap* sh = (ShenandoahHeap *) Universe::heap();
+
+  // Trace any (new) unmarked root references.
+  sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::rescan_roots);
+  prepare_unmarked_root_objs();
+  sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::rescan_roots);
+  sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::drain_satb);
+  {
+    StrongRootsScope scope(_max_conc_worker_id);
+    ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues);
+    // drain_satb_buffers(0, true);
+    FinishDrainSATBBuffersTask drain_satb_buffers(this, &terminator);
+    sh->conc_workers()->set_active_workers(_max_conc_worker_id);
+    sh->conc_workers()->run_task(&drain_satb_buffers);
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::drain_satb);
+  }
+
+  // Finally mark everything else we've got in our queues during the previous steps.
+  {
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::drain_queues);
+    ParallelTaskTerminator terminator(_max_conc_worker_id, _task_queues);
+    SCMConcurrentMarkingTask markingTask = SCMConcurrentMarkingTask(this, &terminator, sh->need_update_refs());
+    sh->conc_workers()->set_active_workers(_max_conc_worker_id);
+    sh->conc_workers()->run_task(&markingTask);
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::drain_queues);
+  }
+
+#ifdef ASSERT
+  for (int i = 0; i < (int) _max_conc_worker_id; i++) {
+    assert(_task_queues->queue(i)->is_empty(), "Should be empty");
+  }
+#endif
+
+  // When we're done marking everything, we process weak references.
+  if (ShenandoahProcessReferences) {
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::weakrefs);
+    weak_refs_work();
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::weakrefs);
+  }
+
+#ifdef ASSERT
+  for (int i = 0; i < (int) _max_conc_worker_id; i++) {
+    assert(_task_queues->queue(i)->is_empty(), "Should be empty");
+  }
+#endif
+
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("Finishing finishMarkFromRoots");
+#ifdef SLOWDEBUG
+    for (int i = 0; i <(int)_max_conc_worker_id; i++) {
+      tty->print("Queue: "INT32_FORMAT":", i);
+      _task_queues->queue(i)->stats.print(tty, 10);
+      tty->cr();
+      _task_queues->queue(i)->stats.verify();
+    }
+#endif
+  }
+
+  // We still need to update (without marking) alive refs in JNI handles.
+  if (ShenandoahProcessReferences && ClassUnloadingWithConcurrentMark) {
+    ShenandoahUpdateAliveRefs cl;
+    ShenandoahIsAliveClosure is_alive;
+    JNIHandles::weak_oops_do(&is_alive, &cl);
+  }
+
+#ifdef ASSERT
+  verify_roots();
+
+  if (ShenandoahDumpHeapAfterConcurrentMark) {
+    sh->ensure_parsability(false);
+    sh->print_all_refs("post-mark");
+  }
+#endif
+}
+
+#ifdef ASSERT
+void ShenandoahVerifyRootsClosure1::do_oop(oop* p) {
+  oop obj = oopDesc::load_heap_oop(p);
+  if (! oopDesc::is_null(obj)) {
+    guarantee(ShenandoahHeap::heap()->is_marked_current(obj), "oop must be marked");
+    guarantee(obj == ShenandoahBarrierSet::resolve_oop_static_not_null(obj), "oop must not be forwarded");
+  }
+}
+
+void ShenandoahConcurrentMark::verify_roots() {
+  ShenandoahVerifyRootsClosure1 cl;
+  CodeBlobToOopClosure blobsCl(&cl, true);
+  CLDToOopClosure cldCl(&cl);
+  ClassLoaderDataGraph::clear_claimed_marks();
+  ShenandoahRootProcessor rp(ShenandoahHeap::heap(), 1);
+  rp.process_roots(&cl, &cl, &cldCl, &cldCl, &cldCl, &blobsCl);
+}
+#endif
+
+class ShenandoahSATBBufferClosure : public SATBBufferClosure {
+private:
+  SCMObjToScanQueue* _queue;
+
+public:
+  ShenandoahSATBBufferClosure(SCMObjToScanQueue* q) :
+    _queue(q)
+  {
+  }
+
+  void do_buffer(void** buffer, size_t size) {
+    // tty->print_cr("draining one satb buffer");
+    for (size_t i = 0; i < size; ++i) {
+      void* entry = buffer[i];
+      oop obj = oop(entry);
+      // tty->print_cr("satb buffer entry: "PTR_FORMAT, p2i((HeapWord*) obj));
+      if (!oopDesc::is_null(obj)) {
+        obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
+        bool pushed = _queue->push(obj);
+        assert(pushed, "overflow queue should always succeed pushing");
+      }
+    }
+  }
+};
+
+class ShenandoahSATBThreadsClosure : public ThreadClosure {
+  ShenandoahSATBBufferClosure* _satb_cl;
+  int _thread_parity;
+
+ public:
+  ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) :
+    _satb_cl(satb_cl),
+    _thread_parity(Threads::thread_claim_parity()) {}
+
+  void do_thread(Thread* thread) {
+    if (thread->is_Java_thread()) {
+      if (thread->claim_oops_do(true, _thread_parity)) {
+        JavaThread* jt = (JavaThread*)thread;
+        jt->satb_mark_queue().apply_closure_and_empty(_satb_cl);
+      }
+    } else if (thread->is_VM_thread()) {
+      if (thread->claim_oops_do(true, _thread_parity)) {
+        JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
+      }
+    }
+  }
+};
+
+void ShenandoahConcurrentMark::drain_satb_buffers(uint worker_id, bool remark) {
+
+  // tty->print_cr("start draining SATB buffers");
+
+  ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+  SCMObjToScanQueue* q = get_queue(worker_id);
+  ShenandoahSATBBufferClosure cl(q);
+
+  SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+  while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
+
+  if (remark) {
+    ShenandoahSATBThreadsClosure tc(&cl);
+    Threads::threads_do(&tc);
+  }
+
+  // tty->print_cr("end draining SATB buffers");
+
+}
+
+bool ShenandoahConcurrentMark::drain_one_satb_buffer(uint worker_id) {
+
+  ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+  SCMObjToScanQueue* q = get_queue(worker_id);
+  ShenandoahSATBBufferClosure cl(q);
+
+  SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+  bool result = satb_mq_set.apply_closure_to_completed_buffer(&cl);
+  return result;
+}
+
+#if TASKQUEUE_STATS
+void ShenandoahConcurrentMark::print_taskqueue_stats_hdr(outputStream* const st) {
+  st->print_raw_cr("GC Task Stats");
+  st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr();
+  st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr();
+}
+
+void ShenandoahConcurrentMark::print_taskqueue_stats(outputStream* const st) const {
+  print_taskqueue_stats_hdr(st);
+  ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+  TaskQueueStats totals;
+  const int n = sh->max_conc_workers();
+  for (int i = 0; i < n; ++i) {
+    st->print(INT32_FORMAT_W(3), i);
+    _task_queues->queue(i)->stats.print(st);
+    st->print("\n");
+    totals += _task_queues->queue(i)->stats;
+  }
+  st->print_raw("tot "); totals.print(st); st->cr();
+  DEBUG_ONLY(totals.verify());
+
+}
+
+void ShenandoahConcurrentMark::print_push_only_taskqueue_stats(outputStream* const st) const {
+  print_taskqueue_stats_hdr(st);
+  ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+  TaskQueueStats totals;
+  const int n = sh->max_conc_workers();
+  for (int i = 0; i < n; ++i) {
+    st->print(INT32_FORMAT_W(3), i);
+    _task_queues->queue(i)->stats.print(st);
+    st->print("\n");
+    totals += _task_queues->queue(i)->stats;
+  }
+  st->print_raw("tot "); totals.print(st); st->cr();
+
+  DEBUG_ONLY(totals.verify_only_pushes());
+}
+
+void ShenandoahConcurrentMark::reset_taskqueue_stats() {
+  ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+  const int n = sh->max_conc_workers();
+  for (int i = 0; i < n; ++i) {
+    _task_queues->queue(i)->stats.reset();
+  }
+}
+#endif // TASKQUEUE_STATS
+
+// Weak Reference Closures
+class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
+  ShenandoahHeap* _sh;
+  ShenandoahConcurrentMark* _scm;
+  uint _worker_id;
+  int _seed;
+
+public:
+  ShenandoahCMDrainMarkingStackClosure(uint worker_id): _worker_id(worker_id), _seed(17) {
+    _sh = (ShenandoahHeap*) Universe::heap();
+    _scm = _sh->concurrentMark();
+  }
+
+
+  void do_void() {
+
+    SCMObjToScanQueue* q = _scm->get_queue(_worker_id);
+    ShenandoahMarkObjsClosure cl(q, _sh->need_update_refs());
+    while (true) {
+      if (!_scm->try_queue(q, &cl) &&
+          !_scm->try_draining_an_satb_buffer(_worker_id) &&
+          !_scm->try_to_steal(_worker_id, &cl, &_seed)) {
+        break;
+      }
+    }
+  }
+};
+
+
+class ShenandoahCMKeepAliveAndDrainClosure: public OopClosure {
+  SCMObjToScanQueue* _queue;
+  ShenandoahHeap* _sh;
+  ShenandoahConcurrentMark* _scm;
+
+  size_t _ref_count;
+
+public:
+  ShenandoahCMKeepAliveAndDrainClosure(SCMObjToScanQueue* q) :
+    _queue(q) {
+    _sh = (ShenandoahHeap*) Universe::heap();
+    _scm = _sh->concurrentMark();
+    _ref_count = 0;
+  }
+
+  virtual void do_oop(oop* p){ do_oop_work(p);}
+  virtual void do_oop(narrowOop* p) {
+    assert(false, "narrowOops Aren't implemented");
+  }
+
+
+  void do_oop_work(oop* p) {
+
+    oop obj;
+    if (_sh->need_update_refs()) {
+      obj = _sh->maybe_update_oop_ref(p);
+    } else {
+      obj = oopDesc::load_heap_oop(p);
+    }
+
+    assert(obj == oopDesc::bs()->resolve_oop(obj), "only get updated oops in weak ref processing");
+
+    if (obj != NULL) {
+      if (Verbose && ShenandoahTraceWeakReferences) {
+        gclog_or_tty->print_cr("\twe're looking at location "
+                               "*"PTR_FORMAT" = "PTR_FORMAT,
+                               p2i(p), p2i((void*) obj));
+        obj->print();
+      }
+      bool pushed = _queue->push(obj);
+      assert(pushed, "overflow queue should always succeed pushing");
+
+      _ref_count++;
+    }
+  }
+
+  size_t ref_count() { return _ref_count; }
+
+};
+
+class ShenandoahRefProcTaskProxy : public AbstractGangTask {
+
+private:
+  AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
+
+public:
+
+  ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task) :
+    AbstractGangTask("Process reference objects in parallel"),
+    _proc_task(proc_task) {
+  }
+
+  void work(uint worker_id) {
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    ShenandoahIsAliveClosure is_alive;
+    ShenandoahCMKeepAliveAndDrainClosure keep_alive(heap->concurrentMark()->get_queue(worker_id));
+    ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id);
+    _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
+  }
+};
+
+class ShenandoahRefEnqueueTaskProxy : public AbstractGangTask {
+
+private:
+  AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task;
+
+public:
+
+  ShenandoahRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) :
+    AbstractGangTask("Enqueue reference objects in parallel"),
+    _enqueue_task(enqueue_task) {
+  }
+
+  void work(uint worker_id) {
+    _enqueue_task.work(worker_id);
+  }
+};
+
+class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
+
+private:
+  WorkGang* _workers;
+
+public:
+
+  ShenandoahRefProcTaskExecutor() : _workers(ShenandoahHeap::heap()->conc_workers()) {
+  }
+
+  // Executes a task using worker threads.
+  void execute(ProcessTask& task) {
+    ShenandoahRefProcTaskProxy proc_task_proxy(task);
+    _workers->run_task(&proc_task_proxy);
+  }
+
+  void execute(EnqueueTask& task) {
+    ShenandoahRefEnqueueTaskProxy enqueue_task_proxy(task);
+    _workers->run_task(&enqueue_task_proxy);
+  }
+};
+
+
+void ShenandoahConcurrentMark::weak_refs_work() {
+   ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+   ReferenceProcessor* rp = sh->ref_processor();
+
+   // Setup collector policy for softref cleaning.
+   bool clear_soft_refs = sh->collector_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/);
+   if (ShenandoahTraceWeakReferences) {
+     tty->print_cr("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs));
+   }
+   rp->setup_policy(clear_soft_refs);
+
+   uint serial_worker_id = 0;
+   ShenandoahIsAliveClosure is_alive;
+   ShenandoahCMKeepAliveAndDrainClosure keep_alive(get_queue(serial_worker_id));
+   ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id);
+   ShenandoahRefProcTaskExecutor par_task_executor;
+   bool processing_is_mt = true;
+   AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
+
+   if (ShenandoahTraceWeakReferences) {
+     gclog_or_tty->print_cr("start processing references");
+   }
+
+   rp->process_discovered_references(&is_alive, &keep_alive,
+                                     &complete_gc, &par_task_executor,
+                                     NULL,
+                                     ShenandoahHeap::heap()->tracer()->gc_id());
+
+   if (ShenandoahTraceWeakReferences) {
+     gclog_or_tty->print_cr("finished processing references, processed "SIZE_FORMAT" refs", keep_alive.ref_count());
+     gclog_or_tty->print_cr("start enqueuing references");
+   }
+
+   rp->enqueue_discovered_references(executor);
+
+   if (ShenandoahTraceWeakReferences) {
+     gclog_or_tty->print_cr("finished enqueueing references");
+   }
+
+   rp->verify_no_references_recorded();
+   assert(!rp->discovery_enabled(), "Post condition");
+
+   if (ClassUnloadingWithConcurrentMark) {
+     // Unload classes and purge SystemDictionary.
+     bool purged_class = SystemDictionary::do_unloading(&is_alive);
+     // Unload nmethods.
+     CodeCache::do_unloading(&is_alive, purged_class);
+     // Prune dead klasses from subklass/sibling/implementor lists.
+     Klass::clean_weak_klass_links(&is_alive);
+     // Delete entries from dead interned strings.
+     StringTable::unlink(&is_alive);
+     // Clean up unreferenced symbols in symbol table.
+     SymbolTable::unlink();
+
+     ClassLoaderDataGraph::purge();
+   }
+}
+
+void ShenandoahConcurrentMark::cancel() {
+  ShenandoahHeap* sh = ShenandoahHeap::heap();
+
+  // Cancel weak-ref discovery.
+  if (ShenandoahProcessReferences) {
+    ReferenceProcessor* rp = sh->ref_processor();
+    rp->abandon_partial_discovery();
+    rp->disable_discovery();
+  }
+
+  // Clean up marking stacks.
+  SCMObjToScanQueueSet* queues = task_queues();
+  for (uint i = 0; i < _max_conc_worker_id; ++i) {
+    SCMObjToScanQueue* task_queue = queues->queue(i);
+    task_queue->set_empty();
+    task_queue->overflow_stack()->clear();
+  }
+
+  // Cancel SATB buffers.
+  JavaThread::satb_mark_queue_set().abandon_partial_marking();
+}
+
+SCMObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
+  worker_id = worker_id % _max_conc_worker_id;
+  return _task_queues->queue(worker_id);
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.hpp
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP
+
+#include "gc/shared/taskqueue.hpp"
+#include "gc/shared/workgroup.hpp"
+
+typedef OverflowTaskQueue<oop, mtGC> OopOverflowTaskQueue;
+typedef Padded<OopOverflowTaskQueue> SCMObjToScanQueue;
+typedef GenericTaskQueueSet<SCMObjToScanQueue, mtGC> SCMObjToScanQueueSet;
+
+class ShenandoahConcurrentMark;
+
+#ifdef ASSERT
+class ShenandoahVerifyRootsClosure1 : public OopClosure {
+  void do_oop(oop* p);
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+};
+#endif
+
+class ShenandoahMarkRefsClosure : public MetadataAwareOopClosure {
+  SCMObjToScanQueue* _queue;
+  ShenandoahHeap* _heap;
+  bool _update_refs;
+  ShenandoahConcurrentMark* _scm;
+
+public:
+  ShenandoahMarkRefsClosure(SCMObjToScanQueue* q, bool update_refs);
+
+  void do_oop(narrowOop* p);
+
+  inline void do_oop(oop* p);
+
+};
+
+class ShenandoahMarkObjsClosure : public ObjectClosure {
+  ShenandoahHeap* _heap;
+  size_t* _live_data;
+  ShenandoahMarkRefsClosure _mark_refs;
+
+public:
+  ShenandoahMarkObjsClosure(SCMObjToScanQueue* q, bool update_refs);
+
+  ~ShenandoahMarkObjsClosure();
+
+  inline void do_object(oop obj);
+};
+
+class ShenandoahConcurrentMark: public CHeapObj<mtGC> {
+
+private:
+  // The per-worker-thread work queues
+  SCMObjToScanQueueSet* _task_queues;
+
+  bool                    _aborted;
+  uint _max_conc_worker_id;
+  ParallelTaskTerminator* _terminator;
+
+public:
+  // We need to do this later when the heap is already created.
+  void initialize();
+
+  void mark_from_roots();
+
+  // Prepares unmarked root objects by marking them and putting
+  // them into the marking task queue.
+  void prepare_unmarked_root_objs();
+  void prepare_unmarked_root_objs_no_derived_ptrs(bool update_refs);
+
+  void finish_mark_from_roots();
+  // Those are only needed public because they're called from closures.
+
+  SCMObjToScanQueue* get_queue(uint worker_id);
+  inline bool try_queue(SCMObjToScanQueue* q, ShenandoahMarkObjsClosure* cl);
+  inline bool try_to_steal(uint worker_id, ShenandoahMarkObjsClosure* cl, int *seed);
+  inline bool try_draining_an_satb_buffer(uint worker_id);
+  void drain_satb_buffers(uint worker_id, bool remark = false);
+  SCMObjToScanQueueSet* task_queues() { return _task_queues;}
+  uint max_conc_worker_id() { return _max_conc_worker_id; }
+
+  void cancel();
+
+private:
+
+#ifdef ASSERT
+  void verify_roots();
+#endif
+
+  bool drain_one_satb_buffer(uint worker_id);
+  void weak_refs_work();
+
+  ParallelTaskTerminator* terminator() { return _terminator;}
+
+#if TASKQUEUE_STATS
+  static void print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty);
+  void print_taskqueue_stats(outputStream* const st = gclog_or_tty) const;
+  void print_push_only_taskqueue_stats(outputStream* const st = gclog_or_tty) const;
+  void reset_taskqueue_stats();
+#endif // TASKQUEUE_STATS
+
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.inline.hpp b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.inline.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentMark.inline.hpp
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
+
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
+#include "memory/iterator.inline.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/prefetch.inline.hpp"
+
+void ShenandoahMarkRefsClosure::do_oop(oop* p) {
+  // We piggy-back reference updating to the marking tasks.
+#ifdef ASSERT
+  oop* old = p;
+#endif
+  oop obj;
+  if (_update_refs) {
+    obj = _heap->maybe_update_oop_ref(p);
+  } else {
+    obj = oopDesc::load_heap_oop(p);
+  }
+  assert(obj == ShenandoahBarrierSet::resolve_oop_static(obj), "need to-space object here");
+
+#ifdef ASSERT
+  if (ShenandoahTraceUpdates) {
+    if (p != old)
+      tty->print_cr("Update "PTR_FORMAT" => "PTR_FORMAT"  to "PTR_FORMAT" => "PTR_FORMAT, p2i(p), p2i((HeapWord*) *p), p2i(old), p2i((HeapWord*) *old));
+    else
+      tty->print_cr("Not updating "PTR_FORMAT" => "PTR_FORMAT"  to "PTR_FORMAT" => "PTR_FORMAT, p2i(p), p2i((HeapWord*) *p), p2i(old), p2i((HeapWord*) *old));
+  }
+#endif
+
+  // NOTE: We used to assert the following here. This does not always work because
+  // a concurrent Java thread could change the the field after we updated it.
+  // oop obj = oopDesc::load_heap_oop(p);
+  // assert(oopDesc::bs()->resolve_oop(obj) == *p, "we just updated the referrer");
+  // assert(obj == NULL || ! _heap->heap_region_containing(obj)->is_dirty(), "must not point to dirty region");
+
+  //  ShenandoahExtendedMarkObjsClosure cl(_heap->ref_processor(), _worker_id);
+  //  ShenandoahMarkObjsClosure mocl(cl, _worker_id);
+
+  if (obj != NULL) {
+    if (_update_refs) {
+      Prefetch::write(obj, 128);
+    } else {
+      Prefetch::read(obj, 128);
+    }
+
+#ifdef ASSERT
+    uint region_idx  = _heap->heap_region_index_containing(obj);
+    ShenandoahHeapRegion* r = _heap->heap_regions()[region_idx];
+    assert(r->bottom() < (HeapWord*) obj && r->top() > (HeapWord*) obj, "object must be in region");
+#endif
+
+    bool pushed = _queue->push(obj);
+    assert(pushed, "overflow queue should always succeed pushing");
+  }
+}
+
+void ShenandoahMarkObjsClosure::do_object(oop obj) {
+
+  assert(obj != NULL, "expect non-null object");
+
+  assert(obj == ShenandoahBarrierSet::resolve_oop_static_not_null(obj), "expect forwarded obj in queue");
+
+#ifdef ASSERT
+  if (_heap->heap_region_containing(obj)->is_in_collection_set()) {
+    tty->print_cr("trying to mark obj: "PTR_FORMAT" (%s) in dirty region: ", p2i((HeapWord*) obj), BOOL_TO_STR(_heap->is_marked_current(obj)));
+    //      _heap->heap_region_containing(obj)->print();
+    //      _heap->print_heap_regions();
+  }
+#endif
+  assert(_heap->cancelled_concgc()
+         || ! _heap->heap_region_containing(obj)->is_in_collection_set(),
+         "we don't want to mark objects in from-space");
+  assert(_heap->is_in(obj), "referenced objects must be in the heap. No?");
+  if (_heap->mark_current(obj)) {
+#ifdef ASSERT
+    if (ShenandoahTraceConcurrentMarking) {
+      tty->print_cr("marked obj: "PTR_FORMAT, p2i((HeapWord*) obj));
+    }
+#endif
+
+    // Calculate liveness of heap region containing object.
+    uint region_idx  = _heap->heap_region_index_containing(obj);
+#ifdef ASSERT
+    ShenandoahHeapRegion* r = _heap->heap_regions()[region_idx];
+    assert(r->bottom() < (HeapWord*) obj && r->top() > (HeapWord*) obj, "object must be in region");
+#endif
+    _live_data[region_idx] += (obj->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE) * HeapWordSize;
+    obj->oop_iterate(&_mark_refs);
+  }
+
+#ifdef ASSERT
+  else {
+    if (ShenandoahTraceConcurrentMarking) {
+      tty->print_cr("failed to mark obj (already marked): "PTR_FORMAT, p2i((HeapWord*) obj));
+    }
+    assert(_heap->is_marked_current(obj), "make sure object is marked");
+  }
+#endif
+}
+
+inline bool ShenandoahConcurrentMark::try_queue(SCMObjToScanQueue* q, ShenandoahMarkObjsClosure* cl) {
+  oop obj;
+  if (q->pop_local(obj)) {
+    assert(obj != NULL, "Can't mark null");
+    cl->do_object(obj);
+    return true;
+  } else if (q->pop_overflow(obj)) {
+    cl->do_object(obj);
+    return true;
+  } else {
+    return false;
+  }
+}
+
+inline bool ShenandoahConcurrentMark::try_to_steal(uint worker_id, ShenandoahMarkObjsClosure* cl, int *seed) {
+  oop obj;
+  if (task_queues()->steal(worker_id, seed, obj)) {
+    cl->do_object(obj);
+    return true;
+  } else
+    return false;
+}
+
+inline bool ShenandoahConcurrentMark:: try_draining_an_satb_buffer(uint worker_id) {
+  return drain_one_satb_buffer(worker_id);
+}
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.cpp b/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.cpp
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahConcurrentThread.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahJNICritical.hpp"
+#include "gc/shenandoah/vm_operations_shenandoah.hpp"
+#include "memory/iterator.hpp"
+#include "memory/universe.hpp"
+#include "runtime/vmThread.hpp"
+
+SurrogateLockerThread* ShenandoahConcurrentThread::_slt = NULL;
+
+ShenandoahConcurrentThread::ShenandoahConcurrentThread() :
+  ConcurrentGCThread(),
+  _epoch(0),
+  _concurrent_mark_started(false),
+  _concurrent_mark_in_progress(false),
+  _do_full_gc(false),
+  _concurrent_mark_aborted(false)
+{
+  create_and_start();
+}
+
+ShenandoahConcurrentThread::~ShenandoahConcurrentThread() {
+  // This is here so that super is called.
+}
+
+void ShenandoahConcurrentThread::run() {
+  initialize_in_thread();
+
+  wait_for_universe_init();
+
+  // Wait until we have the surrogate locker thread in place.
+  {
+    MutexLockerEx x(CGC_lock, true);
+    while(_slt == NULL && !_should_terminate) {
+      CGC_lock->wait(true, 200);
+    }
+  }
+
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+  while (!_should_terminate) {
+    if (_do_full_gc) {
+      {
+        if (_full_gc_cause == GCCause::_allocation_failure) {
+          heap->shenandoahPolicy()->record_allocation_failure_gc();
+        } else {
+          heap->shenandoahPolicy()->record_user_requested_gc();
+        }
+
+        VM_ShenandoahFullGC full_gc;
+        heap->jni_critical()->execute_in_vm_thread(&full_gc);
+      }
+      MonitorLockerEx ml(ShenandoahFullGC_lock);
+      _do_full_gc = false;
+      ml.notify_all();
+    } else if (heap->shenandoahPolicy()->should_start_concurrent_mark(heap->used(),
+                                                               heap->capacity()))
+      {
+
+        if (ShenandoahGCVerbose)
+          tty->print("Capacity = "SIZE_FORMAT" Used = "SIZE_FORMAT"  doing initMark\n", heap->capacity(), heap->used());
+
+        if (ShenandoahGCVerbose) tty->print("Starting a mark");
+
+        VM_ShenandoahInitMark initMark;
+        VMThread::execute(&initMark);
+
+        if (ShenandoahConcurrentMarking) {
+          ShenandoahHeap::heap()->concurrentMark()->mark_from_roots();
+
+          VM_ShenandoahStartEvacuation finishMark;
+          heap->jni_critical()->execute_in_vm_thread(&finishMark);
+        }
+
+        if (cm_has_aborted()) {
+          clear_cm_aborted();
+          assert(heap->is_bitmap_clear(), "need to continue with clear mark bitmap");
+          assert(! heap->concurrent_mark_in_progress(), "concurrent mark must have terminated");
+          continue;
+        }
+        if (! _should_terminate) {
+          // If we're not concurrently evacuating, evacuation is done
+          // from VM_ShenandoahFinishMark within the VMThread above.
+          if (ShenandoahConcurrentEvacuation) {
+            VM_ShenandoahEvacuation evacuation;
+            evacuation.doit();
+          }
+        }
+
+        if (heap->shenandoahPolicy()->update_refs_early() && ! _should_terminate && ! heap->cancelled_concgc()) {
+          if (ShenandoahConcurrentUpdateRefs) {
+            VM_ShenandoahUpdateRefs update_refs;
+            VMThread::execute(&update_refs);
+            heap->update_references();
+          }
+        } else {
+          if (heap->is_evacuation_in_progress()) {
+            heap->set_evacuation_in_progress(false);
+          }
+          heap->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::reset_bitmaps);
+          heap->reset_mark_bitmap();
+          heap->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::reset_bitmaps);
+        }
+
+      } else {
+      Thread::current()->_ParkEvent->park(10) ;
+      // yield();
+    }
+    heap->clear_cancelled_concgc();
+    // Make sure the _do_full_gc flag changes are seen.
+    OrderAccess::storeload();
+  }
+}
+
+void ShenandoahConcurrentThread::do_full_gc(GCCause::Cause cause) {
+
+  assert(Thread::current()->is_Java_thread(), "expect Java thread here");
+
+  MonitorLockerEx ml(ShenandoahFullGC_lock);
+  schedule_full_gc();
+  _full_gc_cause = cause;
+  while (_do_full_gc) {
+    ml.wait();
+    OrderAccess::storeload();
+  }
+  assert(_do_full_gc == false, "expect full GC to have completed");
+}
+
+void ShenandoahConcurrentThread::schedule_full_gc() {
+  _do_full_gc = true;
+}
+
+void ShenandoahConcurrentThread::print() const {
+  print_on(tty);
+}
+
+void ShenandoahConcurrentThread::print_on(outputStream* st) const {
+  st->print("Shenandoah Concurrent Thread");
+  Thread::print_on(st);
+  st->cr();
+}
+
+void ShenandoahConcurrentThread::sleepBeforeNextCycle() {
+  assert(false, "Wake up in the GC thread that never sleeps :-)");
+}
+
+void ShenandoahConcurrentThread::set_cm_started() {
+    assert(!_concurrent_mark_in_progress, "cycle in progress");
+    _concurrent_mark_started = true;
+}
+
+void ShenandoahConcurrentThread::clear_cm_started() {
+    assert(_concurrent_mark_in_progress, "must be starting a cycle");
+    _concurrent_mark_started = false;
+}
+
+bool ShenandoahConcurrentThread::cm_started() {
+  return _concurrent_mark_started;
+}
+
+void ShenandoahConcurrentThread::set_cm_in_progress() {
+  assert(_concurrent_mark_started, "must be starting a cycle");
+  _concurrent_mark_in_progress = true;
+}
+
+void ShenandoahConcurrentThread::clear_cm_in_progress() {
+  assert(!_concurrent_mark_started, "must not be starting a new cycle");
+  _concurrent_mark_in_progress = false;
+}
+
+bool ShenandoahConcurrentThread::cm_in_progress() {
+  return _concurrent_mark_in_progress;
+}
+
+void ShenandoahConcurrentThread::start() {
+  create_and_start();
+}
+
+void ShenandoahConcurrentThread::yield() {
+  _sts.yield();
+}
+
+void ShenandoahConcurrentThread::safepoint_synchronize() {
+  assert(UseShenandoahGC, "just checking");
+  _sts.synchronize();
+}
+
+void ShenandoahConcurrentThread::safepoint_desynchronize() {
+  assert(UseShenandoahGC, "just checking");
+  _sts.desynchronize();
+}
+
+void ShenandoahConcurrentThread::makeSurrogateLockerThread(TRAPS) {
+  assert(UseShenandoahGC, "SLT thread needed only for concurrent GC");
+  assert(THREAD->is_Java_thread(), "must be a Java thread");
+  assert(_slt == NULL, "SLT already created");
+  _slt = SurrogateLockerThread::make(THREAD);
+}
+
+void ShenandoahConcurrentThread::shutdown() {
+  _should_terminate = true;
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.hpp b/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahConcurrentThread.hpp
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTTHREAD_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTTHREAD_HPP
+
+#include "gc/shared/concurrentGCThread.hpp"
+#include "gc/g1/suspendibleThreadSet.hpp"
+#include "gc/shared/gcCause.hpp"
+#include "memory/resourceArea.hpp"
+
+// For now we just want to have a concurrent marking thread.
+// Once we have that working we will build a concurrent evacuation thread.
+
+class ShenandoahConcurrentThread: public ConcurrentGCThread {
+  friend class VMStructs;
+
+ public:
+  virtual void run();
+
+ private:
+  volatile bool                    _concurrent_mark_started;
+  volatile bool                    _concurrent_mark_in_progress;
+  volatile bool                    _concurrent_mark_aborted;
+
+  int _epoch;
+
+  static SurrogateLockerThread* _slt;
+  static SuspendibleThreadSet _sts;
+
+  bool _do_full_gc;
+  GCCause::Cause _full_gc_cause;
+
+  void sleepBeforeNextCycle();
+
+ public:
+  // Constructor
+  ShenandoahConcurrentThread();
+  ~ShenandoahConcurrentThread();
+
+  static void makeSurrogateLockerThread(TRAPS);
+  static SurrogateLockerThread* slt() { return _slt; }
+
+  // Printing
+  void print_on(outputStream* st) const;
+  void print() const;
+
+  void set_cm_started();
+  void clear_cm_started();
+  bool cm_started();
+
+  void set_cm_in_progress();
+  void clear_cm_in_progress();
+  bool cm_in_progress();
+
+  void cm_abort() { _concurrent_mark_aborted = true;}
+  bool cm_has_aborted() { return _concurrent_mark_aborted;}
+  void clear_cm_aborted() { _concurrent_mark_aborted = false;}
+
+  void do_full_gc(GCCause::Cause cause);
+
+  void schedule_full_gc();
+
+  // This flag returns true from the moment a marking cycle is
+  // initiated (during the initial-mark pause when started() is set)
+  // to the moment when the cycle completes (just after the next
+  // marking bitmap has been cleared and in_progress() is
+  // cleared). While this flag is true we will not start another cycle
+  // so that cycles do not overlap. We cannot use just in_progress()
+  // as the CM thread might take some time to wake up before noticing
+  // that started() is set and set in_progress().
+  bool during_cycle()      { return cm_started() || cm_in_progress(); }
+
+  char* name() const { return (char*)"ShenandoahConcurrentThread";}
+  void start();
+  void yield();
+
+  static void safepoint_synchronize();
+  static void safepoint_desynchronize();
+
+  void shutdown();
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTTHREAD_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeap.cpp b/src/share/vm/gc/shenandoah/shenandoahHeap.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeap.cpp
@@ -0,0 +1,2860 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "asm/macroAssembler.hpp"
+
+#include "classfile/symbolTable.hpp"
+#include "classfile/stringTable.hpp"
+
+#include "gc/shared/collectedHeap.inline.hpp"
+#include "gc/shared/cmBitMap.inline.hpp"
+#include "gc/shared/gcHeapSummary.hpp"
+#include "gc/shared/gcTimer.hpp"
+#include "gc/shared/gcTrace.hpp"
+#include "gc/shared/gcTraceTime.hpp"
+#include "gc/shared/isGCActiveMark.hpp"
+
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahHumongous.hpp"
+#include "gc/shenandoah/shenandoahRootProcessor.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahJNICritical.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/vm_operations_shenandoah.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/vmThread.hpp"
+#include "memory/iterator.hpp"
+#include "memory/oopFactory.hpp"
+#include "gc/shared/referenceProcessor.hpp"
+#include "gc/shared/space.inline.hpp"
+#include "gc/shared/threadLocalAllocBuffer.inline.hpp"
+#include "memory/universe.hpp"
+#include "utilities/copy.hpp"
+#include "gc/shared/vmGCOperations.hpp"
+#include "runtime/atomic.inline.hpp"
+
+#define __ masm->
+
+ShenandoahHeap* ShenandoahHeap::_pgc = NULL;
+
+void ShenandoahHeap::print_heap_locations(HeapWord* start, HeapWord* end) {
+  HeapWord* cur = NULL;
+  for (cur = start; cur < end; cur++) {
+    tty->print_cr(PTR_FORMAT" : "PTR_FORMAT, p2i(cur), p2i(*((HeapWord**) cur)));
+  }
+}
+
+void ShenandoahHeap::print_heap_objects(HeapWord* start, HeapWord* end) {
+  HeapWord* cur = NULL;
+  for (cur = start; cur < end; cur = cur + oop(cur)->size()) {
+    oop(cur)->print();
+    print_heap_locations(cur, cur + oop(cur)->size());
+  }
+}
+
+void ShenandoahHeap::print_heap_object(oop p) {
+  HeapWord* hw = (HeapWord*) p;
+  print_heap_locations(hw-1, hw+1+p->size());
+}
+
+
+class PrintHeapRegionsClosure : public
+   ShenandoahHeapRegionClosure {
+private:
+  outputStream* _st;
+public:
+  PrintHeapRegionsClosure() : _st(tty) {}
+  PrintHeapRegionsClosure(outputStream* st) : _st(st) {}
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    r->print_on(_st);
+    return false;
+  }
+};
+
+class PrintHeapObjectsClosure : public ShenandoahHeapRegionClosure {
+public:
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    tty->print_cr("Region "INT32_FORMAT" top = "PTR_FORMAT" used = "SIZE_FORMAT_HEX" free = "SIZE_FORMAT_HEX,
+               r->region_number(), p2i(r->top()), r->used(), r->free());
+
+    ShenandoahHeap::heap()->print_heap_objects(r->bottom(), r->top());
+    return false;
+  }
+};
+
+jint ShenandoahHeap::initialize() {
+  CollectedHeap::pre_initialize();
+
+  size_t init_byte_size = collector_policy()->initial_heap_byte_size();
+  size_t max_byte_size = collector_policy()->max_heap_byte_size();
+  if (ShenandoahGCVerbose)
+    tty->print_cr("init_byte_size = "SIZE_FORMAT","SIZE_FORMAT_HEX"  max_byte_size = "INT64_FORMAT","SIZE_FORMAT_HEX,
+             init_byte_size, init_byte_size, max_byte_size, max_byte_size);
+
+  Universe::check_alignment(max_byte_size,
+                            ShenandoahHeapRegion::RegionSizeBytes,
+                            "shenandoah heap");
+  Universe::check_alignment(init_byte_size,
+                            ShenandoahHeapRegion::RegionSizeBytes,
+                            "shenandoah heap");
+
+  ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
+                                                 Arguments::conservative_max_heap_alignment());
+  initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
+
+  set_barrier_set(new ShenandoahBarrierSet());
+  ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size);
+  _storage.initialize(pgc_rs, init_byte_size);
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("Calling initialize on reserved space base = "PTR_FORMAT" end = "PTR_FORMAT,
+               p2i(pgc_rs.base()), p2i(pgc_rs.base() + pgc_rs.size()));
+  }
+
+  _num_regions = init_byte_size / ShenandoahHeapRegion::RegionSizeBytes;
+  _max_regions = max_byte_size / ShenandoahHeapRegion::RegionSizeBytes;
+  _ordered_regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _max_regions, mtGC);
+  for (size_t i = 0; i < _max_regions; i++) {
+    _ordered_regions[i] = NULL;
+  }
+
+  _initialSize = _num_regions * ShenandoahHeapRegion::RegionSizeBytes;
+  size_t regionSizeWords = ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize;
+  assert(init_byte_size == _initialSize, "tautology");
+  _free_regions = new ShenandoahHeapRegionSet(_max_regions);
+  _collection_set = new ShenandoahHeapRegionSet(_max_regions);
+
+  for (size_t i = 0; i < _num_regions; i++) {
+    ShenandoahHeapRegion* current = new ShenandoahHeapRegion();
+    current->initialize_heap_region((HeapWord*) pgc_rs.base() +
+                                    regionSizeWords * i, regionSizeWords, i);
+    _free_regions->append(current);
+    _ordered_regions[i] = current;
+  }
+  _first_region = _ordered_regions[0];
+  _first_region_bottom = _first_region->bottom();
+  assert((((size_t) _first_region_bottom) & (ShenandoahHeapRegion::RegionSizeBytes - 1)) == 0, err_msg("misaligned heap: "PTR_FORMAT, p2i(_first_region_bottom)));
+
+  _numAllocs = 0;
+
+  if (ShenandoahGCVerbose) {
+    tty->print("All Regions\n");
+    print_heap_regions();
+    tty->print("Free Regions\n");
+    _free_regions->print();
+  }
+
+  // The call below uses stuff (the SATB* things) that are in G1, but probably
+  // belong into a shared location.
+  JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
+                                               SATB_Q_FL_lock,
+                                               20 /*G1SATBProcessCompletedThreshold */,
+                                               Shared_SATB_Q_lock);
+
+  // Reserve space for prev and next bitmap.
+  size_t bitmap_size = CMBitMap::compute_size(heap_rs.size());
+  MemRegion heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize);
+
+  ReservedSpace bitmap(ReservedSpace::allocation_align_size_up(bitmap_size));
+  os::commit_memory_or_exit(bitmap.base(), bitmap.size(), false, err_msg("couldn't allocate mark bitmap"));
+  MemRegion bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
+  _mark_bit_map.initialize(heap_region, bitmap_region);
+
+  _next_mark_bit_map = &_mark_bit_map;
+  reset_mark_bitmap();
+
+  // Initialize fast collection set test structure.
+  _in_cset_fast_test_length = _max_regions;
+  _in_cset_fast_test_base =
+                   NEW_C_HEAP_ARRAY(bool, (size_t) _in_cset_fast_test_length, mtGC);
+  _in_cset_fast_test = _in_cset_fast_test_base -
+               ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
+  clear_cset_fast_test();
+
+  _concurrent_gc_thread = new ShenandoahConcurrentThread();
+  return JNI_OK;
+}
+
+ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
+  CollectedHeap(),
+  _shenandoah_policy(policy),
+  _concurrent_mark_in_progress(false),
+  _evacuation_in_progress(false),
+  _update_references_in_progress(false),
+  _free_regions(NULL),
+  _collection_set(NULL),
+  _bytesAllocSinceCM(0),
+  _bytes_allocated_during_cm(0),
+  _max_allocated_gc(0),
+  _allocated_last_gc(0),
+  _used_start_gc(0),
+  _max_conc_workers((int) MAX2((uint) ConcGCThreads, 1U)),
+  _max_parallel_workers((int) MAX2((uint) ParallelGCThreads, 1U)),
+  _ref_processor(NULL),
+  _in_cset_fast_test(NULL),
+  _in_cset_fast_test_base(NULL),
+  _mark_bit_map(),
+  _cancelled_concgc(false),
+  _need_update_refs(false),
+  _need_reset_bitmaps(false),
+  _jni_critical(new ShenandoahJNICritical())
+
+{
+  if (ShenandoahLogConfig) {
+    tty->print_cr("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads);
+    tty->print_cr("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads);
+    tty->print_cr("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));
+  }
+  _pgc = this;
+  _scm = new ShenandoahConcurrentMark();
+  _used = 0;
+  // This is odd.  They are concurrent gc threads, but they are also task threads.
+  // Framework doesn't allow both.
+  _workers = new WorkGang("Concurrent GC Threads", ParallelGCThreads,
+                            /* are_GC_task_threads */true,
+                            /* are_ConcurrentGC_threads */false);
+  _conc_workers = new WorkGang("Concurrent GC Threads", ConcGCThreads,
+                            /* are_GC_task_threads */true,
+                            /* are_ConcurrentGC_threads */false);
+  if ((_workers == NULL) || (_conc_workers == NULL)) {
+    vm_exit_during_initialization("Failed necessary allocation.");
+  } else {
+    _workers->initialize_workers();
+    _conc_workers->initialize_workers();
+  }
+}
+
+class ResetBitmapTask : public AbstractGangTask {
+private:
+  ShenandoahHeapRegionSet* _regions;
+
+public:
+  ResetBitmapTask(ShenandoahHeapRegionSet* regions) :
+    AbstractGangTask("Parallel Reset Bitmap Task"),
+    _regions(regions) {
+  }
+
+  void work(uint worker_id) {
+    ShenandoahHeapRegion* region = _regions->claim_next();
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    while (region != NULL) {
+      heap->reset_mark_bitmap_range(region->bottom(), region->end());
+      region = _regions->claim_next();
+    }
+  }
+};
+
+void ShenandoahHeap::reset_mark_bitmap() {
+  if (ShenandoahTracePhases) {
+    tty->print_cr("Shenandoah starting concurrent reset bitmaps");
+  }
+  ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions);
+  ResetBitmapTask task = ResetBitmapTask(&regions);
+  conc_workers()->set_active_workers(_max_conc_workers);
+  conc_workers()->run_task(&task);
+  if (ShenandoahTracePhases) {
+    tty->print_cr("Shenandoah finishing concurrent reset bitmaps");
+  }
+}
+
+void ShenandoahHeap::reset_mark_bitmap_range(HeapWord* from, HeapWord* to) {
+  _next_mark_bit_map->clearRange(MemRegion(from, to));
+}
+
+bool ShenandoahHeap::is_bitmap_clear() {
+  HeapWord* start = _ordered_regions[0]->bottom();
+  HeapWord* end = _ordered_regions[_num_regions-1]->end();
+  return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
+}
+
+void ShenandoahHeap::print_on(outputStream* st) const {
+  st->print("Shenandoah Heap");
+  st->print(" total = " SIZE_FORMAT " K, used " SIZE_FORMAT " K ", capacity()/ K, used() /K);
+  st->print("Region size = " SIZE_FORMAT "K ", ShenandoahHeapRegion::RegionSizeBytes / K);
+  if (_concurrent_mark_in_progress) {
+    st->print("marking ");
+  }
+  if (_evacuation_in_progress) {
+    st->print("evacuating ");
+  }
+  if (_update_references_in_progress) {
+    st->print("updating-refs ");
+  }
+  if (_cancelled_concgc) {
+    st->print("cancelled ");
+  }
+  st->print("\n");
+
+  if (Verbose) {
+    print_heap_regions(st);
+  }
+}
+
+class InitGCLABClosure : public ThreadClosure {
+public:
+  void do_thread(Thread* thread) {
+    thread->gclab().initialize(true);
+  }
+};
+
+void ShenandoahHeap::post_initialize() {
+
+  {
+    MutexLockerEx ml(Threads_lock);
+    InitGCLABClosure init_gclabs;
+    for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
+      init_gclabs.do_thread(thread);
+    }
+    gc_threads_do(&init_gclabs);
+  }
+  _scm->initialize();
+
+  ref_processing_init();
+
+  _max_workers = MAX(_max_parallel_workers, _max_conc_workers);
+}
+
+class CalculateUsedRegionClosure : public ShenandoahHeapRegionClosure {
+  size_t sum;
+public:
+
+  CalculateUsedRegionClosure() {
+    sum = 0;
+  }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    sum = sum + r->used();
+    return false;
+  }
+
+  size_t getResult() { return sum;}
+};
+
+size_t ShenandoahHeap::calculateUsed() {
+  CalculateUsedRegionClosure cl;
+  heap_region_iterate(&cl);
+  return cl.getResult();
+}
+
+size_t ShenandoahHeap::calculateFree() {
+  return capacity() - calculateUsed();
+}
+
+void ShenandoahHeap::verify_heap_size_consistency() {
+
+  assert(calculateUsed() == used(),
+         err_msg("heap used size must be consistent heap-used: "SIZE_FORMAT" regions-used: "SIZE_FORMAT, used(), calculateUsed()));
+}
+
+size_t ShenandoahHeap::used() const {
+  return _used;
+}
+
+void ShenandoahHeap::increase_used(size_t bytes) {
+  _used += bytes;
+  // Atomic::add_ptr(bytes, &_used);
+}
+
+void ShenandoahHeap::set_used(size_t bytes) {
+  _used = bytes;
+}
+
+void ShenandoahHeap::decrease_used(size_t bytes) {
+  assert(_used >= bytes, "never decrease heap size by more than we've left");
+  _used -= bytes;
+
+  // Atomic::add_ptr(-bytes, &_used);
+}
+
+size_t ShenandoahHeap::capacity() const {
+  return _num_regions * ShenandoahHeapRegion::RegionSizeBytes;
+
+}
+
+bool ShenandoahHeap::is_maximal_no_gc() const {
+  Unimplemented();
+  return true;
+}
+
+size_t ShenandoahHeap::max_capacity() const {
+  return _max_regions * ShenandoahHeapRegion::RegionSizeBytes;
+}
+
+class IsInRegionClosure : public ShenandoahHeapRegionClosure {
+  const void* _p;
+  bool _result;
+public:
+
+  IsInRegionClosure(const void* p) {
+    _p = p;
+    _result = false;
+  }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    if (r->is_in(_p)) {
+      _result = true;
+      return true;
+    }
+    return false;
+  }
+
+  bool result() { return _result;}
+};
+
+bool ShenandoahHeap::is_in(const void* p) const {
+  //  IsInRegionClosure isIn(p);
+  //  heap_region_iterate(&isIn);
+  //  bool result = isIn.result();
+
+  //  return isIn.result();
+  HeapWord* first_region_bottom = _first_region->bottom();
+  HeapWord* last_region_end = first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * _num_regions;
+  return p > _first_region_bottom && p < last_region_end;
+}
+
+bool ShenandoahHeap::is_in_partial_collection(const void* p ) {
+  Unimplemented();
+  return false;
+}
+
+bool  ShenandoahHeap::is_scavengable(const void* p) {
+  //  nyi();
+  //  return false;
+  return true;
+}
+
+HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
+  if (UseTLAB) {
+  HeapWord* obj = thread->gclab().allocate(size);
+  if (obj != NULL) {
+    return obj;
+  }
+  // Otherwise...
+  return allocate_from_gclab_slow(thread, size);
+  } else {
+    return NULL;
+  }
+}
+
+HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
+  // Retain tlab and allocate object in shared space if
+  // the amount free in the tlab is too large to discard.
+  if (thread->gclab().free() > thread->gclab().refill_waste_limit()) {
+    thread->gclab().record_slow_allocation(size);
+    return NULL;
+  }
+
+  // Discard gclab and allocate a new one.
+  // To minimize fragmentation, the last GCLAB may be smaller than the rest.
+  size_t new_gclab_size = thread->gclab().compute_size(size);
+
+  thread->gclab().clear_before_allocation();
+
+  if (new_gclab_size == 0) {
+    return NULL;
+  }
+
+  // Allocate a new GCLAB...
+  HeapWord* obj = allocate_new_gclab(new_gclab_size);
+  if (obj == NULL) {
+    return NULL;
+  }
+
+  if (ZeroTLAB) {
+    // ..and clear it.
+    Copy::zero_to_words(obj, new_gclab_size);
+  } else {
+    // ...and zap just allocated object.
+#ifdef ASSERT
+    // Skip mangling the space corresponding to the object header to
+    // ensure that the returned space is not considered parsable by
+    // any concurrent GC thread.
+    size_t hdr_size = oopDesc::header_size();
+    Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
+#endif // ASSERT
+  }
+  thread->gclab().fill(obj, obj + size, new_gclab_size);
+  return obj;
+}
+
+HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
+  return allocate_new_tlab(word_size, true);
+}
+
+HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) {
+  return allocate_new_tlab(word_size, false);
+}
+
+HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size, bool mark) {
+  HeapWord* result = allocate_memory(word_size);
+
+  if (result != NULL) {
+    if (mark && (_concurrent_mark_in_progress ||
+                 (shenandoahPolicy()->update_refs_early() && _evacuation_in_progress))) {
+      // We mark the whole tlab here, this way we avoid marking every single
+      // allocated object. We mark it from the 2nd word, because the 1st word is always
+      // the brooks ptr of the first object, and it confuses the fast marked-iterator
+      // if we mark that.
+      _next_mark_bit_map->parMarkRange(MemRegion(result + BrooksPointer::BROOKS_POINTER_OBJ_SIZE,
+                                                 word_size - BrooksPointer::BROOKS_POINTER_OBJ_SIZE));
+    }
+    assert(! heap_region_containing(result)->is_in_collection_set(), "Never allocate in dirty region");
+    _bytesAllocSinceCM += word_size * HeapWordSize;
+
+#ifdef ASSERT
+    if (ShenandoahTraceTLabs)
+      tty->print_cr("allocating new tlab of size "SIZE_FORMAT" at addr "PTR_FORMAT, word_size, p2i(result));
+#endif
+
+  }
+  return result;
+}
+
+ShenandoahHeap* ShenandoahHeap::heap() {
+  assert(_pgc != NULL, "Unitialized access to ShenandoahHeap::heap()");
+  assert(_pgc->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap");
+  return _pgc;
+}
+
+class VM_ShenandoahVerifyHeap: public VM_GC_Operation {
+public:
+  VM_ShenandoahVerifyHeap(unsigned int gc_count_before,
+                   unsigned int full_gc_count_before,
+                   GCCause::Cause cause)
+    : VM_GC_Operation(gc_count_before, cause, full_gc_count_before) { }
+  virtual VMOp_Type type() const { return VMOp_G1CollectFull; }
+  virtual void doit() {
+    if (ShenandoahGCVerbose)
+      tty->print_cr("verifying heap");
+     Universe::heap()->ensure_parsability(false);
+     Universe::verify();
+  }
+  virtual const char* name() const {
+    return "Shenandoah verify trigger";
+  }
+};
+
+class FindEmptyRegionClosure: public ShenandoahHeapRegionClosure {
+  ShenandoahHeapRegion* _result;
+  size_t _required_size;
+public:
+
+  FindEmptyRegionClosure(size_t required_size) : _required_size(required_size) {
+    _result = NULL;
+  }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    if ((! r->is_in_collection_set()) && r->free() >= _required_size) {
+      _result = r;
+      return true;
+    }
+    return false;
+  }
+  ShenandoahHeapRegion* result() { return _result;}
+
+};
+
+HeapWord* ShenandoahHeap::allocate_memory(size_t word_size) {
+  HeapWord* result = NULL;
+  result = allocate_memory_with_lock(word_size);
+
+  if (result == NULL && ! Thread::current()->is_evacuating()) { // Allocation failed, try full-GC, then retry allocation.
+    // tty->print_cr("failed to allocate "SIZE_FORMAT " bytes, free regions:", word_size * HeapWordSize);
+    // _free_regions->print();
+    collect(GCCause::_allocation_failure);
+    result = allocate_memory_with_lock(word_size);
+  }
+
+  return result;
+}
+
+HeapWord* ShenandoahHeap::allocate_memory_with_lock(size_t word_size) {
+  return allocate_memory_shenandoah_lock(word_size);
+}
+
+HeapWord* ShenandoahHeap::allocate_memory_heap_lock(size_t word_size) {
+  ShouldNotReachHere();
+  MutexLocker ml(Heap_lock);
+  return allocate_memory_work(word_size);
+}
+
+HeapWord* ShenandoahHeap::allocate_memory_shenandoah_lock(size_t word_size) {
+  MutexLockerEx ml(ShenandoahHeap_lock, true);
+  return allocate_memory_work(word_size);
+}
+
+ShenandoahHeapRegion* ShenandoahHeap::check_skip_humongous(ShenandoahHeapRegion* region) const {
+  while (region != NULL && region->is_humongous()) {
+    region = _free_regions->get_next();
+  }
+  return region;
+}
+
+ShenandoahHeapRegion* ShenandoahHeap::get_next_region_skip_humongous() const {
+  ShenandoahHeapRegion* next = _free_regions->get_next();
+  return check_skip_humongous(next);
+}
+
+ShenandoahHeapRegion* ShenandoahHeap::get_current_region_skip_humongous() const {
+  ShenandoahHeapRegion* current = _free_regions->current();
+  return check_skip_humongous(current);
+}
+
+
+ShenandoahHeapRegion* ShenandoahHeap::check_grow_heap(ShenandoahHeapRegion* current) {
+  if (current == NULL) {
+    if (grow_heap_by()) {
+      current = _free_regions->get_next();
+      assert(current != NULL, "After successfully growing the heap we should have a region");
+      assert(! current->is_humongous(), "new region must not be humongous");
+    } else {
+      current = NULL; // No more room to make a new region. OOM.
+    }
+  }
+  return current;
+}
+
+ShenandoahHeapRegion* ShenandoahHeap::get_current_region() {
+  ShenandoahHeapRegion* current = get_current_region_skip_humongous();
+  return check_grow_heap(current);
+}
+
+ShenandoahHeapRegion* ShenandoahHeap::get_next_region() {
+  ShenandoahHeapRegion* current = get_next_region_skip_humongous();
+  return check_grow_heap(current);
+}
+
+
+HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size) {
+
+  if (word_size * HeapWordSize > ShenandoahHeapRegion::RegionSizeBytes) {
+    assert(! Thread::current()->is_evacuating(), "no humongous allocation for evacuating thread");
+    return allocate_large_memory(word_size);
+  }
+
+  ShenandoahHeapRegion* my_current_region = get_current_region();
+  if (my_current_region == NULL) {
+    return NULL; // No more room to make a new region. OOM.
+  }
+  assert(my_current_region != NULL, "should have a region at this point");
+
+#ifdef ASSERT
+  if (my_current_region->is_in_collection_set()) {
+    print_heap_regions();
+  }
+#endif
+  assert(! my_current_region->is_in_collection_set(), "never get targetted regions in free-lists");
+  assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
+
+  HeapWord* result;
+
+  result = my_current_region->par_allocate(word_size);
+  while (result == NULL && my_current_region != NULL) {
+    // 2nd attempt. Try next region.
+    size_t remaining = my_current_region->free();
+    my_current_region = get_next_region();
+    if (my_current_region == NULL) {
+      return NULL; // No more room to make a new region. OOM.
+    }
+    _free_regions->decrease_available(remaining);
+    assert(my_current_region != NULL, "should have a region at this point");
+    assert(! my_current_region->is_in_collection_set(), "never get targetted regions in free-lists");
+    assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
+    result = my_current_region->par_allocate(word_size);
+  }
+
+  if (result != NULL) {
+    my_current_region->increase_live_data(word_size * HeapWordSize);
+    increase_used(word_size * HeapWordSize);
+    _free_regions->decrease_available(word_size * HeapWordSize);
+  }
+  return result;
+}
+
+HeapWord* ShenandoahHeap::allocate_large_memory(size_t words) {
+  if (ShenandoahTraceHumongous) {
+    gclog_or_tty->print_cr("allocating humongous object of size: "SIZE_FORMAT" KB", (words * HeapWordSize) / K);
+  }
+
+  uint required_regions = ShenandoahHumongous::required_regions(words * HeapWordSize);
+
+  assert(required_regions <= _max_regions, "sanity check");
+
+  HeapWord* result;
+  ShenandoahHeapRegion* free_regions[required_regions];
+
+  bool success = find_contiguous_free_regions(required_regions, free_regions);
+  if (! success) {
+    success = allocate_contiguous_free_regions(required_regions, free_regions);
+  }
+  if (! success) {
+    result = NULL; // Throw OOM, we cannot allocate the huge object.
+  } else {
+    // Initialize huge object flags in the regions.
+    size_t live = words * HeapWordSize;
+    free_regions[0]->set_humongous_start(true);
+    free_regions[0]->increase_live_data(live);
+
+    for (uint i = 0; i < required_regions; i++) {
+      if (i == 0) {
+        free_regions[0]->set_humongous_start(true);
+      } else {
+        free_regions[i]->set_humongous_continuation(true);
+      }
+      free_regions[i]->set_top(free_regions[i]->end());
+      increase_used(ShenandoahHeapRegion::RegionSizeBytes);
+    }
+    _free_regions->decrease_available(ShenandoahHeapRegion::RegionSizeBytes * required_regions);
+    result = free_regions[0]->bottom();
+  }
+  return result;
+}
+
+bool ShenandoahHeap::find_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions) {
+  if (ShenandoahTraceHumongous) {
+    gclog_or_tty->print_cr("trying to find "UINT32_FORMAT" contiguous free regions", num_free_regions);
+  }
+  uint free_regions_index = 0;
+  for (uint regions_index = 0; regions_index < _num_regions; regions_index++) {
+    // Claim a free region.
+    ShenandoahHeapRegion* region = _ordered_regions[regions_index];
+    bool free = false;
+    if (region != NULL) {
+      if (region->free() == ShenandoahHeapRegion::RegionSizeBytes) {
+        assert(! region->is_humongous(), "don't reuse occupied humongous regions");
+        free = true;
+      }
+    }
+    if (! free) {
+      // Not contiguous, reset search
+      free_regions_index = 0;
+      continue;
+    }
+    assert(free_regions_index < num_free_regions, "array bounds");
+    free_regions[free_regions_index] = region;
+    free_regions_index++;
+
+    if (free_regions_index == num_free_regions) {
+      if (ShenandoahTraceHumongous) {
+        gclog_or_tty->print_cr("found "UINT32_FORMAT" contiguous free regions:", num_free_regions);
+        for (uint i = 0; i < num_free_regions; i++) {
+          gclog_or_tty->print(UINT32_FORMAT": " , i);
+          free_regions[i]->print_on(gclog_or_tty);
+        }
+      }
+      return true;
+    }
+
+  }
+  if (ShenandoahTraceHumongous) {
+    gclog_or_tty->print_cr("failed to find "UINT32_FORMAT" free regions", num_free_regions);
+  }
+  return false;
+}
+
+bool ShenandoahHeap::allocate_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions) {
+  // We need to be smart here to avoid interleaved allocation of regions when concurrently
+  // allocating for large objects. We get the new index into regions array using CAS, where can
+  // subsequently safely allocate new regions.
+  int new_regions_index = ensure_new_regions(num_free_regions);
+  if (new_regions_index == -1) {
+    return false;
+  }
+
+  int last_new_region = new_regions_index + num_free_regions;
+
+  // Now we can allocate new regions at the found index without being scared that
+  // other threads allocate in the same contiguous region.
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("allocate contiguous regions:");
+  }
+  for (int i = new_regions_index; i < last_new_region; i++) {
+    ShenandoahHeapRegion* region = new ShenandoahHeapRegion();
+    HeapWord* start = _first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * i;
+    region->initialize_heap_region(start, ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize, i);
+    _ordered_regions[i] = region;
+    uint index = i - new_regions_index;
+    assert(index < num_free_regions, "array bounds");
+    free_regions[index] = region;
+
+    if (ShenandoahGCVerbose) {
+      region->print();
+    }
+  }
+  return true;
+}
+
+HeapWord* ShenandoahHeap::mem_allocate_locked(size_t size,
+                                              bool* gc_overhead_limit_was_exceeded) {
+
+  // This was used for allocation while holding the Heap_lock.
+  // HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size);
+
+  HeapWord* filler = allocate_memory(BrooksPointer::BROOKS_POINTER_OBJ_SIZE + size);
+  HeapWord* result = filler + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+  if (filler != NULL) {
+    initialize_brooks_ptr(filler, result);
+    _bytesAllocSinceCM += size * HeapWordSize;
+#ifdef ASSERT
+    if (ShenandoahTraceAllocations) {
+      if (*gc_overhead_limit_was_exceeded)
+        tty->print("gc_overhead_limit_was_exceeded");
+      tty->print_cr("mem_allocate_locked object of size "SIZE_FORMAT" uat addr "PTR_FORMAT, size, p2i(result));
+    }
+#endif
+
+    assert(! heap_region_containing(result)->is_in_collection_set(), "never allocate in targetted region");
+    if (_concurrent_mark_in_progress ||
+        (shenandoahPolicy()->update_refs_early() && _evacuation_in_progress)) {
+      mark_current_no_checks(oop(result));
+    }
+
+    return result;
+  } else {
+    tty->print_cr("Out of memory. Requested number of words: "SIZE_FORMAT" used heap: "INT64_FORMAT", bytes allocated since last CM: "INT64_FORMAT, size, used(), _bytesAllocSinceCM);
+    {
+      MutexLockerEx ml(ShenandoahHeap_lock, true);
+      print_heap_regions();
+      tty->print("Printing "SIZE_FORMAT" free regions:\n", _free_regions->length());
+      _free_regions->print();
+    }
+    assert(false, "Out of memory");
+    return NULL;
+  }
+}
+
+class PrintOopContents: public OopClosure {
+public:
+  void do_oop(oop* o) {
+    oop obj = *o;
+    tty->print_cr("References oop "PTR_FORMAT, p2i((HeapWord*) obj));
+    obj->print();
+  }
+
+  void do_oop(narrowOop* o) {
+    assert(false, "narrowOops aren't implemented");
+  }
+};
+
+HeapWord*  ShenandoahHeap::mem_allocate(size_t size,
+                                        bool*  gc_overhead_limit_was_exceeded) {
+
+#ifdef ASSERT
+  if (ShenandoahVerify && _numAllocs > 1000000) {
+    _numAllocs = 0;
+  //   VM_ShenandoahVerifyHeap op(0, 0, GCCause::_allocation_failure);
+  //   if (Thread::current()->is_VM_thread()) {
+  //     op.doit();
+  //   } else {
+  //     // ...and get the VM thread to execute it.
+  //     VMThread::execute(&op);
+  //   }
+  }
+  _numAllocs++;
+#endif
+
+  // MutexLockerEx ml(ShenandoahHeap_lock, true);
+  HeapWord* result = mem_allocate_locked(size, gc_overhead_limit_was_exceeded);
+  return result;
+}
+
+class ParallelEvacuateRegionObjectClosure : public ObjectClosure {
+private:
+  ShenandoahHeap* _heap;
+  Thread* _thread;
+  public:
+  ParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
+    _heap(heap), _thread(Thread::current()) {
+  }
+
+  void do_object(oop p) {
+
+#ifdef ASSERT
+    if (ShenandoahTraceEvacuations) {
+      tty->print_cr("Calling ParallelEvacuateRegionObjectClosure on "PTR_FORMAT, p2i((HeapWord*) p));
+    }
+#endif
+
+    if (_heap->is_marked_current(p) && p == ShenandoahBarrierSet::resolve_oop_static_not_null(p)) {
+      _heap->evacuate_object(p, _thread);
+    }
+  }
+};
+
+//fixme
+void ShenandoahHeap::initialize_brooks_ptr(HeapWord* filler, HeapWord* obj, bool new_obj) {
+  BrooksPointer brooks_ptr = BrooksPointer::get(oop(obj));
+  brooks_ptr.set_forwardee(oop(obj));
+}
+
+void ShenandoahHeap::initialize_brooks_ptr(oop p) {
+  BrooksPointer brooks_ptr = BrooksPointer::get(p);
+  brooks_ptr.set_forwardee(p);
+}
+
+class VerifyEvacuatedObjectClosure : public ObjectClosure {
+
+public:
+
+  void do_object(oop p) {
+    if (ShenandoahHeap::heap()->is_marked_current(p)) {
+      oop p_prime = oopDesc::bs()->resolve_oop(p);
+      assert(p != p_prime, "Should point to evacuated copy");
+#ifdef ASSERT
+      if (p->klass() != p_prime->klass()) {
+        tty->print_cr("copy has different class than original:");
+        p->klass()->print_on(tty);
+        p_prime->klass()->print_on(tty);
+      }
+#endif
+      assert(p->klass() == p_prime->klass(), err_msg("Should have the same class p: "PTR_FORMAT", p_prime: "PTR_FORMAT, p2i((HeapWord*) p), p2i((HeapWord*) p_prime)));
+      //      assert(p->mark() == p_prime->mark(), "Should have the same mark");
+      assert(p->size() == p_prime->size(), "Should be the same size");
+      assert(p_prime == oopDesc::bs()->resolve_oop(p_prime), "One forward once");
+    }
+  }
+};
+
+void ShenandoahHeap::verify_evacuated_region(ShenandoahHeapRegion* from_region) {
+  if (ShenandoahGCVerbose) {
+    tty->print("Verifying From Region\n");
+    from_region->print();
+  }
+
+  VerifyEvacuatedObjectClosure verify_evacuation;
+  from_region->object_iterate_interruptible(&verify_evacuation, false);
+}
+
+void ShenandoahHeap::parallel_evacuate_region(ShenandoahHeapRegion* from_region) {
+
+  assert(from_region->getLiveData() > 0, "all-garbage regions are reclaimed earlier");
+
+  ParallelEvacuateRegionObjectClosure evacuate_region(this);
+
+#ifdef ASSERT
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("parallel_evacuate_region starting from_region "INT32_FORMAT": free_regions = "SIZE_FORMAT,  from_region->region_number(), _free_regions->available_regions());
+  }
+#endif
+
+  marked_object_iterate(from_region, &evacuate_region);
+
+#ifdef ASSERT
+  if (ShenandoahVerify && ! cancelled_concgc()) {
+    verify_evacuated_region(from_region);
+  }
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("parallel_evacuate_region after from_region = "INT32_FORMAT": free_regions = "SIZE_FORMAT, from_region->region_number(), _free_regions->available_regions());
+  }
+#endif
+}
+
+class ParallelEvacuationTask : public AbstractGangTask {
+private:
+  ShenandoahHeap* _sh;
+  ShenandoahHeapRegionSet* _cs;
+
+public:
+  ParallelEvacuationTask(ShenandoahHeap* sh,
+                         ShenandoahHeapRegionSet* cs) :
+    AbstractGangTask("Parallel Evacuation Task"),
+    _cs(cs),
+    _sh(sh) {}
+
+  void work(uint worker_id) {
+
+    ShenandoahHeapRegion* from_hr = _cs->claim_next();
+
+    while (from_hr != NULL) {
+      if (ShenandoahGCVerbose) {
+        tty->print_cr("Thread "INT32_FORMAT" claimed Heap Region "INT32_FORMAT,
+                   worker_id,
+                   from_hr->region_number());
+        from_hr->print();
+      }
+
+      assert(from_hr->getLiveData() > 0, "all-garbage regions are reclaimed early");
+      _sh->parallel_evacuate_region(from_hr);
+
+      if (_sh->cancelled_concgc()) {
+        if (ShenandoahTracePhases) {
+          tty->print_cr("Cancelled concurrent evacuation");
+        }
+        break;
+      }
+      from_hr = _cs->claim_next();
+    }
+
+    Thread::current()->gclab().make_parsable(true);
+  }
+};
+
+class RecycleDirtyRegionsClosure: public ShenandoahHeapRegionClosure {
+private:
+  ShenandoahHeap* _heap;
+  size_t _bytes_reclaimed;
+public:
+  RecycleDirtyRegionsClosure() : _heap(ShenandoahHeap::heap()) {}
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+
+    // If evacuation has been cancelled, we can't recycle regions, we only
+    // clear their collection-set status.
+    if (_heap->cancelled_concgc()) {
+      r->set_is_in_collection_set(false);
+      return false;
+    }
+
+    if (r->is_in_collection_set()) {
+      //      tty->print_cr("recycling region "INT32_FORMAT":", r->region_number());
+      //      r->print_on(tty);
+      //      tty->print_cr(" ");
+      _heap->decrease_used(r->used());
+      _bytes_reclaimed += r->used();
+      r->recycle();
+      _heap->free_regions()->append(r);
+    }
+
+    return false;
+  }
+  size_t bytes_reclaimed() { return _bytes_reclaimed;}
+  void clear_bytes_reclaimed() {_bytes_reclaimed = 0;}
+};
+
+void ShenandoahHeap::recycle_dirty_regions() {
+  RecycleDirtyRegionsClosure cl;
+  cl.clear_bytes_reclaimed();
+
+  heap_region_iterate(&cl);
+
+  _shenandoah_policy->record_bytes_reclaimed(cl.bytes_reclaimed());
+  clear_cset_fast_test();
+}
+
+ShenandoahHeapRegionSet* ShenandoahHeap::free_regions() {
+  return _free_regions;
+}
+
+void ShenandoahHeap::print_heap_regions(outputStream* st) const {
+  PrintHeapRegionsClosure pc1(st);
+  heap_region_iterate(&pc1);
+}
+
+class PrintAllRefsOopClosure: public ExtendedOopClosure {
+private:
+  int _index;
+  const char* _prefix;
+
+public:
+  PrintAllRefsOopClosure(const char* prefix) : _index(0), _prefix(prefix) {}
+
+  void do_oop(oop* p)       {
+    oop o = *p;
+    if (o != NULL) {
+      if (ShenandoahHeap::heap()->is_in(o) && o->is_oop()) {
+        tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT")-> "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT")", _prefix, _index, p2i(p), p2i((HeapWord*) o), BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_current(o)), o->klass()->internal_name(), p2i(o->klass()));
+      } else {
+        //        tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty: %s) -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty (%s))", _prefix, _index, p2i(p), BOOL_TO_STR(ShenandoahHeap::heap()->heap_region_containing(p)->is_in_collection_set()), p2i((HeapWord*) o), BOOL_TO_STR(ShenandoahHeap::heap()->heap_region_containing(o)->is_in_collection_set()));
+        tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty)", _prefix, _index, p2i(p), p2i((HeapWord*) o));
+      }
+    } else {
+      tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT") -> "PTR_FORMAT, _prefix, _index, p2i(p), p2i((HeapWord*) o));
+    }
+    _index++;
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+
+};
+
+class PrintAllRefsObjectClosure : public ObjectClosure {
+  const char* _prefix;
+
+public:
+  PrintAllRefsObjectClosure(const char* prefix) : _prefix(prefix) {}
+
+  void do_object(oop p) {
+    if (ShenandoahHeap::heap()->is_in(p)) {
+        tty->print_cr("%s object "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT") refers to:", _prefix, p2i((HeapWord*) p), BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_current(p)), p->klass()->internal_name(), p2i(p->klass()));
+        PrintAllRefsOopClosure cl(_prefix);
+        p->oop_iterate(&cl);
+      }
+  }
+};
+
+void ShenandoahHeap::print_all_refs(const char* prefix) {
+  tty->print_cr("printing all references in the heap");
+  tty->print_cr("root references:");
+
+  ensure_parsability(false);
+
+  PrintAllRefsOopClosure cl(prefix);
+  roots_iterate(&cl);
+
+  tty->print_cr("heap references:");
+  PrintAllRefsObjectClosure cl2(prefix);
+  object_iterate(&cl2);
+}
+
+class VerifyAfterMarkingOopClosure: public ExtendedOopClosure {
+private:
+  ShenandoahHeap*  _heap;
+
+public:
+  VerifyAfterMarkingOopClosure() :
+    _heap(ShenandoahHeap::heap()) { }
+
+  void do_oop(oop* p)       {
+    oop o = *p;
+    if (o != NULL) {
+      if (! _heap->is_marked_current(o)) {
+        _heap->print_heap_regions();
+        _heap->print_all_refs("post-mark");
+        tty->print_cr("oop not marked, although referrer is marked: "PTR_FORMAT": in_heap: %s, is_marked: %s",
+                      p2i((HeapWord*) o), BOOL_TO_STR(_heap->is_in(o)), BOOL_TO_STR(_heap->is_marked_current(o)));
+        _heap->print_heap_locations((HeapWord*) o, (HeapWord*) o + o->size());
+
+        tty->print_cr("oop class: %s", o->klass()->internal_name());
+        if (_heap->is_in(p)) {
+          oop referrer = oop(_heap->heap_region_containing(p)->block_start_const(p));
+          tty->print_cr("Referrer starts at addr "PTR_FORMAT, p2i((HeapWord*) referrer));
+          referrer->print();
+          _heap->print_heap_locations((HeapWord*) referrer, (HeapWord*) referrer + referrer->size());
+        }
+        tty->print_cr("heap region containing object:");
+        _heap->heap_region_containing(o)->print();
+        tty->print_cr("heap region containing referrer:");
+        _heap->heap_region_containing(p)->print();
+        tty->print_cr("heap region containing forwardee:");
+        _heap->heap_region_containing(oopDesc::bs()->resolve_oop(o))->print();
+      }
+      assert(o->is_oop(), "oop must be an oop");
+      assert(Metaspace::contains(o->klass()), "klass pointer must go to metaspace");
+      if (! (o == oopDesc::bs()->resolve_oop(o))) {
+        tty->print_cr("oops has forwardee: p: "PTR_FORMAT" (%s), o = "PTR_FORMAT" (%s), new-o: "PTR_FORMAT" (%s)", p2i(p), BOOL_TO_STR(_heap->heap_region_containing(p)->is_in_collection_set()), p2i((HeapWord*) o),  BOOL_TO_STR(_heap->heap_region_containing(o)->is_in_collection_set()), p2i((HeapWord*) oopDesc::bs()->resolve_oop(o)), BOOL_TO_STR(_heap->heap_region_containing(oopDesc::bs()->resolve_oop(o))->is_in_collection_set()));
+        tty->print_cr("oop class: %s", o->klass()->internal_name());
+      }
+      assert(o == oopDesc::bs()->resolve_oop(o), "oops must not be forwarded");
+      assert(! _heap->heap_region_containing(o)->is_in_collection_set(), "references must not point to dirty heap regions");
+      assert(_heap->is_marked_current(o), "live oops must be marked current");
+    }
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+
+};
+
+class IterateMarkedCurrentObjectsClosure: public ObjectClosure {
+private:
+  ShenandoahHeap* _heap;
+  ExtendedOopClosure* _cl;
+public:
+  IterateMarkedCurrentObjectsClosure(ExtendedOopClosure* cl) :
+    _heap(ShenandoahHeap::heap()), _cl(cl) {};
+
+  void do_object(oop p) {
+    if (_heap->is_marked_current(p)) {
+      p->oop_iterate(_cl);
+    }
+  }
+
+};
+
+class IterateMarkedObjectsClosure: public ObjectClosure {
+private:
+  ShenandoahHeap* _heap;
+  ExtendedOopClosure* _cl;
+public:
+  IterateMarkedObjectsClosure(ExtendedOopClosure* cl) :
+    _heap(ShenandoahHeap::heap()), _cl(cl) {};
+
+  void do_object(oop p) {
+    if (_heap->is_marked_current(p)) {
+      p->oop_iterate(_cl);
+    }
+  }
+
+};
+
+void ShenandoahHeap::verify_heap_after_marking() {
+
+  verify_heap_size_consistency();
+
+  if (ShenandoahGCVerbose) {
+    tty->print("verifying heap after marking\n");
+  }
+  ensure_parsability(false);
+  VerifyAfterMarkingOopClosure cl;
+  roots_iterate(&cl);
+
+  IterateMarkedCurrentObjectsClosure marked_oops(&cl);
+  object_iterate(&marked_oops);
+}
+
+void ShenandoahHeap::prepare_for_concurrent_evacuation() {
+  if (!cancelled_concgc()) {
+
+    recycle_dirty_regions();
+
+      ensure_parsability(true);
+
+      // NOTE: This needs to be done during a stop the world pause, because
+      // putting regions into the collection set concurrently with Java threads
+      // will create a race. In particular, acmp could fail because when we
+      // resolve the first operand, the containing region might not yet be in
+      // the collection set, and thus return the original oop. When the 2nd
+      // operand gets resolved, the region could be in the collection set
+      // and the oop gets evacuated. If both operands have originally been
+      // the same, we get false negatives.
+      ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions);
+      regions.reclaim_humongous_regions();
+      _collection_set->clear();
+      _free_regions->clear();
+      _shenandoah_policy->choose_collection_and_free_sets(&regions, _collection_set, _free_regions);
+
+      if (PrintGCTimeStamps) {
+        gclog_or_tty->print("Collection set used = " SIZE_FORMAT " K live = " SIZE_FORMAT " K reclaimable = " SIZE_FORMAT " K\n",
+                            _collection_set->used() / K, _collection_set->live_data() / K, _collection_set->garbage() / K);
+      }
+
+      if (_collection_set->length() == 0)
+        cancel_concgc();
+
+      _bytesAllocSinceCM = 0;
+
+      Universe::update_heap_info_at_gc();
+    }
+}
+
+
+class ShenandoahUpdateRootsClosure: public ExtendedOopClosure {
+
+  void do_oop(oop* p)       {
+    ShenandoahHeap::heap()->maybe_update_oop_ref(p);
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+};
+
+void ShenandoahHeap::update_roots() {
+
+  COMPILER2_PRESENT(DerivedPointerTable::clear());
+
+  assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
+
+  ShenandoahUpdateRootsClosure cl;
+  CodeBlobToOopClosure blobsCl(&cl, false);
+  CLDToOopClosure cldCl(&cl);
+
+  ClassLoaderDataGraph::clear_claimed_marks();
+
+  {
+    ShenandoahRootProcessor rp(this, 1);
+    rp.process_all_roots(&cl, &cldCl, &blobsCl);
+    ShenandoahIsAliveClosure is_alive;
+    JNIHandles::weak_oops_do(&is_alive, &cl);
+  }
+
+  COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+}
+
+class ShenandoahUpdateObjectsClosure : public ObjectClosure {
+  ShenandoahHeap* _heap;
+
+public:
+  ShenandoahUpdateObjectsClosure() :
+    _heap(ShenandoahHeap::heap()) {
+  }
+
+  void do_object(oop p) {
+    ShenandoahUpdateRootsClosure refs_cl;
+    assert(ShenandoahHeap::heap()->is_in(p), "only update objects in heap (where else?)");
+
+    if (_heap->is_marked_current(p)) {
+      p->oop_iterate(&refs_cl);
+    }
+  }
+
+};
+
+class ParallelUpdateRefsTask : public AbstractGangTask {
+private:
+  ShenandoahHeapRegionSet* _regions;
+
+public:
+  ParallelUpdateRefsTask(ShenandoahHeapRegionSet* regions) :
+    AbstractGangTask("Parallel Update References Task"),
+  _regions(regions) {
+  }
+
+  void work(uint worker_id) {
+    ShenandoahUpdateObjectsClosure update_refs_cl;
+    ShenandoahHeapRegion* region = _regions->claim_next();
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    while (region != NULL && ! heap->cancelled_concgc()) {
+      if ((! region->is_in_collection_set()) && ! region->is_humongous_continuation()) {
+        heap->marked_object_iterate_careful(region, &update_refs_cl);
+      }
+      heap->reset_mark_bitmap_range(region->bottom(), region->end());
+      region = _regions->claim_next();
+    }
+    if (ShenandoahTracePhases && heap->cancelled_concgc()) {
+      tty->print_cr("Cancelled concurrent update references");
+    }
+  }
+};
+
+class RetireTLABClosure : public ThreadClosure {
+private:
+  bool _retire;
+
+public:
+  RetireTLABClosure(bool retire) : _retire(retire) {
+  }
+
+  void do_thread(Thread* thread) {
+    thread->gclab().make_parsable(_retire);
+  }
+};
+
+void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
+  if (UseTLAB) {
+  CollectedHeap::ensure_parsability(retire_tlabs);
+
+  RetireTLABClosure cl(retire_tlabs);
+  for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
+    cl.do_thread(thread);
+  }
+  gc_threads_do(&cl);
+  }
+}
+
+void ShenandoahHeap::prepare_for_update_references() {
+  ensure_parsability(true);
+
+  ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions);
+  regions.set_concurrent_iteration_safe_limits();
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    set_from_region_protection(false);
+
+    // We need to update the roots so that they are ok for C2 when returning from the safepoint.
+    update_roots();
+
+    set_from_region_protection(true);
+
+  } else {
+    // We need to update the roots so that they are ok for C2 when returning from the safepoint.
+    update_roots();
+  }
+
+  set_update_references_in_progress(true);
+}
+
+void ShenandoahHeap::update_references() {
+
+  ShenandoahHeapRegionSet regions = ShenandoahHeapRegionSet(_num_regions, _ordered_regions, _num_regions);
+  ParallelUpdateRefsTask task = ParallelUpdateRefsTask(&regions);
+  conc_workers()->set_active_workers(_max_conc_workers);
+  _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_uprefs);
+  conc_workers()->run_task(&task);
+  _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_uprefs);
+  conc_workers()->set_active_workers(_max_conc_workers);
+
+  if (! cancelled_concgc()) {
+    VM_ShenandoahUpdateRootRefs update_roots;
+    if (ShenandoahConcurrentUpdateRefs) {
+      VMThread::execute(&update_roots);
+    } else {
+      update_roots.doit();
+    }
+
+    _allocated_last_gc = used() - _used_start_gc;
+    size_t max_allocated_gc = MAX2(_max_allocated_gc, _allocated_last_gc);
+    /*
+      tty->print_cr("prev max_allocated_gc: "SIZE_FORMAT", new max_allocated_gc: "SIZE_FORMAT", allocated_last_gc: "SIZE_FORMAT" diff %f", _max_allocated_gc, max_allocated_gc, _allocated_last_gc, ((double) max_allocated_gc/ (double) _allocated_last_gc));
+    */
+    _max_allocated_gc = max_allocated_gc;
+
+    // Update-references completed, no need to update-refs during marking.
+    set_need_update_refs(false);
+  }
+
+  Universe::update_heap_info_at_gc();
+
+  set_update_references_in_progress(false);
+}
+
+
+class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure {
+private:
+  ShenandoahHeap* _heap;
+  Thread* _thread;
+public:
+  ShenandoahEvacuateUpdateRootsClosure() :
+    _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
+  }
+
+  void do_oop(oop* p) {
+    assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
+
+    oop obj = oopDesc::load_heap_oop(p);
+    if (obj != NULL && _heap->in_cset_fast_test((HeapWord*) obj)) {
+      assert(_heap->is_marked_current(obj), err_msg("only evacuate marked objects %d %d", _heap->is_marked_current(obj), _heap->is_marked_current(ShenandoahBarrierSet::resolve_oop_static_not_null(obj))));
+      oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
+      if (resolved == obj) {
+        resolved = _heap->evacuate_object(obj, _thread);
+      }
+      oopDesc::store_heap_oop(p, resolved);
+    }
+#ifdef ASSERT
+    else if (! oopDesc::is_null(obj)) {
+      // tty->print_cr("not updating root at: "PTR_FORMAT" with object: "PTR_FORMAT", is_in_heap: %s, is_in_cset: %s, is_marked: %s", p2i(p), p2i((HeapWord*) obj), BOOL_TO_STR(_heap->is_in(obj)), BOOL_TO_STR(_heap->in_cset_fast_test(obj)), BOOL_TO_STR(_heap->is_marked_current(obj)));
+    }
+#endif
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+};
+
+class ShenandoahEvacuateUpdateStrongRootsTask : public AbstractGangTask {
+  ShenandoahRootProcessor* _rp;
+public:
+
+  ShenandoahEvacuateUpdateStrongRootsTask(ShenandoahRootProcessor* rp) :
+    AbstractGangTask("Shenandoah evacuate and update strong roots"),
+    _rp(rp)
+  {
+    // Nothing else to do.
+  }
+
+  void work(uint worker_id) {
+    ShenandoahEvacuateUpdateRootsClosure cl;
+    CodeBlobToOopClosure blobsCl(&cl, false);
+    CLDToOopClosure cldCl(&cl);
+
+    _rp->process_all_roots(&cl, &cldCl, &blobsCl);
+  }
+};
+
+class ShenandoahEvacuateUpdateWeakRootsTask : public AbstractGangTask {
+public:
+
+  ShenandoahEvacuateUpdateWeakRootsTask() : AbstractGangTask("Shenandoah evacuate and update weak roots") {
+    // Nothing else to do.
+  }
+
+  void work(uint worker_id) {
+    ShenandoahEvacuateUpdateRootsClosure cl;
+    ShenandoahIsAliveClosure is_alive;
+    JNIHandles::weak_oops_do(&is_alive, &cl);
+
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    if (ShenandoahProcessReferences) {
+      heap->ref_processor()->weak_oops_do(&cl);
+    }
+  }
+};
+
+void ShenandoahHeap::evacuate_and_update_roots() {
+
+  COMPILER2_PRESENT(DerivedPointerTable::clear());
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    set_from_region_protection(false);
+  }
+
+  assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
+  ClassLoaderDataGraph::clear_claimed_marks();
+
+  {
+    ShenandoahRootProcessor rp(this, _max_parallel_workers);
+    ShenandoahEvacuateUpdateStrongRootsTask strong_roots_task(&rp);
+    workers()->set_active_workers(_max_parallel_workers);
+    workers()->run_task(&strong_roots_task);
+  }
+
+  // We process weak roots using only 1 worker thread, multi-threaded weak roots
+  // processing is not implemented yet. We can't use the VMThread itself, because
+  // we need to grab the Heap_lock.
+  {
+    ShenandoahEvacuateUpdateWeakRootsTask weak_roots_task;
+    workers()->set_active_workers(1);
+    workers()->run_task(&weak_roots_task);
+    workers()->set_active_workers(_max_parallel_workers);
+  }
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    set_from_region_protection(true);
+  }
+
+  COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+
+}
+
+
+void ShenandoahHeap::do_evacuation() {
+  assert(Thread::current()->is_VM_thread() || ShenandoahConcurrentEvacuation, "Only evacuate from VMThread unless we do concurrent evacuation");
+
+  parallel_evacuate();
+
+  if (! ShenandoahConcurrentEvacuation) {
+    // We need to make sure that after leaving the safepoint, all
+    // GC roots are up-to-date. This is an assumption built into
+    // the hotspot compilers, especially C2, that allows it to
+    // do optimizations like lifting barriers outside of a loop.
+
+    if (ShenandoahVerifyReadsToFromSpace) {
+      set_from_region_protection(false);
+
+      update_roots();
+
+      set_from_region_protection(true);
+
+    } else {
+      update_roots();
+    }
+  }
+
+  if (ShenandoahVerify && ! cancelled_concgc()) {
+    VM_ShenandoahVerifyHeapAfterEvacuation verify_after_evacuation;
+    if (Thread::current()->is_VM_thread()) {
+      verify_after_evacuation.doit();
+    } else {
+      VMThread::execute(&verify_after_evacuation);
+    }
+  }
+
+}
+
+void ShenandoahHeap::parallel_evacuate() {
+
+  if (! cancelled_concgc()) {
+    assert(Thread::current()->is_VM_thread() || ShenandoahConcurrentEvacuation, "Only evacuate from VMThread unless we do concurrent evacuation");
+
+    if (ShenandoahGCVerbose) {
+      tty->print_cr("starting parallel_evacuate");
+      //    PrintHeapRegionsClosure pc1;
+      //    heap_region_iterate(&pc1);
+    }
+
+    _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_evac);
+
+    if (ShenandoahGCVerbose) {
+      tty->print("Printing all available regions");
+      print_heap_regions();
+    }
+
+    if (ShenandoahPrintCollectionSet) {
+      tty->print("Printing collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->length());
+      _collection_set->print();
+
+      tty->print("Printing free set which contains "SIZE_FORMAT" regions:\n", _free_regions->length());
+      _free_regions->print();
+
+      //    if (_collection_set->length() == 0)
+      //      print_heap_regions();
+    }
+
+    ParallelEvacuationTask evacuationTask = ParallelEvacuationTask(this, _collection_set);
+
+    conc_workers()->set_active_workers(_max_conc_workers);
+    conc_workers()->run_task(&evacuationTask);
+    //workers()->set_active_workers(_max_parallel_workers);
+
+    if (ShenandoahGCVerbose) {
+
+      tty->print("Printing postgc collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->available_regions());
+      _collection_set->print();
+
+      tty->print("Printing postgc free regions which contain "SIZE_FORMAT" free regions:\n", _free_regions->available_regions());
+      _free_regions->print();
+
+      tty->print_cr("finished parallel_evacuate");
+      print_heap_regions();
+
+      tty->print_cr("all regions after evacuation:");
+      print_heap_regions();
+    }
+
+    _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_evac);
+  }
+}
+
+class VerifyEvacuationClosure: public ExtendedOopClosure {
+private:
+  ShenandoahHeap*  _heap;
+  ShenandoahHeapRegion* _from_region;
+
+public:
+  VerifyEvacuationClosure(ShenandoahHeapRegion* from_region) :
+    _heap(ShenandoahHeap::heap()), _from_region(from_region) { }
+
+  void do_oop(oop* p)       {
+    oop heap_oop = oopDesc::load_heap_oop(p);
+    if (! oopDesc::is_null(heap_oop)) {
+      guarantee(! _from_region->is_in(heap_oop), err_msg("no references to from-region allowed after evacuation: "PTR_FORMAT, p2i((HeapWord*) heap_oop)));
+    }
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+
+};
+
+void ShenandoahHeap::roots_iterate(ExtendedOopClosure* cl) {
+
+  assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
+
+  CodeBlobToOopClosure blobsCl(cl, false);
+  CLDToOopClosure cldCl(cl);
+
+  ClassLoaderDataGraph::clear_claimed_marks();
+
+  ShenandoahRootProcessor rp(this, 1);
+  rp.process_all_roots(cl, &cldCl, &blobsCl);
+}
+
+void ShenandoahHeap::weak_roots_iterate(ExtendedOopClosure* cl) {
+  if (ShenandoahProcessReferences) {
+    ref_processor()->weak_oops_do(cl);
+  }
+  ShenandoahAlwaysTrueClosure always_true;
+  JNIHandles::weak_oops_do(&always_true, cl);
+}
+
+void ShenandoahHeap::verify_evacuation(ShenandoahHeapRegion* from_region) {
+
+  VerifyEvacuationClosure rootsCl(from_region);
+  roots_iterate(&rootsCl);
+
+}
+
+bool ShenandoahHeap::supports_tlab_allocation() const {
+  return true;
+}
+
+
+size_t  ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
+  ShenandoahHeapRegion* current = get_current_region_skip_humongous();
+  if (current == NULL)
+    return 0;
+  else if (current->free() > MinTLABSize) {
+    return current->free();
+  } else {
+    return MinTLABSize;
+  }
+}
+
+size_t ShenandoahHeap::max_tlab_size() const {
+  return ShenandoahHeapRegion::RegionSizeBytes;
+}
+
+class ResizeGCLABClosure : public ThreadClosure {
+public:
+  void do_thread(Thread* thread) {
+    thread->gclab().resize();
+  }
+};
+
+void ShenandoahHeap::resize_all_tlabs() {
+  CollectedHeap::resize_all_tlabs();
+
+  if (PrintTLAB && Verbose) {
+    tty->print_cr("Resizing Shenandoah GCLABs...");
+  }
+
+  ResizeGCLABClosure cl;
+  for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
+    cl.do_thread(thread);
+  }
+  gc_threads_do(&cl);
+
+  if (PrintTLAB && Verbose) {
+    tty->print_cr("Done resizing Shenandoah GCLABs...");
+  }
+}
+
+class AccumulateStatisticsGCLABClosure : public ThreadClosure {
+public:
+  void do_thread(Thread* thread) {
+    thread->gclab().accumulate_statistics();
+    thread->gclab().initialize_statistics();
+  }
+};
+
+void ShenandoahHeap::accumulate_statistics_all_gclabs() {
+
+  AccumulateStatisticsGCLABClosure cl;
+  for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
+    cl.do_thread(thread);
+  }
+  gc_threads_do(&cl);
+}
+
+bool  ShenandoahHeap::can_elide_tlab_store_barriers() const {
+  return true;
+}
+
+oop ShenandoahHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) {
+  // Overridden to do nothing.
+  return new_obj;
+}
+
+bool  ShenandoahHeap::can_elide_initializing_store_barrier(oop new_obj) {
+  return true;
+}
+
+bool ShenandoahHeap::card_mark_must_follow_store() const {
+  return false;
+}
+
+bool ShenandoahHeap::supports_heap_inspection() const {
+  return false;
+}
+
+size_t ShenandoahHeap::unsafe_max_alloc() {
+  return ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize;
+}
+
+void ShenandoahHeap::collect(GCCause::Cause cause) {
+  if (GCCause::is_user_requested_gc(cause)) {
+    if (! DisableExplicitGC) {
+      if (ShenandoahTraceFullGC) {
+        gclog_or_tty->print_cr("Shenandoah-full-gc: requested full GC");
+      }
+      cancel_concgc();
+      _concurrent_gc_thread->do_full_gc(cause);
+    }
+  } else if (cause == GCCause::_allocation_failure) {
+
+    if (ShenandoahTraceFullGC) {
+      gclog_or_tty->print_cr("Shenandoah-full-gc: full GC for allocation failure heap free: "SIZE_FORMAT", available: "SIZE_FORMAT, capacity() - used(), free_regions()->available());
+    }
+    cancel_concgc();
+    collector_policy()->set_should_clear_all_soft_refs(true);
+      _concurrent_gc_thread->do_full_gc(cause);
+
+  } else if (cause == GCCause::_gc_locker) {
+
+    if (ShenandoahTraceJNICritical) {
+      gclog_or_tty->print_cr("Resuming deferred evacuation after JNI critical regions");
+    }
+
+    jni_critical()->notify_jni_critical();
+  }
+}
+
+void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
+  //assert(false, "Shouldn't need to do full collections");
+}
+
+AdaptiveSizePolicy* ShenandoahHeap::size_policy() {
+  Unimplemented();
+  return NULL;
+
+}
+
+ShenandoahCollectorPolicy* ShenandoahHeap::collector_policy() const {
+  return _shenandoah_policy;
+}
+
+
+HeapWord* ShenandoahHeap::block_start(const void* addr) const {
+  Space* sp = space_containing(addr);
+  if (sp != NULL) {
+    return sp->block_start(addr);
+  }
+  return NULL;
+}
+
+size_t ShenandoahHeap::block_size(const HeapWord* addr) const {
+  Space* sp = space_containing(addr);
+  assert(sp != NULL, "block_size of address outside of heap");
+  return sp->block_size(addr);
+}
+
+bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
+  Space* sp = space_containing(addr);
+  return sp->block_is_obj(addr);
+}
+
+jlong ShenandoahHeap::millis_since_last_gc() {
+  return 0;
+}
+
+void ShenandoahHeap::prepare_for_verify() {
+  if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
+    ensure_parsability(false);
+  }
+}
+
+void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
+  workers()->print_worker_threads_on(st);
+  conc_workers()->print_worker_threads_on(st);
+}
+
+void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
+  workers()->threads_do(tcl);
+  conc_workers()->threads_do(tcl);
+}
+
+void ShenandoahHeap::print_tracing_info() const {
+  if (PrintGCDetails) {
+    _shenandoah_policy->print_tracing_info();
+  }
+}
+
+class ShenandoahVerifyRootsClosure: public ExtendedOopClosure {
+private:
+  ShenandoahHeap*  _heap;
+  VerifyOption     _vo;
+  bool             _failures;
+public:
+  // _vo == UsePrevMarking -> use "prev" marking information,
+  // _vo == UseNextMarking -> use "next" marking information,
+  // _vo == UseMarkWord    -> use mark word from object header.
+  ShenandoahVerifyRootsClosure(VerifyOption vo) :
+    _heap(ShenandoahHeap::heap()),
+    _vo(vo),
+    _failures(false) { }
+
+  bool failures() { return _failures; }
+
+  void do_oop(oop* p)       {
+    if (*p != NULL) {
+      oop heap_oop = oopDesc::load_heap_oop(p);
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+      if (!obj->is_oop()) {
+        { // Just for debugging.
+          gclog_or_tty->print_cr("Root location "PTR_FORMAT
+                                 "verified "PTR_FORMAT, p2i(p), p2i((void*) obj));
+          //      obj->print_on(gclog_or_tty);
+        }
+      }
+      guarantee(obj->is_oop(), "is_oop");
+    }
+  }
+
+  void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+
+};
+
+class ShenandoahVerifyHeapClosure: public ObjectClosure {
+private:
+  ShenandoahVerifyRootsClosure _rootsCl;
+public:
+  ShenandoahVerifyHeapClosure(ShenandoahVerifyRootsClosure rc) :
+    _rootsCl(rc) {};
+
+  void do_object(oop p) {
+    _rootsCl.do_oop(&p);
+  }
+};
+
+class ShenandoahVerifyKlassClosure: public KlassClosure {
+  OopClosure *_oop_closure;
+ public:
+  ShenandoahVerifyKlassClosure(OopClosure* cl) : _oop_closure(cl) {}
+  void do_klass(Klass* k) {
+    k->oops_do(_oop_closure);
+  }
+};
+
+void ShenandoahHeap::verify(bool silent , VerifyOption vo) {
+  if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
+
+    ShenandoahVerifyRootsClosure rootsCl(vo);
+
+    assert(Thread::current()->is_VM_thread(),
+           "Expected to be executed serially by the VM thread at this point");
+
+    roots_iterate(&rootsCl);
+
+    bool failures = rootsCl.failures();
+    if (ShenandoahGCVerbose)
+      gclog_or_tty->print("verify failures: %s", BOOL_TO_STR(failures));
+
+    ShenandoahVerifyHeapClosure heapCl(rootsCl);
+
+    object_iterate(&heapCl);
+    // TODO: Implement rest of it.
+#ifdef ASSERT_DISABLED
+    verify_live();
+#endif
+  } else {
+    if (!silent) gclog_or_tty->print("(SKIPPING roots, heapRegions, remset) ");
+  }
+}
+size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
+  return _free_regions->available();
+}
+
+class ShenandoahIterateObjectClosureRegionClosure: public ShenandoahHeapRegionClosure {
+  ObjectClosure* _cl;
+public:
+  ShenandoahIterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    r->object_iterate_interruptible(_cl, false);
+    return false;
+  }
+};
+
+class ShenandoahIterateUpdateClosure: public ShenandoahHeapRegionClosure {
+  ObjectClosure* _cl;
+public:
+  ShenandoahIterateUpdateClosure(ObjectClosure *cl) : _cl(cl) {}
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    if ((! r->is_in_collection_set()) && !r->is_humongous_continuation()) {
+      r->object_iterate_interruptible(_cl, false);
+    }
+    return false;
+  }
+};
+
+void ShenandoahHeap::cleanup_after_cancelconcgc() {
+  if (need_update_refs()) {
+  ShenandoahUpdateObjectsClosure update_refs_cl;
+  ShenandoahIterateUpdateClosure blk(&update_refs_cl);
+  heap_region_iterate(&blk, false, false);
+  }
+}
+
+class ShenandoahIterateObjectClosureCarefulRegionClosure: public ShenandoahHeapRegionClosure {
+  ObjectClosureCareful* _cl;
+public:
+  ShenandoahIterateObjectClosureCarefulRegionClosure(ObjectClosureCareful* cl) : _cl(cl) {}
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    r->object_iterate_careful(_cl);
+    return false;
+  }
+};
+
+void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
+  ShenandoahIterateObjectClosureRegionClosure blk(cl);
+  heap_region_iterate(&blk, false, true);
+}
+
+void ShenandoahHeap::object_iterate_careful(ObjectClosureCareful* cl) {
+  ShenandoahIterateObjectClosureCarefulRegionClosure blk(cl);
+  heap_region_iterate(&blk, false, true);
+}
+
+void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
+  Unimplemented();
+}
+
+void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl) {
+  marked_object_iterate(region, cl, region->bottom(), region->top());
+}
+
+void ShenandoahHeap::marked_object_iterate_careful(ShenandoahHeapRegion* region, ObjectClosure* cl) {
+  marked_object_iterate(region, cl, region->bottom(), region->concurrent_iteration_safe_limit());
+}
+
+void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl,
+                                           HeapWord* addr, HeapWord* limit) {
+  addr += BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+  HeapWord* last_addr = NULL;
+  size_t last_size = 0;
+  while (addr < limit) {
+    addr = _next_mark_bit_map->getNextMarkedWordAddress(addr, limit);
+    if (addr < limit) {
+      oop obj = oop(addr);
+      assert(is_marked_current(obj), "object expected to be marked");
+      cl->do_object(obj);
+      last_addr = addr;
+      last_size = obj->size();
+      addr += obj->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+    } else {
+      break;
+    }
+  }
+}
+
+class ShenandoahIterateOopClosureRegionClosure : public ShenandoahHeapRegionClosure {
+  MemRegion _mr;
+  ExtendedOopClosure* _cl;
+  bool _skip_unreachable_objects;
+public:
+  ShenandoahIterateOopClosureRegionClosure(ExtendedOopClosure* cl, bool skip_unreachable_objects) :
+    _cl(cl), _skip_unreachable_objects(skip_unreachable_objects) {}
+  ShenandoahIterateOopClosureRegionClosure(MemRegion mr, ExtendedOopClosure* cl)
+    :_mr(mr), _cl(cl) {}
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    r->oop_iterate_skip_unreachable(_cl, _skip_unreachable_objects);
+    return false;
+  }
+};
+
+void ShenandoahHeap::oop_iterate(ExtendedOopClosure* cl, bool skip_dirty_regions, bool skip_unreachable_objects) {
+  ShenandoahIterateOopClosureRegionClosure blk(cl, skip_unreachable_objects);
+  heap_region_iterate(&blk, skip_dirty_regions, true);
+}
+
+void ShenandoahHeap::oop_iterate(MemRegion mr,
+                                 ExtendedOopClosure* cl) {
+  ShenandoahIterateOopClosureRegionClosure blk(mr, cl);
+  heap_region_iterate(&blk, false, true);
+}
+
+void  ShenandoahHeap::object_iterate_since_last_GC(ObjectClosure* cl) {
+  Unimplemented();
+}
+
+class SpaceClosureRegionClosure: public ShenandoahHeapRegionClosure {
+  SpaceClosure* _cl;
+public:
+  SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    _cl->do_space(r);
+    return false;
+  }
+};
+
+void  ShenandoahHeap::space_iterate(SpaceClosure* cl) {
+  SpaceClosureRegionClosure blk(cl);
+  heap_region_iterate(&blk);
+}
+
+ShenandoahHeapRegion*
+ShenandoahHeap::heap_region_containing(const void* addr) const {
+  uint index = heap_region_index_containing(addr);
+  ShenandoahHeapRegion* result = _ordered_regions[index];
+#ifdef ASSERT
+  if (!(addr >= result->bottom() && addr < result->end())) {
+    tty->print_cr("heap region does not contain address, first_region_bottom: "PTR_FORMAT", real bottom of first region: "PTR_FORMAT", num_regions: "SIZE_FORMAT, p2i(_first_region_bottom), p2i(_ordered_regions[0]->bottom()), _num_regions);
+  }
+#endif
+  assert(addr >= result->bottom() && addr < result->end(), "address must be in found region");
+  return result;
+}
+
+Space*  ShenandoahHeap::space_containing(const void* oop) const {
+  Space* res = heap_region_containing(oop);
+  return res;
+}
+
+void  ShenandoahHeap::gc_prologue(bool b) {
+  Unimplemented();
+}
+
+void  ShenandoahHeap::gc_epilogue(bool b) {
+  Unimplemented();
+}
+
+// Apply blk->doHeapRegion() on all committed regions in address order,
+// terminating the iteration early if doHeapRegion() returns true.
+void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions, bool skip_humongous_continuation) const {
+  for (size_t i = 0; i < _num_regions; i++) {
+    ShenandoahHeapRegion* current  = _ordered_regions[i];
+    if (skip_humongous_continuation && current->is_humongous_continuation()) {
+      continue;
+    }
+    if (skip_dirty_regions && current->is_in_collection_set()) {
+      continue;
+    }
+    if (blk->doHeapRegion(current)) {
+      return;
+    }
+  }
+}
+
+/**
+ * Maybe we need that at some point...
+oop* ShenandoahHeap::resolve_oop_ptr(oop* p) {
+  if (is_in(p) && heap_region_containing(p)->is_dirty()) {
+    // If the reference is in an object in from-space, we need to first
+    // find its to-space counterpart.
+    // TODO: This here is slow (linear search inside region). Make it faster.
+    oop from_space_oop = oop_containing_oop_ptr(p);
+    HeapWord* to_space_obj = (HeapWord*) oopDesc::bs()->resolve_oop(from_space_oop);
+    return (oop*) (to_space_obj + ((HeapWord*) p - ((HeapWord*) from_space_oop)));
+  } else {
+    return p;
+  }
+}
+
+oop ShenandoahHeap::oop_containing_oop_ptr(oop* p) {
+  HeapWord* from_space_ref = (HeapWord*) p;
+  ShenandoahHeapRegion* region = heap_region_containing(from_space_ref);
+  HeapWord* from_space_obj = NULL;
+  for (HeapWord* curr = region->bottom(); curr < from_space_ref; ) {
+    oop curr_obj = (oop) curr;
+    if (curr < from_space_ref && from_space_ref < (curr + curr_obj->size())) {
+      from_space_obj = curr;
+      break;
+    } else {
+      curr += curr_obj->size();
+    }
+  }
+  assert (from_space_obj != NULL, "must not happen");
+  oop from_space_oop = (oop) from_space_obj;
+  assert (from_space_oop->is_oop(), "must be oop");
+  assert(ShenandoahBarrierSet::is_brooks_ptr(oop(((HeapWord*) from_space_oop) - BrooksPointer::BROOKS_POINTER_OBJ_SIZE)), "oop must have a brooks ptr");
+  return from_space_oop;
+}
+ */
+
+class ClearLivenessClosure : public ShenandoahHeapRegionClosure {
+  ShenandoahHeap* sh;
+public:
+  ClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) { }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    r->clearLiveData();
+    return false;
+  }
+};
+
+
+void ShenandoahHeap::start_concurrent_marking() {
+
+  accumulate_statistics_all_tlabs();
+
+  set_concurrent_mark_in_progress(true);
+  // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
+  if (UseTLAB) {
+    ensure_parsability(true);
+  }
+
+  _shenandoah_policy->record_bytes_allocated(_bytesAllocSinceCM);
+  _used_start_gc = used();
+
+#ifdef ASSERT
+  if (ShenandoahDumpHeapBeforeConcurrentMark) {
+    ensure_parsability(false);
+    print_all_refs("pre-mark");
+  }
+#endif
+
+  ClearLivenessClosure clc(this);
+  heap_region_iterate(&clc);
+
+  // print_all_refs("pre -mark");
+
+  // oopDesc::_debug = true;
+
+  concurrentMark()->prepare_unmarked_root_objs();
+
+  //  print_all_refs("pre-mark2");
+}
+
+
+class VerifyLivenessClosure : public ExtendedOopClosure {
+
+  ShenandoahHeap* _sh;
+
+public:
+  VerifyLivenessClosure() : _sh ( ShenandoahHeap::heap() ) {}
+
+  template<class T> void do_oop_nv(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+      guarantee(_sh->heap_region_containing(obj)->is_in_collection_set() == (obj != oopDesc::bs()->resolve_oop(obj)),
+                err_msg("forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s",
+                        BOOL_TO_STR(_sh->heap_region_containing(obj)->is_in_collection_set()),
+                        BOOL_TO_STR(obj != oopDesc::bs()->resolve_oop(obj)))
+                );
+      obj = oopDesc::bs()->resolve_oop(obj);
+      guarantee(! _sh->heap_region_containing(obj)->is_in_collection_set(), "forwarded oops must not point to dirty regions");
+      guarantee(obj->is_oop(), "is_oop");
+      ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+      if (! sh->is_marked_current(obj)) {
+        sh->print_on(tty);
+      }
+      assert(sh->is_marked_current(obj), err_msg("Referenced Objects should be marked obj: "PTR_FORMAT", marked: %s, is_in_heap: %s",
+                                               p2i((HeapWord*) obj), BOOL_TO_STR(sh->is_marked_current(obj)), BOOL_TO_STR(sh->is_in(obj))));
+    }
+  }
+
+  void do_oop(oop* p)       { do_oop_nv(p); }
+  void do_oop(narrowOop* p) { do_oop_nv(p); }
+
+};
+
+void ShenandoahHeap::verify_live() {
+
+  VerifyLivenessClosure cl;
+  roots_iterate(&cl);
+
+  IterateMarkedObjectsClosure marked_oops(&cl);
+  object_iterate(&marked_oops);
+
+}
+
+class VerifyAfterEvacuationClosure : public ExtendedOopClosure {
+
+  ShenandoahHeap* _sh;
+
+public:
+  VerifyAfterEvacuationClosure() : _sh ( ShenandoahHeap::heap() ) {}
+
+  template<class T> void do_oop_nv(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+      guarantee(_sh->heap_region_containing(obj)->is_in_collection_set() == (obj != oopDesc::bs()->resolve_oop(obj)),
+                err_msg("forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s obj-klass: %s, marked: %s",
+                        BOOL_TO_STR(_sh->heap_region_containing(obj)->is_in_collection_set()),
+                        BOOL_TO_STR(obj != oopDesc::bs()->resolve_oop(obj)), obj->klass()->external_name(), BOOL_TO_STR(_sh->is_marked_current(obj)))
+                );
+      obj = oopDesc::bs()->resolve_oop(obj);
+      guarantee(! _sh->heap_region_containing(obj)->is_in_collection_set(), "forwarded oops must not point to dirty regions");
+      guarantee(obj->is_oop(), "is_oop");
+      guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace");
+    }
+  }
+
+  void do_oop(oop* p)       { do_oop_nv(p); }
+  void do_oop(narrowOop* p) { do_oop_nv(p); }
+
+};
+
+class VerifyAfterUpdateRefsClosure : public ExtendedOopClosure {
+
+  ShenandoahHeap* _sh;
+
+public:
+  VerifyAfterUpdateRefsClosure() : _sh ( ShenandoahHeap::heap() ) {}
+
+  template<class T> void do_oop_nv(T* p) {
+    T heap_oop = oopDesc::load_heap_oop(p);
+    if (!oopDesc::is_null(heap_oop)) {
+      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+      guarantee((! _sh->heap_region_containing(obj)->is_in_collection_set()),
+                err_msg("no live reference must point to from-space, is_marked: %s",
+                        BOOL_TO_STR(_sh->is_marked_current(obj))));
+      if (obj != oopDesc::bs()->resolve_oop(obj) && _sh->is_in(p)) {
+        tty->print_cr("top-limit: "PTR_FORMAT", p: "PTR_FORMAT, p2i(_sh->heap_region_containing(p)->concurrent_iteration_safe_limit()), p2i(p));
+      }
+      guarantee(obj == oopDesc::bs()->resolve_oop(obj), "no live reference must point to forwarded object");
+      guarantee(obj->is_oop(), "is_oop");
+      guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace");
+    }
+  }
+
+  void do_oop(oop* p)       { do_oop_nv(p); }
+  void do_oop(narrowOop* p) { do_oop_nv(p); }
+
+};
+
+void ShenandoahHeap::verify_heap_after_evacuation() {
+
+  verify_heap_size_consistency();
+
+  ensure_parsability(false);
+
+  VerifyAfterEvacuationClosure cl;
+  roots_iterate(&cl);
+
+  IterateMarkedCurrentObjectsClosure marked_oops(&cl);
+  object_iterate(&marked_oops);
+
+}
+
+class VerifyRegionsAfterUpdateRefsClosure : public ShenandoahHeapRegionClosure {
+public:
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    assert(! r->is_in_collection_set(), "no region must be in collection set");
+    assert(! ShenandoahHeap::heap()->in_cset_fast_test(r->bottom()), "no region must be in collection set");
+    return false;
+  }
+};
+
+void ShenandoahHeap::verify_regions_after_update_refs() {
+  VerifyRegionsAfterUpdateRefsClosure verify_regions;
+  heap_region_iterate(&verify_regions);
+}
+
+void ShenandoahHeap::verify_heap_after_update_refs() {
+
+  verify_heap_size_consistency();
+
+  ensure_parsability(false);
+
+  VerifyAfterUpdateRefsClosure cl;
+
+  roots_iterate(&cl);
+  weak_roots_iterate(&cl);
+  oop_iterate(&cl, true, true);
+
+}
+
+void ShenandoahHeap::stop_concurrent_marking() {
+  assert(concurrent_mark_in_progress(), "How else could we get here?");
+  if (! cancelled_concgc()) {
+    // If we needed to update refs, and concurrent marking has been cancelled,
+    // we need to finish updating references.
+    set_need_update_refs(false);
+  }
+  set_concurrent_mark_in_progress(false);
+
+  if (ShenandoahGCVerbose) {
+    print_heap_regions();
+  }
+
+#ifdef ASSERT
+  if (ShenandoahVerify && ! _cancelled_concgc) {
+    verify_heap_after_marking();
+  }
+
+#endif
+}
+
+bool ShenandoahHeap::concurrent_mark_in_progress() {
+  return _concurrent_mark_in_progress;
+}
+
+void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
+  if (ShenandoahTracePhases) {
+    if (in_progress) {
+      gclog_or_tty->print_cr("Shenandoah starting concurrent marking, heap used: "SIZE_FORMAT" MB", used() / M);
+    } else {
+      gclog_or_tty->print_cr("Shenandoah finishing concurrent marking, heap used: "SIZE_FORMAT" MB", used() / M);
+    }
+  }
+
+  _concurrent_mark_in_progress = in_progress;
+  JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, ! in_progress);
+}
+
+void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
+  if (ShenandoahTracePhases) {
+    if (ShenandoahConcurrentEvacuation) {
+      if (in_progress) {
+        gclog_or_tty->print_cr("Shenandoah starting concurrent evacuation, heap used: "SIZE_FORMAT" MB", used() / M);
+      } else {
+        gclog_or_tty->print_cr("Shenandoah finishing concurrent evacuation, heap used: "SIZE_FORMAT" MB", used() / M);
+      }
+    } else {
+      if (in_progress) {
+        gclog_or_tty->print_cr("Shenandoah starting non-concurrent evacuation");
+      } else {
+        gclog_or_tty->print_cr("Shenandoah finishing non-concurrent evacuation");
+      }
+    }
+  }
+  JavaThread::set_evacuation_in_progress_all_threads(in_progress);
+  _evacuation_in_progress = in_progress;
+  OrderAccess::fence();
+}
+
+bool ShenandoahHeap::is_evacuation_in_progress() {
+  return _evacuation_in_progress;
+}
+
+bool ShenandoahHeap::is_update_references_in_progress() {
+  return _update_references_in_progress;
+}
+
+void ShenandoahHeap::set_update_references_in_progress(bool update_refs_in_progress) {
+  if (ShenandoahTracePhases) {
+    if (ShenandoahConcurrentUpdateRefs) {
+      if (update_refs_in_progress) {
+        gclog_or_tty->print_cr("Shenandoah starting concurrent reference-updating");
+      } else {
+        gclog_or_tty->print_cr("Shenandoah finishing concurrent reference-updating");
+      }
+    } else {
+      if (update_refs_in_progress) {
+        gclog_or_tty->print_cr("Shenandoah starting non-concurrent reference-updating");
+      } else {
+        gclog_or_tty->print_cr("Shenandoah finishing non-concurrent reference-updating");
+      }
+    }
+  }
+  _update_references_in_progress = update_refs_in_progress;
+}
+
+void ShenandoahHeap::verify_copy(oop p,oop c){
+    assert(p != oopDesc::bs()->resolve_oop(p), "forwarded correctly");
+    assert(oopDesc::bs()->resolve_oop(p) == c, "verify pointer is correct");
+    if (p->klass() != c->klass()) {
+      print_heap_regions();
+    }
+    assert(p->klass() == c->klass(), err_msg("verify class p-size: "INT32_FORMAT" c-size: "INT32_FORMAT, p->size(), c->size()));
+    assert(p->size() == c->size(), "verify size");
+    // Object may have been locked between copy and verification
+    //    assert(p->mark() == c->mark(), "verify mark");
+    assert(c == oopDesc::bs()->resolve_oop(c), "verify only forwarded once");
+  }
+
+void ShenandoahHeap::oom_during_evacuation() {
+  // tty->print_cr("Out of memory during evacuation, cancel evacuation, schedule full GC");
+  // We ran out of memory during evacuation. Cancel evacuation, and schedule a full-GC.
+  collector_policy()->set_should_clear_all_soft_refs(true);
+  concurrent_thread()->schedule_full_gc();
+  cancel_concgc();
+
+  if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) {
+    tty->print_cr("OOM during evacuation. Let Java thread wait until evacuation settlded..");
+    while (_evacuation_in_progress) { // wait.
+      Thread::current()->_ParkEvent->park(1) ;
+    }
+  }
+
+}
+
+void ShenandoahHeap::copy_object(oop p, HeapWord* s) {
+  HeapWord* filler = s;
+  assert(s != NULL, "allocation of brooks pointer must not fail");
+  HeapWord* copy = s + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+
+  guarantee(copy != NULL, "allocation of copy object must not fail");
+  Copy::aligned_disjoint_words((HeapWord*) p, copy, p->size());
+  initialize_brooks_ptr(filler, copy);
+
+#ifdef ASSERT
+  if (ShenandoahTraceEvacuations) {
+    tty->print_cr("copy object from "PTR_FORMAT" to: "PTR_FORMAT, p2i((HeapWord*) p), p2i(copy));
+  }
+#endif
+}
+
+oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
+  ShenandoahHeapRegion* hr;
+  size_t required;
+
+#ifdef ASSERT
+  if (ShenandoahVerifyReadsToFromSpace) {
+    hr = heap_region_containing(p);
+    {
+      hr->memProtectionOff();
+      required  = BrooksPointer::BROOKS_POINTER_OBJ_SIZE + p->size();
+      hr->memProtectionOn();
+    }
+  } else {
+    required  = BrooksPointer::BROOKS_POINTER_OBJ_SIZE + p->size();
+  }
+#else
+    required  = BrooksPointer::BROOKS_POINTER_OBJ_SIZE + p->size();
+#endif
+
+  assert(! heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
+
+  // Don't even attempt to evacuate anything if evacuation has been cancelled.
+  if (_cancelled_concgc) {
+    return ShenandoahBarrierSet::resolve_oop_static(p);
+  }
+
+  bool alloc_from_gclab = true;
+  thread->set_evacuating(true);
+  HeapWord* filler = allocate_from_gclab(thread, required);
+  if (filler == NULL) {
+    filler = allocate_memory(required);
+    alloc_from_gclab = false;
+  }
+  thread->set_evacuating(false);
+
+  if (filler == NULL) {
+    oom_during_evacuation();
+    // If this is a Java thread, it should have waited
+    // until all GC threads are done, and then we
+    // return the forwardee.
+    oop resolved = ShenandoahBarrierSet::resolve_oop_static(p);
+    return resolved;
+  }
+
+  HeapWord* copy = filler + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+
+#ifdef ASSERT
+  if (ShenandoahVerifyReadsToFromSpace) {
+    hr->memProtectionOff();
+    copy_object(p, filler);
+    hr->memProtectionOn();
+  } else {
+    copy_object(p, filler);
+  }
+#else
+    copy_object(p, filler);
+#endif
+
+  HeapWord* result = BrooksPointer::get(p).cas_forwardee((HeapWord*) p, copy);
+
+  oop return_val;
+  if (result == (HeapWord*) p) {
+    return_val = oop(copy);
+
+    if (shenandoahPolicy()->update_refs_early()) {
+      mark_current(return_val);
+    }
+
+#ifdef ASSERT
+    if (ShenandoahTraceEvacuations) {
+      tty->print("Copy of "PTR_FORMAT" to "PTR_FORMAT" succeeded \n", p2i((HeapWord*) p), p2i(copy));
+    }
+    assert(return_val->is_oop(), "expect oop");
+    assert(p->klass() == return_val->klass(), err_msg("Should have the same class p: "PTR_FORMAT", copy: "PTR_FORMAT, p2i((HeapWord*) p), p2i((HeapWord*) copy)));
+#endif
+  }  else {
+    if (alloc_from_gclab) {
+      thread->gclab().rollback(required);
+    }
+#ifdef ASSERT
+    if (ShenandoahTraceEvacuations) {
+      tty->print_cr("Copy of "PTR_FORMAT" to "PTR_FORMAT" failed, use other: "PTR_FORMAT, p2i((HeapWord*) p), p2i(copy), p2i((HeapWord*) result));
+    }
+#endif
+    return_val = (oopDesc*) result;
+  }
+
+  return return_val;
+}
+
+HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
+  HeapWord* result = obj + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+  initialize_brooks_ptr(obj, result);
+  return result;
+}
+
+uint ShenandoahHeap::oop_extra_words() {
+  return BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+}
+
+bool ShenandoahHeap::grow_heap_by() {
+  int new_region_index = ensure_new_regions(1);
+  if (new_region_index != -1) {
+    ShenandoahHeapRegion* new_region = new ShenandoahHeapRegion();
+    HeapWord* start = _first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * new_region_index;
+    new_region->initialize_heap_region(start, ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize, new_region_index);
+    if (ShenandoahGCVerbose) {
+      tty->print_cr("allocating new region at index: "INT32_FORMAT, new_region_index);
+      new_region->print();
+    }
+    _ordered_regions[new_region_index] = new_region;
+    _free_regions->append(new_region);
+    return true;
+  } else {
+    return false;
+  }
+}
+
+int ShenandoahHeap::ensure_new_regions(int new_regions) {
+
+  size_t num_regions = _num_regions;
+  size_t new_num_regions = num_regions + new_regions;
+  if (new_num_regions >= _max_regions) {
+    // Not enough regions left.
+    return -1;
+  }
+
+  size_t expand_size = new_regions * ShenandoahHeapRegion::RegionSizeBytes;
+  if (ShenandoahGCVerbose) {
+    tty->print_cr("expanding storage by "SIZE_FORMAT_HEX" bytes, for "INT32_FORMAT" new regions", expand_size, new_regions);
+  }
+  bool success = _storage.expand_by(expand_size);
+  assert(success, "should always be able to expand by requested size");
+
+  _num_regions = new_num_regions;
+
+  return num_regions;
+
+}
+
+bool  ShenandoahIsAliveClosure:: do_object_b(oop obj) {
+
+  ShenandoahHeap* sh = ShenandoahHeap::heap();
+  if (sh->need_update_refs()) {
+    obj = ShenandoahBarrierSet::resolve_oop_static(obj);
+  }
+
+#ifdef ASSERT
+  if (obj != ShenandoahBarrierSet::resolve_oop_static(obj)) {
+    ShenandoahHeap* sh = ShenandoahHeap::heap();
+  }
+#endif
+  assert(obj == ShenandoahBarrierSet::resolve_oop_static(obj), "needs to be in to-space");
+
+    HeapWord* addr = (HeapWord*) obj;
+
+    if (ShenandoahTraceWeakReferences) {
+
+      if (addr != NULL) {
+        if(sh->is_in(addr)) {
+          if (sh->is_obj_ill(obj)) {
+            HandleMark hm;
+            tty->print_cr("ShenandoahIsAliveClosure Found an ill object "PTR_FORMAT, p2i((HeapWord*) obj));
+            obj->print();
+          }
+          else
+            tty->print_cr("found a healthy object "PTR_FORMAT, p2i((HeapWord*) obj));
+
+        } else {
+          tty->print_cr("found an object outside the heap "PTR_FORMAT, p2i((HeapWord*) obj));
+        }
+      } else {
+        tty->print_cr("found a null object "PTR_FORMAT, p2i((HeapWord*) obj));
+      }
+    }
+
+    return addr != NULL && sh->is_marked_current(obj); //(!sh->is_in(addr) || !sh->is_obj_ill(obj));
+}
+
+void ShenandoahHeap::ref_processing_init() {
+  MemRegion mr = reserved_region();
+
+  // Concurrent Mark ref processor
+//   _ref_processor =
+//     new ReferenceProcessor(mr,    // span
+//                            ParallelRefProcEnabled && (ParallelGCThreads > 1),
+//                                 // mt processing
+//                            (int) ParallelGCThreads,
+//                                 // degree of mt processing
+//                            (ParallelGCThreads > 1) || (ConcGCThreads > 1),
+//                                 // mt discovery
+//                            (int) MAX2(ParallelGCThreads, ConcGCThreads),
+//                                 // degree of mt discovery
+//                            false,
+//                                 // Reference discovery is not atomic
+//                         &isAlive);
+//                                 // is alive closure
+//                                 // (for efficiency/performance)
+  _ref_processor =
+    new ReferenceProcessor(mr,    // span
+                           ParallelRefProcEnabled && (ConcGCThreads > 1),
+                           // mt processing
+                           (int) ConcGCThreads,
+                           // degree of mt processing
+                           (ConcGCThreads > 1),
+                           // mt discovery
+                           (int) ConcGCThreads,
+                           // degree of mt discovery
+                           false,
+                           // Reference discovery is not atomic
+                           &isAlive);
+  // is alive closure
+  // (for efficiency/performance)
+
+
+
+}
+
+#ifdef ASSERT
+void ShenandoahHeap::set_from_region_protection(bool protect) {
+  for (uint i = 0; i < _num_regions; i++) {
+    ShenandoahHeapRegion* region = _ordered_regions[i];
+    if (region != NULL && region->is_in_collection_set()) {
+      if (protect) {
+        region->memProtectionOn();
+      } else {
+        region->memProtectionOff();
+      }
+    }
+  }
+}
+#endif
+
+void ShenandoahHeap::acquire_pending_refs_lock() {
+  _concurrent_gc_thread->slt()->manipulatePLL(SurrogateLockerThread::acquirePLL);
+}
+
+void ShenandoahHeap::release_pending_refs_lock() {
+  _concurrent_gc_thread->slt()->manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL);
+}
+
+ShenandoahHeapRegion** ShenandoahHeap::heap_regions() {
+  return _ordered_regions;
+}
+
+size_t ShenandoahHeap::num_regions() {
+  return _num_regions;
+}
+
+size_t ShenandoahHeap::max_regions() {
+  return _max_regions;
+}
+
+GCTracer* ShenandoahHeap::tracer() {
+  return collector_policy()->tracer();
+}
+
+size_t ShenandoahHeap::tlab_used(Thread* thread) const {
+  return _free_regions->used();
+}
+
+void ShenandoahHeap::cancel_concgc() {
+  // only report it once
+  if (!_cancelled_concgc) {
+    if (ShenandoahTracePhases) {
+      tty->print_cr("Cancelling GC");
+    }
+    _cancelled_concgc = true;
+    OrderAccess::fence();
+    _shenandoah_policy->report_concgc_cancelled();
+  }
+
+  if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) {
+    while (_evacuation_in_progress) { // wait.
+      Thread::current()->_ParkEvent->park(1) ;
+    }
+  }
+}
+
+bool ShenandoahHeap::cancelled_concgc() {
+  bool cancelled = _cancelled_concgc;
+  return cancelled;
+}
+
+void ShenandoahHeap::clear_cancelled_concgc() {
+  _cancelled_concgc = false;
+}
+
+int ShenandoahHeap::max_workers() {
+  return _max_workers;
+}
+
+int ShenandoahHeap::max_parallel_workers() {
+  return _max_parallel_workers;
+}
+int ShenandoahHeap::max_conc_workers() {
+  return _max_conc_workers;
+}
+
+void ShenandoahHeap::shutdown() {
+  // We set this early here, to let GC threads terminate before we ask the concurrent thread
+  // to terminate, which would otherwise block until all GC threads come to finish normally.
+  _cancelled_concgc = true;
+  _concurrent_gc_thread->shutdown();
+  cancel_concgc();
+}
+
+class ShenandoahStringSymbolTableUnlinkTask : public AbstractGangTask {
+private:
+  BoolObjectClosure* _is_alive;
+  int _initial_string_table_size;
+  int _initial_symbol_table_size;
+
+  bool  _process_strings;
+  int _strings_processed;
+  int _strings_removed;
+
+  bool  _process_symbols;
+  int _symbols_processed;
+  int _symbols_removed;
+
+public:
+  ShenandoahStringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) :
+    AbstractGangTask("String/Symbol Unlinking"),
+    _is_alive(is_alive),
+    _process_strings(process_strings), _strings_processed(0), _strings_removed(0),
+    _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) {
+
+    _initial_string_table_size = StringTable::the_table()->table_size();
+    _initial_symbol_table_size = SymbolTable::the_table()->table_size();
+    if (process_strings) {
+      StringTable::clear_parallel_claimed_index();
+    }
+    if (process_symbols) {
+      SymbolTable::clear_parallel_claimed_index();
+    }
+  }
+
+  ~ShenandoahStringSymbolTableUnlinkTask() {
+    guarantee(!_process_strings || StringTable::parallel_claimed_index() >= _initial_string_table_size,
+              err_msg("claim value %d after unlink less than initial string table size %d",
+                      StringTable::parallel_claimed_index(), _initial_string_table_size));
+    guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size,
+              err_msg("claim value %d after unlink less than initial symbol table size %d",
+                      SymbolTable::parallel_claimed_index(), _initial_symbol_table_size));
+
+    if (G1TraceStringSymbolTableScrubbing) {
+      gclog_or_tty->print_cr("Cleaned string and symbol table, "
+                             "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, "
+                             "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed",
+                             strings_processed(), strings_removed(),
+                             symbols_processed(), symbols_removed());
+    }
+  }
+
+  void work(uint worker_id) {
+    int strings_processed = 0;
+    int strings_removed = 0;
+    int symbols_processed = 0;
+    int symbols_removed = 0;
+    if (_process_strings) {
+      StringTable::possibly_parallel_unlink(_is_alive, &strings_processed, &strings_removed);
+      Atomic::add(strings_processed, &_strings_processed);
+      Atomic::add(strings_removed, &_strings_removed);
+    }
+    if (_process_symbols) {
+      SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed);
+      Atomic::add(symbols_processed, &_symbols_processed);
+      Atomic::add(symbols_removed, &_symbols_removed);
+    }
+  }
+
+  size_t strings_processed() const { return (size_t)_strings_processed; }
+  size_t strings_removed()   const { return (size_t)_strings_removed; }
+
+  size_t symbols_processed() const { return (size_t)_symbols_processed; }
+  size_t symbols_removed()   const { return (size_t)_symbols_removed; }
+};
+
+void ShenandoahHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) {
+
+  workers()->set_active_workers(_max_parallel_workers);
+  ShenandoahStringSymbolTableUnlinkTask shenandoah_unlink_task(is_alive, process_strings, process_symbols);
+  workers()->run_task(&shenandoah_unlink_task);
+
+  //  if (G1StringDedup::is_enabled()) {
+  //    G1StringDedup::unlink(is_alive);
+  //  }
+}
+
+bool ShenandoahHeap::is_obj_ill(const oop obj) const {
+  return ! is_marked_current(obj);
+}
+
+void ShenandoahHeap::set_need_update_refs(bool need_update_refs) {
+  _need_update_refs = need_update_refs;
+}
+
+ShenandoahJNICritical* ShenandoahHeap::jni_critical() {
+  return _jni_critical;
+}
+
+ShenandoahHeapRegion* ShenandoahHeap::next_compaction_region(const ShenandoahHeapRegion* r) {
+  int region_idx = r->region_number() + 1;
+  ShenandoahHeapRegion* next = _ordered_regions[region_idx];
+  guarantee(next->region_number() == region_idx, "region number must match");
+  while (next->is_humongous()) {
+    region_idx = next->region_number() + 1;
+    next = _ordered_regions[region_idx];
+    guarantee(next->region_number() == region_idx, "region number must match");
+  }
+  return next;
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeap.hpp b/src/share/vm/gc/shenandoah/shenandoahHeap.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeap.hpp
@@ -0,0 +1,464 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
+
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
+#include "gc/shenandoah/shenandoahConcurrentThread.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
+
+#include "gc/shared/cmBitMap.hpp"
+#include "gc/g1/heapRegionBounds.inline.hpp"
+
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/space.hpp"
+#include "oops/oop.hpp"
+#include "oops/markOop.hpp"
+
+
+class SpaceClosure;
+class GCTracer;
+
+class ShenandoahJNICritical;
+
+class ShenandoahJNICritical;
+
+class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
+public:
+  bool do_object_b(oop p) { return true; }
+};
+
+
+class ShenandoahIsAliveClosure: public BoolObjectClosure {
+
+public:
+  bool do_object_b(oop obj);
+};
+
+
+class ShenandoahHeapRegionClosure : public StackObj {
+  bool _complete;
+  void incomplete() {_complete = false;}
+
+public:
+  ShenandoahHeapRegionClosure(): _complete(true) {}
+
+  // typically called on each region until it returns true;
+  virtual bool doHeapRegion(ShenandoahHeapRegion* r) = 0;
+
+  bool complete() { return _complete;}
+};
+
+// A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
+// It uses a new pauseless GC algorithm based on Brooks pointers.
+// Derived from G1
+
+//
+// CollectedHeap
+//    SharedHeap
+//      ShenandoahHeap
+
+class ShenandoahHeap : public CollectedHeap {
+
+private:
+
+  static ShenandoahHeap* _pgc;
+  ShenandoahCollectorPolicy* _shenandoah_policy;
+  VirtualSpace _storage;
+  ShenandoahHeapRegion* _first_region;
+  HeapWord* _first_region_bottom;
+  // Ordered array of regions  (name confusing with _regions)
+  ShenandoahHeapRegion** _ordered_regions;
+
+  // Sortable array of regions
+  ShenandoahHeapRegionSet* _free_regions;
+  ShenandoahHeapRegionSet* _collection_set;
+  ShenandoahHeapRegion* _currentAllocationRegion;
+  ShenandoahConcurrentMark* _scm;
+
+
+
+  ShenandoahConcurrentThread* _concurrent_gc_thread;
+
+  size_t _num_regions;
+  size_t _max_regions;
+  size_t _initialSize;
+#ifndef NDEBUG
+  uint _numAllocs;
+#endif
+  WorkGangBarrierSync barrierSync;
+  int _max_parallel_workers;
+  int _max_conc_workers;
+  int _max_workers;
+
+  WorkGang* _conc_workers;
+  WorkGang* _workers;
+
+
+  volatile size_t _used;
+
+  CMBitMap _mark_bit_map;
+  CMBitMap* _next_mark_bit_map;
+
+  bool* _in_cset_fast_test;
+  bool* _in_cset_fast_test_base;
+  uint _in_cset_fast_test_length;
+
+  bool _cancelled_concgc;
+
+  ShenandoahJNICritical* _jni_critical;
+
+public:
+  size_t _bytesAllocSinceCM;
+  size_t _bytes_allocated_during_cm;
+  size_t _bytes_allocated_during_cm_start;
+  size_t _max_allocated_gc;
+  size_t _allocated_last_gc;
+  size_t _used_start_gc;
+
+public:
+  ShenandoahHeap(ShenandoahCollectorPolicy* policy);
+  HeapWord* allocate_from_gclab(Thread* thread, size_t size);
+  HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
+  HeapWord* allocate_new_tlab(size_t word_size);
+  HeapWord* allocate_new_gclab(size_t word_size);
+private:
+  HeapWord* allocate_new_tlab(size_t word_size, bool mark);
+public:
+  HeapWord* allocate_memory(size_t word_size);
+
+  bool find_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions);
+  bool allocate_contiguous_free_regions(uint num_free_regions, ShenandoahHeapRegion** free_regions);
+
+  // For now we are ignoring eden.
+  inline bool should_alloc_in_eden(size_t size) { return false;}
+  void print_on(outputStream* st) const ;
+
+  ShenandoahHeap::Name kind() const {
+    return CollectedHeap::ShenandoahHeap;
+  }
+
+  static ShenandoahHeap* heap();
+
+  ShenandoahCollectorPolicy *shenandoahPolicy() { return _shenandoah_policy;}
+
+  jint initialize();
+  static size_t conservative_max_heap_alignment() {
+    return HeapRegionBounds::max_size();
+  }
+
+  void post_initialize();
+  size_t capacity() const;
+  size_t used() const;
+  bool is_maximal_no_gc() const;
+  size_t max_capacity() const;
+  virtual bool is_in(const void* p) const;
+  bool is_in_partial_collection(const void* p);
+  bool is_scavengable(const void* addr);
+  virtual HeapWord* mem_allocate(size_t size, bool* what);
+  HeapWord* mem_allocate_locked(size_t size, bool* what);
+  virtual size_t unsafe_max_alloc();
+  bool can_elide_tlab_store_barriers() const;
+  virtual oop new_store_pre_barrier(JavaThread* thread, oop new_obj);
+  bool can_elide_initializing_store_barrier(oop new_obj);
+  bool card_mark_must_follow_store() const;
+  bool supports_heap_inspection() const;
+  void collect(GCCause::Cause);
+  void do_full_collection(bool clear_all_soft_refs);
+  AdaptiveSizePolicy* size_policy();
+  ShenandoahCollectorPolicy* collector_policy() const;
+
+  void ensure_parsability(bool retire_tlabs);
+
+  void add_free_region(ShenandoahHeapRegion* r) {_free_regions->append(r);}
+  void clear_free_regions() {_free_regions->clear();}
+
+  void oop_iterate(ExtendedOopClosure* cl, bool skip_dirty_regions,
+                   bool skip_unreachable_objects);
+  void oop_iterate(ExtendedOopClosure* cl) {
+    oop_iterate(cl, false, false);
+  }
+
+  void roots_iterate(ExtendedOopClosure* cl);
+  void weak_roots_iterate(ExtendedOopClosure* cl);
+
+  void object_iterate(ObjectClosure* cl);
+  void object_iterate_careful(ObjectClosureCareful* cl);
+  void object_iterate_no_from_space(ObjectClosure* cl);
+  void safe_object_iterate(ObjectClosure* cl);
+
+  void marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl);
+  void marked_object_iterate_careful(ShenandoahHeapRegion* region, ObjectClosure* cl);
+private:
+  void marked_object_iterate(ShenandoahHeapRegion* region, ObjectClosure* cl, HeapWord* start, HeapWord* limit);
+
+public:
+  HeapWord* block_start(const void* addr) const;
+  size_t block_size(const HeapWord* addr) const;
+  bool block_is_obj(const HeapWord* addr) const;
+  jlong millis_since_last_gc();
+  void prepare_for_verify();
+  void print_gc_threads_on(outputStream* st) const;
+  void gc_threads_do(ThreadClosure* tcl) const;
+  void print_tracing_info() const;
+  void verify(bool silent,  VerifyOption vo);
+  bool supports_tlab_allocation() const;
+  virtual size_t tlab_capacity(Thread *thr) const;
+  void oop_iterate(MemRegion mr, ExtendedOopClosure* ecl);
+  void object_iterate_since_last_GC(ObjectClosure* cl);
+  void space_iterate(SpaceClosure* scl);
+  virtual size_t unsafe_max_tlab_alloc(Thread *thread) const;
+  virtual size_t max_tlab_size() const;
+
+  void resize_all_tlabs();
+  void accumulate_statistics_all_gclabs();
+
+  HeapWord* tlab_post_allocation_setup(HeapWord* obj);
+
+  uint oop_extra_words();
+
+#ifndef CC_INTERP
+  void compile_prepare_oop(MacroAssembler* masm, Register obj = rax);
+#endif
+
+  Space* space_containing(const void* oop) const;
+  void gc_prologue(bool b);
+  void gc_epilogue(bool b);
+
+  void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions = false, bool skip_humongous_continuation = false) const;
+  ShenandoahHeapRegion* heap_region_containing(const void* addr) const;
+  inline uint heap_region_index_containing(const void* addr) const;
+
+/**
+ * Maybe we need that at some point...
+
+  oop* resolve_oop_ptr(oop* p);
+
+  oop oop_containing_oop_ptr(oop* p);
+
+*/
+
+  void temp();
+
+  volatile unsigned int _concurrent_mark_in_progress;
+
+  volatile unsigned int _evacuation_in_progress;
+  volatile bool _update_references_in_progress;
+  bool _need_update_refs;
+  bool _need_reset_bitmaps;
+
+  void start_concurrent_marking();
+  void stop_concurrent_marking();
+  ShenandoahConcurrentMark* concurrentMark() { return _scm;}
+  ShenandoahConcurrentThread* concurrent_thread() { return _concurrent_gc_thread; }
+
+  ShenandoahJNICritical* jni_critical();
+
+  size_t bump_object_age(HeapWord* start, HeapWord* end);
+
+  inline bool mark_current(oop obj) const;
+  inline bool mark_current_no_checks(oop obj) const;
+  inline bool is_marked_current(oop obj) const;
+
+  ReferenceProcessor* _ref_processor;
+  bool is_marked_prev(oop obj) const;
+
+  bool is_obj_ill(const oop obj) const;
+
+  void reset_mark_bitmap();
+  void reset_mark_bitmap_range(HeapWord* from, HeapWord* to);
+
+  bool is_bitmap_clear();
+
+  void mark_object_live(oop obj, bool enqueue);
+
+  void prepare_for_concurrent_evacuation();
+  void do_evacuation();
+  void parallel_evacuate();
+
+  void initialize_brooks_ptr(HeapWord* brooks_ptr, HeapWord* object, bool new_obj = true);
+  void initialize_brooks_ptr(oop p);
+
+  inline oop maybe_update_oop_ref(oop* p);
+  void evacuate_region(ShenandoahHeapRegion* from_region, ShenandoahHeapRegion* to_region);
+  void parallel_evacuate_region(ShenandoahHeapRegion* from_region);
+  void verify_evacuated_region(ShenandoahHeapRegion* from_region);
+
+  void print_heap_regions(outputStream* st = tty) const;
+
+  void print_all_refs(const char* prefix);
+
+  void print_heap_objects(HeapWord* start, HeapWord* end);
+  void print_heap_locations(HeapWord* start, HeapWord* end);
+  void print_heap_object(oop p);
+
+  oop  evacuate_object(oop src, Thread* thread);
+  bool is_in_collection_set(const void* p) {
+    return heap_region_containing(p)->is_in_collection_set();
+  }
+
+  void copy_object(oop p, HeapWord* s);
+  void verify_copy(oop p, oop c);
+  //  void assign_brooks_pointer(oop p, HeapWord* filler, HeapWord* copy);
+  void verify_heap_size_consistency();
+  void verify_heap_after_marking();
+  void verify_heap_after_evacuation();
+  void verify_heap_after_update_refs();
+  void verify_regions_after_update_refs();
+
+  static ByteSize ordered_regions_offset() { return byte_offset_of(ShenandoahHeap, _ordered_regions); }
+  static ByteSize first_region_bottom_offset() { return byte_offset_of(ShenandoahHeap, _first_region_bottom); }
+
+  void cleanup_after_cancelconcgc();
+  void increase_used(size_t bytes);
+  void decrease_used(size_t bytes);
+  void set_used(size_t bytes);
+
+  int ensure_new_regions(int num_new_regions);
+
+  void set_evacuation_in_progress(bool in_progress);
+  bool is_evacuation_in_progress();
+
+  bool is_update_references_in_progress();
+  void set_update_references_in_progress(bool update_refs_in_progress);
+
+  inline bool need_update_refs() const;
+  void set_need_update_refs(bool update_refs);
+
+  ReferenceProcessor* ref_processor() { return _ref_processor;}
+  virtual void ref_processing_init();
+  ShenandoahIsAliveClosure isAlive;
+  void evacuate_and_update_roots();
+  void prepare_for_update_references();
+
+  void update_references();
+
+  ShenandoahHeapRegionSet* free_regions();
+
+  void update_roots();
+
+  void acquire_pending_refs_lock();
+  void release_pending_refs_lock();
+
+  int max_workers();
+  int max_conc_workers();
+  int max_parallel_workers();
+  WorkGang* conc_workers() const{ return _conc_workers;}
+  WorkGang* workers() const{ return _workers;}
+
+  ShenandoahHeapRegion** heap_regions();
+  size_t num_regions();
+  size_t max_regions();
+
+  ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
+
+  void recycle_dirty_regions();
+
+  void register_region_with_in_cset_fast_test(ShenandoahHeapRegion* r) {
+    assert(_in_cset_fast_test_base != NULL, "sanity");
+    assert(r->is_in_collection_set(), "invariant");
+    uint index = r->region_number();
+    assert(index < _in_cset_fast_test_length, "invariant");
+    assert(!_in_cset_fast_test_base[index], "invariant");
+    _in_cset_fast_test_base[index] = true;
+  }
+  bool in_cset_fast_test(HeapWord* obj) {
+    assert(_in_cset_fast_test != NULL, "sanity");
+    if (is_in(obj)) {
+      // no need to subtract the bottom of the heap from obj,
+      // _in_cset_fast_test is biased
+      uintx index = ((uintx) obj) >> ShenandoahHeapRegion::RegionSizeShift;
+      bool ret = _in_cset_fast_test[index];
+      // let's make sure the result is consistent with what the slower
+      // test returns
+      assert( ret || !is_in_collection_set(obj), "sanity");
+      assert(!ret ||  is_in_collection_set(obj), "sanity");
+      return ret;
+    } else {
+      return false;
+    }
+  }
+
+  static address in_cset_fast_test_addr() {
+    return (address) (ShenandoahHeap::heap()->_in_cset_fast_test);
+  }
+
+  void clear_cset_fast_test() {
+    assert(_in_cset_fast_test_base != NULL, "sanity");
+    memset(_in_cset_fast_test_base, false,
+           (size_t) _in_cset_fast_test_length * sizeof(bool));
+  }
+
+  GCTracer* tracer();
+  ShenandoahHeapRegionSet* collection_set() { return _collection_set; }
+  size_t tlab_used(Thread* ignored) const;
+
+private:
+
+  bool grow_heap_by();
+
+  void verify_evacuation(ShenandoahHeapRegion* from_region);
+  void set_concurrent_mark_in_progress(bool in_progress);
+
+  void oom_during_evacuation();
+  void cancel_concgc();
+public:
+  bool cancelled_concgc();
+  void clear_cancelled_concgc();
+
+  void shutdown();
+
+  bool concurrent_mark_in_progress();
+  size_t calculateUsed();
+  size_t calculateFree();
+
+private:
+  void verify_live();
+  void verify_liveness_after_concurrent_mark();
+
+  HeapWord* allocate_memory_with_lock(size_t word_size);
+  HeapWord* allocate_memory_heap_lock(size_t word_size);
+  HeapWord* allocate_memory_shenandoah_lock(size_t word_size);
+  HeapWord* allocate_memory_work(size_t word_size);
+  HeapWord* allocate_large_memory(size_t word_size);
+  ShenandoahHeapRegion* check_skip_humongous(ShenandoahHeapRegion* region) const;
+  ShenandoahHeapRegion* get_next_region_skip_humongous() const;
+  ShenandoahHeapRegion* get_current_region_skip_humongous() const;
+  ShenandoahHeapRegion* check_grow_heap(ShenandoahHeapRegion* current);
+  ShenandoahHeapRegion* get_next_region();
+  ShenandoahHeapRegion* get_current_region();
+
+  void set_from_region_protection(bool protect);
+
+public:
+  // Delete entries for dead interned string and clean up unreferenced symbols
+  // in symbol table, possibly in parallel.
+  void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true);
+
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeap.inline.hpp b/src/share/vm/gc/shenandoah/shenandoahHeap.inline.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeap.inline.hpp
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
+
+#include "gc/shared/cmBitMap.inline.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "runtime/atomic.inline.hpp"
+
+/*
+ * Marks the object. Returns true if the object has not been marked before and has
+ * been marked by this thread. Returns false if the object has already been marked,
+ * or if a competing thread succeeded in marking this object.
+ */
+inline bool ShenandoahHeap::mark_current(oop obj) const {
+#ifdef ASSERT
+  if (obj != oopDesc::bs()->resolve_oop(obj)) {
+    tty->print_cr("heap region containing obj:");
+    ShenandoahHeapRegion* obj_region = heap_region_containing(obj);
+    obj_region->print();
+    tty->print_cr("heap region containing forwardee:");
+    ShenandoahHeapRegion* forward_region = heap_region_containing(oopDesc::bs()->resolve_oop(obj));
+    forward_region->print();
+  }
+#endif
+
+  assert(obj == oopDesc::bs()->resolve_oop(obj), "only mark forwarded copy of objects");
+  return mark_current_no_checks(obj);
+}
+
+inline bool ShenandoahHeap::mark_current_no_checks(oop obj) const {
+  return _next_mark_bit_map->parMark((HeapWord*) obj);
+}
+
+inline bool ShenandoahHeap::is_marked_current(oop obj) const {
+  return _next_mark_bit_map->isMarked((HeapWord*) obj);
+}
+
+inline bool ShenandoahHeap::need_update_refs() const {
+  return _need_update_refs;
+}
+
+inline uint ShenandoahHeap::heap_region_index_containing(const void* addr) const {
+  uintptr_t region_start = ((uintptr_t) addr); // & ~(ShenandoahHeapRegion::RegionSizeBytes - 1);
+  uintptr_t index = (region_start - (uintptr_t) _first_region_bottom) >> ShenandoahHeapRegion::RegionSizeShift;
+#ifdef ASSERT
+  if (!(index < _num_regions)) {
+    tty->print_cr("heap region does not contain address, first_region_bottom: "PTR_FORMAT", real bottom of first region: "PTR_FORMAT", num_regions: "SIZE_FORMAT", region_size: "SIZE_FORMAT, p2i(_first_region_bottom), p2i(_ordered_regions[0]->bottom()), _num_regions, ShenandoahHeapRegion::RegionSizeBytes);
+  }
+#endif
+  assert(index < _num_regions, "heap region index must be in range");
+  return index;
+}
+
+oop ShenandoahHeap::maybe_update_oop_ref(oop* p) {
+
+  assert((! is_in(p)) || (! heap_region_containing(p)->is_in_collection_set()),
+         "never update refs in from-space, unless evacuation has been cancelled");
+
+  oop heap_oop = oopDesc::load_heap_oop(p); // read p
+  if (! oopDesc::is_null(heap_oop)) {
+
+#ifdef ASSERT
+    if (! is_in(heap_oop)) {
+      print_heap_regions();
+      tty->print_cr("object not in heap: "PTR_FORMAT", referenced by: "PTR_FORMAT, p2i((HeapWord*) heap_oop), p2i(p));
+      assert(is_in(heap_oop), "object must be in heap");
+    }
+#endif
+    assert(is_in(heap_oop), "only ever call this on objects in the heap");
+    assert((! (is_in(p) && heap_region_containing(p)->is_in_collection_set())), "we don't want to update references in from-space");
+    oop forwarded_oop = ShenandoahBarrierSet::resolve_oop_static_not_null(heap_oop); // read brooks ptr
+    if (forwarded_oop != heap_oop) {
+      // tty->print_cr("updating old ref: "PTR_FORMAT" pointing to "PTR_FORMAT" to new ref: "PTR_FORMAT, p2i(p), p2i(heap_oop), p2i(forwarded_oop));
+      assert(forwarded_oop->is_oop(), "oop required");
+      assert(is_in(forwarded_oop), "forwardee must be in heap");
+      assert(! heap_region_containing(forwarded_oop)->is_in_collection_set(), "forwardee must not be in collection set");
+      // If this fails, another thread wrote to p before us, it will be logged in SATB and the
+      // reference be updated later.
+      oop result = (oop) Atomic::cmpxchg_ptr(forwarded_oop, p, heap_oop);
+
+      if (result == heap_oop) { // CAS successful.
+          return forwarded_oop;
+      } else {
+        return result;
+      }
+    } else {
+      return forwarded_oop;
+    }
+    /*
+      else {
+      tty->print_cr("not updating ref: "PTR_FORMAT, p2i(heap_oop));
+      }
+    */
+  }
+  return NULL;
+}
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeapRegion.cpp b/src/share/vm/gc/shenandoah/shenandoahHeapRegion.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegion.cpp
@@ -0,0 +1,357 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "memory/allocation.hpp"
+#include "gc/g1/heapRegionBounds.inline.hpp"
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shared/space.inline.hpp"
+#include "memory/universe.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "runtime/os.hpp"
+
+size_t ShenandoahHeapRegion::RegionSizeShift = 0;
+size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
+
+jint ShenandoahHeapRegion::initialize_heap_region(HeapWord* start,
+                                                  size_t regionSizeWords, int index) {
+
+  reserved = MemRegion((HeapWord*) start, regionSizeWords);
+  ContiguousSpace::initialize(reserved, true, false);
+  liveData = 0;
+  _is_in_collection_set = false;
+  _region_number = index;
+#ifdef ASSERT
+  _mem_protection_level = 1; // Off, level 1.
+#endif
+  return JNI_OK;
+}
+
+int ShenandoahHeapRegion::region_number() const {
+  return _region_number;
+}
+
+bool ShenandoahHeapRegion::rollback_allocation(uint size) {
+  set_top(top() - size);
+  return true;
+}
+
+void ShenandoahHeapRegion::clearLiveData() {
+  setLiveData(0);
+}
+
+void ShenandoahHeapRegion::setLiveData(size_t s) {
+  Atomic::store_ptr(s, (intptr_t*) &liveData);
+}
+
+void ShenandoahHeapRegion::increase_live_data(size_t s) {
+  size_t new_live_data = Atomic::add(s, &liveData);
+  assert(new_live_data <= used() || is_humongous(), "can't have more live data than used");
+}
+
+size_t ShenandoahHeapRegion::getLiveData() const {
+  return liveData;
+}
+
+size_t ShenandoahHeapRegion::garbage() const {
+  assert(used() >= getLiveData() || is_humongous(), err_msg("Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT, getLiveData(), used()));
+  size_t result = used() - getLiveData();
+  return result;
+}
+
+bool ShenandoahHeapRegion::is_in_collection_set() const {
+  return _is_in_collection_set;
+}
+
+#include <sys/mman.h>
+
+#ifdef ASSERT
+
+void ShenandoahHeapRegion::memProtectionOn() {
+  /*
+  tty->print_cr("protect memory on region level: "INT32_FORMAT, _mem_protection_level);
+  print(tty);
+  */
+  MutexLockerEx ml(ShenandoahMemProtect_lock, true);
+  assert(_mem_protection_level >= 1, "invariant");
+
+  if (--_mem_protection_level == 0) {
+    if (ShenandoahVerifyWritesToFromSpace) {
+      assert(! ShenandoahVerifyReadsToFromSpace, "can't verify from-space reads when verifying from-space writes");
+      os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_READ);
+    } else {
+      assert(ShenandoahVerifyReadsToFromSpace, "need to be verifying reads here");
+      assert(! ShenandoahConcurrentEvacuation, "concurrent evacuation needs to be turned off for verifying from-space-reads");
+      os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_NONE);
+    }
+  }
+}
+
+void ShenandoahHeapRegion::memProtectionOff() {
+  /*
+  tty->print_cr("unprotect memory on region level: "INT32_FORMAT, _mem_protection_level);
+  print(tty);
+  */
+  MutexLockerEx ml(ShenandoahMemProtect_lock, true);
+  assert(_mem_protection_level >= 0, "invariant");
+  if (_mem_protection_level++ == 0) {
+    os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_RW);
+  }
+}
+
+#endif
+
+void ShenandoahHeapRegion::set_is_in_collection_set(bool b) {
+  assert(! (is_humongous() && b), "never ever enter a humongous region into the collection set");
+
+  _is_in_collection_set = b;
+
+  if (b) {
+    // tty->print_cr("registering region in fast-cset");
+    // print();
+    ShenandoahHeap::heap()->register_region_with_in_cset_fast_test(this);
+  }
+
+#ifdef ASSERT
+  if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) {
+    if (b) {
+      memProtectionOn();
+      assert(_mem_protection_level == 0, "need to be protected here");
+    } else {
+      assert(_mem_protection_level == 0, "need to be protected here");
+      memProtectionOff();
+    }
+  }
+#endif
+}
+
+ByteSize ShenandoahHeapRegion::is_in_collection_set_offset() {
+  return byte_offset_of(ShenandoahHeapRegion, _is_in_collection_set);
+}
+
+void ShenandoahHeapRegion::print_on(outputStream* st) const {
+  st->print_cr("ShenandoahHeapRegion: "PTR_FORMAT"/"INT32_FORMAT, p2i(this), _region_number);
+
+  if (is_in_collection_set())
+    st->print("C");
+  if (is_humongous_start()) {
+    st->print("H");
+  }
+  if (is_humongous_continuation()) {
+    st->print("h");
+  }
+  //else
+    st->print(" ");
+
+  st->print_cr("live = "SIZE_FORMAT" garbage = "SIZE_FORMAT" bottom = "PTR_FORMAT" end = "PTR_FORMAT" top = "PTR_FORMAT,
+               getLiveData(), garbage(), p2i(bottom()), p2i(end()), p2i(top()));
+}
+
+
+class SkipUnreachableObjectToOopClosure: public ObjectClosure {
+  ExtendedOopClosure* _cl;
+  bool _skip_unreachable_objects;
+  ShenandoahHeap* _heap;
+
+public:
+  SkipUnreachableObjectToOopClosure(ExtendedOopClosure* cl, bool skip_unreachable_objects) :
+    _cl(cl), _skip_unreachable_objects(skip_unreachable_objects), _heap(ShenandoahHeap::heap()) {}
+
+  void do_object(oop obj) {
+
+    if ((! _skip_unreachable_objects) || _heap->is_marked_current(obj)) {
+      if (_skip_unreachable_objects) {
+        assert(_heap->is_marked_current(obj), "obj must be live");
+      }
+      obj->oop_iterate(_cl);
+    }
+
+  }
+};
+
+void ShenandoahHeapRegion::object_iterate_interruptible(ObjectClosure* blk, bool allow_cancel) {
+  HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  while (p < top() && !(allow_cancel && heap->cancelled_concgc())) {
+    blk->do_object(oop(p));
+#ifdef ASSERT
+    if (ShenandoahVerifyReadsToFromSpace) {
+      memProtectionOff();
+      p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+      memProtectionOn();
+    } else {
+      p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+    }
+#else
+      p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+#endif
+  }
+}
+
+HeapWord* ShenandoahHeapRegion::object_iterate_careful(ObjectClosureCareful* blk) {
+  HeapWord * limit = concurrent_iteration_safe_limit();
+  assert(limit <= top(), "sanity check");
+  for (HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; p < limit;) {
+    size_t size = blk->do_object_careful(oop(p));
+    if (size == 0) {
+      return p;  // failed at p
+    } else {
+      p += size + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+    }
+  }
+  return NULL; // all done
+}
+
+void ShenandoahHeapRegion::oop_iterate_skip_unreachable(ExtendedOopClosure* cl, bool skip_unreachable_objects) {
+  SkipUnreachableObjectToOopClosure cl2(cl, skip_unreachable_objects);
+  object_iterate_interruptible(&cl2, false);
+}
+
+void ShenandoahHeapRegion::fill_region() {
+  ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
+
+  if (free() > (BrooksPointer::BROOKS_POINTER_OBJ_SIZE + CollectedHeap::min_fill_size())) {
+    HeapWord* filler = allocate(BrooksPointer::BROOKS_POINTER_OBJ_SIZE);
+    HeapWord* obj = allocate(end() - top());
+    sh->fill_with_object(obj, end() - obj);
+    sh->initialize_brooks_ptr(filler, obj);
+  }
+}
+
+void ShenandoahHeapRegion::set_humongous_start(bool start) {
+  _humongous_start = start;
+}
+
+void ShenandoahHeapRegion::set_humongous_continuation(bool continuation) {
+  _humongous_continuation = continuation;
+}
+
+bool ShenandoahHeapRegion::is_humongous() const {
+  return _humongous_start || _humongous_continuation;
+}
+
+bool ShenandoahHeapRegion::is_humongous_start() const {
+  return _humongous_start;
+}
+
+bool ShenandoahHeapRegion::is_humongous_continuation() const {
+  return _humongous_continuation;
+}
+
+void ShenandoahHeapRegion::do_reset() {
+  ContiguousSpace::initialize(reserved, true, false);
+  clearLiveData();
+  _humongous_start = false;
+  _humongous_continuation = false;
+}
+
+void ShenandoahHeapRegion::recycle() {
+  do_reset();
+  set_is_in_collection_set(false);
+}
+
+void ShenandoahHeapRegion::reset() {
+  assert(_mem_protection_level == 1, "needs to be unprotected here");
+  do_reset();
+  _is_in_collection_set = false;
+}
+
+HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
+  assert(MemRegion(bottom(), end()).contains(p),
+         err_msg("p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")",
+                 p2i(p), p2i(bottom()), p2i(end())));
+  if (p >= top()) {
+    return top();
+  } else {
+    HeapWord* last = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+    HeapWord* cur = last;
+    while (cur <= p) {
+      last = cur;
+      cur += oop(cur)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+    }
+    assert(oop(last)->is_oop(),
+           err_msg(PTR_FORMAT" should be an object start", p2i(last)));
+    return last;
+  }
+}
+
+void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
+  uintx region_size = ShenandoahHeapRegionSize;
+  if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
+    size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
+    region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(),
+                       (uintx) HeapRegionBounds::min_size());
+  }
+
+  int region_size_log = log2_long((jlong) region_size);
+  // Recalculate the region size to make sure it's a power of
+  // 2. This means that region_size is the largest power of 2 that's
+  // <= what we've calculated so far.
+  region_size = ((uintx)1 << region_size_log);
+
+  // Now make sure that we don't go over or under our limits.
+  if (region_size < HeapRegionBounds::min_size()) {
+    region_size = HeapRegionBounds::min_size();
+  } else if (region_size > HeapRegionBounds::max_size()) {
+    region_size = HeapRegionBounds::max_size();
+  }
+
+  // And recalculate the log.
+  region_size_log = log2_long((jlong) region_size);
+
+  // Now, set up the globals.
+  guarantee(RegionSizeShift == 0, "we should only set it once");
+  RegionSizeShift = region_size_log;
+
+  guarantee(RegionSizeBytes == 0, "we should only set it once");
+  RegionSizeBytes = (size_t)region_size;
+
+  if (ShenandoahLogConfig) {
+    tty->print_cr("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes);
+    tty->print_cr("Region size shift: "SIZE_FORMAT, RegionSizeShift);
+    tty->print_cr("Initial number of regions: "SIZE_FORMAT, initial_heap_size / RegionSizeBytes);
+    tty->print_cr("Maximum number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes);
+  }
+}
+
+CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const {
+  return ShenandoahHeap::heap()->next_compaction_region(this);
+}
+
+void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) {
+  scan_and_forward(this, cp);
+}
+
+void ShenandoahHeapRegion::adjust_pointers() {
+  // Check first is there is any work to do.
+  if (used() == 0) {
+    return;   // Nothing to do.
+  }
+
+  scan_and_adjust_pointers(this);
+}
+
+void ShenandoahHeapRegion::compact() {
+  scan_and_compact(this);
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeapRegion.hpp b/src/share/vm/gc/shenandoah/shenandoahHeapRegion.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegion.hpp
@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP
+
+#include "gc/shared/space.hpp"
+#include "memory/universe.hpp"
+#include "utilities/sizes.hpp"
+
+class ShenandoahHeapRegion : public ContiguousSpace {
+
+  // Allow scan_and_forward to call (private) overrides for auxiliary functions on this class
+  template <typename SpaceType>
+  friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp);
+  template <typename SpaceType>
+  friend void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space);
+  template <typename SpaceType>
+  friend void CompactibleSpace::scan_and_compact(SpaceType* space);
+
+private:
+  // Auxiliary functions for scan_and_forward support.
+  // See comments for CompactibleSpace for more information.
+  inline HeapWord* scan_limit() const {
+    return top();
+  }
+
+  inline bool scanned_block_is_obj(const HeapWord* addr) const {
+    return true; // Always true, since scan_limit is top
+  }
+
+  inline size_t scanned_block_size(const HeapWord* addr) const {
+    oop obj = oop(addr+1);
+    size_t size = obj->size() + 1;
+    return size;
+  }
+
+    // Auxiliary functions for scan_and_{forward,adjust_pointers,compact} support.
+  inline size_t adjust_obj_size(size_t size) const {
+    return size + 1;
+  }
+
+  inline size_t obj_size(const HeapWord* addr) const {
+    return oop(addr+1)->size() + 1;
+  }
+
+  inline oop make_oop(HeapWord* addr) const {
+    return oop(addr+1);
+  }
+public:
+  static size_t RegionSizeBytes;
+  static size_t RegionSizeShift;
+
+private:
+  int _region_number;
+  volatile size_t liveData;
+  MemRegion reserved;
+  bool _is_in_collection_set;
+
+  bool _humongous_start;
+  bool _humongous_continuation;
+
+#ifdef ASSERT
+  int _mem_protection_level;
+#endif
+
+public:
+  static void setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size);
+
+  jint initialize_heap_region(HeapWord* start, size_t regionSize, int index);
+
+
+  int region_number() const;
+
+  // Roll back the previous allocation of an object with specified size.
+  // Returns TRUE when successful, FALSE if not successful or not supported.
+  bool rollback_allocation(uint size);
+
+  void clearLiveData();
+  void setLiveData(size_t s);
+  void increase_live_data(size_t s);
+
+  size_t getLiveData() const;
+
+  void print_on(outputStream* st) const;
+
+  size_t garbage() const;
+
+  void recycle();
+  void reset();
+
+  void oop_iterate_skip_unreachable(ExtendedOopClosure* cl, bool skip_unreachable_objects);
+
+  void object_iterate_interruptible(ObjectClosure* blk, bool allow_cancel);
+
+  HeapWord* object_iterate_careful(ObjectClosureCareful* cl);
+
+  HeapWord* block_start_const(const void* p) const;
+
+  // Just before GC we need to fill the current region.
+  void fill_region();
+
+  bool is_in_collection_set() const;
+
+  void set_is_in_collection_set(bool b);
+
+  void set_humongous_start(bool start);
+  void set_humongous_continuation(bool continuation);
+
+  bool is_humongous() const;
+  bool is_humongous_start() const;
+  bool is_humongous_continuation() const;
+
+#ifdef ASSERT
+  void memProtectionOn();
+  void memProtectionOff();
+#endif
+
+  static ByteSize is_in_collection_set_offset();
+  // The following are for humongous regions.  We need to save the
+  markOop saved_mark_word;
+  void save_mark_word(oop obj) {saved_mark_word = obj->mark();}
+  markOop mark_word() {return saved_mark_word;}
+
+  virtual CompactibleSpace* next_compaction_space() const;
+
+  // Override for scan_and_forward support.
+  void prepare_for_compaction(CompactPoint* cp);
+  void adjust_pointers();
+  void compact();
+
+  virtual oop compact_oop(HeapWord* addr) const {
+    return oop(addr + 1);
+  }
+private:
+  void do_reset();
+
+};
+
+
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.cpp b/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.cpp
@@ -0,0 +1,374 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
+#include "gc/shenandoah/shenandoahHumongous.hpp"
+#include "memory/resourceArea.hpp"
+#include "utilities/quickSort.hpp"
+
+ShenandoahHeapRegionSet::ShenandoahHeapRegionSet(size_t max_regions) :
+  _max_regions(max_regions),
+  _regions(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, max_regions, mtGC)),
+  _garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2),
+  _free_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2),
+  _available(0), _used(0)
+{
+
+  _next = &_regions[0];
+  _current = NULL;
+  _next_free = &_regions[0];
+}
+
+ShenandoahHeapRegionSet::ShenandoahHeapRegionSet(size_t max_regions, ShenandoahHeapRegion** regions, size_t num_regions) :
+  _max_regions(num_regions),
+  _regions(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, max_regions, mtGC)),
+  _garbage_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2),
+  _free_threshold(ShenandoahHeapRegion::RegionSizeBytes / 2) {
+
+  // Make copy of the regions array so that we can sort without destroying the original.
+  memcpy(_regions, regions, sizeof(ShenandoahHeapRegion*) * num_regions);
+
+  _next = &_regions[0];
+  _current = NULL;
+  _next_free = &_regions[num_regions];
+}
+
+ShenandoahHeapRegionSet::~ShenandoahHeapRegionSet() {
+  FREE_C_HEAP_ARRAY(ShenandoahHeapRegion*, _regions);
+}
+
+int compareHeapRegionsByGarbage(ShenandoahHeapRegion* a, ShenandoahHeapRegion* b) {
+  if (a == NULL) {
+    if (b == NULL) {
+      return 0;
+    } else {
+      return 1;
+    }
+  } else if (b == NULL) {
+    return -1;
+  }
+
+  size_t garbage_a = a->garbage();
+  size_t garbage_b = b->garbage();
+
+  if (garbage_a > garbage_b)
+    return -1;
+  else if (garbage_a < garbage_b)
+    return 1;
+  else return 0;
+}
+
+ShenandoahHeapRegion* ShenandoahHeapRegionSet::current() {
+  ShenandoahHeapRegion** current = _current;
+  if (current == NULL) {
+    return get_next();
+  } else {
+    return *(limit_region(current));
+  }
+}
+
+size_t ShenandoahHeapRegionSet::length() {
+  return _next_free - _regions;
+}
+
+size_t ShenandoahHeapRegionSet::available_regions() {
+  return (_regions + _max_regions) - _next_free;
+}
+
+void ShenandoahHeapRegionSet::append(ShenandoahHeapRegion* region) {
+  assert(_next_free < _regions + _max_regions, "need space for additional regions");
+  assert(SafepointSynchronize::is_at_safepoint() || ShenandoahHeap_lock->owned_by_self() || ! Universe::is_fully_initialized(), "only append regions to list while world is stopped");
+
+  // Grab next slot.
+  ShenandoahHeapRegion** next_free = _next_free;
+  _next_free++;
+
+  // Insert new region into slot.
+  *next_free = region;
+
+  _available += region->free();
+}
+
+void ShenandoahHeapRegionSet::clear() {
+  _current = NULL;
+  _next = _regions;
+  _next_free = _regions;
+  _available = 0;
+  _used = 0;
+}
+
+ShenandoahHeapRegion* ShenandoahHeapRegionSet::claim_next() {
+  ShenandoahHeapRegion** next = (ShenandoahHeapRegion**) Atomic::add_ptr(sizeof(ShenandoahHeapRegion**), &_next);
+  next--;
+  if (next < _next_free) {
+    return *next;
+  } else {
+    return NULL;
+  }
+}
+
+ShenandoahHeapRegion* ShenandoahHeapRegionSet::get_next() {
+
+  ShenandoahHeapRegion** next = _next;
+  if (next < _next_free) {
+    _current = next;
+    _next++;
+    return *next;
+  } else {
+    return NULL;
+  }
+}
+
+ShenandoahHeapRegion** ShenandoahHeapRegionSet::limit_region(ShenandoahHeapRegion** region) {
+  if (region >= _next_free) {
+    return NULL;
+  } else {
+    return region;
+  }
+}
+
+void ShenandoahHeapRegionSet::print() {
+  for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) {
+    if (i == _current) {
+      tty->print_cr("C->");
+    }
+    if (i == _next) {
+      tty->print_cr("N->");
+    }
+    (*i)->print();
+  }
+}
+
+void ShenandoahHeapRegionSet::choose_collection_and_free_sets(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set) {
+  col_set->choose_collection_set(_regions, length());
+  free_set->choose_free_set(_regions, length());
+  //  assert(col_set->length() > 0 && free_set->length() > 0, "Better have some regions in the collection and free sets");
+
+}
+
+void ShenandoahHeapRegionSet::choose_collection_and_free_sets_min_garbage(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set, size_t min_garbage) {
+  col_set->choose_collection_set_min_garbage(_regions, length(), min_garbage);
+  free_set->choose_free_set(_regions, length());
+  //  assert(col_set->length() > 0 && free_set->length() > 0, "Better have some regions in the collection and free sets");
+}
+
+void ShenandoahHeapRegionSet::choose_collection_set(ShenandoahHeapRegion** regions, size_t length) {
+
+  clear();
+
+  assert(length <= _max_regions, "must not blow up array");
+
+  ShenandoahHeapRegion** tmp = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, length, mtGC);
+
+  memcpy(tmp, regions, sizeof(ShenandoahHeapRegion*) * length);
+
+  QuickSort::sort<ShenandoahHeapRegion*>(tmp, length, compareHeapRegionsByGarbage, false);
+
+  ShenandoahHeapRegion** r = tmp;
+  ShenandoahHeapRegion** end = tmp + length;
+
+  // We don't want the current allocation region in the collection set because a) it is still being allocated into and b) This is where the write barriers will allocate their copies.
+
+  while (r < end) {
+    ShenandoahHeapRegion* region = *r;
+    if (region->garbage() > _garbage_threshold && ! region->is_humongous()) {
+      //      tty->print("choose region %d with garbage = " SIZE_FORMAT " and live = " SIZE_FORMAT " and _garbage_threshold = " SIZE_FORMAT "\n",
+      //                 region->region_number(), region->garbage(), region->getLiveData(), _garbage_threshold);
+
+      assert(! region->is_humongous(), "no humongous regions in collection set");
+
+      if (region->getLiveData() == 0) {
+        // We can recycle it right away and put it in the free set.
+        ShenandoahHeap::heap()->decrease_used(region->used());
+        region->recycle();
+      } else {
+        append(region);
+        region->set_is_in_collection_set(true);
+      }
+      //    } else {
+      //      tty->print("rejected region %d with garbage = " SIZE_FORMAT " and live = " SIZE_FORMAT " and _garbage_threshold = " SIZE_FORMAT "\n",
+      //                 region->region_number(), region->garbage(), region->getLiveData(), _garbage_threshold);
+    }
+    r++;
+  }
+
+  FREE_C_HEAP_ARRAY(ShenandoahHeapRegion*, tmp);
+
+}
+
+void ShenandoahHeapRegionSet::choose_collection_set_min_garbage(ShenandoahHeapRegion** regions, size_t length, size_t min_garbage) {
+
+  clear();
+
+  assert(length <= _max_regions, "must not blow up array");
+
+  ShenandoahHeapRegion** tmp = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, length, mtGC);
+
+  memcpy(tmp, regions, sizeof(ShenandoahHeapRegion*) * length);
+
+  QuickSort::sort<ShenandoahHeapRegion*>(tmp, length, compareHeapRegionsByGarbage, false);
+
+  ShenandoahHeapRegion** r = tmp;
+  ShenandoahHeapRegion** end = tmp + length;
+
+  // We don't want the current allocation region in the collection set because a) it is still being allocated into and b) This is where the write barriers will allocate their copies.
+
+  size_t garbage = 0;
+  while (r < end && garbage < min_garbage) {
+    ShenandoahHeapRegion* region = *r;
+    if (region->garbage() > _garbage_threshold && ! region->is_humongous()) {
+      append(region);
+      garbage += region->garbage();
+      region->set_is_in_collection_set(true);
+    }
+    r++;
+  }
+
+  FREE_C_HEAP_ARRAY(ShenandoahHeapRegion*, tmp);
+
+  /*
+  tty->print_cr("choosen region with "SIZE_FORMAT" garbage given "SIZE_FORMAT" min_garbage", garbage, min_garbage);
+  */
+}
+
+
+void ShenandoahHeapRegionSet::choose_free_set(ShenandoahHeapRegion** regions, size_t length) {
+
+  clear();
+  ShenandoahHeapRegion** end = regions + length;
+
+  for (ShenandoahHeapRegion** r = regions; r < end; r++) {
+    ShenandoahHeapRegion* region = *r;
+    if ((! region->is_in_collection_set())
+        && (! region->is_humongous())) {
+      append(region);
+    }
+  }
+}
+
+void ShenandoahHeapRegionSet::reclaim_humongous_regions() {
+
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  for (ShenandoahHeapRegion** r = _regions; r < _next_free; r++) {
+    // We can immediately reclaim humongous objects/regions that are no longer reachable.
+    ShenandoahHeapRegion* region = *r;
+    if (region->is_humongous_start()) {
+      oop humongous_obj = oop(region->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE);
+      if (! heap->is_marked_current(humongous_obj)) {
+        reclaim_humongous_region_at(r);
+      }
+    }
+  }
+
+}
+
+void ShenandoahHeapRegionSet::reclaim_humongous_region_at(ShenandoahHeapRegion** r) {
+  assert((*r)->is_humongous_start(), "reclaim regions starting with the first one");
+
+  oop humongous_obj = oop((*r)->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE);
+  size_t size = humongous_obj->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE;
+  uint required_regions = ShenandoahHumongous::required_regions(size * HeapWordSize);
+
+  if (ShenandoahTraceHumongous) {
+    tty->print_cr("reclaiming "UINT32_FORMAT" humongous regions for object of size: "SIZE_FORMAT" words", required_regions, size);
+  }
+
+  assert((*r)->getLiveData() == 0, "liveness must be zero");
+
+  for (ShenandoahHeapRegion** i = r; i < r + required_regions; i++) {
+    ShenandoahHeapRegion* region = *i;
+
+    assert(i == r ? region->is_humongous_start() : region->is_humongous_continuation(),
+           "expect correct humongous start or continuation");
+
+    if (ShenandoahTraceHumongous) {
+      region->print();
+    }
+
+    region->reset();
+    ShenandoahHeap::heap()->decrease_used(ShenandoahHeapRegion::RegionSizeBytes);
+  }
+}
+
+void ShenandoahHeapRegionSet::set_concurrent_iteration_safe_limits() {
+  for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) {
+    ShenandoahHeapRegion* region = *i;
+    region->set_concurrent_iteration_safe_limit(region->top());
+  }
+}
+
+size_t ShenandoahHeapRegionSet::garbage() {
+  size_t garbage = 0;
+  for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) {
+    ShenandoahHeapRegion* region = *i;
+    garbage += region->garbage();
+  }
+  return garbage;
+}
+
+size_t ShenandoahHeapRegionSet::used() {
+  size_t used = 0;
+  for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) {
+    ShenandoahHeapRegion* region = *i;
+    used += region->used();
+  }
+  return used;
+}
+
+size_t ShenandoahHeapRegionSet::live_data() {
+  size_t live = 0;
+  for (ShenandoahHeapRegion** i = _regions; i < _next_free; i++) {
+    ShenandoahHeapRegion* region = *i;
+    live += region->getLiveData();
+  }
+  return live;
+}
+
+void ShenandoahHeapRegionSet::decrease_available(size_t num_bytes) {
+  assert(_available >= num_bytes, "can't use more than available");
+  _available -= num_bytes;
+  _used += num_bytes;
+}
+
+size_t ShenandoahHeapRegionSet::available() const {
+#ifdef ASSERT
+  // Need to grab the lock here, because the different counters are updated
+  // within the lock. Otherwise we sometimes get inconsistent results.
+  {
+    MutexLockerEx ml(ShenandoahHeap_lock, true);
+    assert(ShenandoahHeap::heap()->capacity() - ShenandoahHeap::heap()->used()>= _available,
+           err_msg("must not be > heap free, capacity: "SIZE_FORMAT", used: "SIZE_FORMAT", available: "SIZE_FORMAT,
+                   ShenandoahHeap::heap()->capacity(), ShenandoahHeap::heap()->used(), _available)
+           );
+  }
+#endif
+  return _available;
+}
+
+size_t ShenandoahHeapRegionSet::used() const {
+  assert(ShenandoahHeap::heap()->used() >= _used, "must not be > heap used");
+  return _used;
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.hpp b/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHeapRegionSet.hpp
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGIONSET_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGIONSET_HPP
+
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+
+
+class ShenandoahHeapRegionSet : public CHeapObj<mtGC> {
+private:
+  ShenandoahHeapRegion** _regions;
+  // current region to be returned from get_next()
+  ShenandoahHeapRegion** _current;
+  ShenandoahHeapRegion** _next;
+
+  // last inserted region.
+  ShenandoahHeapRegion** _next_free;
+  ShenandoahHeapRegion** _concurrent_next_free;
+
+  // Maximum size of the set.
+  const size_t _max_regions;
+
+  size_t _garbage_threshold;
+  size_t _free_threshold;
+
+  size_t _available;
+  size_t _used;
+
+  void choose_collection_set(ShenandoahHeapRegion** regions, size_t length);
+  void choose_collection_set_min_garbage(ShenandoahHeapRegion** regions, size_t length, size_t min_garbage);
+  void choose_free_set(ShenandoahHeapRegion** regions, size_t length);
+
+public:
+  ShenandoahHeapRegionSet(size_t max_regions);
+
+  ShenandoahHeapRegionSet(size_t max_regions, ShenandoahHeapRegion** regions, size_t num_regions);
+
+  ~ShenandoahHeapRegionSet();
+
+  void set_garbage_threshold(size_t minimum_garbage) { _garbage_threshold = minimum_garbage;}
+  void set_free_threshold(size_t minimum_free) { _free_threshold = minimum_free;}
+
+  /**
+   * Appends a region to the set. This is implemented to be concurrency-safe.
+   */
+  void append(ShenandoahHeapRegion* region);
+
+  void clear();
+
+  size_t length();
+  size_t used_regions() {
+    return _current - _regions;
+  }
+  size_t available_regions();
+  void print();
+
+  size_t garbage();
+  size_t used();
+  size_t live_data();
+  size_t reclaimed() {return _reclaimed;}
+
+  /**
+   * Returns a pointer to the current region.
+   */
+   ShenandoahHeapRegion* current();
+
+  /**
+   * Gets the next region for allocation (from free-list).
+   * If multiple threads are competing, one will succeed to
+   * increment to the next region, the others will fail and return
+   * the region that the succeeding thread got.
+   */
+  ShenandoahHeapRegion* get_next();
+
+  /**
+   * Claims next region for processing. This is implemented to be concurrency-safe.
+   */
+  ShenandoahHeapRegion* claim_next();
+
+  void choose_collection_and_free_sets(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set);
+  void choose_collection_and_free_sets_min_garbage(ShenandoahHeapRegionSet* col_set, ShenandoahHeapRegionSet* free_set, size_t min_garbage);
+
+  // Check for unreachable humongous regions and reclaim them.
+  void reclaim_humongous_regions();
+
+  void set_concurrent_iteration_safe_limits();
+
+  void decrease_available(size_t num_bytes);
+
+  size_t available() const;
+  size_t used() const;
+
+private:
+  void reclaim_humongous_region_at(ShenandoahHeapRegion** r);
+
+  ShenandoahHeapRegion** limit_region(ShenandoahHeapRegion** region);
+  size_t _reclaimed;
+
+};
+
+#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAPREGIONSET_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahHumongous.hpp b/src/share/vm/gc/shenandoah/shenandoahHumongous.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahHumongous.hpp
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHUMONGOUS_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHUMONGOUS_HPP
+
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+
+class ShenandoahHumongous : public AllStatic {
+
+public:
+  static uint required_regions(size_t bytes) {
+    return (bytes + ShenandoahHeapRegion::RegionSizeBytes - 1) / ShenandoahHeapRegion::RegionSizeBytes;
+  }
+};
+
+#endif
diff --git a/src/share/vm/gc/shenandoah/shenandoahJNICritical.cpp b/src/share/vm/gc/shenandoah/shenandoahJNICritical.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahJNICritical.cpp
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahJNICritical.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+
+#include "gc/shared/gcLocker.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "runtime/thread.hpp"
+#include "runtime/vmThread.hpp"
+
+class VM_ShenandoahJNICriticalOperation : public VM_Operation {
+private:
+  VM_Operation* _target;
+public:
+  VM_ShenandoahJNICriticalOperation(VM_Operation* target);
+  VMOp_Type type() const;
+  bool doit_prologue();
+  void doit_epilogue();
+  void doit();
+  const char* name() const;
+};
+
+ShenandoahJNICritical::ShenandoahJNICritical() : _op_waiting_for_jni_critical(NULL) {
+}
+
+/*
+ * This is called by the Java thread who leaves the last JNI critical block.
+ */
+void ShenandoahJNICritical::notify_jni_critical() {
+  assert(Thread::current()->is_Java_thread(), "call only from Java thread");
+  assert(_op_waiting_for_jni_critical != NULL, "must be waiting for jni critical notification");
+
+  MonitorLockerEx ml(ShenandoahJNICritical_lock, true);
+
+  VMThread::execute(_op_waiting_for_jni_critical);
+  _op_waiting_for_jni_critical = NULL;
+
+  ml.notify_all();
+
+}
+
+/*
+ * This is called by the VM thread, if it determines that the task must wait
+ * for JNI critical regions to be left.
+ */
+void ShenandoahJNICritical::set_waiting_for_jni_before_gc(VM_Operation* op) {
+  assert(Thread::current()->is_VM_thread(), "call only from VM thread");
+  _op_waiting_for_jni_critical = op;
+}
+
+/**
+ * This is called by the Shenandoah concurrent thread in order
+ * to execute a VM_Operation on the VM thread, that needs to perform
+ * a JNI critical region check.
+ */
+void ShenandoahJNICritical::execute_in_vm_thread(VM_Operation* op) {
+  MonitorLockerEx ml(ShenandoahJNICritical_lock, true);
+  VM_ShenandoahJNICriticalOperation jni_op(op);
+  VMThread::execute(&jni_op);
+  while (_op_waiting_for_jni_critical != NULL) {
+    ml.wait(true);
+  }
+}
+
+
+VM_ShenandoahJNICriticalOperation::VM_ShenandoahJNICriticalOperation(VM_Operation* target)
+  : _target(target) {
+}
+
+VM_Operation::VMOp_Type VM_ShenandoahJNICriticalOperation::type() const {
+  return _target->type();
+}
+
+const char* VM_ShenandoahJNICriticalOperation::name() const {
+  return _target->name();
+}
+
+bool VM_ShenandoahJNICriticalOperation::doit_prologue() {
+  return _target->doit_prologue();
+}
+
+void VM_ShenandoahJNICriticalOperation::doit_epilogue() {
+  _target->doit_epilogue();
+}
+
+void VM_ShenandoahJNICriticalOperation::doit() {
+  if (! GC_locker::check_active_before_gc()) {
+    _target->doit();
+  } else {
+
+    if (ShenandoahTraceJNICritical) {
+      gclog_or_tty->print_cr("Deferring JNI critical op because of active JNI critical regions");
+    }
+
+    // This makes the GC background thread wait, and kick off evacuation as
+    // soon as JNI notifies us that critical regions have all been left.
+    ShenandoahHeap *sh = ShenandoahHeap::heap();
+    sh->jni_critical()->set_waiting_for_jni_before_gc(this);
+  }
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahJNICritical.hpp b/src/share/vm/gc/shenandoah/shenandoahJNICritical.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahJNICritical.hpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHJNICRITICAL_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHJNICRITICAL_HPP
+
+#include "gc/shared/vmGCOperations.hpp"
+#include "memory/allocation.hpp"
+
+class ShenandoahJNICritical : public CHeapObj<mtGC> {
+private:
+  VM_Operation* _op_waiting_for_jni_critical;
+
+public:
+  ShenandoahJNICritical();
+  void notify_jni_critical();
+  void set_waiting_for_jni_before_gc(VM_Operation* op);
+  void execute_in_vm_thread(VM_Operation* op);
+};
+
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHJNICRITICAL_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahMarkCompact.cpp b/src/share/vm/gc/shenandoah/shenandoahMarkCompact.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahMarkCompact.cpp
@@ -0,0 +1,453 @@
+/*
+ * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "code/codeCache.hpp"
+#include "gc/shared/isGCActiveMark.hpp"
+#include "gc/shenandoah/brooksPointer.hpp"
+#include "gc/shenandoah/shenandoahMarkCompact.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahRootProcessor.hpp"
+#include "gc/shenandoah/vm_operations_shenandoah.hpp"
+#include "gc/serial/markSweep.inline.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/biasedLocking.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/copy.hpp"
+#include "gc/shared/taskqueue.inline.hpp"
+#include "gc/shared/workgroup.hpp"
+
+
+
+void ShenandoahMarkCompact::allocate_stacks() {
+  MarkSweep::_preserved_count_max = 0;
+  MarkSweep::_preserved_marks = NULL;
+  MarkSweep::_preserved_count = 0;
+}
+
+void ShenandoahMarkCompact::do_mark_compact() {
+
+  COMPILER2_PRESENT(DerivedPointerTable::clear());
+
+  ShenandoahHeap* _heap = ShenandoahHeap::heap();
+
+  assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
+  IsGCActiveMark is_active;
+
+  assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
+  assert(_heap->is_bitmap_clear(), "require cleared bitmap");
+  assert(!_heap->concurrent_mark_in_progress(), "can't do full-GC while marking is in progress");
+  assert(!_heap->is_evacuation_in_progress(), "can't do full-GC while evacuation is in progress");
+  assert(!_heap->is_update_references_in_progress(), "can't do full-GC while updating of references is in progress");
+  BarrierSet* _old_barrier_set = oopDesc::bs();
+
+  oopDesc::set_bs(new ShenandoahMarkCompactBarrierSet());
+
+  _heap->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::full_gc);
+
+  // We need to clear the is_in_collection_set flag in all regions.
+  ShenandoahHeapRegion** regions = _heap->heap_regions();
+  size_t num_regions = _heap->num_regions();
+  for (size_t i = 0; i < num_regions; i++) {
+    regions[i]->set_is_in_collection_set(false);
+  }
+  _heap->clear_cset_fast_test();
+
+  /*
+  if (ShenandoahVerify) {
+    // Full GC should only be called between regular concurrent cycles, therefore
+    // those verifications should be valid.
+    _heap->verify_heap_after_evacuation();
+    _heap->verify_heap_after_update_refs();
+  }
+  */
+
+  if (ShenandoahTraceFullGC) {
+    gclog_or_tty->print_cr("Shenandoah-full-gc: start with heap used: "SIZE_FORMAT" MB", _heap->used() / M);
+    gclog_or_tty->print_cr("Shenandoah-full-gc: phase 1: marking the heap");
+    // _heap->print_heap_regions();
+  }
+
+  if (UseTLAB) {
+    _heap->ensure_parsability(true);
+  }
+
+  _heap->cleanup_after_cancelconcgc();
+
+  ReferenceProcessor* rp = _heap->ref_processor();
+
+  // hook up weak ref data so it can be used during Mark-Sweep
+  assert(MarkSweep::ref_processor() == NULL, "no stomping");
+  assert(rp != NULL, "should be non-NULL");
+  assert(rp == ShenandoahHeap::heap()->ref_processor(), "Precondition");
+  bool clear_all_softrefs = true;  //fixme
+  MarkSweep::_ref_processor = rp;
+  rp->setup_policy(clear_all_softrefs);
+
+  CodeCache::gc_prologue();
+  allocate_stacks();
+
+  // We should save the marks of the currently locked biased monitors.
+  // The marking doesn't preserve the marks of biased objects.
+  BiasedLocking::preserve_marks();
+
+  phase1_mark_heap();
+
+  if (ShenandoahTraceFullGC) {
+    gclog_or_tty->print_cr("Shenandoah-full-gc: phase 2: calculating target addresses");
+  }
+  phase2_calculate_target_addresses();
+
+  if (ShenandoahTraceFullGC) {
+    gclog_or_tty->print_cr("Shenandoah-full-gc: phase 3: updating references");
+  }
+
+  // Don't add any more derived pointers during phase3
+  COMPILER2_PRESENT(DerivedPointerTable::set_active(false));
+
+  phase3_update_references();
+
+  if (ShenandoahTraceFullGC) {
+    gclog_or_tty->print_cr("Shenandoah-full-gc: phase 4: compacting objects");
+  }
+
+  phase4_compact_objects();
+
+
+  MarkSweep::restore_marks();
+  BiasedLocking::restore_marks();
+  GenMarkSweep::deallocate_stacks();
+
+  CodeCache::gc_epilogue();
+  JvmtiExport::gc_epilogue();
+
+  // refs processing: clean slate
+  MarkSweep::_ref_processor = NULL;
+
+
+  if (ShenandoahVerify) {
+    _heap->verify_heap_after_evacuation();
+    _heap->verify_heap_after_update_refs();
+  }
+
+  if (ShenandoahTraceFullGC) {
+    gclog_or_tty->print_cr("Shenandoah-full-gc: finish with heap used: "SIZE_FORMAT" MB", _heap->used() / M);
+  }
+
+  _heap->_bytesAllocSinceCM = 0;
+
+  oopDesc::set_bs(_old_barrier_set);
+
+  _heap->set_need_update_refs(false);
+
+  COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+
+  _heap->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::full_gc);
+}
+
+class UpdateRefsClosure: public ExtendedOopClosure {
+public:
+  virtual void do_oop(oop* p) {
+    oop obj = oopDesc::load_heap_oop(p);
+    if (! oopDesc::is_null(obj)) {
+      ShenandoahBarrierSet::resolve_and_update_oop_static(p, obj);
+    }
+  }
+  virtual void do_oop(narrowOop* p) {
+    Unimplemented();
+  }
+};
+
+void ShenandoahMarkCompact::phase1_mark_heap() {
+  ShenandoahHeap* _heap = ShenandoahHeap::heap();
+  ReferenceProcessor* rp = _heap->ref_processor();
+
+  MarkSweep::_ref_processor = rp;
+
+  // First, update _all_ references in GC roots to point to to-space.
+  {
+    // Need cleared claim bits for the roots processing
+    /*
+    ClassLoaderDataGraph::clear_claimed_marks();
+    UpdateRefsClosure uprefs;
+    CLDToOopClosure cld_uprefs(&uprefs);
+    CodeBlobToOopClosure code_uprefs(&uprefs, CodeBlobToOopClosure::FixRelocations);
+    ShenandoahRootProcessor rp(_heap, 1);
+    rp.process_all_roots(&uprefs,
+                         &cld_uprefs,
+                         &code_uprefs);
+    */
+  }
+
+  {
+    MarkingCodeBlobClosure follow_code_closure(&MarkSweep::follow_root_closure, CodeBlobToOopClosure::FixRelocations);
+    // Need cleared claim bits for the roots processing
+    ClassLoaderDataGraph::clear_claimed_marks();
+    ShenandoahRootProcessor rp(_heap, 1);
+    rp.process_strong_roots(&MarkSweep::follow_root_closure,
+                            &MarkSweep::follow_cld_closure,
+                            &follow_code_closure);
+
+    // Also update (without marking) weak CLD refs, in case they're reachable.
+    UpdateRefsClosure uprefs;
+    CLDToOopClosure cld_uprefs(&uprefs);
+    ClassLoaderDataGraph::roots_cld_do(NULL, &cld_uprefs);
+
+    // Same for weak JNI handles.
+    ShenandoahAlwaysTrueClosure always_true;
+    JNIHandles::weak_oops_do(&always_true, &uprefs);
+  }
+
+  _heap->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::weakrefs);
+  bool clear_soft_refs = false; //fixme
+  rp->setup_policy(clear_soft_refs);
+
+  const ReferenceProcessorStats& stats =
+    rp->process_discovered_references(&MarkSweep::is_alive,
+                                      &MarkSweep::keep_alive,
+                                      &MarkSweep::follow_stack_closure,
+                                      NULL,
+                                      NULL,
+                                      _heap->tracer()->gc_id());
+
+  //     heap->tracer()->report_gc_reference_stats(stats);
+
+  _heap->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::weakrefs);
+
+  // Unload classes and purge the SystemDictionary.
+  bool purged_class = SystemDictionary::do_unloading(&MarkSweep::is_alive);
+
+  // Unload nmethods.
+  CodeCache::do_unloading(&MarkSweep::is_alive, purged_class);
+
+  // Prune dead klasses from subklass/sibling/implementor lists.
+  Klass::clean_weak_klass_links(&MarkSweep::is_alive);
+
+  // Delete entries for dead interned string and clean up unreferenced symbols in symbol table.
+  _heap->unlink_string_and_symbol_table(&MarkSweep::is_alive);
+
+  if (VerifyDuringGC) {
+    HandleMark hm;  // handle scope
+    COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact);
+    //    Universe::heap()->prepare_for_verify();
+    _heap->prepare_for_verify();
+    // Note: we can verify only the heap here. When an object is
+    // marked, the previous value of the mark word (including
+    // identity hash values, ages, etc) is preserved, and the mark
+    // word is set to markOop::marked_value - effectively removing
+    // any hash values from the mark word. These hash values are
+    // used when verifying the dictionaries and so removing them
+    // from the mark word can make verification of the dictionaries
+    // fail. At the end of the GC, the original mark word values
+    // (including hash values) are restored to the appropriate
+    // objects.
+    if (!VerifySilently) {
+      gclog_or_tty->print(" VerifyDuringGC:(full)[Verifying ");
+    }
+    //    Universe::heap()->verify(VerifySilently, VerifyOption_G1UseMarkWord);
+    _heap->verify(VerifySilently, VerifyOption_G1UseMarkWord);
+    if (!VerifySilently) {
+      gclog_or_tty->print_cr("]");
+    }
+  }
+}
+
+class ShenandoahPrepareForCompaction : public ShenandoahHeapRegionClosure {
+  CompactPoint _cp;
+  ShenandoahHeap* _heap;
+  bool _dead_humongous;
+
+public:
+  ShenandoahPrepareForCompaction() :
+    _heap(ShenandoahHeap::heap()),
+    _dead_humongous(false) {
+  }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    // We need to save the contents
+    if (!r->is_humongous()) {
+      if (_cp.space == NULL) {
+        _cp.space = r;
+        _cp.threshold = r->initialize_threshold();
+      }
+      _dead_humongous = false;
+      r->prepare_for_compaction(&_cp);
+    }  else {
+      if (r->is_humongous_start()) {
+        oop obj = oop(r->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE);
+        if (obj->is_gc_marked()) {
+          obj->forward_to(obj);
+          _dead_humongous = false;
+        } else {
+          if (_cp.space == NULL) {
+            _cp.space = r;
+            _cp.threshold = r->initialize_threshold();
+          }
+          _dead_humongous = true;
+          guarantee(r->region_number() >= ((ShenandoahHeapRegion*)_cp.space)->region_number(),
+                    "only reset regions that are not yet used for compaction");
+          r->reset();
+          r->prepare_for_compaction(&_cp);
+        }
+      } else {
+        assert(r->is_humongous_continuation(), "expect humongous continuation");
+        if (_dead_humongous) {
+          guarantee(r->region_number() > ((ShenandoahHeapRegion*)_cp.space)->region_number(),
+                    "only reset regions that are not yet used for compaction");
+          r->reset();
+          r->prepare_for_compaction(&_cp);
+        }
+      }
+    }
+    return false;
+  }
+};
+
+void ShenandoahMarkCompact::phase2_calculate_target_addresses() {
+  ShenandoahPrepareForCompaction prepare;
+  ShenandoahHeap::heap()->heap_region_iterate(&prepare);
+}
+
+
+class ShenandoahMarkCompactAdjustPointersClosure : public ShenandoahHeapRegionClosure {
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    if (r->is_humongous()) {
+      if (r->is_humongous_start()) {
+        // We must adjust the pointers on the single H object.
+        oop obj = oop(r->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE);
+        assert(obj->is_gc_marked(), "should be marked");
+        // point all the oops to the new location
+        MarkSweep::adjust_pointers(obj);
+      }
+    } else {
+      r->adjust_pointers();
+    }
+    return false;
+  }
+};
+
+void ShenandoahMarkCompact::phase3_update_references() {
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+    // Need cleared claim bits for the roots processing
+  ClassLoaderDataGraph::clear_claimed_marks();
+
+  CodeBlobToOopClosure adjust_code_closure(&MarkSweep::adjust_pointer_closure,
+                                           CodeBlobToOopClosure::FixRelocations);
+
+  {
+    ShenandoahRootProcessor rp(heap, 1);
+    rp.process_all_roots(&MarkSweep::adjust_pointer_closure,
+                         &MarkSweep::adjust_cld_closure,
+                         &adjust_code_closure);
+  }
+
+  assert(MarkSweep::ref_processor() == heap->ref_processor(), "Sanity");
+
+  // Now adjust pointers in remaining weak roots.  (All of which should
+  // have been cleared if they pointed to non-surviving objects.)
+  heap->weak_roots_iterate(&MarkSweep::adjust_pointer_closure);
+
+  //  if (G1StringDedup::is_enabled()) {
+  //    G1StringDedup::oops_do(&MarkSweep::adjust_pointer_closure);
+  //  }
+
+  MarkSweep::adjust_marks();
+
+  ShenandoahMarkCompactAdjustPointersClosure apc;
+  heap->heap_region_iterate(&apc);
+}
+
+class ShenandoahCleanupObjectClosure : public ObjectClosure {
+  void  do_object(oop p) {
+    ShenandoahHeap::heap()->initialize_brooks_ptr(p);
+  }
+};
+
+class CompactObjectsClosure : public ShenandoahHeapRegionClosure {
+
+public:
+
+  CompactObjectsClosure() {
+  }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    if (r->is_humongous()) {
+      if (r->is_humongous_start()) {
+        oop obj = oop(r->bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE);
+        assert(obj->is_gc_marked(), "expect marked humongous object");
+        obj->init_mark();
+      }
+    } else {
+      r->compact();
+    }
+
+    return false;
+  }
+
+};
+
+class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
+  size_t _live;
+  ShenandoahHeap* _heap;
+public:
+
+  ShenandoahPostCompactClosure() : _live(0), _heap(ShenandoahHeap::heap()) {
+    _heap->clear_free_regions();
+  }
+
+  bool doHeapRegion(ShenandoahHeapRegion* r) {
+    if (r->is_humongous()) {
+      _live += ShenandoahHeapRegion::RegionSizeBytes;
+
+    } else {
+      size_t live = r->used();
+      if (live == 0) {
+        r->recycle();
+        _heap->add_free_region(r);
+      }
+      r->setLiveData(live);
+      _live += live;
+    }
+
+    return false;
+  }
+
+  size_t getLive() { return _live;}
+
+};
+
+void ShenandoahMarkCompact::phase4_compact_objects() {
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  CompactObjectsClosure coc;
+  heap->heap_region_iterate(&coc);
+
+  ShenandoahCleanupObjectClosure cleanup;
+  heap->object_iterate(&cleanup);
+
+  ShenandoahPostCompactClosure post_compact;
+  heap->heap_region_iterate(&post_compact);
+
+  heap->set_used(post_compact.getLive());
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahMarkCompact.hpp b/src/share/vm/gc/shenandoah/shenandoahMarkCompact.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahMarkCompact.hpp
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP
+
+#include "gc/serial/genMarkSweep.hpp"
+#include "gc/shared/taskqueue.hpp"
+#include "gc/shared/workgroup.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+
+class HeapWord;
+class ShenandoahMarkCompactBarrierSet;
+
+/**
+ * This implements full-GC (e.g. when invoking System.gc() ) using a
+ * mark-compact algorithm. It's implemented in four phases:
+ *
+ * 1. Mark all live objects of the heap by traversing objects starting at GC roots.
+ * 2. Calculate the new location of each live object. This is done by sequentially scanning
+ *    the heap, keeping track of a next-location-pointer, which is then written to each
+ *    object's brooks ptr field.
+ * 3. Update all references. This is implemented by another scan of the heap, and updates
+ *    all references in live objects by what's stored in the target object's brooks ptr.
+ * 3. Compact the heap by copying all live objects to their new location.
+ */
+
+class ShenandoahMarkCompact: AllStatic {
+
+public:
+
+  static void do_mark_compact();
+
+private:
+
+  static void phase1_mark_heap();
+  static void phase2_calculate_target_addresses();
+  static void phase3_update_references();
+  static void phase4_compact_objects();
+  static void finish_compaction(HeapWord* last_addr);
+
+  static void allocate_stacks();
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahRootProcessor.cpp b/src/share/vm/gc/shenandoah/shenandoahRootProcessor.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahRootProcessor.cpp
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "classfile/stringTable.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "code/codeCache.hpp"
+#include "gc/shenandoah/shenandoahRootProcessor.hpp"
+#include "memory/allocation.inline.hpp"
+#include "runtime/fprofiler.hpp"
+#include "runtime/mutex.hpp"
+#include "services/management.hpp"
+
+ShenandoahRootProcessor::ShenandoahRootProcessor(ShenandoahHeap* heap, uint n_workers) :
+  _process_strong_tasks(new SubTasksDone(SHENANDOAH_RP_PS_NumElements)),
+  _srs(n_workers)
+{
+}
+
+void ShenandoahRootProcessor::process_roots(OopClosure* strong_oops,
+                                            OopClosure* weak_oops,
+                                            CLDClosure* strong_clds,
+                                            CLDClosure* weak_clds,
+                                            CLDClosure* thread_stack_clds,
+                                            CodeBlobClosure* strong_code) {
+  process_java_roots(strong_oops, thread_stack_clds, strong_clds, weak_clds, strong_code, 0);
+  process_vm_roots(strong_oops, weak_oops, 0);
+
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_CodeCache_oops_do)) {
+    CodeCache::blobs_do(strong_code);
+  }
+
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_JNIHandles_weak_oops_do)) {
+    ShenandoahAlwaysTrueClosure always_true;
+    JNIHandles::weak_oops_do(&always_true, weak_oops);
+  }
+
+  _process_strong_tasks->all_tasks_completed(n_workers());
+}
+
+void ShenandoahRootProcessor::process_strong_roots(OopClosure* oops,
+                                           CLDClosure* clds,
+                                           CodeBlobClosure* blobs) {
+
+  process_java_roots(oops, clds, clds, NULL, blobs, 0);
+  process_vm_roots(oops, NULL, 0);
+
+  _process_strong_tasks->all_tasks_completed(n_workers());
+}
+
+void ShenandoahRootProcessor::process_all_roots(OopClosure* oops,
+                                        CLDClosure* clds,
+                                        CodeBlobClosure* blobs) {
+
+  process_java_roots(oops, NULL, clds, clds, NULL, 0);
+  process_vm_roots(oops, oops, 0);
+
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_CodeCache_oops_do)) {
+    CodeCache::blobs_do(blobs);
+  }
+
+  _process_strong_tasks->all_tasks_completed(n_workers());
+}
+
+void ShenandoahRootProcessor::process_java_roots(OopClosure* strong_roots,
+                                                 CLDClosure* thread_stack_clds,
+                                                 CLDClosure* strong_clds,
+                                                 CLDClosure* weak_clds,
+                                                 CodeBlobClosure* strong_code,
+                                                 uint worker_i)
+{
+  //assert(thread_stack_clds == NULL || weak_clds == NULL, "There is overlap between those, only one may be set");
+  // Iterating over the CLDG and the Threads are done early to allow us to
+  // first process the strong CLDs and nmethods and then, after a barrier,
+  // let the thread process the weak CLDs and nmethods.
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_ClassLoaderDataGraph_oops_do)) {
+    ClassLoaderDataGraph::roots_cld_do(strong_clds, weak_clds);
+  }
+
+  bool is_par = n_workers() > 1;
+  ResourceMark rm;
+  Threads::possibly_parallel_oops_do(is_par, strong_roots, thread_stack_clds, strong_code);
+}
+
+void ShenandoahRootProcessor::process_vm_roots(OopClosure* strong_roots,
+                                               OopClosure* weak_roots,
+                                               uint worker_i)
+{
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_Universe_oops_do)) {
+    Universe::oops_do(strong_roots);
+  }
+
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_JNIHandles_oops_do)) {
+    JNIHandles::oops_do(strong_roots);
+  }
+  if (!_process_strong_tasks-> is_task_claimed(SHENANDOAH_RP_PS_ObjectSynchronizer_oops_do)) {
+    ObjectSynchronizer::oops_do(strong_roots);
+  }
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_FlatProfiler_oops_do)) {
+    FlatProfiler::oops_do(strong_roots);
+  }
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_Management_oops_do)) {
+    Management::oops_do(strong_roots);
+  }
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_jvmti_oops_do)) {
+    JvmtiExport::oops_do(strong_roots);
+  }
+  if (!_process_strong_tasks->is_task_claimed(SHENANDOAH_RP_PS_SystemDictionary_oops_do)) {
+    SystemDictionary::roots_oops_do(strong_roots, weak_roots);
+  }
+  // All threads execute the following. A specific chunk of buckets
+  // from the StringTable are the individual tasks.
+  if (weak_roots != NULL) {
+    StringTable::possibly_parallel_oops_do(weak_roots);
+  }
+}
+
+uint ShenandoahRootProcessor::n_workers() const {
+  return _srs.n_threads();
+}
diff --git a/src/share/vm/gc/shenandoah/shenandoahRootProcessor.hpp b/src/share/vm/gc/shenandoah/shenandoahRootProcessor.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahRootProcessor.hpp
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_HPP
+
+#include "gc/shared/strongRootsScope.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/mutex.hpp"
+
+class CLDClosure;
+class CodeBlobClosure;
+class G1CollectedHeap;
+class G1GCPhaseTimes;
+class G1ParPushHeapRSClosure;
+class Monitor;
+class OopClosure;
+class SubTasksDone;
+
+class ShenandoahRootProcessor : public StackObj {
+  SubTasksDone* _process_strong_tasks;
+  StrongRootsScope _srs;
+
+  enum Shenandoah_process_roots_tasks {
+    SHENANDOAH_RP_PS_Universe_oops_do,
+    SHENANDOAH_RP_PS_JNIHandles_oops_do,
+    SHENANDOAH_RP_PS_JNIHandles_weak_oops_do,
+    SHENANDOAH_RP_PS_ObjectSynchronizer_oops_do,
+    SHENANDOAH_RP_PS_FlatProfiler_oops_do,
+    SHENANDOAH_RP_PS_Management_oops_do,
+    SHENANDOAH_RP_PS_SystemDictionary_oops_do,
+    SHENANDOAH_RP_PS_ClassLoaderDataGraph_oops_do,
+    SHENANDOAH_RP_PS_jvmti_oops_do,
+    SHENANDOAH_RP_PS_CodeCache_oops_do,
+    SHENANDOAH_RP_PS_filter_satb_buffers,
+    SHENANDOAH_RP_PS_refProcessor_oops_do,
+    // Leave this one last.
+    SHENANDOAH_RP_PS_NumElements
+  };
+
+  void process_java_roots(OopClosure* scan_non_heap_roots,
+                          CLDClosure* thread_stack_clds,
+                          CLDClosure* scan_strong_clds,
+                          CLDClosure* scan_weak_clds,
+                          CodeBlobClosure* scan_strong_code,
+                          uint worker_i);
+
+  void process_vm_roots(OopClosure* scan_non_heap_roots,
+                        OopClosure* scan_non_heap_weak_roots,
+                        uint worker_i);
+
+public:
+  ShenandoahRootProcessor(ShenandoahHeap* heap, uint n_workers);
+
+  void process_roots(OopClosure* strong_oops,
+                     OopClosure* weak_oops,
+                     CLDClosure* strong_clds,
+                     CLDClosure* weak_clds,
+                     CLDClosure* thread_stack_clds,
+                     CodeBlobClosure* strong_code);
+
+  // Apply oops, clds and blobs to all strongly reachable roots in the system
+  void process_strong_roots(OopClosure* oops,
+                            CLDClosure* clds,
+                            CodeBlobClosure* blobs);
+
+  // Apply oops, clds and blobs to strongly and weakly reachable roots in the system
+  void process_all_roots(OopClosure* oops,
+                         CLDClosure* clds,
+                         CodeBlobClosure* blobs);
+
+  // Number of worker threads used by the root processor.
+  uint n_workers() const;
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_HPP
diff --git a/src/share/vm/gc/shenandoah/shenandoahRuntime.cpp b/src/share/vm/gc/shenandoah/shenandoahRuntime.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahRuntime.cpp
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "oops/oop.inline.hpp"
+
+JRT_LEAF(bool, ShenandoahRuntime::compare_and_swap_object(HeapWord* addr, oopDesc* newval, oopDesc* old))
+  bool success;
+  oop expected;
+  do {
+    expected = old;
+    old = oopDesc::atomic_compare_exchange_oop(newval, addr, expected, true);
+    success  = (old == expected);
+  } while ((! success) && oopDesc::bs()->resolve_oop(old) == oopDesc::bs()->resolve_oop(expected));
+
+  return success;
+JRT_END
diff --git a/src/share/vm/gc/shenandoah/shenandoahRuntime.hpp b/src/share/vm/gc/shenandoah/shenandoahRuntime.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/shenandoahRuntime.hpp
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP
+
+#include "oops/oop.hpp"
+
+class ShenandoahRuntime : AllStatic {
+public:
+  static bool compare_and_swap_object(HeapWord* adr, oopDesc* newval, oopDesc* expected);
+};
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP
diff --git a/src/share/vm/gc/shenandoah/vm_operations_shenandoah.cpp b/src/share/vm/gc/shenandoah/vm_operations_shenandoah.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/vm_operations_shenandoah.cpp
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahMarkCompact.hpp"
+#include "gc/shenandoah/vm_operations_shenandoah.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+
+VM_Operation::VMOp_Type VM_ShenandoahInitMark::type() const {
+  return VMOp_ShenandoahInitMark;
+}
+
+const char* VM_ShenandoahInitMark::name() const {
+  return "Shenandoah Initial Marking";
+}
+
+void VM_ShenandoahInitMark::doit() {
+  ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+  sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::init_mark);
+
+  assert(sh->is_bitmap_clear(), "need clear marking bitmap");
+
+  if (ShenandoahGCVerbose)
+    tty->print("vm_ShenandoahInitMark\n");
+  sh->start_concurrent_marking();
+  if (UseTLAB) {
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::resize_tlabs);
+    sh->resize_all_tlabs();
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::resize_tlabs);
+  }
+
+  sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::init_mark);
+
+  if (! ShenandoahConcurrentMarking) {
+    sh->concurrentMark()->mark_from_roots();
+    VM_ShenandoahStartEvacuation finishMark;
+    finishMark.doit();
+  }
+}
+
+VM_Operation::VMOp_Type VM_ShenandoahFullGC::type() const {
+  return VMOp_ShenandoahFullGC;
+}
+
+void VM_ShenandoahFullGC::doit() {
+
+  ShenandoahMarkCompact::do_mark_compact();
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  if (UseTLAB) {
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::resize_tlabs);
+    sh->resize_all_tlabs();
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::resize_tlabs);
+  }
+}
+
+const char* VM_ShenandoahFullGC::name() const {
+  return "Shenandoah Full GC";
+}
+
+
+bool VM_ShenandoahReferenceOperation::doit_prologue() {
+  ShenandoahHeap *sh = (ShenandoahHeap*) Universe::heap();
+  sh->acquire_pending_refs_lock();
+  return true;
+}
+
+void VM_ShenandoahReferenceOperation::doit_epilogue() {
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  sh->release_pending_refs_lock();
+}
+
+void VM_ShenandoahStartEvacuation::doit() {
+
+  // We need to do the finish mark here, so that a JNI critical region
+  // can't divide it from evacuation start. It is critical that we
+  // evacuate roots right after finishing marking, so that we don't
+  // get unmarked objects in the roots.
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  if (!sh->cancelled_concgc()) {
+    if (ShenandoahGCVerbose)
+      tty->print("vm_ShenandoahFinalMark\n");
+
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::final_mark);
+    sh->concurrentMark()->finish_mark_from_roots();
+    sh->stop_concurrent_marking();
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_mark);
+
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::prepare_evac);
+    sh->prepare_for_concurrent_evacuation();
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::prepare_evac);
+
+    if (!sh->cancelled_concgc()){
+      sh->set_evacuation_in_progress(true);
+
+      // From here on, we need to update references.
+      sh->set_need_update_refs(true);
+
+      if (! ShenandoahConcurrentEvacuation) {
+        VM_ShenandoahEvacuation evacuation;
+        evacuation.doit();
+      } else {
+        if (!sh->cancelled_concgc()) {
+          sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::init_evac);
+          sh->evacuate_and_update_roots();
+          sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::init_evac);
+        }
+      }
+    } else {
+      sh->free_regions()->set_concurrent_iteration_safe_limits();
+      //      sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::prepare_evac);
+      //      sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_mark);
+    }
+  } else {
+    sh->concurrentMark()->cancel();
+    sh->stop_concurrent_marking();
+  }
+}
+
+VM_Operation::VMOp_Type VM_ShenandoahStartEvacuation::type() const {
+  return VMOp_ShenandoahStartEvacuation;
+}
+
+const char* VM_ShenandoahStartEvacuation::name() const {
+  return "Start shenandoah evacuation";
+}
+
+VM_Operation::VMOp_Type VM_ShenandoahVerifyHeapAfterEvacuation::type() const {
+  return VMOp_ShenandoahVerifyHeapAfterEvacuation;
+}
+
+const char* VM_ShenandoahVerifyHeapAfterEvacuation::name() const {
+  return "Shenandoah verify heap after evacuation";
+}
+
+void VM_ShenandoahVerifyHeapAfterEvacuation::doit() {
+
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  sh->verify_heap_after_evacuation();
+
+}
+
+VM_Operation::VMOp_Type VM_ShenandoahEvacuation::type() const {
+  return VMOp_ShenandoahEvacuation;
+}
+
+const char* VM_ShenandoahEvacuation::name() const {
+  return "Shenandoah evacuation";
+}
+
+void VM_ShenandoahEvacuation::doit() {
+  if (ShenandoahGCVerbose)
+    tty->print("vm_ShenandoahEvacuation\n");
+
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  sh->do_evacuation();
+
+  if (! ShenandoahConcurrentUpdateRefs) {
+    assert(! ShenandoahConcurrentEvacuation, "turn off concurrent evacuation");
+    sh->prepare_for_update_references();
+    sh->update_references();
+  }
+}
+/*
+  VM_Operation::VMOp_Type VM_ShenandoahVerifyHeapAfterUpdateRefs::type() const {
+  return VMOp_ShenandoahVerifyHeapAfterUpdateRefs;
+  }
+
+  const char* VM_ShenandoahVerifyHeapAfterUpdateRefs::name() const {
+  return "Shenandoah verify heap after updating references";
+  }
+
+  void VM_ShenandoahVerifyHeapAfterUpdateRefs::doit() {
+
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  sh->verify_heap_after_update_refs();
+
+  }
+*/
+VM_Operation::VMOp_Type VM_ShenandoahUpdateRootRefs::type() const {
+  return VMOp_ShenandoahUpdateRootRefs;
+}
+
+const char* VM_ShenandoahUpdateRootRefs::name() const {
+  return "Shenandoah update root references";
+}
+
+void VM_ShenandoahUpdateRootRefs::doit() {
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  if (! sh->cancelled_concgc()) {
+
+    if (ShenandoahGCVerbose)
+      tty->print("vm_ShenandoahUpdateRootRefs\n");
+
+
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::final_uprefs);
+
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::update_roots);
+
+    sh->update_roots();
+
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::update_roots);
+
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_uprefs);
+  }
+
+  sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::recycle_regions);
+  sh->recycle_dirty_regions();
+  sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::recycle_regions);
+
+  if (ShenandoahVerify && ! sh->cancelled_concgc()) {
+    sh->verify_heap_after_update_refs();
+    sh->verify_regions_after_update_refs();
+  }
+#ifdef ASSERT
+  if (! ShenandoahVerify) {
+    assert(sh->is_bitmap_clear(), "need cleared bitmap here");
+  }
+#endif
+
+}
+
+VM_Operation::VMOp_Type VM_ShenandoahUpdateRefs::type() const {
+  return VMOp_ShenandoahUpdateRefs;
+}
+
+const char* VM_ShenandoahUpdateRefs::name() const {
+  return "Shenandoah update references";
+}
+
+void VM_ShenandoahUpdateRefs::doit() {
+  ShenandoahHeap *sh = ShenandoahHeap::heap();
+  if (!sh->cancelled_concgc()) {
+
+    if (ShenandoahGCVerbose)
+      tty->print("vm_ShenandoahUpdateRefs\n");
+
+    sh->shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::final_evac);
+    sh->set_evacuation_in_progress(false);
+    sh->prepare_for_update_references();
+    assert(ShenandoahConcurrentUpdateRefs, "only do this when concurrent update references is turned on");
+    sh->shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::final_evac);
+  }
+}
diff --git a/src/share/vm/gc/shenandoah/vm_operations_shenandoah.hpp b/src/share/vm/gc/shenandoah/vm_operations_shenandoah.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/gc/shenandoah/vm_operations_shenandoah.hpp
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_VM_OPERATIONS_SHENANDOAH_HPP
+#define SHARE_VM_GC_SHENANDOAH_VM_OPERATIONS_SHENANDOAH_HPP
+
+#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
+#include "gc/shared/vmGCOperations.hpp"
+
+// VM_operations for the Shenandoah Collector.
+// For now we are just doing two pauses.  The initial marking pause, and the final finish up marking and perform evacuation pause.
+//    VM_ShenandoahInitMark
+//    VM_ShenandoahFinishMark
+
+class VM_ShenandoahInitMark: public VM_Operation {
+
+public:
+  virtual VMOp_Type type() const;
+  virtual void doit();
+
+  virtual const char* name() const;
+};
+
+class VM_ShenandoahReferenceOperation : public VM_Operation {
+  bool doit_prologue();
+  void doit_epilogue();
+
+};
+
+class VM_ShenandoahStartEvacuation: public VM_ShenandoahReferenceOperation {
+
+ public:
+  VMOp_Type type() const;
+  void doit();
+  const char* name() const;
+
+};
+
+class VM_ShenandoahFullGC : public VM_ShenandoahReferenceOperation {
+ public:
+  VMOp_Type type() const;
+  void doit();
+  const char* name() const;
+};
+
+class VM_ShenandoahVerifyHeapAfterEvacuation: public VM_Operation {
+
+ public:
+  virtual VMOp_Type type() const;
+  virtual void doit();
+
+  virtual const char* name() const;
+
+};
+
+class VM_ShenandoahEvacuation: public VM_Operation {
+
+ public:
+  virtual VMOp_Type type() const;
+  virtual void doit();
+
+  virtual const char* name() const;
+
+};
+
+/*
+class VM_ShenandoahVerifyHeapAfterUpdateRefs: public VM_Operation {
+
+ public:
+  virtual VMOp_Type type() const;
+  virtual void doit();
+
+  virtual const char* name() const;
+
+};
+*/
+class VM_ShenandoahUpdateRootRefs: public VM_Operation {
+
+ public:
+  virtual VMOp_Type type() const;
+  virtual void doit();
+
+  virtual const char* name() const;
+
+};
+
+class VM_ShenandoahUpdateRefs: public VM_Operation {
+
+ public:
+  virtual VMOp_Type type() const;
+  virtual void doit();
+
+  virtual const char* name() const;
+
+};
+
+#endif //SHARE_VM_GC_SHENANDOAH_VM_OPERATIONS_SHENANDOAH_HPP
diff --git a/src/share/vm/interpreter/bytecodeInterpreter.cpp b/src/share/vm/interpreter/bytecodeInterpreter.cpp
--- a/src/share/vm/interpreter/bytecodeInterpreter.cpp
+++ b/src/share/vm/interpreter/bytecodeInterpreter.cpp
@@ -1900,7 +1900,7 @@
         BasicObjectLock* limit = istate->monitor_base();
         BasicObjectLock* most_recent = (BasicObjectLock*) istate->stack_base();
         while (most_recent != limit ) {
-          if ((most_recent)->obj() == lockee) {
+          if (most_recent->obj() == lockee) {
             BasicLock* lock = most_recent->lock();
             markOop header = lock->displaced_header();
             most_recent->set_obj(NULL);
@@ -2152,7 +2152,7 @@
           Klass* k_entry = (Klass*) entry;
           assert(k_entry->oop_is_instance(), "Should be InstanceKlass");
           InstanceKlass* ik = (InstanceKlass*) k_entry;
-          if ( ik->is_initialized() && ik->can_be_fastpath_allocated() ) {
+          if (ik->is_initialized() && ik->can_be_fastpath_allocated() ) {
             size_t obj_size = ik->size_helper();
             oop result = NULL;
             // If the TLAB isn't pre-zeroed then we'll have to do it
diff --git a/src/share/vm/interpreter/interpreterRuntime.cpp b/src/share/vm/interpreter/interpreterRuntime.cpp
--- a/src/share/vm/interpreter/interpreterRuntime.cpp
+++ b/src/share/vm/interpreter/interpreterRuntime.cpp
@@ -615,7 +615,7 @@
   if (PrintBiasedLockingStatistics) {
     Atomic::inc(BiasedLocking::slow_path_entry_count_addr());
   }
-  Handle h_obj(thread, elem->obj());
+  Handle h_obj(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(elem->obj()));
   assert(Universe::heap()->is_in_reserved_or_null(h_obj()),
          "must be NULL or an object");
   if (UseBiasedLocking) {
@@ -637,7 +637,7 @@
 #ifdef ASSERT
   thread->last_frame().interpreter_frame_verify_monitor(elem);
 #endif
-  Handle h_obj(thread, elem->obj());
+  Handle h_obj(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(elem->obj()));
   assert(Universe::heap()->is_in_reserved_or_null(h_obj()),
          "must be NULL or an object");
   if (elem == NULL || h_obj()->is_unlocked()) {
diff --git a/src/share/vm/memory/oopFactory.cpp b/src/share/vm/memory/oopFactory.cpp
--- a/src/share/vm/memory/oopFactory.cpp
+++ b/src/share/vm/memory/oopFactory.cpp
@@ -40,6 +40,7 @@
 typeArrayOop oopFactory::new_charArray(const char* utf8_str, TRAPS) {
   int length = utf8_str == NULL ? 0 : UTF8::unicode_length(utf8_str);
   typeArrayOop result = new_charArray(length, CHECK_NULL);
+  result = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(result));
   if (length > 0) {
     UTF8::convert_to_unicode(utf8_str, result->char_at_addr(0), length);
   }
diff --git a/src/share/vm/memory/universe.cpp b/src/share/vm/memory/universe.cpp
--- a/src/share/vm/memory/universe.cpp
+++ b/src/share/vm/memory/universe.cpp
@@ -75,6 +75,8 @@
 #include "utilities/macros.hpp"
 #include "utilities/preserveException.hpp"
 #if INCLUDE_ALL_GCS
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/cms/cmsCollectorPolicy.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectorPolicy.hpp"
@@ -703,6 +705,9 @@
     fatal("UseG1GC not supported in this VM.");
   } else if (UseConcMarkSweepGC) {
     fatal("UseConcMarkSweepGC not supported in this VM.");
+  } else if (UseShenandoahGC) {
+    fatal("UseShenandoahGC not supported in this VM.");
+  }
 #else
   if (UseParallelGC) {
     return Universe::create_heap_with_policy<ParallelScavengeHeap, GenerationSizer>();
@@ -710,6 +715,8 @@
     return Universe::create_heap_with_policy<G1CollectedHeap, G1CollectorPolicy>();
   } else if (UseConcMarkSweepGC) {
     return Universe::create_heap_with_policy<GenCollectedHeap, ConcurrentMarkSweepPolicy>();
+  } else if (UseShenandoahGC) {
+    return Universe::create_heap_with_policy<ShenandoahHeap, ShenandoahCollectorPolicy>();
 #endif
   } else if (UseSerialGC) {
     return Universe::create_heap_with_policy<GenCollectedHeap, MarkSweepPolicy>();
diff --git a/src/share/vm/oops/cpCache.cpp b/src/share/vm/oops/cpCache.cpp
--- a/src/share/vm/oops/cpCache.cpp
+++ b/src/share/vm/oops/cpCache.cpp
@@ -283,7 +283,7 @@
   // the lock, so that when the losing writer returns, he can use the linked
   // cache entry.
 
-  objArrayHandle resolved_references = cpool->resolved_references();
+  objArrayHandle resolved_references = objArrayHandle(objArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(cpool->resolved_references())));
   // Use the resolved_references() lock for this cpCache entry.
   // resolved_references are created for all classes with Invokedynamic, MethodHandle
   // or MethodType constant pool cache entries.
diff --git a/src/share/vm/oops/instanceKlass.cpp b/src/share/vm/oops/instanceKlass.cpp
--- a/src/share/vm/oops/instanceKlass.cpp
+++ b/src/share/vm/oops/instanceKlass.cpp
@@ -471,6 +471,7 @@
 void InstanceKlass::eager_initialize_impl(instanceKlassHandle this_k) {
   EXCEPTION_MARK;
   oop init_lock = this_k->init_lock();
+  init_lock = oopDesc::bs()->resolve_and_maybe_copy_oop(init_lock);
   ObjectLocker ol(init_lock, THREAD, init_lock != NULL);
 
   // abort if someone beat us to the initialization
@@ -617,6 +618,7 @@
   // verification & rewriting
   {
     oop init_lock = this_k->init_lock();
+    init_lock = oopDesc::bs()->resolve_and_maybe_copy_oop(init_lock);
     ObjectLocker ol(init_lock, THREAD, init_lock != NULL);
     // rewritten will have been set if loader constraint error found
     // on an earlier link attempt
@@ -750,6 +752,7 @@
   // Step 1
   {
     oop init_lock = this_k->init_lock();
+    init_lock = oopDesc::bs()->resolve_and_maybe_copy_oop(init_lock);
     ObjectLocker ol(init_lock, THREAD, init_lock != NULL);
 
     Thread *self = THREAD; // it's passed the current thread
@@ -882,6 +885,7 @@
 
 void InstanceKlass::set_initialization_state_and_notify_impl(instanceKlassHandle this_k, ClassState state, TRAPS) {
   oop init_lock = this_k->init_lock();
+  init_lock = oopDesc::bs()->resolve_and_maybe_copy_oop(init_lock);
   ObjectLocker ol(init_lock, THREAD, init_lock != NULL);
   this_k->set_init_state(state);
   this_k->fence_and_clear_init_lock();
@@ -2254,7 +2258,7 @@
 }
 
 address InstanceKlass::static_field_addr(int offset) {
-  return (address)(offset + InstanceMirrorKlass::offset_of_static_fields() + cast_from_oop<intptr_t>(java_mirror()));
+  return (address)(offset + InstanceMirrorKlass::offset_of_static_fields() + cast_from_oop<intptr_t>(oopDesc::bs()->resolve_and_maybe_copy_oop(java_mirror())));
 }
 
 
diff --git a/src/share/vm/oops/instanceRefKlass.cpp b/src/share/vm/oops/instanceRefKlass.cpp
--- a/src/share/vm/oops/instanceRefKlass.cpp
+++ b/src/share/vm/oops/instanceRefKlass.cpp
@@ -91,7 +91,7 @@
 
 bool InstanceRefKlass::owns_pending_list_lock(JavaThread* thread) {
   if (java_lang_ref_Reference::pending_list_lock() == NULL) return false;
-  Handle h_lock(thread, java_lang_ref_Reference::pending_list_lock());
+  Handle h_lock(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(java_lang_ref_Reference::pending_list_lock()));
   return ObjectSynchronizer::current_thread_holds_lock(thread, h_lock);
 }
 
@@ -104,7 +104,7 @@
   // to hold the pending list lock. We want to free this handle.
   HandleMark hm;
 
-  Handle h_lock(THREAD, java_lang_ref_Reference::pending_list_lock());
+  Handle h_lock(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(java_lang_ref_Reference::pending_list_lock()));
   ObjectSynchronizer::fast_enter(h_lock, pending_list_basic_lock, false, THREAD);
   assert(ObjectSynchronizer::current_thread_holds_lock(
            JavaThread::current(), h_lock),
@@ -122,7 +122,7 @@
   // to hold the pending list lock. We want to free this handle.
   HandleMark hm;
 
-  Handle h_lock(THREAD, java_lang_ref_Reference::pending_list_lock());
+  Handle h_lock(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(java_lang_ref_Reference::pending_list_lock()));
   assert(ObjectSynchronizer::current_thread_holds_lock(
            JavaThread::current(), h_lock),
          "Lock should be held");
diff --git a/src/share/vm/oops/instanceRefKlass.inline.hpp b/src/share/vm/oops/instanceRefKlass.inline.hpp
--- a/src/share/vm/oops/instanceRefKlass.inline.hpp
+++ b/src/share/vm/oops/instanceRefKlass.inline.hpp
@@ -46,7 +46,10 @@
   ReferenceProcessor* rp = closure->_ref_processor;
   if (!oopDesc::is_null(heap_oop)) {
     oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
-    if (!referent->is_gc_marked() && (rp != NULL) &&
+    if (UseShenandoahGC) {
+      referent = ShenandoahBarrierSet::resolve_and_update_oop_static(referent_addr, referent);
+    }
+    if ((UseShenandoahGC || !referent->is_gc_marked()) && (rp != NULL) &&
         rp->discover_reference(obj, reference_type())) {
       return;
     } else if (contains(referent_addr)) {
diff --git a/src/share/vm/oops/objArrayKlass.cpp b/src/share/vm/oops/objArrayKlass.cpp
--- a/src/share/vm/oops/objArrayKlass.cpp
+++ b/src/share/vm/oops/objArrayKlass.cpp
@@ -298,6 +298,10 @@
   if (length==0) {
     return;
   }
+
+  s = arrayOop(oopDesc::bs()->resolve_oop(s));
+  d = arrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(d));
+
   if (UseCompressedOops) {
     narrowOop* const src = objArrayOop(s)->obj_at_addr<narrowOop>(src_pos);
     narrowOop* const dst = objArrayOop(d)->obj_at_addr<narrowOop>(dst_pos);
diff --git a/src/share/vm/oops/objArrayOop.hpp b/src/share/vm/oops/objArrayOop.hpp
--- a/src/share/vm/oops/objArrayOop.hpp
+++ b/src/share/vm/oops/objArrayOop.hpp
@@ -25,6 +25,7 @@
 #ifndef SHARE_VM_OOPS_OBJARRAYOOP_HPP
 #define SHARE_VM_OOPS_OBJARRAYOOP_HPP
 
+#include "gc/shared/barrierSet.hpp"
 #include "gc/shared/specialized_oop_closures.hpp"
 #include "oops/arrayOop.hpp"
 
@@ -82,10 +83,11 @@
   oop obj_at(int index) const;
 
   void obj_at_put(int index, oop value) {
+    objArrayOop p = (objArrayOop) oopDesc::bs()->resolve_and_maybe_copy_oop(this);
     if (UseCompressedOops) {
-      oop_store(obj_at_addr<narrowOop>(index), value);
+      oop_store(p->obj_at_addr<narrowOop>(index), value);
     } else {
-      oop_store(obj_at_addr<oop>(index), value);
+      oop_store(p->obj_at_addr<oop>(index), value);
     }
   }
 
diff --git a/src/share/vm/oops/objArrayOop.inline.hpp b/src/share/vm/oops/objArrayOop.inline.hpp
--- a/src/share/vm/oops/objArrayOop.inline.hpp
+++ b/src/share/vm/oops/objArrayOop.inline.hpp
@@ -30,12 +30,13 @@
 #include "runtime/globals.hpp"
 
 inline oop objArrayOopDesc::obj_at(int index) const {
+  objArrayOop p = (objArrayOop) oopDesc::bs()->resolve_oop((oop) this);
   // With UseCompressedOops decode the narrow oop in the objArray to an
   // uncompressed oop.  Otherwise this is simply a "*" operator.
   if (UseCompressedOops) {
-    return load_decode_heap_oop(obj_at_addr<narrowOop>(index));
+    return load_decode_heap_oop(p->obj_at_addr<narrowOop>(index));
   } else {
-    return load_decode_heap_oop(obj_at_addr<oop>(index));
+    return load_decode_heap_oop(p->obj_at_addr<oop>(index));
   }
 }
 
diff --git a/src/share/vm/oops/oop.cpp b/src/share/vm/oops/oop.cpp
--- a/src/share/vm/oops/oop.cpp
+++ b/src/share/vm/oops/oop.cpp
@@ -100,7 +100,7 @@
   // slow case; we have to acquire the micro lock in order to locate the header
   ResetNoHandleMark rnm; // Might be called from LEAF/QUICK ENTRY
   HandleMark hm;
-  Handle object(this);
+  Handle object(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
   return ObjectSynchronizer::identity_hash_value_for(object);
 }
 
diff --git a/src/share/vm/oops/oop.inline.hpp b/src/share/vm/oops/oop.inline.hpp
--- a/src/share/vm/oops/oop.inline.hpp
+++ b/src/share/vm/oops/oop.inline.hpp
@@ -280,28 +280,35 @@
 // In order to put or get a field out of an instance, must first check
 // if the field has been compressed and uncompress it.
 inline oop oopDesc::obj_field(int offset) const {
+  oop p = bs()->resolve_oop((oop) this);
   return UseCompressedOops ?
-    load_decode_heap_oop(obj_field_addr<narrowOop>(offset)) :
-    load_decode_heap_oop(obj_field_addr<oop>(offset));
+    load_decode_heap_oop(p->obj_field_addr<narrowOop>(offset)) :
+    load_decode_heap_oop(p->obj_field_addr<oop>(offset));
 }
 
 inline void oopDesc::obj_field_put(int offset, oop value) {
-  UseCompressedOops ? oop_store(obj_field_addr<narrowOop>(offset), value) :
-                      oop_store(obj_field_addr<oop>(offset),       value);
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  value = bs()->resolve_oop(value);
+  UseCompressedOops ? oop_store(p->obj_field_addr<narrowOop>(offset), value) :
+                      oop_store(p->obj_field_addr<oop>(offset),       value);
 }
 
 inline Metadata* oopDesc::metadata_field(int offset) const {
-  return *metadata_field_addr(offset);
+  oop p = bs()->resolve_oop((oop) this);
+  return *p->metadata_field_addr(offset);
 }
 
 inline void oopDesc::metadata_field_put(int offset, Metadata* value) {
-  *metadata_field_addr(offset) = value;
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->metadata_field_addr(offset) = value;
 }
 
 inline void oopDesc::obj_field_put_raw(int offset, oop value) {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  value = bs()->resolve_oop(value);
   UseCompressedOops ?
-    encode_store_heap_oop(obj_field_addr<narrowOop>(offset), value) :
-    encode_store_heap_oop(obj_field_addr<oop>(offset),       value);
+    encode_store_heap_oop(p->obj_field_addr<narrowOop>(offset), value) :
+    encode_store_heap_oop(p->obj_field_addr<oop>(offset),       value);
 }
 inline void oopDesc::obj_field_put_volatile(int offset, oop value) {
   OrderAccess::release();
@@ -309,72 +316,183 @@
   OrderAccess::fence();
 }
 
-inline jbyte oopDesc::byte_field(int offset) const                  { return (jbyte) *byte_field_addr(offset);    }
-inline void oopDesc::byte_field_put(int offset, jbyte contents)     { *byte_field_addr(offset) = (jint) contents; }
+inline jbyte oopDesc::byte_field(int offset) const                  {
+  oop p = bs()->resolve_oop((oop) this);
+  return (jbyte) *p->byte_field_addr(offset);
+}
+inline void oopDesc::byte_field_put(int offset, jbyte contents)     {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->byte_field_addr(offset) = (jint) contents;
+}
 
-inline jboolean oopDesc::bool_field(int offset) const               { return (jboolean) *bool_field_addr(offset); }
-inline void oopDesc::bool_field_put(int offset, jboolean contents)  { *bool_field_addr(offset) = (jint) contents; }
+inline jboolean oopDesc::bool_field(int offset) const               {
+  oop p = bs()->resolve_oop((oop) this);
+  return (jboolean) *p->bool_field_addr(offset);
+}
+inline void oopDesc::bool_field_put(int offset, jboolean contents)  {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->bool_field_addr(offset) = (jint) contents;
+}
 
-inline jchar oopDesc::char_field(int offset) const                  { return (jchar) *char_field_addr(offset);    }
-inline void oopDesc::char_field_put(int offset, jchar contents)     { *char_field_addr(offset) = (jint) contents; }
+inline jchar oopDesc::char_field(int offset) const                  {
+  oop p = bs()->resolve_oop((oop) this);
+  return (jchar) *p->char_field_addr(offset);
+}
+inline void oopDesc::char_field_put(int offset, jchar contents)     {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->char_field_addr(offset) = (jint) contents;
+}
 
-inline jint oopDesc::int_field(int offset) const                    { return *int_field_addr(offset);        }
-inline void oopDesc::int_field_put(int offset, jint contents)       { *int_field_addr(offset) = contents;    }
+inline jint oopDesc::int_field(int offset) const                    {
+  oop p = bs()->resolve_oop((oop) this);
+  return *p->int_field_addr(offset);
+}
+inline void oopDesc::int_field_put(int offset, jint contents)       {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->int_field_addr(offset) = contents;
+}
 
-inline jshort oopDesc::short_field(int offset) const                { return (jshort) *short_field_addr(offset);  }
-inline void oopDesc::short_field_put(int offset, jshort contents)   { *short_field_addr(offset) = (jint) contents;}
+inline jshort oopDesc::short_field(int offset) const                {
+  oop p = bs()->resolve_oop((oop) this);
+  return (jshort) *p->short_field_addr(offset);
+}
+inline void oopDesc::short_field_put(int offset, jshort contents)   {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->short_field_addr(offset) = (jint) contents;
+}
 
-inline jlong oopDesc::long_field(int offset) const                  { return *long_field_addr(offset);       }
-inline void oopDesc::long_field_put(int offset, jlong contents)     { *long_field_addr(offset) = contents;   }
+inline jlong oopDesc::long_field(int offset) const                  {
+  oop p = bs()->resolve_oop((oop) this);
+  return *p->long_field_addr(offset);
+}
+inline void oopDesc::long_field_put(int offset, jlong contents)     {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->long_field_addr(offset) = contents;
+}
 
-inline jfloat oopDesc::float_field(int offset) const                { return *float_field_addr(offset);      }
-inline void oopDesc::float_field_put(int offset, jfloat contents)   { *float_field_addr(offset) = contents;  }
+inline jfloat oopDesc::float_field(int offset) const                {
+  oop p = bs()->resolve_oop((oop) this);
+  return *p->float_field_addr(offset);
+}
+inline void oopDesc::float_field_put(int offset, jfloat contents)   {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->float_field_addr(offset) = contents;
+}
 
-inline jdouble oopDesc::double_field(int offset) const              { return *double_field_addr(offset);     }
-inline void oopDesc::double_field_put(int offset, jdouble contents) { *double_field_addr(offset) = contents; }
+inline jdouble oopDesc::double_field(int offset) const              {
+  oop p = bs()->resolve_oop((oop) this);
+  return *p->double_field_addr(offset);
+}
+inline void oopDesc::double_field_put(int offset, jdouble contents) {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->double_field_addr(offset) = contents;
+}
 
-inline address oopDesc::address_field(int offset) const              { return *address_field_addr(offset);     }
-inline void oopDesc::address_field_put(int offset, address contents) { *address_field_addr(offset) = contents; }
+inline address oopDesc::address_field(int offset) const              {
+  oop p = bs()->resolve_oop((oop) this);
+  return *p->address_field_addr(offset);
+}
+inline void oopDesc::address_field_put(int offset, address contents) {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  *p->address_field_addr(offset) = contents;
+}
 
 inline oop oopDesc::obj_field_acquire(int offset) const {
+  oop p = bs()->resolve_oop((oop) this);
   return UseCompressedOops ?
              decode_heap_oop((narrowOop)
-               OrderAccess::load_acquire(obj_field_addr<narrowOop>(offset)))
+               OrderAccess::load_acquire(p->obj_field_addr<narrowOop>(offset)))
            : decode_heap_oop((oop)
-               OrderAccess::load_ptr_acquire(obj_field_addr<oop>(offset)));
+               OrderAccess::load_ptr_acquire(p->obj_field_addr<oop>(offset)));
 }
 inline void oopDesc::release_obj_field_put(int offset, oop value) {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  value = bs()->resolve_oop(value);
   UseCompressedOops ?
-    oop_store((volatile narrowOop*)obj_field_addr<narrowOop>(offset), value) :
-    oop_store((volatile oop*)      obj_field_addr<oop>(offset),       value);
+    oop_store((volatile narrowOop*)p->obj_field_addr<narrowOop>(offset), value) :
+    oop_store((volatile oop*)      p->obj_field_addr<oop>(offset),       value);
 }
 
-inline jbyte oopDesc::byte_field_acquire(int offset) const                  { return OrderAccess::load_acquire(byte_field_addr(offset));     }
-inline void oopDesc::release_byte_field_put(int offset, jbyte contents)     { OrderAccess::release_store(byte_field_addr(offset), contents); }
+inline jbyte oopDesc::byte_field_acquire(int offset) const                  {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->byte_field_addr(offset));
+}
+inline void oopDesc::release_byte_field_put(int offset, jbyte contents)     {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->byte_field_addr(offset), contents);
+}
 
-inline jboolean oopDesc::bool_field_acquire(int offset) const               { return OrderAccess::load_acquire(bool_field_addr(offset));     }
-inline void oopDesc::release_bool_field_put(int offset, jboolean contents)  { OrderAccess::release_store(bool_field_addr(offset), contents); }
+inline jboolean oopDesc::bool_field_acquire(int offset) const               {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->bool_field_addr(offset));
+}
+inline void oopDesc::release_bool_field_put(int offset, jboolean contents)  {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->bool_field_addr(offset), contents);
+}
 
-inline jchar oopDesc::char_field_acquire(int offset) const                  { return OrderAccess::load_acquire(char_field_addr(offset));     }
-inline void oopDesc::release_char_field_put(int offset, jchar contents)     { OrderAccess::release_store(char_field_addr(offset), contents); }
+inline jchar oopDesc::char_field_acquire(int offset) const                  {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->char_field_addr(offset));
+}
+inline void oopDesc::release_char_field_put(int offset, jchar contents)     {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->char_field_addr(offset), contents);
+}
 
-inline jint oopDesc::int_field_acquire(int offset) const                    { return OrderAccess::load_acquire(int_field_addr(offset));      }
-inline void oopDesc::release_int_field_put(int offset, jint contents)       { OrderAccess::release_store(int_field_addr(offset), contents);  }
+inline jint oopDesc::int_field_acquire(int offset) const                    {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->int_field_addr(offset));
+}
+inline void oopDesc::release_int_field_put(int offset, jint contents)       {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->int_field_addr(offset), contents);
+}
 
-inline jshort oopDesc::short_field_acquire(int offset) const                { return (jshort)OrderAccess::load_acquire(short_field_addr(offset)); }
-inline void oopDesc::release_short_field_put(int offset, jshort contents)   { OrderAccess::release_store(short_field_addr(offset), contents);     }
+inline jshort oopDesc::short_field_acquire(int offset) const                {
+  oop p = bs()->resolve_oop((oop) this);
+  return (jshort)OrderAccess::load_acquire(p->short_field_addr(offset));
+}
+inline void oopDesc::release_short_field_put(int offset, jshort contents)   {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->short_field_addr(offset), contents);
+}
 
-inline jlong oopDesc::long_field_acquire(int offset) const                  { return OrderAccess::load_acquire(long_field_addr(offset));       }
-inline void oopDesc::release_long_field_put(int offset, jlong contents)     { OrderAccess::release_store(long_field_addr(offset), contents);   }
+inline jlong oopDesc::long_field_acquire(int offset) const                  {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->long_field_addr(offset));
+}
+inline void oopDesc::release_long_field_put(int offset, jlong contents)     {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->long_field_addr(offset), contents);
+}
 
-inline jfloat oopDesc::float_field_acquire(int offset) const                { return OrderAccess::load_acquire(float_field_addr(offset));      }
-inline void oopDesc::release_float_field_put(int offset, jfloat contents)   { OrderAccess::release_store(float_field_addr(offset), contents);  }
+inline jfloat oopDesc::float_field_acquire(int offset) const                {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->float_field_addr(offset));
+}
+inline void oopDesc::release_float_field_put(int offset, jfloat contents)   {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->float_field_addr(offset), contents);
+}
 
-inline jdouble oopDesc::double_field_acquire(int offset) const              { return OrderAccess::load_acquire(double_field_addr(offset));     }
-inline void oopDesc::release_double_field_put(int offset, jdouble contents) { OrderAccess::release_store(double_field_addr(offset), contents); }
+inline jdouble oopDesc::double_field_acquire(int offset) const              {
+  oop p = bs()->resolve_oop((oop) this);
+  return OrderAccess::load_acquire(p->double_field_addr(offset));
+}
+inline void oopDesc::release_double_field_put(int offset, jdouble contents) {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store(p->double_field_addr(offset), contents);
+}
 
-inline address oopDesc::address_field_acquire(int offset) const             { return (address) OrderAccess::load_ptr_acquire(address_field_addr(offset)); }
-inline void oopDesc::release_address_field_put(int offset, address contents) { OrderAccess::release_store_ptr(address_field_addr(offset), contents); }
+inline address oopDesc::address_field_acquire(int offset) const             {
+  oop p = bs()->resolve_oop((oop) this);
+  return (address) OrderAccess::load_ptr_acquire(p->address_field_addr(offset));
+}
+inline void oopDesc::release_address_field_put(int offset, address contents) {
+  oop p = bs()->resolve_and_maybe_copy_oop(this);
+  OrderAccess::release_store_ptr(p->address_field_addr(offset), contents);
+}
 
 inline int oopDesc::size_given_klass(Klass* klass)  {
   int lh = klass->layout_helper();
diff --git a/src/share/vm/oops/typeArrayKlass.cpp b/src/share/vm/oops/typeArrayKlass.cpp
--- a/src/share/vm/oops/typeArrayKlass.cpp
+++ b/src/share/vm/oops/typeArrayKlass.cpp
@@ -148,6 +148,9 @@
   if (length == 0)
     return;
 
+  s = arrayOop(oopDesc::bs()->resolve_oop(s));
+  d = arrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(d));
+
   // This is an attempt to make the copy_array fast.
   int l2es = log2_element_size();
   int ihs = array_header_in_bytes() / wordSize;
diff --git a/src/share/vm/oops/typeArrayOop.hpp b/src/share/vm/oops/typeArrayOop.hpp
--- a/src/share/vm/oops/typeArrayOop.hpp
+++ b/src/share/vm/oops/typeArrayOop.hpp
@@ -25,6 +25,7 @@
 #ifndef SHARE_VM_OOPS_TYPEARRAYOOP_HPP
 #define SHARE_VM_OOPS_TYPEARRAYOOP_HPP
 
+#include "gc/shared/barrierSet.hpp"
 #include "oops/arrayOop.hpp"
 #include "oops/typeArrayKlass.hpp"
 #include "runtime/orderAccess.inline.hpp"
@@ -92,50 +93,116 @@
     return &double_base()[which];
   }
 
-  jbyte byte_at(int which) const                  { return *byte_at_addr(which); }
-  void byte_at_put(int which, jbyte contents)     { *byte_at_addr(which) = contents; }
+  jbyte byte_at(int which) const                  {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->byte_at_addr(which);
+  }
+  void byte_at_put(int which, jbyte contents)     {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->byte_at_addr(which) = contents;
+  }
 
-  jboolean bool_at(int which) const               { return *bool_at_addr(which); }
-  void bool_at_put(int which, jboolean contents)  { *bool_at_addr(which) = contents; }
+  jboolean bool_at(int which) const               {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->bool_at_addr(which);
+  }
+  void bool_at_put(int which, jboolean contents)  {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->bool_at_addr(which) = contents;
+  }
 
-  jchar char_at(int which) const                  { return *char_at_addr(which); }
-  void char_at_put(int which, jchar contents)     { *char_at_addr(which) = contents; }
+  jchar char_at(int which) const                  {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->char_at_addr(which);
+  }
+  void char_at_put(int which, jchar contents)     {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->char_at_addr(which) = contents;
+  }
 
-  jint int_at(int which) const                    { return *int_at_addr(which); }
-  void int_at_put(int which, jint contents)       { *int_at_addr(which) = contents; }
+  jint int_at(int which) const                    {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->int_at_addr(which);
+  }
+  void int_at_put(int which, jint contents)       {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->int_at_addr(which) = contents;
+  }
 
-  jshort short_at(int which) const                { return *short_at_addr(which); }
-  void short_at_put(int which, jshort contents)   { *short_at_addr(which) = contents; }
+  jshort short_at(int which) const                {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->short_at_addr(which);
+  }
+  void short_at_put(int which, jshort contents)   {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->short_at_addr(which) = contents;
+  }
 
-  jushort ushort_at(int which) const              { return *ushort_at_addr(which); }
-  void ushort_at_put(int which, jushort contents) { *ushort_at_addr(which) = contents; }
+  jushort ushort_at(int which) const              {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->ushort_at_addr(which);
+  }
+  void ushort_at_put(int which, jushort contents) {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->ushort_at_addr(which) = contents;
+  }
 
-  jlong long_at(int which) const                  { return *long_at_addr(which); }
-  void long_at_put(int which, jlong contents)     { *long_at_addr(which) = contents; }
+  jlong long_at(int which) const                  {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->long_at_addr(which);
+  }
+  void long_at_put(int which, jlong contents)     {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->long_at_addr(which) = contents;
+  }
 
-  jfloat float_at(int which) const                { return *float_at_addr(which); }
-  void float_at_put(int which, jfloat contents)   { *float_at_addr(which) = contents; }
+  jfloat float_at(int which) const                {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->float_at_addr(which);
+  }
+  void float_at_put(int which, jfloat contents)   {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->float_at_addr(which) = contents;
+  }
 
-  jdouble double_at(int which) const              { return *double_at_addr(which); }
-  void double_at_put(int which, jdouble contents) { *double_at_addr(which) = contents; }
+  jdouble double_at(int which) const              {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return *p->double_at_addr(which);
+  }
+  void double_at_put(int which, jdouble contents) {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->double_at_addr(which) = contents;
+  }
 
-  jbyte byte_at_acquire(int which) const              { return OrderAccess::load_acquire(byte_at_addr(which)); }
-  void release_byte_at_put(int which, jbyte contents) { OrderAccess::release_store(byte_at_addr(which), contents); }
+  jbyte byte_at_acquire(int which) const              {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return OrderAccess::load_acquire(p->byte_at_addr(which));
+  }
+  void release_byte_at_put(int which, jbyte contents) {
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    OrderAccess::release_store(p->byte_at_addr(which), contents);
+  }
 
   // Java thinks metadata arrays are just arrays of either long or int, since
   // there doesn't seem to be T_ADDRESS, so this is a bit of unfortunate
   // casting
 #ifdef _LP64
   Metadata* metadata_at(int which) const {
-    return (Metadata*)*long_at_addr(which); }
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return (Metadata*)*p->long_at_addr(which);
+  }
   void metadata_at_put(int which, Metadata* contents) {
-    *long_at_addr(which) = (long)contents;
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->long_at_addr(which) = (long)contents;
   }
 #else
   Metadata* metadata_at(int which) const {
-    return (Metadata*)*int_at_addr(which); }
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_oop((oop) this));
+    return (Metadata*)*p->int_at_addr(which);
+  }
   void metadata_at_put(int which, Metadata* contents) {
-    *int_at_addr(which) = (int)contents;
+    typeArrayOop p = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(this));
+    *p->int_at_addr(which) = (int)contents;
   }
 #endif // _LP64
 
diff --git a/src/share/vm/opto/arraycopynode.cpp b/src/share/vm/opto/arraycopynode.cpp
--- a/src/share/vm/opto/arraycopynode.cpp
+++ b/src/share/vm/opto/arraycopynode.cpp
@@ -254,6 +254,12 @@
       return false;
     }
 
+    if (UseShenandoahGC && dest_elem == T_OBJECT) {
+      // TODO: Disabled. We need to resolve the loaded values before storing them into
+      // the copy object. Or call out to the clone barrier that takes care of it.
+      return false;
+    }
+
     value_type = ary_src->elem();
 
     base_src = src;
diff --git a/src/share/vm/opto/callGenerator.cpp b/src/share/vm/opto/callGenerator.cpp
--- a/src/share/vm/opto/callGenerator.cpp
+++ b/src/share/vm/opto/callGenerator.cpp
@@ -985,7 +985,7 @@
     assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state");
     SafePointNode* new_map = kit.map();
     assert(old_io  == new_map->i_o(), "generate_predicate should not change i_o");
-    assert(old_mem == new_map->memory(), "generate_predicate should not change memory");
+    assert(old_mem == new_map->memory() || UseShenandoahGC, "generate_predicate should not change memory");
     assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions");
 #endif
     if (!kit.stopped()) {
diff --git a/src/share/vm/opto/classes.cpp b/src/share/vm/opto/classes.cpp
--- a/src/share/vm/opto/classes.cpp
+++ b/src/share/vm/opto/classes.cpp
@@ -45,6 +45,7 @@
 #include "opto/node.hpp"
 #include "opto/opaquenode.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/subnode.hpp"
 #include "opto/vectornode.hpp"
 
diff --git a/src/share/vm/opto/classes.hpp b/src/share/vm/opto/classes.hpp
--- a/src/share/vm/opto/classes.hpp
+++ b/src/share/vm/opto/classes.hpp
@@ -228,6 +228,9 @@
 macro(RoundFloat)
 macro(SafePoint)
 macro(SafePointScalarObject)
+macro(ShenandoahReadBarrier)
+macro(ShenandoahWriteBarrier)
+macro(ShenandoahWBMemProj)
 macro(SCMemProj)
 macro(SinD)
 macro(SqrtD)
diff --git a/src/share/vm/opto/compile.cpp b/src/share/vm/opto/compile.cpp
--- a/src/share/vm/opto/compile.cpp
+++ b/src/share/vm/opto/compile.cpp
@@ -61,6 +61,7 @@
 #include "opto/phaseX.hpp"
 #include "opto/rootnode.hpp"
 #include "opto/runtime.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/stringopts.hpp"
 #include "opto/type.hpp"
 #include "opto/vectornode.hpp"
@@ -757,7 +758,8 @@
       StartNode* s = new StartNode(root(), tf()->domain());
       initial_gvn()->set_type_bottom(s);
       init_start(s);
-      if (method()->intrinsic_id() == vmIntrinsics::_Reference_get && UseG1GC) {
+      if (method()->intrinsic_id() == vmIntrinsics::_Reference_get
+          && (UseG1GC || UseShenandoahGC)) {
         // With java.lang.ref.reference.get() we must go through the
         // intrinsic when G1 is enabled - even when get() is the root
         // method of the compile - so that, if necessary, the value in
@@ -1434,6 +1436,9 @@
         tj = TypeInstPtr::MARK;
         ta = TypeAryPtr::RANGE; // generic ignored junk
         ptr = TypePtr::BotPTR;
+      } else if (offset == -8) {
+        // Need to distinguish brooks ptr as is.
+        tj = ta = TypeAryPtr::make(ptr,ta->ary(),ta->klass(),false,offset);
       } else {                  // Random constant offset into array body
         offset = Type::OffsetBot;   // Flatten constant access into array body
         tj = ta = TypeAryPtr::make(ptr,ta->ary(),ta->klass(),false,offset);
@@ -1498,7 +1503,7 @@
       if (!is_known_inst) { // Do it only for non-instance types
         tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, NULL, offset);
       }
-    } else if (offset < 0 || offset >= k->size_helper() * wordSize) {
+    } else if ((offset != -8) && (offset < 0 || offset >= k->size_helper() * wordSize)) {
       // Static fields are in the space above the normal instance
       // fields in the java.lang.Class instance.
       if (to->klass() != ciEnv::current()->Class_klass()) {
@@ -1596,7 +1601,8 @@
           (offset == Type::OffsetBot && tj == TypePtr::BOTTOM) ||
           (offset == oopDesc::mark_offset_in_bytes() && tj->base() == Type::AryPtr) ||
           (offset == oopDesc::klass_offset_in_bytes() && tj->base() == Type::AryPtr) ||
-          (offset == arrayOopDesc::length_offset_in_bytes() && tj->base() == Type::AryPtr)  ,
+          (offset == arrayOopDesc::length_offset_in_bytes() && tj->base() == Type::AryPtr) ||
+          (offset == -8 && tj->base() == Type::AryPtr && UseShenandoahGC),
           "For oops, klasses, raw offset must be constant; for arrays the offset is never known" );
   assert( tj->ptr() != TypePtr::TopPTR &&
           tj->ptr() != TypePtr::AnyNull &&
@@ -2850,7 +2856,7 @@
         Node *m = wq.at(next);
         for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
           Node* use = m->fast_out(i);
-          if (use->is_Mem() || use->is_EncodeNarrowPtr()) {
+          if (use->is_Mem() || use->is_EncodeNarrowPtr() || use->is_ShenandoahBarrier()) {
             use->ensure_control_or_add_prec(n->in(0));
           } else if (use->in(0) == NULL) {
             switch(use->Opcode()) {
@@ -3174,6 +3180,11 @@
       n->set_req(MemBarNode::Precedent, top());
     }
     break;
+  case Op_ShenandoahReadBarrier:
+    break;
+  case Op_ShenandoahWriteBarrier:
+    n->set_req(ShenandoahBarrierNode::Memory, immutable_memory());
+    break;
   default:
     assert( !n->is_Call(), "" );
     assert( !n->is_Mem(), "" );
@@ -3540,7 +3551,7 @@
 // Currently supported:
 // - G1 pre-barriers (see GraphKit::g1_write_barrier_pre())
 void Compile::verify_barriers() {
-  if (UseG1GC) {
+  if (UseG1GC || UseShenandoahGC) {
     // Verify G1 pre-barriers
     const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + PtrQueue::byte_offset_of_active());
 
diff --git a/src/share/vm/opto/escape.cpp b/src/share/vm/opto/escape.cpp
--- a/src/share/vm/opto/escape.cpp
+++ b/src/share/vm/opto/escape.cpp
@@ -36,6 +36,7 @@
 #include "opto/phaseX.hpp"
 #include "opto/movenode.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/shenandoahSupport.hpp"
 
 ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn) :
   _nodes(C->comp_arena(), C->unique(), C->unique(), NULL),
@@ -530,7 +531,7 @@
           // Pointer stores in G1 barriers looks like unsafe access.
           // Ignore such stores to be able scalar replace non-escaping
           // allocations.
-          if (UseG1GC && adr->is_AddP()) {
+          if ((UseG1GC || UseShenandoahGC) && adr->is_AddP()) {
             Node* base = get_addp_base(adr);
             if (base->Opcode() == Op_LoadP &&
                 base->in(MemNode::Address)->is_AddP()) {
@@ -572,6 +573,12 @@
       add_java_object(n, PointsToNode::ArgEscape);
       break;
     }
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
+      // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
+      // It doesn't escape.
+      add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
+      break;
     default:
       ; // Do nothing for nodes not related to EA.
   }
@@ -766,6 +773,12 @@
       }
       break;
     }
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
+      // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
+      // It doesn't escape.
+      add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
+      break;
     default: {
       // This method should be called only for EA specific nodes which may
       // miss some edges when they were created.
@@ -885,7 +898,7 @@
   } else {
     // An other type of call, assume the worst case:
     // returned value is unknown and globally escapes.
-    assert(call->Opcode() == Op_CallDynamicJava, "add failed case check");
+    assert(call->Opcode() == Op_CallDynamicJava || call->Opcode() == Op_CallLeaf, "add failed case check");
     map_ideal_node(call, phantom_obj);
   }
 }
@@ -961,6 +974,9 @@
                 (call->as_CallLeaf()->_name != NULL &&
                  (strcmp(call->as_CallLeaf()->_name, "g1_wb_pre")  == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "g1_wb_post") == 0 ||
+                  strcmp(call->as_CallLeaf()->_name, "shenandoah_clone_barrier")  == 0 ||
+                  strcmp(call->as_CallLeaf()->_name, "shenandoah_read_barrier")  == 0 ||
+                  strcmp(call->as_CallLeaf()->_name, "shenandoah_cas_obj")  == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32") == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32C") == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "updateBytesAdler32") == 0 ||
@@ -2054,6 +2070,9 @@
     } else if (adr_type->isa_aryptr()) {
       if (offset == arrayOopDesc::length_offset_in_bytes()) {
         // Ignore array length load.
+      } else if (UseShenandoahGC && offset == -8) {
+        // Shenandoah read barrier.
+        bt = T_ARRAY;
       } else if (find_second_addp(n, n->in(AddPNode::Base)) != NULL) {
         // Ignore first AddP.
       } else {
@@ -2642,6 +2661,7 @@
     prev = result;
     if (result == start_mem)
       break;  // hit one of our sentinels
+    assert(result->Opcode() != Op_ShenandoahWBMemProj, "unexpected memory slice");
     if (result->is_Mem()) {
       const Type *at = igvn->type(result->in(MemNode::Address));
       if (at == Type::TOP)
@@ -3005,6 +3025,7 @@
                n->is_CheckCastPP() ||
                n->is_EncodeP() ||
                n->is_DecodeN() ||
+               n->is_ShenandoahBarrier() ||
                (n->is_ConstraintCast() && n->Opcode() == Op_CastPP)) {
       if (visited.test_set(n->_idx)) {
         assert(n->is_Phi(), "loops only through Phi's");
@@ -3075,6 +3096,7 @@
                  use->is_CheckCastPP() ||
                  use->is_EncodeNarrowPtr() ||
                  use->is_DecodeNarrowPtr() ||
+                 use->is_ShenandoahBarrier() ||
                  (use->is_ConstraintCast() && use->Opcode() == Op_CastPP)) {
         alloc_worklist.append_if_missing(use);
 #ifdef ASSERT
@@ -3099,7 +3121,8 @@
         if (!(op == Op_CmpP || op == Op_Conv2B ||
               op == Op_CastP2X || op == Op_StoreCM ||
               op == Op_FastLock || op == Op_AryEq || op == Op_StrComp ||
-              op == Op_StrEquals || op == Op_StrIndexOf)) {
+              op == Op_StrEquals || op == Op_StrIndexOf ||
+              op == Op_ShenandoahWBMemProj)) {
           n->dump();
           use->dump();
           assert(false, "EA: missing allocation reference path");
diff --git a/src/share/vm/opto/graphKit.cpp b/src/share/vm/opto/graphKit.cpp
--- a/src/share/vm/opto/graphKit.cpp
+++ b/src/share/vm/opto/graphKit.cpp
@@ -41,6 +41,7 @@
 #include "opto/parse.hpp"
 #include "opto/rootnode.hpp"
 #include "opto/runtime.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "runtime/deoptimization.hpp"
 #include "runtime/sharedRuntime.hpp"
 
@@ -815,7 +816,7 @@
 // Helper function for enforcing certain bytecodes to reexecute if
 // deoptimization happens
 static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) {
-  ciMethod* cur_method = jvms->method();
+  ciMethod* cur_method = jvms->has_method() ? jvms->method() : NULL;
   int       cur_bci   = jvms->bci();
   if (cur_method != NULL && cur_bci != InvocationEntryBci) {
     Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
@@ -1153,6 +1154,9 @@
   // Special-case a fresh allocation to avoid building nodes:
   Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
   if (akls != NULL)  return akls;
+  if (ShenandoahVerifyReadsToFromSpace) {
+    obj = shenandoah_read_barrier(obj);
+  }
   Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
   return _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS));
 }
@@ -1163,6 +1167,10 @@
   AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array, &_gvn);
   Node *alen;
   if (alloc == NULL) {
+    if (ShenandoahVerifyReadsToFromSpace) {
+      array = shenandoah_read_barrier(array);
+    }
+
     Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes());
     alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS));
   } else {
@@ -1519,6 +1527,7 @@
   set_control(ctl);
   switch (bs->kind()) {
     case BarrierSet::G1SATBCTLogging:
+    case BarrierSet::ShenandoahBarrierSet:
       g1_write_barrier_pre(do_load, obj, adr, adr_idx, val, val_type, pre_val, bt);
       break;
 
@@ -1537,6 +1546,7 @@
   BarrierSet* bs = Universe::heap()->barrier_set();
   switch (bs->kind()) {
     case BarrierSet::G1SATBCTLogging:
+    case BarrierSet::ShenandoahBarrierSet:
       return true; // Can move it if no safepoint
 
     case BarrierSet::CardTableForRS:
@@ -1571,6 +1581,7 @@
       break;
 
     case BarrierSet::ModRef:
+    case BarrierSet::ShenandoahBarrierSet:
       break;
 
     default      :
@@ -1678,6 +1689,9 @@
   uint nargs = call->method()->arg_size();
   for (uint i = 0; i < nargs; i++) {
     Node* arg = argument(i);
+    if (ShenandoahVerifyReadsToFromSpace && call->is_CallDynamicJava() && i == 0) {
+      arg = shenandoah_read_barrier(arg);
+    }
     call->init_req(i + TypeFunc::Parms, arg);
   }
 }
@@ -2919,6 +2933,10 @@
     }
   }
 
+  if (ShenandoahVerifyReadsToFromSpace) {
+    not_null_obj = shenandoah_read_barrier(not_null_obj);
+  }
+
   // Load the object's klass
   Node* obj_klass = load_object_klass(not_null_obj);
 
@@ -3000,6 +3018,10 @@
   Node* null_ctl = top();
   Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);
 
+  if (ShenandoahVerifyReadsToFromSpace) {
+    not_null_obj = shenandoah_read_barrier(not_null_obj);
+  }
+
   // If not_null_obj is dead, only null-path is taken
   if (stopped()) {              // Doing instance-of on a NULL?
     set_control(null_ctl);
@@ -3149,6 +3171,8 @@
 
   assert(dead_locals_are_killed(), "should kill locals before sync. point");
 
+  obj = shenandoah_write_barrier(obj);
+
   // Box the stack location
   Node* box = _gvn.transform(new BoxLockNode(next_monitor()));
   Node* mem = reset_memory();
@@ -3622,6 +3646,10 @@
   if (ptr == NULL) {     // reduce dumb test in callers
     return NULL;
   }
+
+  // Attempt to see through Shenandoah barriers.
+  ptr = ShenandoahBarrierNode::skip_through_barrier(ptr);
+
   if (ptr->is_CheckCastPP()) { // strip only one raw-to-oop cast
     ptr = ptr->in(1);
     if (ptr == NULL) return NULL;
@@ -4274,6 +4302,9 @@
                                                        false, NULL, 0);
     const TypePtr* offset_field_type = string_type->add_offset(offset_offset);
     int offset_field_idx = C->get_alias_index(offset_field_type);
+
+    str = shenandoah_read_barrier(str);
+
     return make_load(ctrl,
                      basic_plus_adr(str, str, offset_offset),
                      TypeInt::INT, T_INT, offset_field_idx, MemNode::unordered);
@@ -4289,6 +4320,9 @@
                                                        false, NULL, 0);
     const TypePtr* count_field_type = string_type->add_offset(count_offset);
     int count_field_idx = C->get_alias_index(count_field_type);
+
+    str = shenandoah_read_barrier(str);
+
     return make_load(ctrl,
                      basic_plus_adr(str, str, count_offset),
                      TypeInt::INT, T_INT, count_field_idx, MemNode::unordered);
@@ -4306,6 +4340,9 @@
                                                    TypeAry::make(TypeInt::CHAR,TypeInt::POS),
                                                    ciTypeArrayKlass::make(T_CHAR), true, 0);
   int value_field_idx = C->get_alias_index(value_field_type);
+
+  str = shenandoah_read_barrier(str);
+
   Node* load = make_load(ctrl, basic_plus_adr(str, str, value_offset),
                          value_type, T_OBJECT, value_field_idx, MemNode::unordered);
   // String.value field is known to be @Stable.
@@ -4321,7 +4358,11 @@
                                                      false, NULL, 0);
   const TypePtr* offset_field_type = string_type->add_offset(offset_offset);
   int offset_field_idx = C->get_alias_index(offset_field_type);
-  store_to_memory(ctrl, basic_plus_adr(str, offset_offset),
+
+  // TODO: Use incoming ctrl.
+  str = shenandoah_write_barrier(str);
+
+  store_to_memory(UseShenandoahGC ? control() : ctrl, basic_plus_adr(str, offset_offset),
                   value, T_INT, offset_field_idx, MemNode::unordered);
 }
 
@@ -4331,7 +4372,11 @@
                                                      false, NULL, 0);
   const TypePtr* value_field_type = string_type->add_offset(value_offset);
 
-  store_oop_to_object(ctrl, str,  basic_plus_adr(str, value_offset), value_field_type,
+  // TODO: Use incoming ctrl.
+  str = shenandoah_write_barrier(str);
+  value = shenandoah_read_barrier_nomem(value);
+
+  store_oop_to_object(UseShenandoahGC ? control() : ctrl, str,  basic_plus_adr(str, value_offset), value_field_type,
       value, TypeAryPtr::CHARS, T_OBJECT, MemNode::unordered);
 }
 
@@ -4341,7 +4386,11 @@
                                                      false, NULL, 0);
   const TypePtr* count_field_type = string_type->add_offset(count_offset);
   int count_field_idx = C->get_alias_index(count_field_type);
-  store_to_memory(ctrl, basic_plus_adr(str, count_offset),
+
+  // TODO: Use incoming ctrl.
+  str = shenandoah_write_barrier(str);
+
+  store_to_memory(UseShenandoahGC ? control() : ctrl, basic_plus_adr(str, count_offset),
                   value, T_INT, count_field_idx, MemNode::unordered);
 }
 
@@ -4350,3 +4399,195 @@
   // assumption of CCP analysis.
   return _gvn.transform(new CastPPNode(ary, ary_type->cast_to_stable(true)));
 }
+
+Node* GraphKit::shenandoah_read_barrier(Node* obj) {
+  return shenandoah_read_barrier_impl(obj, false, true);
+}
+
+Node* GraphKit::shenandoah_read_barrier_nomem(Node* obj) {
+  return shenandoah_read_barrier_impl(obj, false, false);
+}
+
+Node* GraphKit::shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem) {
+
+  if (UseShenandoahGC && ShenandoahReadBarrier) {
+    const Type* obj_type = obj->bottom_type();
+    if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
+      // tty->print_cr("killed barrier for NULL object");
+      return obj;
+    }
+    const TypePtr* adr_type = obj_type->is_ptr()->add_offset(-8);
+    Node* mem = use_mem ? memory(adr_type) : immutable_memory();
+
+    if (! ShenandoahBarrierNode::needs_barrier(&_gvn, NULL, obj, mem)) {
+      // We know it is null, no barrier needed.
+      return obj;
+    }
+
+
+    if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
+
+      // We don't know if it's null or not. Need null-check.
+      enum { _not_null_path = 1, _null_path, PATH_LIMIT };
+      RegionNode* region = new RegionNode(PATH_LIMIT);
+      Node*       phi    = new PhiNode(region, obj_type);
+      Node* null_ctrl = top();
+      Node* not_null_obj = null_check_oop(obj, &null_ctrl);
+
+      region->init_req(_null_path, null_ctrl);
+      phi   ->init_req(_null_path, obj);
+
+      Node* ctrl = use_ctrl ? control() : NULL;
+      ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj);
+      Node* n = _gvn.transform(rb);
+
+      region->init_req(_not_null_path, control());
+      phi   ->init_req(_not_null_path, n);
+
+      set_control(_gvn.transform(region));
+      record_for_igvn(region);
+      return _gvn.transform(phi);
+
+    } else {
+      // We know it is not null. Simple barrier is sufficient.
+      Node* ctrl = use_ctrl ? control() : NULL;
+      ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj);
+      Node* n = _gvn.transform(rb);
+      record_for_igvn(n);
+      return n;
+    }
+
+  } else {
+    return obj;
+  }
+}
+
+Node* GraphKit::shenandoah_write_barrier(Node* obj) {
+
+  if (UseShenandoahGC && ShenandoahWriteBarrier) {
+
+    if (! ShenandoahBarrierNode::needs_barrier(&_gvn, NULL, obj, NULL)) {
+      return obj;
+    }
+    const Type* obj_type = obj->bottom_type();
+    const TypePtr* adr_type = obj_type->is_ptr()->add_offset(-8);
+    // tty->print_cr("memory at:");
+    // adr_type->dump();
+    // tty->print_cr("\n");
+    // memory(adr_type)->dump();
+    if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
+      // We don't know if it's null or not. Need null-check.
+      enum { _not_null_path = 1, _null_path, PATH_LIMIT };
+      RegionNode* region = new RegionNode(PATH_LIMIT);
+      Node*       phi    = new PhiNode(region, obj_type);
+      Node*    memphi    = PhiNode::make(region, memory(adr_type), Type::MEMORY, C->alias_type(adr_type)->adr_type());
+
+      Node* prev_mem = memory(adr_type);
+      Node* null_ctrl = top();
+      Node* not_null_obj = null_check_oop(obj, &null_ctrl);
+
+      region->init_req(_null_path, null_ctrl);
+      phi   ->init_req(_null_path, null());
+      memphi->init_req(_null_path, prev_mem);
+
+      ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(NULL, memory(adr_type), not_null_obj);
+      Node* n = _gvn.transform(wb);
+      if (n == wb) { // New barrier needs memory projection.
+        Node* proj = _gvn.transform(new ShenandoahWBMemProjNode(n));
+        set_memory(proj, adr_type);
+      }
+
+      region->init_req(_not_null_path, control());
+      phi   ->init_req(_not_null_path, n);
+      memphi->init_req(_not_null_path, memory(adr_type));
+
+      set_control(_gvn.transform(region));
+      record_for_igvn(region);
+      set_memory(_gvn.transform(memphi), adr_type);
+
+      Node* res_val = _gvn.transform(phi);
+      // replace_in_map(obj, res_val);
+      return res_val;
+    } else {
+      // We know it is not null. Simple barrier is sufficient.
+      ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(NULL, memory(adr_type), obj);
+      Node* n = _gvn.transform(wb);
+      if (n == wb) {
+        Node* proj = _gvn.transform(new ShenandoahWBMemProjNode(wb));
+        set_memory(proj, adr_type);
+      }
+      // replace_in_map(obj, n);
+      record_for_igvn(n);
+      return n;
+    }
+
+  } else {
+    return obj;
+  }
+}
+
+void GraphKit::shenandoah_acmp_barrier(Node*& a, Node*& b) {
+  if (UseShenandoahGC) {
+    const Type* a_type = a->bottom_type();
+    const Type* b_type = b->bottom_type();
+    if (a_type->higher_equal(TypePtr::NULL_PTR) || b_type->higher_equal(TypePtr::NULL_PTR)) {
+      // We know one arg is gonna be null. No need for barriers.
+      // tty->print_cr("eliminate acmp barrier on null");
+      return;
+    }
+    /*
+    if ((!a_type->isa_oopptr()) || (!b_type->isa_oopptr())) {
+      a_type->dump();
+      b_type->dump();
+    }
+    */
+    if (a_type->is_oopptr()->const_oop() != NULL && b_type->is_oopptr()->const_oop() != NULL ) {
+      // We know one arg is inlined constant. No need for barriers.
+      // tty->print_cr("eliminate acmp barrier on constant");
+      return;
+    }
+    if (a->Opcode() == Op_ShenandoahWriteBarrier && b->Opcode() == Op_ShenandoahWriteBarrier) {
+      // We know one arg is already write-barrier'd. No need for barriers.
+      // tty->print_cr("eliminate acmp barrier on write barrier");
+      return;
+    }
+    if (AllocateNode::Ideal_allocation(a, &_gvn) != NULL || AllocateNode::Ideal_allocation(b, &_gvn) != NULL) {
+      // We know one arg is already in to-space. No need for barriers.
+      // tty->print_cr("eliminate acmp barrier on new obj");
+      return;
+    }
+
+    enum { _equal = 1, _not_equal, PATH_LIMIT };
+    RegionNode* region = new RegionNode(PATH_LIMIT);
+    PhiNode* phiA = PhiNode::make(region, a);
+    PhiNode* phiB = PhiNode::make(region, b);
+
+    Node* cmp = _gvn.transform(new CmpPNode(b, a));
+    Node* tst = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
+
+    // TODO: Use profiling data.
+    IfNode* iff = create_and_map_if(control(), tst, PROB_FAIR, COUNT_UNKNOWN);
+    Node* iftrue = _gvn.transform(new IfTrueNode(iff));
+    Node* iffalse = _gvn.transform(new IfFalseNode(iff));
+
+    // Equal path: Use original values.
+    region->init_req(_equal, iftrue);
+    phiA->init_req(_equal, a);
+    phiB->init_req(_equal, b);
+
+    // Unequal path: retry after read barriers.
+    set_control(iffalse);
+    a = shenandoah_read_barrier_impl(a, true, true);
+    b = shenandoah_read_barrier_impl(b, true, true);
+
+    region->init_req(_not_equal, control());
+    phiA->init_req(_not_equal, a);
+    phiB->init_req(_not_equal, b);
+
+    set_control(_gvn.transform(region));
+    record_for_igvn(region);
+
+    a = _gvn.transform(phiA);
+    b = _gvn.transform(phiB);
+  }
+}
diff --git a/src/share/vm/opto/graphKit.hpp b/src/share/vm/opto/graphKit.hpp
--- a/src/share/vm/opto/graphKit.hpp
+++ b/src/share/vm/opto/graphKit.hpp
@@ -894,6 +894,13 @@
 
   // Produce new array node of stable type
   Node* cast_array_to_stable(Node* ary, const TypeAryPtr* ary_type);
+
+  Node* shenandoah_read_barrier(Node* obj);
+  Node* shenandoah_read_barrier_nomem(Node* obj);
+  Node* shenandoah_write_barrier(Node* obj);
+  void shenandoah_acmp_barrier(Node*& a, Node*& b);
+private:
+  Node* shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem);
 };
 
 // Helper class to support building of control flow branches. Upon
diff --git a/src/share/vm/opto/ifg.cpp b/src/share/vm/opto/ifg.cpp
--- a/src/share/vm/opto/ifg.cpp
+++ b/src/share/vm/opto/ifg.cpp
@@ -535,7 +535,7 @@
       // The method add_input_to_liveout() keeps such nodes alive (put them on liveout list)
       // when it sees SCMemProj node in a block. Unfortunately SCMemProj node could be placed
       // in block in such order that KILL MachProj nodes are processed first.
-      if (def->has_out_with(Op_SCMemProj)) {
+      if (def->has_out_with(Op_SCMemProj) || def->has_out_with(Op_ShenandoahWBMemProj)) {
         return false;
       }
     }
@@ -683,7 +683,7 @@
   JVMState* jvms = n->jvms();
   uint debug_start = jvms ? jvms->debug_start() : 999999;
 
-  for (uint k = ((n->Opcode() == Op_SCMemProj) ? 0:1); k < n->req(); k++) {
+  for (uint k = ((n->Opcode() == Op_SCMemProj || n->Opcode() == Op_ShenandoahWBMemProj) ? 0:1); k < n->req(); k++) {
     Node* def = n->in(k);
     uint lid = _lrg_map.live_range_id(def);
     if (!lid) {
diff --git a/src/share/vm/opto/lcm.cpp b/src/share/vm/opto/lcm.cpp
--- a/src/share/vm/opto/lcm.cpp
+++ b/src/share/vm/opto/lcm.cpp
@@ -174,6 +174,8 @@
     case Op_LoadRange:
     case Op_LoadD_unaligned:
     case Op_LoadL_unaligned:
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
       assert(mach->in(2) == val, "should be address");
       break;
     case Op_StoreB:
@@ -380,7 +382,7 @@
   // Should be DU safe because no edge updates.
   for (DUIterator_Fast jmax, j = best->fast_outs(jmax); j < jmax; j++) {
     Node* n = best->fast_out(j);
-    if( n->is_MachProj() ) {
+    if( n->is_MachProj() || n->Opcode() == Op_ShenandoahWBMemProj) {
       get_block_for_node(n)->find_remove(n);
       block->add_inst(n);
       map_node_to_block(n, block);
diff --git a/src/share/vm/opto/library_call.cpp b/src/share/vm/opto/library_call.cpp
--- a/src/share/vm/opto/library_call.cpp
+++ b/src/share/vm/opto/library_call.cpp
@@ -28,6 +28,7 @@
 #include "classfile/vmSymbols.hpp"
 #include "compiler/compileBroker.hpp"
 #include "compiler/compileLog.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
 #include "oops/objArrayKlass.hpp"
 #include "opto/addnode.hpp"
 #include "opto/arraycopynode.hpp"
@@ -46,6 +47,7 @@
 #include "opto/opaquenode.hpp"
 #include "opto/parse.hpp"
 #include "opto/runtime.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/subnode.hpp"
 #include "prims/nativeLookup.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -972,9 +974,19 @@
 //------------------------------inline_string_equals------------------------
 bool LibraryCallKit::inline_string_equals() {
   Node* receiver = null_check_receiver();
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    receiver = shenandoah_read_barrier(receiver);
+  }
+
   // NOTE: Do not null check argument for String.equals() because spec
   // allows to specify NULL as argument.
   Node* argument = this->argument(1);
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    argument = shenandoah_read_barrier(argument);
+  }
+
   if (stopped()) {
     return true;
   }
@@ -1023,6 +1035,11 @@
 
     // Get start addr of receiver
     Node* receiver_val    = load_String_value(no_ctrl, receiver);
+
+    if (ShenandoahVerifyReadsToFromSpace) {
+      receiver_val = shenandoah_read_barrier(receiver_val);
+    }
+
     Node* receiver_offset = load_String_offset(no_ctrl, receiver);
     Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
 
@@ -1031,6 +1048,11 @@
 
     // Get start addr of argument
     Node* argument_val    = load_String_value(no_ctrl, argument);
+
+    if (ShenandoahVerifyReadsToFromSpace) {
+      argument_val = shenandoah_read_barrier(argument_val);
+    }
+
     Node* argument_offset = load_String_offset(no_ctrl, argument);
     Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
 
@@ -1067,6 +1089,10 @@
 bool LibraryCallKit::inline_array_equals() {
   Node* arg1 = argument(0);
   Node* arg2 = argument(1);
+
+  arg1 = shenandoah_read_barrier(arg1);
+  arg2 = shenandoah_read_barrier(arg2);
+
   set_result(_gvn.transform(new AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2)));
   return true;
 }
@@ -2152,7 +2178,7 @@
   // runtime filters that guard the pre-barrier code.
   // Also add memory barrier for non volatile load from the referent field
   // to prevent commoning of loads across safepoint.
-  if (!UseG1GC && !need_mem_bar)
+  if (!(UseG1GC || UseShenandoahGC) && !need_mem_bar)
     return;
 
   // Some compile time checks.
@@ -2341,6 +2367,18 @@
   if (!is_native_ptr) {
     // The base is either a Java object or a value produced by Unsafe.staticFieldBase
     Node* base = argument(1);  // type: oop
+    if (UseShenandoahGC) {
+      // Note: if we don't null-check here, we generate a read barrier with a built-in
+      // null-check. This will later be attempted to be split on the phi, which
+      // results in a load on a NULL-based address on the null-path, which blows up.
+      // It will go away when we do late-insertion of read barriers.
+      base = null_check(base);
+    }
+    if (is_store) {
+      base = shenandoah_write_barrier(base);
+    } else {
+      base = shenandoah_read_barrier(base);
+    }
     // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
     offset = argument(2);  // type: long
     // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
@@ -2492,6 +2530,7 @@
     if (type != T_OBJECT ) {
       (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile);
     } else {
+      val = shenandoah_read_barrier_nomem(val);
       // Possibly an oop being stored to Java heap or native memory
       if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
         // oop to Java heap.
@@ -2621,6 +2660,8 @@
     return true;
   }
 
+  base = shenandoah_write_barrier(base);
+
   // Build field offset expression.
   // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
   // to be plain byte offsets, which are also the same as those accepted
@@ -2662,6 +2703,7 @@
   // For now, we handle only those cases that actually exist: ints,
   // longs, and Object. Adding others should be straightforward.
   Node* load_store;
+  Node* result;
   switch(type) {
   case T_INT:
     if (kind == LS_xadd) {
@@ -2673,6 +2715,7 @@
     } else {
       ShouldNotReachHere();
     }
+    result = load_store;
     break;
   case T_LONG:
     if (kind == LS_xadd) {
@@ -2684,6 +2727,7 @@
     } else {
       ShouldNotReachHere();
     }
+    result = load_store;
     break;
   case T_OBJECT:
     // Transformation of a value which could be NULL pointer (CastPP #NULL)
@@ -2692,6 +2736,8 @@
     if (_gvn.type(newval) == TypePtr::NULL_PTR)
       newval = _gvn.makecon(TypePtr::NULL_PTR);
 
+    newval = shenandoah_read_barrier_nomem(newval);
+
     // Reference stores need a store barrier.
     if (kind == LS_xchg) {
       // If pre-barrier must execute before the oop store, old value will require do_load here.
@@ -2727,14 +2773,77 @@
         load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr,
                                                                 newval_enc, oldval_enc));
       }
+      result = load_store;
     } else
 #endif
     {
       if (kind == LS_xchg) {
         load_store = _gvn.transform(new GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
+        result = load_store;
       } else {
         assert(kind == LS_cmpxchg, "wrong LoadStore operation");
         load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval));
+        result = load_store;
+
+        if (UseShenandoahGC) {
+          // if (! success)
+          Node* cmp_true = _gvn.transform(new CmpINode(load_store, intcon(1)));
+          Node* tst_true = _gvn.transform(new BoolNode(cmp_true, BoolTest::eq));
+          IfNode* iff = create_and_map_if(control(), tst_true, PROB_LIKELY_MAG(2), COUNT_UNKNOWN);
+          Node* iftrue = _gvn.transform(new IfTrueNode(iff));
+          Node* iffalse = _gvn.transform(new IfFalseNode(iff));
+
+          enum { _success_path = 1, _fail_path, _shenandoah_path, PATH_LIMIT };
+          RegionNode* region = new RegionNode(PATH_LIMIT);
+          Node*       phi    = new PhiNode(region, TypeInt::BOOL);
+          // success -> return result of CAS1.
+          region->init_req(_success_path, iftrue);
+          phi   ->init_req(_success_path, load_store);
+
+          // failure
+          set_control(iffalse);
+
+          // if (read_barrier(expected) == read_barrier(old)
+          oldval = shenandoah_read_barrier(oldval);
+
+          // Load old value from memory. We shuold really use what we get back from the CAS,
+          // if we can.
+          Node* current = make_load(control(), adr, TypeInstPtr::BOTTOM, type, MemNode::unordered);
+          // read_barrier(old)
+          Node* new_current = shenandoah_read_barrier(current);
+
+          Node* chk = _gvn.transform(new CmpPNode(new_current, oldval));
+          Node* test = _gvn.transform(new BoolNode(chk, BoolTest::eq));
+
+          IfNode* iff2 = create_and_map_if(control(), test, PROB_UNLIKELY_MAG(2), COUNT_UNKNOWN);
+          Node* iftrue2 = _gvn.transform(new IfTrueNode(iff2));
+          Node* iffalse2 = _gvn.transform(new IfFalseNode(iff2));
+
+          // If they are not equal, it's a legitimate failure and we return the result of CAS1.
+          region->init_req(_fail_path, iffalse2);
+          phi   ->init_req(_fail_path, load_store);
+
+          // Otherwise we retry with old.
+          set_control(iftrue2);
+
+          Node *call = make_runtime_call(RC_LEAF | RC_NO_IO,
+                                         OptoRuntime::shenandoah_cas_obj_Type(),
+                                         CAST_FROM_FN_PTR(address, ShenandoahRuntime::compare_and_swap_object),
+                                         "shenandoah_cas_obj",
+                                         NULL,
+                                         adr, newval, current);
+
+          Node* retval = _gvn.transform(new ProjNode(call, TypeFunc::Parms + 0));
+
+          region->init_req(_shenandoah_path, control());
+          phi   ->init_req(_shenandoah_path, retval);
+
+          set_control(_gvn.transform(region));
+          record_for_igvn(region);
+          phi = _gvn.transform(phi);
+          result = phi;
+        }
+
       }
     }
     if (kind == LS_cmpxchg) {
@@ -2744,14 +2853,14 @@
       // critical path, while CAS failure path can use the penalty for going through unlikely
       // path as backoff. Which is still better than doing a store barrier there.
       IdealKit ideal(this);
-      ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); {
+      ideal.if_then(result, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); {
         sync_kit(ideal);
-        post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
+        post_barrier(ideal.ctrl(), result, base, adr, alias_idx, newval, T_OBJECT, true);
         ideal.sync_kit(this);
       } ideal.end_if();
       final_sync(ideal);
     } else {
-      post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
+      post_barrier(control(), result, base, adr, alias_idx, newval, T_OBJECT, true);
     }
     break;
   default:
@@ -2768,7 +2877,7 @@
   if (type == T_OBJECT && kind == LS_xchg) {
 #ifdef _LP64
     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
-      load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
+      result = _gvn.transform(new DecodeNNode(result, result->get_ptr_type()));
     }
 #endif
     if (can_move_pre_barrier()) {
@@ -2777,7 +2886,7 @@
       // gets inserted between them.
       pre_barrier(false /* do_load */,
                   control(), NULL, NULL, max_juint, NULL, NULL,
-                  load_store /* pre_val */,
+                  result /* pre_val */,
                   T_OBJECT);
     }
   }
@@ -2786,8 +2895,8 @@
   insert_mem_bar(Op_MemBarCPUOrder);
   insert_mem_bar(Op_MemBarAcquire);
 
-  assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
-  set_result(load_store);
+  assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
+  set_result(result);
   return true;
 }
 
@@ -2831,6 +2940,8 @@
     return true;
   }
 
+  base = shenandoah_write_barrier(base);
+
   // Build field offset expression.
   assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
   // 32-bit machines ignore the high half of long offsets
@@ -2845,8 +2956,10 @@
   // Ensure that the store is atomic for longs:
   const bool require_atomic_access = true;
   Node* store;
-  if (type == T_OBJECT) // reference stores need a store barrier.
+  if (type == T_OBJECT) { // reference stores need a store barrier.
+    val = shenandoah_read_barrier_nomem(val);
     store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release);
+  }
   else {
     store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access);
   }
@@ -3168,6 +3281,11 @@
   enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
 
   Node* mirror = argument(0);
+
+  if (ShenandoahVerifyReadsToFromSpace) {
+    mirror = shenandoah_read_barrier(mirror);
+  }
+
   Node* obj    = top();
 
   switch (id) {
@@ -3175,6 +3293,9 @@
     // nothing is an instance of a primitive type
     prim_return_value = intcon(0);
     obj = argument(1);
+    if (ShenandoahVerifyReadsToFromSpace) {
+      obj = shenandoah_read_barrier(obj);
+    }
     break;
   case vmIntrinsics::_getModifiers:
     prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
@@ -3422,6 +3543,15 @@
   Node* args[2];                // two java.lang.Class mirrors: superc, subc
   args[0] = argument(0);
   args[1] = argument(1);
+
+  // We need write barriers here, because for primitive types we later compare
+  // the two Class objects using ==, and those would give false negatives
+  // if one obj is in from-space, and one in to-space.
+  // TODO: Consider doing improved == comparison that only needs read barriers
+  // on the false-path.
+  args[0] = shenandoah_write_barrier(args[0]);
+  args[1] = shenandoah_write_barrier(args[1]);
+
   Node* klasses[2];             // corresponding Klasses: superk, subk
   klasses[0] = klasses[1] = top();
 
@@ -3728,6 +3858,8 @@
       Node* orig_tail = _gvn.transform(new SubINode(orig_length, start));
       Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
 
+      original = shenandoah_read_barrier(original);
+
       // Generate a direct call to the right arraycopy function(s).
       // We know the copy is disjoint but we might not know if the
       // oop stores need checking.
@@ -3909,6 +4041,10 @@
     result_val->init_req(_null_path, _gvn.intcon(0));
   }
 
+  if (ShenandoahVerifyReadsToFromSpace) {
+    obj = shenandoah_read_barrier(obj);
+  }
+
   // Unconditionally null?  Then return right away.
   if (stopped()) {
     set_control( result_reg->in(_null_path));
@@ -4224,6 +4360,9 @@
   assert(Unsafe_field_offset_to_byte_offset(11) == 11,
          "fieldOffset must be byte-scaled");
 
+  src_ptr = shenandoah_read_barrier(src_ptr);
+  dst_ptr = shenandoah_write_barrier(dst_ptr);
+
   Node* src = make_unsafe_address(src_ptr, src_off);
   Node* dst = make_unsafe_address(dst_ptr, dst_off);
 
@@ -4252,6 +4391,8 @@
   Node* raw_obj = alloc_obj->in(1);
   assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
 
+  obj = shenandoah_read_barrier(obj);
+
   AllocateNode* alloc = NULL;
   if (ReduceBulkZeroing) {
     // We will be completely responsible for initializing this object -
@@ -4309,6 +4450,15 @@
     set_all_memory(n);
   }
 
+  if (UseShenandoahGC) {
+    // Make sure that references in the cloned object are updated for Shenandoah.
+    make_runtime_call(RC_LEAF|RC_NO_FP,
+                      OptoRuntime::shenandoah_clone_barrier_Type(),
+                      CAST_FROM_FN_PTR(address, SharedRuntime::shenandoah_clone_barrier),
+                      "shenandoah_clone_barrier", TypePtr::BOTTOM,
+                      alloc_obj);
+  }
+
   // If necessary, emit some card marks afterwards.  (Non-arrays only.)
   if (card_mark) {
     assert(!is_array, "");
@@ -4435,6 +4585,9 @@
         if (is_obja != NULL) {
           PreserveJVMState pjvms2(this);
           set_control(is_obja);
+
+          obj = shenandoah_read_barrier(obj);
+
           // Generate a direct call to the right arraycopy function(s).
           Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
           ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
@@ -4683,6 +4836,8 @@
   Node* dest_offset = argument(3);  // type: int
   Node* length      = argument(4);  // type: int
 
+  src = shenandoah_read_barrier(src);
+  dest = shenandoah_write_barrier(dest);
 
   // Check for allocation before we add nodes that would confuse
   // tightly_coupled_allocation()
@@ -4922,6 +5077,8 @@
   if (stopped())             return NULL;  // no fast path
   if (C->AliasLevel() == 0)  return NULL;  // no MergeMems around
 
+  ptr = ShenandoahBarrierNode::skip_through_barrier(ptr);
+
   AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
   if (alloc == NULL)  return NULL;
 
@@ -5001,6 +5158,9 @@
   Node *dst_offset  = argument(3);
   Node *length      = argument(4);
 
+  src = shenandoah_read_barrier(src);
+  dst = shenandoah_write_barrier(dst);
+
   const Type* src_type = src->Value(&_gvn);
   const Type* dst_type = dst->Value(&_gvn);
   const TypeAryPtr* top_src = src_type->isa_aryptr();
@@ -5050,6 +5210,10 @@
   Node* ylen = argument(3);
   Node* z    = argument(4);
 
+  x = shenandoah_read_barrier(x);
+  y = shenandoah_read_barrier(y);
+  z = shenandoah_write_barrier(z);
+
   const Type* x_type = x->Value(&_gvn);
   const Type* y_type = y->Value(&_gvn);
   const TypeAryPtr* top_x = x_type->isa_aryptr();
@@ -5150,6 +5314,9 @@
   Node* z    = argument(2);
   Node* zlen = argument(3);
 
+  x = shenandoah_read_barrier(x);
+  z = shenandoah_write_barrier(z);
+
   const Type* x_type = x->Value(&_gvn);
   const Type* z_type = z->Value(&_gvn);
   const TypeAryPtr* top_x = x_type->isa_aryptr();
@@ -5197,6 +5364,9 @@
   Node* len      = argument(3);
   Node* k        = argument(4);
 
+  in = shenandoah_read_barrier(in);
+  out = shenandoah_write_barrier(out);
+
   const Type* out_type = out->Value(&_gvn);
   const Type* in_type = in->Value(&_gvn);
   const TypeAryPtr* top_out = out_type->isa_aryptr();
@@ -5246,6 +5416,11 @@
   Node* inv  = argument(4);
   Node* m    = argument(6);
 
+  a = shenandoah_read_barrier(a);
+  b = shenandoah_read_barrier(b);
+  n = shenandoah_read_barrier(n);
+  m = shenandoah_write_barrier(m);
+
   const Type* a_type = a->Value(&_gvn);
   const TypeAryPtr* top_a = a_type->isa_aryptr();
   const Type* b_type = b->Value(&_gvn);
@@ -5305,6 +5480,10 @@
   Node* inv  = argument(3);
   Node* m    = argument(5);
 
+  a = shenandoah_read_barrier(a);
+  n = shenandoah_read_barrier(n);
+  m = shenandoah_write_barrier(m);
+
   const Type* a_type = a->Value(&_gvn);
   const TypeAryPtr* top_a = a_type->isa_aryptr();
   const Type* n_type = a->Value(&_gvn);
@@ -5391,6 +5570,8 @@
   Node* offset  = argument(2); // type: int
   Node* length  = argument(3); // type: int
 
+  src = shenandoah_read_barrier(src);
+
   const Type* src_type = src->Value(&_gvn);
   const TypeAryPtr* top_src = src_type->isa_aryptr();
   if (top_src  == NULL || top_src->klass()  == NULL) {
@@ -5493,10 +5674,12 @@
   }
 
   // 'src_start' points to src array + scaled offset
+  src = shenandoah_read_barrier(src);
   Node* src_start = array_element_address(src, offset, src_elem);
 
   // static final int[] byteTable in class CRC32C
   Node* table = get_table_from_crc32c_class(callee()->holder());
+  table = shenandoah_read_barrier(table);
   Node* table_start = array_element_address(table, intcon(0), T_INT);
 
   // We assume that range check is done by caller.
@@ -5540,6 +5723,7 @@
 
   // static final int[] byteTable in class CRC32C
   Node* table = get_table_from_crc32c_class(callee()->holder());
+  table = shenandoah_read_barrier(table);
   Node* table_start = array_element_address(table, intcon(0), T_INT);
 
   // Call the stub.
@@ -5583,6 +5767,7 @@
   }
 
   // 'src_start' points to src array + scaled offset
+  src = shenandoah_read_barrier(src);
   Node* src_start = array_element_address(src, offset, src_elem);
 
   // We assume that range check is done by caller.
@@ -5645,6 +5830,10 @@
   Node* reference_obj = null_check_receiver();
   if (stopped()) return true;
 
+  if (ShenandoahVerifyReadsToFromSpace) {
+    reference_obj = shenandoah_read_barrier(reference_obj);
+  }
+
   Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
 
   ciInstanceKlass* klass = env()->Object_klass();
@@ -5693,6 +5882,8 @@
     fromObj = makecon(tip);
   }
 
+  fromObj = shenandoah_read_barrier(fromObj);
+
   // Next code  copied from Parse::do_get_xxx():
 
   // Compute address and memory type.
@@ -5753,6 +5944,10 @@
   Node* dest            = argument(3);
   Node* dest_offset     = argument(4);
 
+  // Resolve src and dest arrays for ShenandoahGC.
+  src = shenandoah_read_barrier(src);
+  dest = shenandoah_write_barrier(dest);
+
   // (1) src and dest are arrays.
   const Type* src_type = src->Value(&_gvn);
   const Type* dest_type = dest->Value(&_gvn);
@@ -5821,6 +6016,10 @@
   Node* dest                       = argument(4);
   Node* dest_offset                = argument(5);
 
+  // Resolve src and dest arrays for ShenandoahGC.
+  src = shenandoah_read_barrier(src);
+  dest = shenandoah_write_barrier(dest);
+
   // (1) src and dest are arrays.
   const Type* src_type = src->Value(&_gvn);
   const Type* dest_type = dest->Value(&_gvn);
@@ -5865,6 +6064,9 @@
 
   // similarly, get the start address of the r vector
   Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
+
+  objRvec = shenandoah_write_barrier(objRvec);
+
   if (objRvec == NULL) return false;
   Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
 
@@ -5900,6 +6102,8 @@
   assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
   if (objAESCryptKey == NULL) return (Node *) NULL;
 
+  objAESCryptKey = shenandoah_read_barrier(objAESCryptKey);
+
   // now have the array, need to get the start address of the K array
   Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
   return k_start;
diff --git a/src/share/vm/opto/loopPredicate.cpp b/src/share/vm/opto/loopPredicate.cpp
--- a/src/share/vm/opto/loopPredicate.cpp
+++ b/src/share/vm/opto/loopPredicate.cpp
@@ -441,7 +441,7 @@
           // loop, it was marked invariant but n is only invariant if
           // it depends only on that test. Otherwise, unless that test
           // is out of the loop, it's not invariant.
-          if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) {
+          if (n->Opcode() == Op_ShenandoahWBMemProj || n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) {
             _invariant.set(n->_idx); // I am a invariant too
           }
         }
diff --git a/src/share/vm/opto/loopTransform.cpp b/src/share/vm/opto/loopTransform.cpp
--- a/src/share/vm/opto/loopTransform.cpp
+++ b/src/share/vm/opto/loopTransform.cpp
@@ -927,6 +927,7 @@
   // Check for 'n' being pinned in the backedge.
   if( n->in(0) && n->in(0) == back_ctrl ) {
     assert(clones.find(n->_idx) == NULL, "dead loop");
+    assert(n->Opcode() != Op_ShenandoahWriteBarrier, "no wbs yet");
     x = n->clone();             // Clone a copy of 'n' to preheader
     clones.push(x, n->_idx);
     x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
diff --git a/src/share/vm/opto/loopUnswitch.cpp b/src/share/vm/opto/loopUnswitch.cpp
--- a/src/share/vm/opto/loopUnswitch.cpp
+++ b/src/share/vm/opto/loopUnswitch.cpp
@@ -180,6 +180,7 @@
     ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
     while (worklist.size() > 0) {
       Node* use = worklist.pop();
+      assert(use->Opcode() != Op_ShenandoahWriteBarrier, "not with wbs yet");
       Node* nuse = use->clone();
       nuse->set_req(0, invar_proj);
       _igvn.replace_input_of(use, 1, nuse);
diff --git a/src/share/vm/opto/loopnode.cpp b/src/share/vm/opto/loopnode.cpp
--- a/src/share/vm/opto/loopnode.cpp
+++ b/src/share/vm/opto/loopnode.cpp
@@ -3089,7 +3089,7 @@
         ++i;
         if (in == NULL) continue;
         if (in->pinned() && !in->is_CFG())
-          set_ctrl(in, in->in(0));
+          set_ctrl(in, in->Opcode() == Op_ShenandoahWBMemProj ? in->in(0)->in(0) : in->in(0));
         int is_visited = visited.test_set( in->_idx );
         if (!has_node(in)) {  // No controlling input yet?
           assert( !in->is_CFG(), "CFG Node with no controlling input?" );
@@ -3274,7 +3274,7 @@
         }
       } else {
         Node *sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0);
-        assert(sctrl != NULL || s->outcnt() == 0, "must have control");
+        assert(sctrl != NULL || s->outcnt() == 0 || s->is_ShenandoahBarrier(), "must have control");
         if (sctrl != NULL && !sctrl->is_top() && is_dominator(early, sctrl)) {
           LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n);
         }
@@ -3495,6 +3495,8 @@
     case Op_StrEquals:
     case Op_StrIndexOf:
     case Op_AryEq:
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
       pinned = false;
     }
     if( pinned ) {
diff --git a/src/share/vm/opto/loopnode.hpp b/src/share/vm/opto/loopnode.hpp
--- a/src/share/vm/opto/loopnode.hpp
+++ b/src/share/vm/opto/loopnode.hpp
@@ -1062,6 +1062,8 @@
   Node *split_thru_region( Node *n, Node *region );
   // Split Node 'n' through merge point if there is enough win.
   Node *split_thru_phi( Node *n, Node *region, int policy );
+  void split_mem_thru_phi(Node*, Node* r, Node* phi);
+
   // Found an If getting its condition-code input from a Phi in the
   // same block.  Split thru the Region.
   void do_split_if( Node *iff );
diff --git a/src/share/vm/opto/loopopts.cpp b/src/share/vm/opto/loopopts.cpp
--- a/src/share/vm/opto/loopopts.cpp
+++ b/src/share/vm/opto/loopopts.cpp
@@ -33,6 +33,7 @@
 #include "opto/movenode.hpp"
 #include "opto/opaquenode.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/subnode.hpp"
 
 //=============================================================================
@@ -67,6 +68,9 @@
     if (region->in(i) == C->top()) {
       x = C->top();             // Dead path?  Use a dead data op
     } else {
+#ifdef ASSERT
+      if (n->is_ShenandoahBarrier()) { n->as_ShenandoahBarrier()->check_invariants(); }
+#endif
       x = n->clone();           // Else clone up the data op
       the_clone = x;            // Remember for possible deletion.
       // Alter data node to use pre-phi inputs
@@ -111,21 +115,26 @@
       // otherwise it will be not updated during igvn->transform since
       // igvn->type(x) is set to x->Value() already.
       x->raise_bottom_type(t);
-      Node *y = x->Identity(&_igvn);
-      if (y != x) {
-        wins++;
-        x = y;
-      } else {
-        y = _igvn.hash_find(x);
-        if (y) {
+      if (x->Opcode() != Op_ShenandoahWriteBarrier) {
+        Node *y = x->Identity(&_igvn);
+        if (y != x) {
           wins++;
           x = y;
         } else {
-          // Else x is a new node we are keeping
-          // We do not need register_new_node_with_optimizer
-          // because set_type has already been called.
-          _igvn._worklist.push(x);
+          y = _igvn.hash_find(x);
+          if (y) {
+            wins++;
+            x = y;
+          } else {
+            // Else x is a new node we are keeping
+            // We do not need register_new_node_with_optimizer
+            // because set_type has already been called.
+            _igvn._worklist.push(x);
+          }
         }
+      } else {
+        // wins++;
+        _igvn._worklist.push(x);
       }
     }
     if (x != the_clone && the_clone != NULL)
@@ -194,6 +203,38 @@
   return phi;
 }
 
+void PhaseIdealLoop::split_mem_thru_phi(Node* n, Node* r, Node* phi) {
+  if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+#ifdef ASSERT
+    n->as_ShenandoahBarrier()->check_invariants();
+#endif
+    if (n->has_out_with(Op_ShenandoahWBMemProj)) {
+      Node* old_mem_phi = n->in(ShenandoahBarrierNode::Memory);
+      assert(r->is_Region(), "need region to control phi");
+      assert(phi->is_Phi(), "expect phi");
+      Node* memphi = PhiNode::make(r, old_mem_phi, Type::MEMORY, C->alias_type(n->adr_type())->adr_type());
+      for (uint i = 1; i < r->req(); i++) {
+        Node* wb = phi->in(i);
+        if (wb->Opcode() == Op_ShenandoahWriteBarrier) {
+          assert(! wb->has_out_with(Op_ShenandoahWBMemProj), "new clone does not have mem proj");
+          Node* new_proj = new ShenandoahWBMemProjNode(wb);
+          register_new_node(new_proj, r->in(i));
+          memphi->set_req(i, new_proj);
+        } else {
+          if (old_mem_phi->is_Phi() && old_mem_phi->in(0) == r) {
+            memphi->set_req(i, old_mem_phi->in(i));
+          }
+        }
+      }
+      register_new_node(memphi, r);
+      Node* old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj);
+      assert(old_mem_out != NULL, "expect memory projection");
+      _igvn.replace_node(old_mem_out, memphi);
+    }
+    assert(! n->has_out_with(Op_ShenandoahWBMemProj), "no more memory outs");
+  }
+}
+
 //------------------------------dominated_by------------------------------------
 // Replace the dominated test with an obvious true or false.  Place it on the
 // IGVN worklist for later cleanup.  Move control-dependent data Nodes on the
@@ -934,12 +975,18 @@
 
   // Found a Phi to split thru!
   // Replace 'n' with the new phi
+  split_mem_thru_phi(n, n_blk, phi);
   _igvn.replace_node( n, phi );
   // Moved a load around the loop, 'en-registering' something.
   if (n_blk->is_Loop() && n->is_Load() &&
       !phi->in(LoopNode::LoopBackControl)->is_Load())
     C->set_major_progress();
 
+  // Moved a barrier around the loop, 'en-registering' something.
+  if (n_blk->is_Loop() && n->is_ShenandoahBarrier() &&
+      !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier())
+    C->set_major_progress();
+
   return phi;
 }
 
@@ -1082,9 +1129,11 @@
 
     // Found a Phi to split thru!
     // Replace 'n' with the new phi
+    assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
     _igvn.replace_node( n, phi );
 
     // Now split the bool up thru the phi
+    assert(bol->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
     Node *bolphi = split_thru_phi( bol, n_ctrl, -1 );
     guarantee(bolphi != NULL, "null boolean phi node");
 
@@ -1096,6 +1145,7 @@
 
     // Conditional-move?  Must split up now
     if( !iff->is_If() ) {
+      assert(iff->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
       Node *cmovphi = split_thru_phi( iff, n_ctrl, -1 );
       _igvn.replace_node( iff, cmovphi );
       return;
@@ -1229,7 +1279,7 @@
             // to fold a StoreP and an AddP together (as part of an
             // address expression) and the AddP and StoreP have
             // different controls.
-            if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
+            if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier()) _igvn._worklist.yank(x);
           }
           _igvn.remove_dead_node(n);
         }
@@ -1502,6 +1552,16 @@
   uint i;
   for( i = 0; i < loop->_body.size(); i++ ) {
     Node *old = loop->_body.at(i);
+#ifdef ASSERT
+    if (old->Opcode() == Op_ShenandoahWriteBarrier) {
+      Node* memproj = old->find_out_with(Op_ShenandoahWBMemProj);
+      assert(memproj == NULL || (loop->is_member(get_loop(has_ctrl(memproj) ? get_ctrl(memproj) : memproj))), "WB's mem-proj must be in loop too");
+    }
+    if (old->Opcode() == Op_ShenandoahWBMemProj) {
+      Node* wb = old->in(0);
+      assert(loop->is_member(get_loop(has_ctrl(wb) ? get_ctrl(wb) : wb)), "WB must be in loop too");
+    }
+#endif
     Node *nnn = old->clone();
     old_new.map( old->_idx, nnn );
     if (C->do_vector_loop()) {
@@ -1692,12 +1752,13 @@
         // private Phi and those Phis need to be merged here.
         Node *phi;
         if( prev->is_Region() ) {
-          if( idx == 0 ) {      // Updating control edge?
+          if( idx == 0  && use->Opcode() != Op_ShenandoahWBMemProj) {      // Updating control edge?
             phi = prev;         // Just use existing control
           } else {              // Else need a new Phi
             phi = PhiNode::make( prev, old );
             // Now recursively fix up the new uses of old!
-            for( uint i = 1; i < prev->req(); i++ ) {
+            uint first = use->Opcode() != Op_ShenandoahWBMemProj ? 1 : 0;
+            for( uint i = first; i < prev->req(); i++ ) {
               worklist.push(phi); // Onto worklist once for each 'old' input
             }
           }
@@ -1705,7 +1766,7 @@
           // Get new RegionNode merging old and new loop exits
           prev = old_new[prev->_idx];
           assert( prev, "just made this in step 7" );
-          if( idx == 0 ) {      // Updating control edge?
+          if( idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) {      // Updating control edge?
             phi = prev;         // Just use existing control
           } else {              // Else need a new Phi
             // Make a new Phi merging data values properly
@@ -2188,6 +2249,7 @@
 int PhaseIdealLoop::clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist ) {
   int cloned = 0;
   assert(worklist.size() == 0, "should be empty");
+  assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
   for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
     Node* use = n->fast_out(j);
     if( !loop->is_member(get_loop(has_ctrl(use) ? get_ctrl(use) : use)) ) {
@@ -2204,6 +2266,7 @@
     assert(j < use->req(), "must be there");
 
     // clone "n" and insert it between the inputs of "n" and the use outside the loop
+    assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers");
     Node* n_clone = n->clone();
     _igvn.replace_input_of(use, j, n_clone);
     cloned++;
@@ -2250,6 +2313,7 @@
   }
   if (worklist.size() > 0) {
     // clone "n" and insert it between inputs of "n" and the use
+    assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers");
     Node* n_clone = n->clone();
     loop->_body.push(n_clone);
     _igvn.register_new_node_with_optimizer(n_clone);
diff --git a/src/share/vm/opto/macro.cpp b/src/share/vm/opto/macro.cpp
--- a/src/share/vm/opto/macro.cpp
+++ b/src/share/vm/opto/macro.cpp
@@ -42,6 +42,7 @@
 #include "opto/phaseX.hpp"
 #include "opto/rootnode.hpp"
 #include "opto/runtime.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/subnode.hpp"
 #include "opto/type.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -933,6 +934,9 @@
           field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
         }
       }
+      if (field_val->isa_ShenandoahBarrier()) {
+        field_val = field_val->in(ShenandoahBarrierNode::ValueIn);
+      }
       sfpt->add_req(field_val);
     }
     JVMState *jvms = sfpt->jvms();
@@ -1411,6 +1415,13 @@
 
     transform_later(old_eden_top);
     // Add to heap top to get a new heap top
+
+    if (UseShenandoahGC) {
+      // Allocate one word more for the Shenandoah brooks pointer.
+      size_in_bytes = new AddLNode(size_in_bytes, _igvn.MakeConX(8));
+      transform_later(size_in_bytes);
+    }
+
     Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
     transform_later(new_eden_top);
     // Check for needing a GC; compare against heap end
@@ -1501,6 +1512,13 @@
                                    0, new_alloc_bytes, T_LONG);
     }
 
+    if (UseShenandoahGC) {
+      // Bump up object by one word. The preceding word is used for
+      // the Shenandoah brooks pointer.
+      fast_oop = new AddPNode(top(), fast_oop, _igvn.MakeConX(8));
+      transform_later(fast_oop);
+    }
+
     InitializeNode* init = alloc->initialization();
     fast_oop_rawmem = initialize_object(alloc,
                                         fast_oop_ctrl, fast_oop_rawmem, fast_oop,
@@ -1778,6 +1796,11 @@
       header_size = Klass::layout_helper_header_size(k->layout_helper());
   }
 
+  if (UseShenandoahGC) {
+    // Initialize Shenandoah brooks pointer to point to the object itself.
+    rawmem = make_store(control, rawmem, object, -8, object, T_OBJECT);
+  }
+
   // Clear the object body, if necessary.
   if (init == NULL) {
     // The init has somehow disappeared; be cautious and clear everything.
diff --git a/src/share/vm/opto/macro.hpp b/src/share/vm/opto/macro.hpp
--- a/src/share/vm/opto/macro.hpp
+++ b/src/share/vm/opto/macro.hpp
@@ -42,6 +42,7 @@
   Node* intcon(jint con)        const { return _igvn.intcon(con); }
   Node* longcon(jlong con)      const { return _igvn.longcon(con); }
   Node* makecon(const Type *t)  const { return _igvn.makecon(t); }
+
   Node* basic_plus_adr(Node* base, int offset) {
     return (offset == 0)? base: basic_plus_adr(base, MakeConX(offset));
   }
diff --git a/src/share/vm/opto/macroArrayCopy.cpp b/src/share/vm/opto/macroArrayCopy.cpp
--- a/src/share/vm/opto/macroArrayCopy.cpp
+++ b/src/share/vm/opto/macroArrayCopy.cpp
@@ -547,7 +547,7 @@
     // At this point we know we do not need type checks on oop stores.
 
     // Let's see if we need card marks:
-    if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks()) {
+    if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks() && ! UseShenandoahGC) {
       // If we do not need card marks, copy using the jint or jlong stub.
       copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
       assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
diff --git a/src/share/vm/opto/matcher.cpp b/src/share/vm/opto/matcher.cpp
--- a/src/share/vm/opto/matcher.cpp
+++ b/src/share/vm/opto/matcher.cpp
@@ -2156,6 +2156,11 @@
       case Op_SafePoint:
         mem_op = true;
         break;
+      case Op_ShenandoahReadBarrier:
+      case Op_ShenandoahWriteBarrier:
+        mem_op = true;
+        set_shared(n);
+        break;
       default:
         if( n->is_Store() ) {
           // Do match stores, despite no ideal reg
diff --git a/src/share/vm/opto/memnode.cpp b/src/share/vm/opto/memnode.cpp
--- a/src/share/vm/opto/memnode.cpp
+++ b/src/share/vm/opto/memnode.cpp
@@ -41,6 +41,7 @@
 #include "opto/narrowptrnode.hpp"
 #include "opto/phaseX.hpp"
 #include "opto/regmask.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "utilities/copy.hpp"
 
 // Portions of code courtesy of Clifford Click
@@ -1000,6 +1001,7 @@
         (tp != NULL) && tp->is_ptr_to_boxed_value()) {
       intptr_t ignore = 0;
       Node* base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ignore);
+      base = ShenandoahBarrierNode::skip_through_barrier(base);
       if (base != NULL && base->is_Proj() &&
           base->as_Proj()->_con == TypeFunc::Parms &&
           base->in(0)->is_CallStaticJava() &&
@@ -1603,7 +1605,7 @@
     // as to alignment, which will therefore produce the smallest
     // possible base offset.
     const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
-    const bool off_beyond_header = ((uint)off >= (uint)min_base_off);
+    const bool off_beyond_header = (off != -8 || !UseShenandoahGC) && ((uint)off >= (uint)min_base_off);
 
     // Try to constant-fold a stable array element.
     if (FoldStableValues && ary->is_stable() && ary->const_oop() != NULL) {
diff --git a/src/share/vm/opto/multnode.cpp b/src/share/vm/opto/multnode.cpp
--- a/src/share/vm/opto/multnode.cpp
+++ b/src/share/vm/opto/multnode.cpp
@@ -31,6 +31,7 @@
 #include "opto/opcodes.hpp"
 #include "opto/phaseX.hpp"
 #include "opto/regmask.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/type.hpp"
 
 //=============================================================================
@@ -141,6 +142,7 @@
   if (n->is_Mach())    return;  // mach. projs. are not type-safe
   if (n->is_Start())   return;  // alas, starts can have mach. projs. also
   if (_con == SCMemProjNode::SCMEMPROJCON ) return;
+  if (_con == ShenandoahWBMemProjNode::SWBMEMPROJCON ) return;
   const Type* t = n->bottom_type();
   if (t == Type::TOP)  return;  // multi is dead
   assert(_con < t->is_tuple()->cnt(), "ProjNode::_con must be in range");
diff --git a/src/share/vm/opto/node.hpp b/src/share/vm/opto/node.hpp
--- a/src/share/vm/opto/node.hpp
+++ b/src/share/vm/opto/node.hpp
@@ -130,6 +130,7 @@
 class RootNode;
 class SafePointNode;
 class SafePointScalarObjectNode;
+class ShenandoahBarrierNode;
 class StartNode;
 class State;
 class StoreNode;
@@ -621,6 +622,7 @@
       DEFINE_CLASS_ID(EncodeNarrowPtr, Type, 6)
         DEFINE_CLASS_ID(EncodeP, EncodeNarrowPtr, 0)
         DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1)
+      DEFINE_CLASS_ID(ShenandoahBarrier, Type, 7)
 
     DEFINE_CLASS_ID(Proj,  Node, 3)
       DEFINE_CLASS_ID(CatchProj, Proj, 0)
@@ -800,6 +802,7 @@
   DEFINE_CLASS_QUERY(Root)
   DEFINE_CLASS_QUERY(SafePoint)
   DEFINE_CLASS_QUERY(SafePointScalarObject)
+  DEFINE_CLASS_QUERY(ShenandoahBarrier)
   DEFINE_CLASS_QUERY(Start)
   DEFINE_CLASS_QUERY(Store)
   DEFINE_CLASS_QUERY(Sub)
diff --git a/src/share/vm/opto/output.cpp b/src/share/vm/opto/output.cpp
--- a/src/share/vm/opto/output.cpp
+++ b/src/share/vm/opto/output.cpp
@@ -907,8 +907,8 @@
             !method->is_synchronized() ||
             method->is_native() ||
             num_mon > 0 ||
-            !GenerateSynchronizationCode,
-            "monitors must always exist for synchronized methods");
+            !GenerateSynchronizationCode || (UseShenandoahGC && jvms->bci() < 0),
+            err_msg("monitors must always exist for synchronized methods, bci: %d", jvms->bci()));
 
     // Build the growable array of ScopeValues for exp stack
     GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
diff --git a/src/share/vm/opto/parse.hpp b/src/share/vm/opto/parse.hpp
--- a/src/share/vm/opto/parse.hpp
+++ b/src/share/vm/opto/parse.hpp
@@ -483,7 +483,7 @@
   // Helper function to generate array store
   void array_store(BasicType etype);
   // Helper function to compute array addressing
-  Node* array_addressing(BasicType type, int vals, const Type* *result2=NULL);
+  Node* array_addressing(BasicType type, int vals, bool is_store, const Type* *result2=NULL);
 
   void rtm_deopt();
 
diff --git a/src/share/vm/opto/parse2.cpp b/src/share/vm/opto/parse2.cpp
--- a/src/share/vm/opto/parse2.cpp
+++ b/src/share/vm/opto/parse2.cpp
@@ -50,7 +50,7 @@
 //---------------------------------array_load----------------------------------
 void Parse::array_load(BasicType elem_type) {
   const Type* elem = Type::TOP;
-  Node* adr = array_addressing(elem_type, 0, &elem);
+  Node* adr = array_addressing(elem_type, 0, false, &elem);
   if (stopped())  return;     // guaranteed null or range check
   dec_sp(2);                  // Pop array and index
   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(elem_type);
@@ -61,7 +61,7 @@
 
 //--------------------------------array_store----------------------------------
 void Parse::array_store(BasicType elem_type) {
-  Node* adr = array_addressing(elem_type, 1);
+  Node* adr = array_addressing(elem_type, 1, true);
   if (stopped())  return;     // guaranteed null or range check
   Node* val = pop();
   dec_sp(2);                  // Pop array and index
@@ -72,7 +72,7 @@
 
 //------------------------------array_addressing-------------------------------
 // Pull array and index from the stack.  Compute pointer-to-element.
-Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) {
+Node* Parse::array_addressing(BasicType type, int vals, bool is_store, const Type* *result2) {
   Node *idx   = peek(0+vals);   // Get from stack without popping
   Node *ary   = peek(1+vals);   // in case of exception
 
@@ -158,6 +158,12 @@
   // Check for always knowing you are throwing a range-check exception
   if (stopped())  return top();
 
+  if (is_store) {
+    ary = shenandoah_write_barrier(ary);
+  } else {
+    ary = shenandoah_read_barrier(ary);
+  }
+
   Node* ptr = array_element_address(ary, idx, type, sizetype);
 
   if (result2 != NULL)  *result2 = elemtype;
@@ -1696,14 +1702,14 @@
   case Bytecodes::_faload: array_load(T_FLOAT);  break;
   case Bytecodes::_aaload: array_load(T_OBJECT); break;
   case Bytecodes::_laload: {
-    a = array_addressing(T_LONG, 0);
+    a = array_addressing(T_LONG, 0, false);
     if (stopped())  return;     // guaranteed null or range check
     dec_sp(2);                  // Pop array and index
     push_pair(make_load(control(), a, TypeLong::LONG, T_LONG, TypeAryPtr::LONGS, MemNode::unordered));
     break;
   }
   case Bytecodes::_daload: {
-    a = array_addressing(T_DOUBLE, 0);
+    a = array_addressing(T_DOUBLE, 0, false);
     if (stopped())  return;     // guaranteed null or range check
     dec_sp(2);                  // Pop array and index
     push_pair(make_load(control(), a, Type::DOUBLE, T_DOUBLE, TypeAryPtr::DOUBLES, MemNode::unordered));
@@ -1715,7 +1721,7 @@
   case Bytecodes::_sastore: array_store(T_SHORT); break;
   case Bytecodes::_fastore: array_store(T_FLOAT); break;
   case Bytecodes::_aastore: {
-    d = array_addressing(T_OBJECT, 1);
+    d = array_addressing(T_OBJECT, 1, true);
     if (stopped())  return;     // guaranteed null or range check
     array_store_check();
     c = pop();                  // Oop to store
@@ -1723,12 +1729,17 @@
     a = pop();                  // the array itself
     const TypeOopPtr* elemtype  = _gvn.type(a)->is_aryptr()->elem()->make_oopptr();
     const TypeAryPtr* adr_type = TypeAryPtr::OOPS;
+    // Note: We don't need a write barrier for Shenandoah on a here, because
+    // a is not used except for an assert. The address d already has the
+    // write barrier. Adding a barrier on a only results in additional code
+    // being generated.
+    c = shenandoah_read_barrier_nomem(c);
     Node* store = store_oop_to_array(control(), a, d, adr_type, c, elemtype, T_OBJECT,
                                      StoreNode::release_if_reference(T_OBJECT));
     break;
   }
   case Bytecodes::_lastore: {
-    a = array_addressing(T_LONG, 2);
+    a = array_addressing(T_LONG, 2, true);
     if (stopped())  return;     // guaranteed null or range check
     c = pop_pair();
     dec_sp(2);                  // Pop array and index
@@ -1736,7 +1747,7 @@
     break;
   }
   case Bytecodes::_dastore: {
-    a = array_addressing(T_DOUBLE, 2);
+    a = array_addressing(T_DOUBLE, 2, true);
     if (stopped())  return;     // guaranteed null or range check
     c = pop_pair();
     dec_sp(2);                  // Pop array and index
@@ -2276,6 +2287,7 @@
     maybe_add_safepoint(iter().get_dest());
     a = pop();
     b = pop();
+    shenandoah_acmp_barrier(a, b);
     c = _gvn.transform( new CmpPNode(b, a) );
     c = optimize_cmp_with_klass(c);
     do_if(btest, c);
diff --git a/src/share/vm/opto/parse3.cpp b/src/share/vm/opto/parse3.cpp
--- a/src/share/vm/opto/parse3.cpp
+++ b/src/share/vm/opto/parse3.cpp
@@ -162,6 +162,10 @@
   // Compute address and memory type.
   int offset = field->offset_in_bytes();
   const TypePtr* adr_type = C->alias_type(field)->adr_type();
+
+  // Insert read barrier for Shenandoah.
+  obj = shenandoah_read_barrier(obj);
+
   Node *adr = basic_plus_adr(obj, obj, offset);
   BasicType bt = field->layout_type();
 
@@ -244,6 +248,9 @@
   // another volatile read.
   if (is_vol)  insert_mem_bar(Op_MemBarRelease);
 
+  // Insert write barrier for Shenandoah.
+  obj = shenandoah_write_barrier(obj);
+
   // Compute address and memory type.
   int offset = field->offset_in_bytes();
   const TypePtr* adr_type = C->alias_type(field)->adr_type();
@@ -273,6 +280,9 @@
     } else {
       field_type = TypeOopPtr::make_from_klass(field->type()->as_klass());
     }
+
+    val = shenandoah_read_barrier_nomem(val);
+
     store = store_oop_to_object(control(), obj, adr, adr_type, val, field_type, bt, mo);
   } else {
     bool needs_atomic_access = is_vol || AlwaysAtomicAccesses;
diff --git a/src/share/vm/opto/parseHelper.cpp b/src/share/vm/opto/parseHelper.cpp
--- a/src/share/vm/opto/parseHelper.cpp
+++ b/src/share/vm/opto/parseHelper.cpp
@@ -144,6 +144,11 @@
   Node *idx = peek(1);
   Node *ary = peek(2);
 
+  if (ShenandoahVerifyReadsToFromSpace) {
+    obj = shenandoah_read_barrier(obj);
+    ary = shenandoah_read_barrier(ary);
+  }
+
   if (_gvn.type(obj) == TypePtr::NULL_PTR) {
     // There's never a type check on null values.
     // This cutout lets us avoid the uncommon_trap(Reason_array_check)
diff --git a/src/share/vm/opto/phaseX.cpp b/src/share/vm/opto/phaseX.cpp
--- a/src/share/vm/opto/phaseX.cpp
+++ b/src/share/vm/opto/phaseX.cpp
@@ -34,6 +34,7 @@
 #include "opto/phaseX.hpp"
 #include "opto/regalloc.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/shenandoahSupport.hpp"
 
 //=============================================================================
 #define NODE_HASH_MINIMUM_SIZE    255
@@ -565,6 +566,38 @@
 
 #endif
 
+bool PhaseTransform::eqv(const Node* n1, const Node* n2) const {
+  if (UseShenandoahGC) {
+
+    if (n1 == n2) return true;
+
+    if (n1 == NULL || n2 == NULL) return false;
+
+    if (n1->is_AddP() && n2->is_AddP()) {
+      Node* addp1 = n1->as_AddP();
+      Node* base1 = addp1->in(AddPNode::Base);
+      Node* addr1 = addp1->in(AddPNode::Address);
+      Node* offs1 = addp1->in(AddPNode::Offset);
+
+      Node* addp2 = n2->as_AddP();
+      Node* base2 = addp2->in(AddPNode::Base);
+      Node* addr2 = addp2->in(AddPNode::Address);
+      Node* offs2 = addp2->in(AddPNode::Offset);
+
+      if (base1 == addr1 && base2 == addr2) {
+
+        addr1 = ShenandoahBarrierNode::skip_through_barrier(addr1);
+        addr2 = ShenandoahBarrierNode::skip_through_barrier(addr2);
+
+        if (addr1 == addr2 && offs1 == offs2) return true;
+      }
+
+    }
+    return false;
+  } else {
+    return n1 == n2;
+  }
+}
 
 //=============================================================================
 //------------------------------PhaseValues------------------------------------
@@ -1190,6 +1223,9 @@
     NOT_PRODUCT(set_progress();)
     Node* con = makecon(t);     // Make a constant
     add_users_to_worklist(k);
+    if (k->Opcode() == Op_ShenandoahWriteBarrier) {
+      assert(con->is_top(), "can only replace barrier with top");
+    }
     subsume_node(k, con);       // Everybody using k now uses con
     return con;
   }
diff --git a/src/share/vm/opto/phaseX.hpp b/src/share/vm/opto/phaseX.hpp
--- a/src/share/vm/opto/phaseX.hpp
+++ b/src/share/vm/opto/phaseX.hpp
@@ -190,6 +190,8 @@
   // _nodes is used in varying ways by subclasses, which define local accessors
 
 public:
+  virtual PhaseIterGVN *is_IterGVN() { return 0; }
+
   // Get a previously recorded type for the node n.
   // This type must already have been recorded.
   // If you want the type of a very new (untransformed) node,
@@ -257,7 +259,7 @@
   // Return whether two Nodes are equivalent.
   // Must not be recursive, since the recursive version is built from this.
   // For pessimistic optimizations this is simply pointer equivalence.
-  bool eqv(const Node* n1, const Node* n2) const { return n1 == n2; }
+  bool eqv(const Node* n1, const Node* n2) const;
 
   // For pessimistic passes, the return type must monotonically narrow.
   // For optimistic  passes, the return type must monotonically widen.
diff --git a/src/share/vm/opto/runtime.cpp b/src/share/vm/opto/runtime.cpp
--- a/src/share/vm/opto/runtime.cpp
+++ b/src/share/vm/opto/runtime.cpp
@@ -34,6 +34,7 @@
 #include "code/vtableStubs.hpp"
 #include "compiler/compileBroker.hpp"
 #include "compiler/oopMap.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "gc/shared/barrierSet.hpp"
@@ -91,6 +92,7 @@
 address OptoRuntime::_multianewarray4_Java                        = NULL;
 address OptoRuntime::_multianewarray5_Java                        = NULL;
 address OptoRuntime::_multianewarrayN_Java                        = NULL;
+address OptoRuntime::_shenandoah_write_barrier_Java               = NULL;
 address OptoRuntime::_g1_wb_pre_Java                              = NULL;
 address OptoRuntime::_g1_wb_post_Java                             = NULL;
 address OptoRuntime::_vtable_must_compile_Java                    = NULL;
@@ -142,6 +144,7 @@
   gen(env, _multianewarray4_Java           , multianewarray4_Type         , multianewarray4_C               ,    0 , true , false, false);
   gen(env, _multianewarray5_Java           , multianewarray5_Type         , multianewarray5_C               ,    0 , true , false, false);
   gen(env, _multianewarrayN_Java           , multianewarrayN_Type         , multianewarrayN_C               ,    0 , true , false, false);
+  gen(env, _shenandoah_write_barrier_Java  , shenandoah_write_barrier_Type, ShenandoahBarrierSet::resolve_and_maybe_copy_oop_c2, 0, false, false, false);
   gen(env, _g1_wb_pre_Java                 , g1_wb_pre_Type               , SharedRuntime::g1_wb_pre        ,    0 , false, false, false);
   gen(env, _g1_wb_post_Java                , g1_wb_post_Type              , SharedRuntime::g1_wb_post       ,    0 , false, false, false);
   gen(env, _complete_monitor_locking_Java  , complete_monitor_enter_Type  , SharedRuntime::complete_monitor_locking_C, 0, false, false, false);
@@ -420,7 +423,7 @@
   ResourceMark rm;
   jint len = dims->length();
   assert(len > 0, "Dimensions array should contain data");
-  jint *j_dims = typeArrayOop(dims)->int_at_addr(0);
+  jint *j_dims = typeArrayOop(oopDesc::bs()->resolve_oop(dims))->int_at_addr(0);
   jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
   Copy::conjoint_jints_atomic(j_dims, c_dims, len);
 
@@ -435,6 +438,7 @@
   // the dominant fast-path is to simply return.
   // Relatedly, it's critical that notify/notifyAll be fast in order to
   // reduce lock hold times.
+  obj = oopDesc::bs()->resolve_and_maybe_copy_oop(obj);
   if (!SafepointSynchronize::is_synchronizing()) {
     if (ObjectSynchronizer::quick_notify(obj, thread, false)) {
       return;
@@ -453,6 +457,7 @@
 
 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread *thread))
 
+  obj = oopDesc::bs()->resolve_and_maybe_copy_oop(obj);
   if (!SafepointSynchronize::is_synchronizing() ) {
     if (ObjectSynchronizer::quick_notify(obj, thread, true)) {
       return;
@@ -591,6 +596,33 @@
   return TypeFunc::make(domain, range);
 }
 
+const TypeFunc *OptoRuntime::shenandoah_clone_barrier_Type() {
+  const Type **fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(0);
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc *OptoRuntime::shenandoah_cas_obj_Type() {
+  const Type **fields = TypeTuple::fields(3);
+  fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Address
+  fields[TypeFunc::Parms+1] = TypeInstPtr::BOTTOM;  // New value
+  fields[TypeFunc::Parms+2] = TypeInstPtr::BOTTOM;  // Expected value
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInt::BOOL;
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
 const TypeFunc *OptoRuntime::uncommon_trap_Type() {
   // create input type (domain)
   const Type **fields = TypeTuple::fields(1);
@@ -1520,6 +1552,37 @@
   return TypeFunc::make(domain,range);
 }
 
+/**
+ * oop resolve_and_maybe_copy_oop_static(oop obj)
+ * oop resolve_oop_static(oop obj)
+ */
+const TypeFunc* OptoRuntime::shenandoah_barrier_Type(const Type* type) {
+  // create input type (domain)
+  const Type** fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = type;
+
+  const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // result type needed
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = type;
+  const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc* OptoRuntime::shenandoah_write_barrier_Type() {
+  // create input type (domain)
+  const Type** fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL;
+
+  const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // result type needed
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL;
+  const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
+  return TypeFunc::make(domain, range);
+}
 
 JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer(oopDesc* obj, JavaThread* thread))
   assert(obj->is_oop(), "must be a valid oop");
diff --git a/src/share/vm/opto/runtime.hpp b/src/share/vm/opto/runtime.hpp
--- a/src/share/vm/opto/runtime.hpp
+++ b/src/share/vm/opto/runtime.hpp
@@ -141,6 +141,7 @@
   static address _multianewarray4_Java;
   static address _multianewarray5_Java;
   static address _multianewarrayN_Java;
+  static address _shenandoah_write_barrier_Java;
   static address _g1_wb_pre_Java;
   static address _g1_wb_post_Java;
   static address _vtable_must_compile_Java;
@@ -246,6 +247,7 @@
   static address multianewarray4_Java()                  { return _multianewarray4_Java; }
   static address multianewarray5_Java()                  { return _multianewarray5_Java; }
   static address multianewarrayN_Java()                  { return _multianewarrayN_Java; }
+  static address shenandoah_write_barrier_Java()         { return _shenandoah_write_barrier_Java; }
   static address g1_wb_pre_Java()                        { return _g1_wb_pre_Java; }
   static address g1_wb_post_Java()                       { return _g1_wb_post_Java; }
   static address vtable_must_compile_stub()              { return _vtable_must_compile_Java; }
@@ -290,6 +292,8 @@
   static const TypeFunc* multianewarrayN_Type(); // multianewarray
   static const TypeFunc* g1_wb_pre_Type();
   static const TypeFunc* g1_wb_post_Type();
+  static const TypeFunc* shenandoah_clone_barrier_Type();
+  static const TypeFunc* shenandoah_cas_obj_Type();
   static const TypeFunc* complete_monitor_enter_Type();
   static const TypeFunc* complete_monitor_exit_Type();
   static const TypeFunc* monitor_notify_Type();
@@ -353,6 +357,10 @@
   static const TypeFunc* dtrace_method_entry_exit_Type();
   static const TypeFunc* dtrace_object_alloc_Type();
 
+  // Shenandoah support
+  static const TypeFunc* shenandoah_barrier_Type(const Type* type);
+  static const TypeFunc* shenandoah_write_barrier_Type();
+
 # ifdef ENABLE_ZAP_DEAD_LOCALS
   static const TypeFunc* zap_dead_locals_Type();
 # endif
diff --git a/src/share/vm/opto/shenandoahSupport.cpp b/src/share/vm/opto/shenandoahSupport.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/opto/shenandoahSupport.cpp
@@ -0,0 +1,603 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "opto/callnode.hpp"
+#include "opto/movenode.hpp"
+#include "opto/phaseX.hpp"
+#include "opto/shenandoahSupport.hpp"
+
+Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
+  if (n->is_ShenandoahBarrier()) {
+    return n->in(ValueIn);
+  } else {
+    return n;
+  }
+}
+
+bool ShenandoahBarrierNode::needs_barrier(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem) {
+  Unique_Node_List visited;
+  return needs_barrier_impl(phase, orig, n, rb_mem, visited);
+}
+
+bool ShenandoahBarrierNode::needs_barrier_impl(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem, Unique_Node_List &visited) {
+
+  if (visited.member(n)) {
+    return false; // Been there.
+  }
+  visited.push(n);
+
+  if (n->is_Allocate()) {
+    // tty->print_cr("killed barrier for newly allocated object");
+    return false;
+  }
+
+  if (n->is_CallJava()) {
+    return true;
+  }
+
+  const Type* type = phase->type(n);
+  if (type->higher_equal(TypePtr::NULL_PTR)) {
+    // tty->print_cr("killed barrier for NULL object");
+    return false;
+  }
+  if (type->isa_oopptr() && type->is_oopptr()->const_oop() != NULL) {
+    // tty->print_cr("killed barrier for constant object");
+    return false;
+  }
+
+  if (n->is_CheckCastPP() || n->is_ConstraintCast()) {
+    return needs_barrier_impl(phase, orig, n->in(1), rb_mem, visited);
+  }
+  if (n->is_Parm()) {
+    return true;
+  }
+  if (n->is_Proj()) {
+    return needs_barrier_impl(phase, orig, n->in(0), rb_mem, visited);
+  }
+  if (n->is_Phi()) {
+    bool need_barrier = false;
+    for (uint i = 1; i < n->req() && ! need_barrier; i++) {
+      Node* input = n->in(i);
+      if (input == NULL) {
+        need_barrier = true; // Phi not complete yet?
+      } else if (needs_barrier_impl(phase, orig, input, rb_mem, visited)) {
+        need_barrier = true;
+      }
+    }
+    return need_barrier;
+  }
+  if (n->is_CMove()) {
+    return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, visited) ||
+           needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, visited);
+  }
+  if (n->Opcode() == Op_CreateEx) {
+    return true;
+  }
+  if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+    // tty->print_cr("skipped barrier for chained write barrier object");
+    return false;
+  }
+  if (n->Opcode() == Op_ShenandoahReadBarrier) {
+    if (rb_mem == n->in(Memory)) {
+      // tty->print_cr("Eliminated chained read barrier");
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  if (n->Opcode() == Op_LoadP) {
+    return true;
+  }
+#ifdef ASSERT
+  tty->print("need barrier on?: "); n->dump();
+#endif
+  return true;
+}
+
+bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseTransform* phase,
+                                                         Node* b1,
+                                                         Node* b2,
+                                                         Node* current,
+                                                         Unique_Node_List &visited) {
+  if (current == NULL) {
+    return false; // Incomplete phi. Try again later.
+  } else if (visited.member(current)) {
+    // We have already seen it.
+    return true;
+  }
+  visited.push(current);
+
+  if (current == b1) {
+    return true;
+  } else if (current == phase->C->immutable_memory()) {
+    return false;
+  } else if (current->isa_Phi()) {
+    bool dominates = true;
+    for (uint i = 1; i < current->req() && dominates == true; i++) {
+      Node* in = current->in(i);
+      dominates = dominates && dominates_memory_rb_impl(phase, b1, b2, in, visited);
+    }
+    return dominates;
+  } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
+    const Type* in_type = current->bottom_type();
+    const Type* this_type = b2->bottom_type();
+    if (is_independent(in_type, this_type)) {
+      Node* in = current->in(Memory);
+      return dominates_memory_rb_impl(phase, b1, b2, in, visited);
+    } else {
+      return false;
+    }
+  } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
+    Node* in = current->in(0);
+    return dominates_memory_rb_impl(phase, b1, b2, in, visited);
+  } else if (current->is_top()) {
+    return true; // Dead path
+  } else if (current->is_Proj()) {
+    return dominates_memory_rb_impl(phase, b1, b2, current->in(0), visited);
+  } else if (current->is_Call()) {
+    return false; // TODO: Maybe improve by looking at the call's memory effects?
+  } else if (current->is_MemBar()) {
+    return false; // TODO: Do we need to stop at *any* membar?
+  } else if (current->is_MergeMem()) {
+    // if (true) return false;
+    // tty->print_cr("current == mergemem: "); current->dump();
+    const TypePtr* adr_type = phase->type(b2)->is_ptr()->add_offset(-8);
+    uint alias_idx = phase->C->get_alias_index(adr_type);
+    Node* mem_in = current->as_MergeMem()->memory_at(alias_idx);
+    return dominates_memory_rb_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited);
+  } else {
+    // tty->print_cr("what else can we see here:");
+#ifdef ASSERT
+    current->dump();
+#endif
+    ShouldNotReachHere();
+    return false;
+  }
+}
+
+bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseTransform* phase, Node* b1, Node* b2) {
+  Unique_Node_List visited;
+  return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), visited);
+}
+
+bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) const {
+  assert(in_type->isa_oopptr(), "expect oop ptr");
+  assert(this_type->isa_oopptr(), "expect oop ptr");
+  /*
+  if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) {
+#ifdef ASSERT
+    tty->print_cr("not oopptr");
+    tty->print("in:   "); in_type->dump(); tty->print_cr(" ");
+    tty->print("this: "); this_type->dump(); tty->print_cr(" ");
+#endif
+    return false;
+  }
+  */
+
+  ciKlass* in_kls = in_type->is_oopptr()->klass();
+  ciKlass* this_kls = this_type->is_oopptr()->klass();
+  if ((!in_kls->is_subclass_of(this_kls)) &&
+      (!this_kls->is_subclass_of(in_kls))) {
+#ifdef ASSERT
+    // tty->print_cr("independent: ");
+    // tty->print("in:   "); in_kls->print(); tty->print_cr(" ");
+    // tty->print("this: "); this_kls->print(); tty->print_cr(" ");
+#endif
+    return true;
+  }
+#ifdef ASSERT
+  // tty->print_cr("possibly dependend?");
+  // tty->print("in:   "); in_type->dump(); tty->print_cr(" ");
+  // tty->print("this: "); this_type->dump(); tty->print_cr(" ");
+#endif
+  return false;
+}
+
+Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+
+  if (! can_reshape) {
+    return NULL;
+  }
+
+  if (in(Memory) == phase->C->immutable_memory()) return NULL;
+
+  // If memory input is a MergeMem, take the appropriate slice out of it.
+  Node* mem_in = in(Memory);
+  if (mem_in->isa_MergeMem()) {
+    const TypePtr* adr_type = bottom_type()->is_ptr()->add_offset(-8);
+    uint alias_idx = phase->C->get_alias_index(adr_type);
+    mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
+    set_req(Memory, mem_in);
+    return this;
+  }
+
+  Node* input = in(Memory);
+  if (input->Opcode() == Op_ShenandoahWBMemProj) {
+    input = input->in(0);
+    if (input->is_top()) return NULL; // Dead path.
+    assert(input->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
+    const Type* in_type = phase->type(input);
+    const Type* this_type = phase->type(this);
+    if (is_independent(in_type, this_type)) {
+      phase->igvn_rehash_node_delayed(input);
+      set_req(Memory, input->in(Memory));
+      return this;
+    }
+  }
+  return NULL;
+}
+
+bool ShenandoahBarrierNode::has_barrier_users(Node* n, Unique_Node_List &visited) {
+  if (visited.member(n)) {
+    return false;
+  }
+  visited.push(n);
+
+  bool has_users = false;
+  for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax && ! has_users; j++) {
+    Node* o = n->fast_out(j);
+    if (o->Opcode() == Op_ShenandoahReadBarrier ||
+        o->Opcode() == Op_ShenandoahWriteBarrier) {
+      // tty->print("counting barrier"); o->dump();
+      has_users = true;
+    } else if (o->isa_Phi()) {
+      has_users = has_barrier_users(o, visited);
+    } else if (o->Opcode() == Op_MergeMem) {
+      // Not a user. ?
+    } else {
+      // tty->print_cr("unknown user"); o->dump();
+      ShouldNotReachHere();
+    }
+  }
+  return has_users;
+}
+
+Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+
+  if (! can_reshape) return NULL;
+
+  if (in(Memory) == phase->C->immutable_memory()) return NULL;
+
+  Node* mem_in = in(Memory);
+  if (mem_in->isa_MergeMem()) {
+    const TypePtr* adr_type = bottom_type()->is_ptr()->add_offset(-8);
+    uint alias_idx = phase->C->get_alias_index(adr_type);
+    mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
+    set_req(Memory, mem_in);
+    return this;
+  }
+
+  Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj);
+  if (mem_proj == NULL) {
+    // tty->print_cr("no mem proj: kill input mem"); dump();
+    set_req(Memory, phase->C->immutable_memory());
+    return this;
+  }
+
+  // if (true) return NULL;
+
+  Unique_Node_List visited;
+  if (! has_barrier_users(mem_proj, visited)) {
+    // tty->print_cr("no users of mem projection. kill input mem"); dump();
+    phase->igvn_rehash_node_delayed(in(Memory));
+    set_req(Memory, phase->C->immutable_memory());
+
+    // tty->print_cr("reshaped wb: ");
+    // dump();
+    return this;
+  }
+  // tty->print_cr("leave mem proj alone");
+  return NULL;
+}
+/*
+bool ShenandoahBarrierNode::dominates_control_impl(PhaseTransform* phase,
+                                                   Node* c1,
+                                                   Node* c2,
+                                                   Node* current,
+                                                   Unique_Node_List & visited) {
+  if (current == NULL) {
+    return true;
+  } else if (visited.member(current)) {
+    return true;
+  }
+  visited.push(current);
+
+  current->dump();
+  ShouldNotReachHere();
+}
+*/
+
+bool ShenandoahBarrierNode::dominates_control(PhaseTransform* phase,
+                                              Node* c1,
+                                              Node* c2) {
+  if (c1 == c2) {
+    return true;
+  }
+  if (c1 == NULL) {
+    return true;
+  }
+#ifdef ASSERT
+  tty->print("c1: "); c1->dump(2);
+  tty->print("c2: "); c2->dump(2);
+#endif
+  ShouldNotReachHere();
+  return false;
+}
+
+bool ShenandoahBarrierNode::dominates_memory_impl(PhaseTransform* phase,
+                                                  Node* b1,
+                                                  Node* b2,
+                                                  Node* current,
+                                                  Unique_Node_List &visited) {
+  /*
+  tty->print_cr("phistack:");
+  for (int x = 0; x < phistack->length(); x++) {
+    tty->print("-->");
+    phistack->at(x)->dump();
+  }
+  */
+  if (current == NULL) {
+    return false;
+  } else if (visited.member(current)) {
+    // We have already seen it.
+    return true;
+  }
+
+  visited.push(current);
+
+  if (current == b1) {
+    // tty->print_cr("current == b1: "); current->dump();
+    return true;
+  } else if (current == b2) {
+    // tty->print_cr("current == b2: "); current->dump();
+    return false;
+  } else if (current == phase->C->immutable_memory()) {
+    // tty->print_cr("current == immutable_memory: "); current->dump();
+    return false;
+  } else if (current->isa_Phi()) {
+    // tty->print_cr("current == phi: "); current->dump();
+    bool dominates = true;
+    for (uint i = 1; i < current->req() && dominates == true; i++) {
+      Node* in = current->in(i);
+      dominates = dominates && dominates_memory_impl(phase, b1, b2, in, visited);
+    }
+    return dominates;
+  } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
+    // tty->print_cr("current == wb: "); current->dump();
+    // Follow through memory input.
+    Node* in = current->in(Memory);
+    return dominates_memory_impl(phase, b1, b2, in, visited);
+  } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
+    // tty->print_cr("current == wb-memproj: "); current->dump();
+    // Follow through memory input.
+    Node* in = current->in(0);
+    return dominates_memory_impl(phase, b1, b2, in, visited);
+  } else if (current->is_top()) {
+    return true; // Dead path
+  } else if (current->is_Proj()) {
+    // tty->print_cr("current == proj: "); current->dump();
+    return dominates_memory_impl(phase, b1, b2, current->in(0), visited);
+  } else if (current->is_Call()) {
+    // tty->print_cr("current == call: "); current->dump();
+    return dominates_memory_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited);
+  } else if (current->is_MemBar()) {
+    // tty->print_cr("current == membar: "); current->dump();
+    return dominates_memory_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited);
+  } else if (current->is_MergeMem()) {
+    // tty->print_cr("current == mergemem: "); current->dump();
+    const TypePtr* adr_type = phase->type(b2)->is_ptr()->add_offset(-8);
+    uint alias_idx = phase->C->get_alias_index(adr_type);
+    Node* mem_in = current->as_MergeMem()->memory_at(alias_idx);
+    return dominates_memory_impl(phase, b1, b2, current->in(TypeFunc::Memory), visited);
+  } else {
+    // tty->print_cr("what else can we see here:");
+#ifdef ASSERT
+    current->dump();
+#endif
+    ShouldNotReachHere();
+    return false;
+  }
+}
+
+/**
+ * Determines if b1 dominates b2 through memory inputs. It returns true if:
+ * - b1 can be reached by following each branch in b2's memory input (through phis, etc)
+ * - or we get back to b2 (i.e. through a loop) without seeing b1
+ * In all other cases, (in particular, if we reach immutable_memory without having seen b1)
+ * we return false.
+ */
+bool ShenandoahBarrierNode::dominates_memory(PhaseTransform* phase, Node* b1, Node* b2) {
+  Unique_Node_List visited;
+  return dominates_memory_impl(phase, b1->in(Memory), b2, b2->in(Memory), visited);
+}
+
+Node* ShenandoahBarrierNode::Identity_impl(PhaseTransform* phase) {
+  Node* n = in(ValueIn);
+
+  Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL;
+  if (! needs_barrier(phase, this, n, rb_mem)) {
+    return n;
+  }
+
+  // tty->print_cr("find sibling for: "); dump(2);
+  // Try to find a write barrier sibling with identical inputs that we can fold into.
+  for (DUIterator i = n->outs(); n->has_out(i); i++) {
+    Node* sibling = n->out(i);
+    if (sibling == this) {
+      continue;
+    }
+    /*
+    assert(sibling->Opcode() != Op_ShenandoahWriteBarrier ||
+           Opcode() != Op_ShenandoahWriteBarrier || hash() == sibling->hash(),
+           "if this is a write barrier, then sibling can't be write barrier too");
+    */
+    if (sibling->Opcode() != Op_ShenandoahWriteBarrier) {
+      continue;
+    }
+    /*
+    if (sibling->outcnt() == 0) {
+      // Some dead node.
+      continue;
+    }
+    */
+    assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
+    assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity");
+    // tty->print_cr("candidate: "); sibling->dump();
+
+    if (dominates_control(phase, sibling->in(Control), in(Control)) &&
+        dominates_memory(phase, sibling, this)) {
+
+      /*
+      tty->print_cr("matched barrier:");
+      sibling->dump();
+      tty->print_cr("for: ");
+      dump();
+      */
+      return sibling;
+    }
+
+    /*
+    tty->print_cr("couldn't match candidate:");
+    sibling->dump(2);
+    */
+  }
+  /*
+  tty->print_cr("couldn't match barrier to any:");
+  dump();
+  */
+  return this;
+}
+
+Node* ShenandoahBarrierNode::Identity(PhaseTransform* phase) {
+
+  Node* replacement = Identity_impl(phase);
+  if (replacement != this) {
+    // If we have a memory projection, we first need to make it go away.
+    Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj);
+    if (mem_proj != NULL) {
+      phase->igvn_rehash_node_delayed(mem_proj);
+      return this;
+    }
+  }
+  return replacement;
+}
+
+Node* ShenandoahReadBarrierNode::Identity(PhaseTransform* phase) {
+
+  // if (true) return this;
+
+  // tty->print("optimizing rb: "); dump();
+  Node* id = ShenandoahBarrierNode::Identity(phase);
+
+  if (id == this && phase->is_IterGVN()) {
+    Node* n = in(ValueIn);
+    // No success in super call. Try to combine identical read barriers.
+    for (DUIterator i = n->outs(); n->has_out(i); i++) {
+      Node* sibling = n->out(i);
+      if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) {
+        continue;
+      }
+      assert(sibling->in(ValueIn)  == in(ValueIn), "sanity");
+      if (phase->is_IterGVN()->hash_find(sibling) &&
+          sibling->bottom_type() == bottom_type() &&
+          sibling->in(Control) == in(Control) &&
+          dominates_memory_rb(phase, sibling, this)) {
+        /*
+        if (in(Memory) != sibling->in(Memory)) {
+          tty->print_cr("interesting rb-fold");
+          dump();
+          sibling->dump();
+        }
+        */
+        return sibling;
+      }
+    }
+  }
+  return id;
+}
+
+const Type* ShenandoahBarrierNode::Value(PhaseTransform* phase) const {
+  // Either input is TOP ==> the result is TOP
+  const Type *t1 = phase->type(in(Memory));
+  if (t1 == Type::TOP) return Type::TOP;
+  const Type *t2 = phase->type(in(ValueIn));
+  if( t2 == Type::TOP ) return Type::TOP;
+
+  Node* input = in(ValueIn);
+  const Type* type = phase->type(input);
+  return type;
+}
+
+#ifdef ASSERT
+uint ShenandoahBarrierNode::num_mem_projs() {
+  uint num_mem_proj = 0;
+  for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
+    Node* use = fast_out(i);
+    if (use->Opcode() == Op_ShenandoahWBMemProj) {
+      num_mem_proj++;
+    }
+  }
+  return num_mem_proj;
+}
+
+void ShenandoahBarrierNode::check_invariants() {
+  // if (Opcode() == Op_ShenandoahWriteBarrier) {
+  //   uint mem_projs = num_mem_projs();
+  //   if (mem_projs > 1) {
+  //     tty->print_cr("num_mem_proj: %u", mem_projs);
+  //     dump(2);
+  //   }
+  //   assert(mem_projs <= 1, "exactly 1 memory projection");
+  // }
+}
+#endif
+
+Node* ShenandoahWBMemProjNode::Identity(PhaseTransform* phase) {
+
+  Node* wb = in(0);
+  if (wb->is_top()) return phase->C->top(); // Dead path.
+
+  assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
+  if (wb->as_ShenandoahBarrier()->Identity_impl(phase) != wb) {
+    // If the parent write barrier would go away, make this mem proj go away first.
+    // Poke parent to give it a chance to go away too.
+    phase->igvn_rehash_node_delayed(wb);
+    return wb->in(ShenandoahBarrierNode::Memory);
+  }
+
+  // We can't do the below unless the graph is fully constructed.
+  if (! phase->is_IterGVN()) {
+    return this;
+  }
+
+  // If the mem projection has no barrier users, it's not needed anymore.
+  Unique_Node_List visited;
+  if (! ShenandoahWriteBarrierNode::has_barrier_users(this, visited)) {
+    // tty->print_cr("mem proj has no users. kill"); dump();
+    phase->igvn_rehash_node_delayed(wb);
+    return wb->in(ShenandoahBarrierNode::Memory);
+  }
+
+  return this;
+}
diff --git a/src/share/vm/opto/shenandoahSupport.hpp b/src/share/vm/opto/shenandoahSupport.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/opto/shenandoahSupport.hpp
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_OPTO_SHENANDOAH_SUPPORT_HPP
+#define SHARE_VM_OPTO_SHENANDOAH_SUPPORT_HPP
+
+#include "memory/allocation.hpp"
+#include "opto/addnode.hpp"
+#include "opto/memnode.hpp"
+#include "opto/multnode.hpp"
+#include "opto/node.hpp"
+
+class PhaseTransform;
+
+
+class ShenandoahBarrierNode : public TypeNode {
+public:
+
+public:
+  enum { Control,
+         Memory,
+         ValueIn
+  };
+
+  ShenandoahBarrierNode(Node* ctrl, Node* mem, Node* obj)
+    : TypeNode(obj->bottom_type(), 3) {
+
+    init_req(Control, ctrl);
+    init_req(Memory, mem);
+    init_req(ValueIn, obj);
+
+    init_class_id(Class_ShenandoahBarrier);
+  }
+
+  static Node* skip_through_barrier(Node* n);
+
+  virtual const class TypePtr* adr_type() const {
+    //const TypePtr* adr_type = in(MemNode::Address)->bottom_type()->is_ptr();
+    const TypePtr* adr_type = bottom_type()->is_ptr()->add_offset(-8);
+    assert(adr_type->offset() == -8, "sane offset");
+    assert(Compile::current()->alias_type(adr_type)->is_rewritable(), "brooks ptr must be rewritable");
+    return adr_type;
+  }
+
+  virtual uint  ideal_reg() const { return Op_RegP; }
+  virtual uint match_edge(uint idx) const {
+    return idx >= ValueIn;
+  }
+  virtual uint size_of() const { return sizeof(*this); }
+
+  virtual Node* Identity(PhaseTransform* phase);
+  Node* Identity_impl(PhaseTransform* phase);
+
+  virtual const Type* Value(PhaseTransform* phase) const;
+  virtual bool depends_only_on_test() const {
+    return true;
+  };
+#ifdef ASSERT
+  void check_invariants();
+  uint num_mem_projs();
+#endif
+
+  static bool needs_barrier(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem);
+
+  static bool has_barrier_users(Node* n, Unique_Node_List &visited);
+
+private:
+  static bool needs_barrier_impl(PhaseTransform* phase, Node* orig, Node* n, Node* rb_mem, Unique_Node_List &visited);
+
+
+  bool dominates_control(PhaseTransform* phase, Node* c1, Node* c2);
+  bool dominates_memory(PhaseTransform* phase, Node* b1, Node* b2);
+  bool dominates_memory_impl(PhaseTransform* phase, Node* b1, Node* b2, Node* current, Unique_Node_List &visisted);
+};
+
+class ShenandoahReadBarrierNode : public ShenandoahBarrierNode {
+public:
+  ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj)
+    : ShenandoahBarrierNode(ctrl, mem, obj) {
+  }
+
+  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
+  virtual Node* Identity(PhaseTransform* phase);
+  virtual int Opcode() const;
+
+private:
+  bool is_independent(const Type* in_type, const Type* this_type) const;
+  bool dominates_memory_rb(PhaseTransform* phase, Node* b1, Node* b2);
+  bool dominates_memory_rb_impl(PhaseTransform* phase, Node* b1, Node* b2, Node* current, Unique_Node_List &visited);
+};
+
+class ShenandoahWriteBarrierNode : public ShenandoahBarrierNode {
+public:
+  ShenandoahWriteBarrierNode(Node* ctrl, Node* mem, Node* obj)
+    : ShenandoahBarrierNode(ctrl, mem, obj) {
+    //tty->print("new wb: "); dump();
+  }
+
+  virtual int Opcode() const;
+  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
+
+  // virtual void set_req( uint i, Node *n ) {
+  //   if (i == MemNode::Memory) { assert(n == Compiler::current()->immutable_memory(), "set only immutable mem on wb"); }
+  //   Node::set_req(i, n);
+  // }
+};
+
+class ShenandoahWBMemProjNode : public ProjNode {
+public:
+  enum {SWBMEMPROJCON = (uint)-3};
+  ShenandoahWBMemProjNode(Node *src) : ProjNode( src, SWBMEMPROJCON) {
+    assert(src->Opcode() == Op_ShenandoahWriteBarrier || src->is_Mach(), "epxect wb");
+#ifdef ASSERT
+    in(0)->as_ShenandoahBarrier()->check_invariants();
+#endif
+  }
+  virtual Node* Identity(PhaseTransform* phase);
+
+  virtual int Opcode() const;
+  virtual bool      is_CFG() const  { return false; }
+  virtual const Type *bottom_type() const {return Type::MEMORY;}
+  virtual const TypePtr *adr_type() const {
+    Node* ctrl = in(0);
+    if (ctrl == NULL)  return NULL; // node is dead
+    assert(ctrl->Opcode() == Op_ShenandoahWriteBarrier || ctrl->is_Mach(), "expect wb");
+    return ctrl->adr_type();
+  }
+
+  virtual uint ideal_reg() const { return 0;} // memory projections don't have a register
+  virtual const Type *Value( PhaseTransform *phase ) const {
+    return bottom_type();
+  }
+#ifndef PRODUCT
+  virtual void dump_spec(outputStream *st) const {};
+#endif
+};
+
+#endif // SHARE_VM_OPTO_SHENANDOAH_SUPPORT_HPP
diff --git a/src/share/vm/opto/split_if.cpp b/src/share/vm/opto/split_if.cpp
--- a/src/share/vm/opto/split_if.cpp
+++ b/src/share/vm/opto/split_if.cpp
@@ -32,6 +32,7 @@
 //------------------------------split_thru_region------------------------------
 // Split Node 'n' through merge point.
 Node *PhaseIdealLoop::split_thru_region( Node *n, Node *region ) {
+  assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers");
   uint wins = 0;
   assert( n->is_CFG(), "" );
   assert( region->is_Region(), "" );
@@ -73,7 +74,8 @@
   if( n->is_Phi() ) return false; // Local PHIs are expected
 
   // Recursively split-up inputs
-  for (uint i = 1; i < n->req(); i++) {
+  uint first_input = n->Opcode() == Op_ShenandoahWBMemProj ? 0 : 1;
+  for (uint i = first_input; i < n->req(); i++) {
     if( split_up( n->in(i), blk1, blk2 ) ) {
       // Got split recursively and self went dead?
       if (n->outcnt() == 0)
@@ -216,6 +218,7 @@
   register_new_node(phi, blk1);
 
   // Remove cloned-up value from optimizer; use phi instead
+  split_mem_thru_phi(n, blk1, phi);
   _igvn.replace_node( n, phi );
 
   // (There used to be a self-recursive call to split_up() here,
diff --git a/src/share/vm/opto/stringopts.cpp b/src/share/vm/opto/stringopts.cpp
--- a/src/share/vm/opto/stringopts.cpp
+++ b/src/share/vm/opto/stringopts.cpp
@@ -1603,6 +1603,7 @@
       char_alloc->maybe_set_complete(_gvn);
 
       // Now copy the string representations into the final char[]
+      char_array = __ shenandoah_write_barrier(char_array);
       Node* start = __ intcon(0);
       for (int argi = 0; argi < sc->num_arguments(); argi++) {
         Node* arg = sc->argument(argi);
diff --git a/src/share/vm/opto/type.cpp b/src/share/vm/opto/type.cpp
--- a/src/share/vm/opto/type.cpp
+++ b/src/share/vm/opto/type.cpp
@@ -2832,7 +2832,7 @@
 
         if (klass() == ciEnv::current()->Class_klass() &&
             (_offset == java_lang_Class::klass_offset_in_bytes() ||
-             _offset == java_lang_Class::array_klass_offset_in_bytes())) {
+             _offset == java_lang_Class::array_klass_offset_in_bytes()) || UseShenandoahGC) {
           // Special hidden fields from the Class.
           assert(this->isa_instptr(), "must be an instance ptr.");
           _is_ptr_to_narrowoop = false;
diff --git a/src/share/vm/prims/jni.cpp b/src/share/vm/prims/jni.cpp
--- a/src/share/vm/prims/jni.cpp
+++ b/src/share/vm/prims/jni.cpp
@@ -336,7 +336,7 @@
   }
   ResourceMark rm(THREAD);
   ClassFileStream st((u1*) buf, bufLen, NULL);
-  Handle class_loader (THREAD, JNIHandles::resolve(loaderRef));
+  Handle class_loader (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(loaderRef)));
 
   if (UsePerfData && !class_loader.is_null()) {
     // check whether the current caller thread holds the lock or not.
@@ -575,6 +575,10 @@
 
   oop sub_mirror   = JNIHandles::resolve_non_null(sub);
   oop super_mirror = JNIHandles::resolve_non_null(super);
+
+  sub_mirror = oopDesc::bs()->resolve_and_maybe_copy_oop(sub_mirror);
+  super_mirror = oopDesc::bs()->resolve_and_maybe_copy_oop(super_mirror);
+
   if (java_lang_Class::is_primitive(sub_mirror) ||
       java_lang_Class::is_primitive(super_mirror)) {
     jboolean ret = (sub_mirror == super_mirror);
@@ -817,7 +821,9 @@
   HOTSPOT_JNI_ISSAMEOBJECT_ENTRY(env, r1, r2);
 
   oop a = JNIHandles::resolve(r1);
+  a = oopDesc::bs()->resolve_and_maybe_copy_oop(a);
   oop b = JNIHandles::resolve(r2);
+  b = oopDesc::bs()->resolve_and_maybe_copy_oop(b);
   jboolean ret = (a == b) ? JNI_TRUE : JNI_FALSE;
 
   HOTSPOT_JNI_ISSAMEOBJECT_RETURN(ret);
@@ -2062,7 +2068,7 @@
   // If G1 is enabled and we are accessing the value of the referent
   // field in a reference object then we need to register a non-null
   // referent with the SATB barrier.
-  if (UseG1GC) {
+  if (UseG1GC || UseShenandoahGC) {
     bool needs_barrier = false;
 
     if (ret != NULL &&
@@ -2467,7 +2473,7 @@
  HOTSPOT_JNI_GETSTRINGCHARS_ENTRY(env, string, (uintptr_t *) isCopy);
   jchar* buf = NULL;
   oop s = JNIHandles::resolve_non_null(string);
-  typeArrayOop s_value = java_lang_String::value(s);
+  typeArrayOop s_value = typeArrayOop(oopDesc::bs()->resolve_oop(java_lang_String::value(s)));
   if (s_value != NULL) {
     int s_len = java_lang_String::length(s);
     int s_offset = java_lang_String::offset(s);
@@ -2566,7 +2572,7 @@
 JNI_QUICK_ENTRY(jsize, jni_GetArrayLength(JNIEnv *env, jarray array))
   JNIWrapper("GetArrayLength");
  HOTSPOT_JNI_GETARRAYLENGTH_ENTRY(env, array);
-  arrayOop a = arrayOop(JNIHandles::resolve_non_null(array));
+  arrayOop a = arrayOop(oopDesc::bs()->resolve_oop(JNIHandles::resolve_non_null(array)));
   assert(a->is_array(), "must be array");
   jsize ret = a->length();
  HOTSPOT_JNI_GETARRAYLENGTH_RETURN(ret);
@@ -2715,6 +2721,7 @@
   EntryProbe; \
   /* allocate an chunk of memory in c land */ \
   typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(array)); \
+  a = typeArrayOop(oopDesc::bs()->resolve_oop(a)); \
   ElementType* result; \
   int len = a->length(); \
   if (len == 0) { \
@@ -2773,6 +2780,7 @@
   JNIWrapper("Release" XSTR(Result) "ArrayElements"); \
   EntryProbe; \
   typeArrayOop a = typeArrayOop(JNIHandles::resolve_non_null(array)); \
+  a = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(a)); \
   int len = a->length(); \
   if (len != 0) {   /* Empty array:  nothing to free or copy. */  \
     if ((mode == 0) || (mode == JNI_COMMIT)) { \
@@ -2823,6 +2831,7 @@
   EntryProbe; \
   DT_VOID_RETURN_MARK(Get##Result##ArrayRegion); \
   typeArrayOop src = typeArrayOop(JNIHandles::resolve_non_null(array)); \
+  src = typeArrayOop(oopDesc::bs()->resolve_oop(src)); \
   if (start < 0 || len < 0 || ((unsigned int)start + (unsigned int)len > (unsigned int)src->length())) { \
     THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); \
   } else { \
@@ -2873,6 +2882,7 @@
   EntryProbe; \
   DT_VOID_RETURN_MARK(Set##Result##ArrayRegion); \
   typeArrayOop dst = typeArrayOop(JNIHandles::resolve_non_null(array)); \
+  dst = typeArrayOop(oopDesc::bs()->resolve_and_maybe_copy_oop(dst)); \
   if (start < 0 || len < 0 || ((unsigned int)start + (unsigned int)len > (unsigned int)dst->length())) { \
     THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); \
   } else { \
@@ -3071,7 +3081,7 @@
     THROW_(vmSymbols::java_lang_NullPointerException(), JNI_ERR);
   }
 
-  Handle obj(thread, JNIHandles::resolve_non_null(jobj));
+  Handle obj(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve_non_null(jobj)));
   ObjectSynchronizer::jni_enter(obj, CHECK_(JNI_ERR));
   ret = JNI_OK;
   return ret;
@@ -3090,7 +3100,7 @@
     THROW_(vmSymbols::java_lang_NullPointerException(), JNI_ERR);
   }
 
-  Handle obj(THREAD, JNIHandles::resolve_non_null(jobj));
+  Handle obj(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve_non_null(jobj)));
   ObjectSynchronizer::jni_exit(obj(), CHECK_(JNI_ERR));
 
   ret = JNI_OK;
@@ -3109,6 +3119,7 @@
  HOTSPOT_JNI_GETSTRINGREGION_ENTRY(env, string, start, len, buf);
   DT_VOID_RETURN_MARK(GetStringRegion);
   oop s = JNIHandles::resolve_non_null(string);
+  s = oopDesc::bs()->resolve_oop(s);
   int s_len = java_lang_String::length(s);
   if (start < 0 || len < 0 || start + len > s_len) {
     THROW(vmSymbols::java_lang_StringIndexOutOfBoundsException());
@@ -3116,6 +3127,7 @@
     if (len > 0) {
       int s_offset = java_lang_String::offset(s);
       typeArrayOop s_value = java_lang_String::value(s);
+      s_value = typeArrayOop(oopDesc::bs()->resolve_oop(s_value));
       memcpy(buf, s_value->char_at_addr(s_offset+start), sizeof(jchar)*len);
     }
   }
@@ -3156,6 +3168,7 @@
     *isCopy = JNI_FALSE;
   }
   oop a = JNIHandles::resolve_non_null(array);
+  a = oopDesc::bs()->resolve_and_maybe_copy_oop(a);
   assert(a->is_array(), "just checking");
   BasicType type;
   if (a->is_objArray()) {
@@ -3188,6 +3201,7 @@
   oop s = JNIHandles::resolve_non_null(string);
   int s_len = java_lang_String::length(s);
   typeArrayOop s_value = java_lang_String::value(s);
+  s_value = typeArrayOop(oopDesc::bs()->resolve_oop(s_value));
   int s_offset = java_lang_String::offset(s);
   const jchar* ret;
   if (s_len > 0) {
diff --git a/src/share/vm/prims/jvm.cpp b/src/share/vm/prims/jvm.cpp
--- a/src/share/vm/prims/jvm.cpp
+++ b/src/share/vm/prims/jvm.cpp
@@ -551,13 +551,15 @@
 JVM_ENTRY(jint, JVM_IHashCode(JNIEnv* env, jobject handle))
   JVMWrapper("JVM_IHashCode");
   // as implemented in the classic virtual machine; return 0 if object is NULL
-  return handle == NULL ? 0 : ObjectSynchronizer::FastHashCode (THREAD, JNIHandles::resolve_non_null(handle)) ;
+  return handle == NULL ? 0 : ObjectSynchronizer::FastHashCode (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve_non_null(handle))) ;
 JVM_END
 
 
 JVM_ENTRY(void, JVM_MonitorWait(JNIEnv* env, jobject handle, jlong ms))
   JVMWrapper("JVM_MonitorWait");
-  Handle obj(THREAD, JNIHandles::resolve_non_null(handle));
+  oop o = JNIHandles::resolve_non_null(handle);
+  o = oopDesc::bs()->resolve_and_maybe_copy_oop(o);
+  Handle obj(THREAD, o);
   JavaThreadInObjectWaitState jtiows(thread, ms != 0);
   if (JvmtiExport::should_post_monitor_wait()) {
     JvmtiExport::post_monitor_wait((JavaThread *)THREAD, (oop)obj(), ms);
@@ -574,21 +576,21 @@
 
 JVM_ENTRY(void, JVM_MonitorNotify(JNIEnv* env, jobject handle))
   JVMWrapper("JVM_MonitorNotify");
-  Handle obj(THREAD, JNIHandles::resolve_non_null(handle));
+  Handle obj(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve_non_null(handle)));
   ObjectSynchronizer::notify(obj, CHECK);
 JVM_END
 
 
 JVM_ENTRY(void, JVM_MonitorNotifyAll(JNIEnv* env, jobject handle))
   JVMWrapper("JVM_MonitorNotifyAll");
-  Handle obj(THREAD, JNIHandles::resolve_non_null(handle));
+  Handle obj(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve_non_null(handle)));
   ObjectSynchronizer::notifyall(obj, CHECK);
 JVM_END
 
 
 JVM_ENTRY(jobject, JVM_Clone(JNIEnv* env, jobject handle))
   JVMWrapper("JVM_Clone");
-  Handle obj(THREAD, JNIHandles::resolve_non_null(handle));
+  Handle obj(THREAD, oopDesc::bs()->resolve_oop(JNIHandles::resolve_non_null(handle)));
   const KlassHandle klass (THREAD, obj->klass());
   JvmtiVMObjectAllocEventCollector oam;
 
@@ -629,6 +631,7 @@
   // of oops.  We know objects are aligned on a minimum of an jlong boundary.
   // The same is true of StubRoutines::object_copy and the various oop_copy
   // variants, and of the code generated by the inline_native_clone intrinsic.
+  assert(new_obj_oop == oopDesc::bs()->resolve_oop(new_obj_oop), "expect to-space copy");
   assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
   Copy::conjoint_jlongs_atomic((jlong*)obj(), (jlong*)new_obj_oop,
                                (size_t)align_object_size(size) / HeapWordsPerLong);
@@ -895,7 +898,7 @@
 
   ResourceMark rm(THREAD);
   ClassFileStream st((u1*) buf, len, (char *)source);
-  Handle class_loader (THREAD, JNIHandles::resolve(loader));
+  Handle class_loader (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(loader)));
   if (UsePerfData) {
     is_lock_held_by_thread(class_loader,
                            ClassLoader::sync_JVMDefineClassLockFreeCounter(),
@@ -959,7 +962,7 @@
   // Security Note:
   //   The Java level wrapper will perform the necessary security check allowing
   //   us to pass the NULL as the initiating class loader.
-  Handle h_loader(THREAD, JNIHandles::resolve(loader));
+  Handle h_loader(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(loader)));
   if (UsePerfData) {
     is_lock_held_by_thread(h_loader,
                            ClassLoader::sync_JVMFindLoadedClassLockFreeCounter(),
@@ -3028,7 +3031,7 @@
   if (obj == NULL) {
     THROW_(vmSymbols::java_lang_NullPointerException(), JNI_FALSE);
   }
-  Handle h_obj(THREAD, JNIHandles::resolve(obj));
+  Handle h_obj(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)));
   return ObjectSynchronizer::current_thread_holds_lock((JavaThread*)THREAD, h_obj);
 JVM_END
 
diff --git a/src/share/vm/prims/jvmtiEnv.cpp b/src/share/vm/prims/jvmtiEnv.cpp
--- a/src/share/vm/prims/jvmtiEnv.cpp
+++ b/src/share/vm/prims/jvmtiEnv.cpp
@@ -467,7 +467,7 @@
     // lock the loader
     Thread* thread = Thread::current();
     HandleMark hm;
-    Handle loader_lock = Handle(thread, SystemDictionary::system_loader_lock());
+    Handle loader_lock = Handle(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(SystemDictionary::system_loader_lock()));
 
     ObjectLocker ol(loader_lock, thread);
 
@@ -513,7 +513,7 @@
 
     // lock the loader
     Thread* THREAD = Thread::current();
-    Handle loader = Handle(THREAD, SystemDictionary::java_system_loader());
+    Handle loader = Handle(THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(SystemDictionary::java_system_loader()));
 
     ObjectLocker ol(loader, THREAD);
 
diff --git a/src/share/vm/prims/unsafe.cpp b/src/share/vm/prims/unsafe.cpp
--- a/src/share/vm/prims/unsafe.cpp
+++ b/src/share/vm/prims/unsafe.cpp
@@ -121,6 +121,10 @@
   if (p != NULL) {
     assert(byte_offset >= 0 && byte_offset <= (jlong)MAX_OBJECT_SIZE, "sane offset");
     if (byte_offset == (jint)byte_offset) {
+      // We need to preemptively evacuate the object here to make the comparison
+      // in the assert below not give false negatives in case the object
+      // gets moved by concurrent threads while executing this code.
+      p = oopDesc::bs()->resolve_and_maybe_copy_oop(p);
       void* ptr_plus_disp = (address)p + byte_offset;
       assert((void*)p->obj_field_addr<oop>((jint)byte_offset) == ptr_plus_disp,
              "raw [ptr+disp] must be consistent with oop::field_base");
@@ -155,14 +159,17 @@
 
 #define GET_FIELD(obj, offset, type_name, v) \
   oop p = JNIHandles::resolve(obj); \
+  p = oopDesc::bs()->resolve_oop(p); \
   type_name v = *(type_name*)index_oop_from_field_offset_long(p, offset)
 
 #define SET_FIELD(obj, offset, type_name, x) \
   oop p = JNIHandles::resolve(obj); \
+  p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); \
   *(type_name*)index_oop_from_field_offset_long(p, offset) = x
 
 #define GET_FIELD_VOLATILE(obj, offset, type_name, v) \
   oop p = JNIHandles::resolve(obj); \
+  p = oopDesc::bs()->resolve_oop(p); \
   if (support_IRIW_for_not_multiple_copy_atomic_cpu) { \
     OrderAccess::fence(); \
   } \
@@ -170,6 +177,7 @@
 
 #define SET_FIELD_VOLATILE(obj, offset, type_name, x) \
   oop p = JNIHandles::resolve(obj); \
+  p = oopDesc::bs()->resolve_and_maybe_copy_oop(p); \
   OrderAccess::release_store_fence((volatile type_name*)index_oop_from_field_offset_long(p, offset), x);
 
 
@@ -181,6 +189,7 @@
 UNSAFE_ENTRY(jobject, Unsafe_GetObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset))
   UnsafeWrapper("Unsafe_GetObject");
   oop p = JNIHandles::resolve(obj);
+  p = oopDesc::bs()->resolve_oop(p);
   oop v;
   if (UseCompressedOops) {
     narrowOop n = *(narrowOop*)index_oop_from_field_offset_long(p, offset);
@@ -193,7 +202,7 @@
   // We could be accessing the referent field in a reference
   // object. If G1 is enabled then we need to register non-null
   // referent with the SATB barrier.
-  if (UseG1GC) {
+  if (UseG1GC || UseShenandoahGC) {
     bool needs_barrier = false;
 
     if (ret != NULL) {
@@ -218,8 +227,8 @@
 
 UNSAFE_ENTRY(void, Unsafe_SetObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h))
   UnsafeWrapper("Unsafe_SetObject");
-  oop x = JNIHandles::resolve(x_h);
-  oop p = JNIHandles::resolve(obj);
+  oop x = oopDesc::bs()->resolve_oop(JNIHandles::resolve(x_h));
+  oop p = oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj));
   if (UseCompressedOops) {
     oop_store((narrowOop*)index_oop_from_field_offset_long(p, offset), x);
   } else {
@@ -230,6 +239,7 @@
 UNSAFE_ENTRY(jobject, Unsafe_GetObjectVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset))
   UnsafeWrapper("Unsafe_GetObjectVolatile");
   oop p = JNIHandles::resolve(obj);
+  p = oopDesc::bs()->resolve_oop(p);
   void* addr = index_oop_from_field_offset_long(p, offset);
   volatile oop v;
   if (UseCompressedOops) {
@@ -246,6 +256,8 @@
   UnsafeWrapper("Unsafe_SetObjectVolatile");
   oop x = JNIHandles::resolve(x_h);
   oop p = JNIHandles::resolve(obj);
+  x = oopDesc::bs()->resolve_oop(x);
+  p = oopDesc::bs()->resolve_and_maybe_copy_oop(p);
   void* addr = index_oop_from_field_offset_long(p, offset);
   OrderAccess::release();
   if (UseCompressedOops) {
@@ -312,7 +324,7 @@
       return v;
     }
     else {
-      Handle p (THREAD, JNIHandles::resolve(obj));
+      Handle p (THREAD, oopDesc::bs()->resolve_oop(JNIHandles::resolve(obj)));
       jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset));
       MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag);
       jlong value = Atomic::load(addr);
@@ -328,7 +340,7 @@
       SET_FIELD_VOLATILE(obj, offset, jlong, x);
     }
     else {
-      Handle p (THREAD, JNIHandles::resolve(obj));
+      Handle p (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)));
       jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset));
       MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag);
       Atomic::store(x, addr);
@@ -435,6 +447,8 @@
   UnsafeWrapper("Unsafe_SetOrderedObject");
   oop x = JNIHandles::resolve(x_h);
   oop p = JNIHandles::resolve(obj);
+  x = oopDesc::bs()->resolve_oop(x);
+  p = oopDesc::bs()->resolve_and_maybe_copy_oop(p);
   void* addr = index_oop_from_field_offset_long(p, offset);
   OrderAccess::release();
   if (UseCompressedOops) {
@@ -456,7 +470,7 @@
       SET_FIELD_VOLATILE(obj, offset, jlong, x);
     }
     else {
-      Handle p (THREAD, JNIHandles::resolve(obj));
+      Handle p (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)));
       jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset));
       MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag);
       Atomic::store(x, addr);
@@ -635,6 +649,7 @@
     THROW(vmSymbols::java_lang_IllegalArgumentException());
   }
   oop base = JNIHandles::resolve(obj);
+  base = oopDesc::bs()->resolve_and_maybe_copy_oop(base);
   void* p = index_oop_from_field_offset_long(base, offset);
   Copy::fill_to_memory_atomic(p, sz, value);
 UNSAFE_END
@@ -650,6 +665,8 @@
   }
   oop srcp = JNIHandles::resolve(srcObj);
   oop dstp = JNIHandles::resolve(dstObj);
+  srcp = oopDesc::bs()->resolve_oop(srcp);
+  dstp = oopDesc::bs()->resolve_and_maybe_copy_oop(dstp);
   if (dstp != NULL && !dstp->is_typeArray()) {
     // NYI:  This works only for non-oop arrays at present.
     // Generalizing it would be reasonable, but requires card marking.
@@ -1082,12 +1099,23 @@
 
 UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSwapObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject e_h, jobject x_h))
   UnsafeWrapper("Unsafe_CompareAndSwapObject");
+  // We are about to write to this entry so check to see if we need to copy it.
+  oop p = oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj));
+  HeapWord* addr = (HeapWord *)index_oop_from_field_offset_long(p, offset);
   oop x = JNIHandles::resolve(x_h);
-  oop e = JNIHandles::resolve(e_h);
-  oop p = JNIHandles::resolve(obj);
-  HeapWord* addr = (HeapWord *)index_oop_from_field_offset_long(p, offset);
-  oop res = oopDesc::atomic_compare_exchange_oop(x, addr, e, true);
-  jboolean success  = (res == e);
+  x = oopDesc::bs()->resolve_oop(x);
+  oop old = JNIHandles::resolve(e_h);
+  jboolean success;
+  if (UseShenandoahGC) {
+    oop expected;
+    do {
+      expected = old;
+      old = oopDesc::atomic_compare_exchange_oop(x, addr, expected, true);
+      success  = (old == expected);
+    } while ((! success) && oopDesc::bs()->resolve_oop(old) == oopDesc::bs()->resolve_oop(expected));
+  } else {
+    success = (old == oopDesc::atomic_compare_exchange_oop(x, addr, old, true));
+  }
   if (success)
     update_barrier_set((void*)addr, x);
   return success;
@@ -1095,14 +1123,15 @@
 
 UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSwapInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x))
   UnsafeWrapper("Unsafe_CompareAndSwapInt");
-  oop p = JNIHandles::resolve(obj);
+  // We are about to write to this entry so check to see if we need to copy it.
+  oop p = oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj));
   jint* addr = (jint *) index_oop_from_field_offset_long(p, offset);
   return (jint)(Atomic::cmpxchg(x, addr, e)) == e;
 UNSAFE_END
 
 UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSwapLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x))
   UnsafeWrapper("Unsafe_CompareAndSwapLong");
-  Handle p (THREAD, JNIHandles::resolve(obj));
+  Handle p (THREAD, oopDesc::bs()->resolve_and_maybe_copy_oop(JNIHandles::resolve(obj)));
   jlong* addr = (jlong*)(index_oop_from_field_offset_long(p(), offset));
 #ifdef SUPPORTS_NATIVE_CX8
   return (jlong)(Atomic::cmpxchg(x, addr, e)) == e;
diff --git a/src/share/vm/runtime/arguments.cpp b/src/share/vm/runtime/arguments.cpp
--- a/src/share/vm/runtime/arguments.cpp
+++ b/src/share/vm/runtime/arguments.cpp
@@ -51,6 +51,7 @@
 #include "utilities/macros.hpp"
 #include "utilities/stringUtils.hpp"
 #if INCLUDE_ALL_GCS
+#include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/cms/compactibleFreeListSpace.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/parallel/parallelScavengeHeap.hpp"
@@ -1471,6 +1472,11 @@
   // the only value that can override MaxHeapSize if we are
   // to use UseCompressedOops is InitialHeapSize.
   size_t max_heap_size = MAX2(MaxHeapSize, InitialHeapSize);
+  if (UseShenandoahGC && FLAG_IS_DEFAULT(UseCompressedOops)) {
+    warning("Compressed Oops not supported with ShenandoahGC");
+    FLAG_SET_ERGO(bool, UseCompressedOops, false);
+    FLAG_SET_ERGO(bool, UseCompressedClassPointers, false);
+  }
 
   if (max_heap_size <= max_heap_for_compressed_oops()) {
 #if !defined(COMPILER1) || defined(TIERED)
@@ -1529,6 +1535,10 @@
     heap_alignment = ParallelScavengeHeap::conservative_max_heap_alignment();
   } else if (UseG1GC) {
     heap_alignment = G1CollectedHeap::conservative_max_heap_alignment();
+  } else if (UseShenandoahGC) {
+    // TODO: This sucks. Can't we have a clean interface to call the GC's collector
+    // policy for this?
+    heap_alignment = ShenandoahHeap::conservative_max_heap_alignment();
   }
 #endif // INCLUDE_ALL_GCS
   _conservative_max_heap_alignment = MAX4(heap_alignment,
@@ -1687,6 +1697,21 @@
   }
 }
 
+void Arguments::set_shenandoah_gc_flags() {
+  FLAG_SET_DEFAULT(UseDynamicNumberOfGCThreads, true);
+  FLAG_SET_DEFAULT(ParallelGCThreads,
+                   Abstract_VM_Version::parallel_worker_threads());
+
+  if (FLAG_IS_DEFAULT(ConcGCThreads)) {
+    uint conc_threads = MAX2((uint) 1, ParallelGCThreads);
+    FLAG_SET_DEFAULT(ConcGCThreads, conc_threads);
+  }
+
+  if (FLAG_IS_DEFAULT(ParallelRefProcEnabled)) {
+    FLAG_SET_DEFAULT(ParallelRefProcEnabled, true);
+  }
+}
+
 #if !INCLUDE_ALL_GCS
 #ifdef ASSERT
 static bool verify_serial_gc_flags() {
@@ -1706,6 +1731,8 @@
     set_cms_and_parnew_gc_flags();
   } else if (UseG1GC) {
     set_g1_gc_flags();
+  } else if (UseShenandoahGC) {
+    set_shenandoah_gc_flags();
   }
   check_deprecated_gc_flags();
   if (AssumeMP && !UseSerialGC) {
diff --git a/src/share/vm/runtime/arguments.hpp b/src/share/vm/runtime/arguments.hpp
--- a/src/share/vm/runtime/arguments.hpp
+++ b/src/share/vm/runtime/arguments.hpp
@@ -334,6 +334,8 @@
   static void set_parallel_gc_flags();
   // Garbage-First (UseG1GC)
   static void set_g1_gc_flags();
+  // Shenandoah GC (UseShenandoahGC)
+  static void set_shenandoah_gc_flags();
   // GC ergonomics
   static void set_conservative_max_heap_alignment();
   static void set_use_compressed_oops();
@@ -609,7 +611,7 @@
 };
 
 bool Arguments::gc_selected() {
-  return UseConcMarkSweepGC || UseG1GC || UseParallelGC || UseParallelOldGC || UseSerialGC;
+  return UseConcMarkSweepGC || UseG1GC || UseParallelGC || UseParallelOldGC || UseSerialGC || UseShenandoahGC;
 }
 
 // Disable options not supported in this release, with a warning if they
diff --git a/src/share/vm/runtime/basicLock.hpp b/src/share/vm/runtime/basicLock.hpp
--- a/src/share/vm/runtime/basicLock.hpp
+++ b/src/share/vm/runtime/basicLock.hpp
@@ -62,8 +62,12 @@
 
  public:
   // Manipulation
-  oop      obj() const                                { return _obj;  }
-  void set_obj(oop obj)                               { _obj = obj; }
+  oop      obj() const                                {
+    return _obj;
+  }
+  void set_obj(oop obj)                               {
+    _obj = obj;
+  }
   BasicLock* lock()                                   { return &_lock; }
 
   // Note: Use frame::interpreter_frame_monitor_size() for the size of BasicObjectLocks
diff --git a/src/share/vm/runtime/biasedLocking.cpp b/src/share/vm/runtime/biasedLocking.cpp
--- a/src/share/vm/runtime/biasedLocking.cpp
+++ b/src/share/vm/runtime/biasedLocking.cpp
@@ -146,6 +146,7 @@
 
 
 static BiasedLocking::Condition revoke_bias(oop obj, bool allow_rebias, bool is_bulk, JavaThread* requesting_thread) {
+  assert(obj == oopDesc::bs()->resolve_oop(obj), "expect to-space copy");
   markOop mark = obj->mark();
   if (!mark->has_bias_pattern()) {
     if (TraceBiasedLocking) {
@@ -325,7 +326,7 @@
                                                                    bool attempt_rebias_of_object,
                                                                    JavaThread* requesting_thread) {
   assert(SafepointSynchronize::is_at_safepoint(), "must be done at safepoint");
-
+  assert(o == oopDesc::bs()->resolve_oop(o), "expect to-space copy");
   if (TraceBiasedLocking) {
     tty->print_cr("* Beginning bulk revocation (kind == %s) because of object "
                   INTPTR_FORMAT " , mark " INTPTR_FORMAT " , type %s",
@@ -367,6 +368,7 @@
           if ((owner->klass() == k_o) && mark->has_bias_pattern()) {
             // We might have encountered this object already in the case of recursive locking
             assert(mark->bias_epoch() == prev_epoch || mark->bias_epoch() == cur_epoch, "error in bias epoch adjustment");
+            assert(owner == oopDesc::bs()->resolve_oop(owner), "expect to-space copy");
             owner->set_mark(mark->set_bias_epoch(cur_epoch));
           }
         }
@@ -531,6 +533,8 @@
 BiasedLocking::Condition BiasedLocking::revoke_and_rebias(Handle obj, bool attempt_rebias, TRAPS) {
   assert(!SafepointSynchronize::is_at_safepoint(), "must not be called while at safepoint");
 
+  assert(obj() == oopDesc::bs()->resolve_oop(obj()), "must be to-space copy");
+
   // We can revoke the biases of anonymously-biased objects
   // efficiently enough that we should not cause these revocations to
   // update the heuristics because doing so may cause unwanted bulk
@@ -645,6 +649,7 @@
 void BiasedLocking::revoke_at_safepoint(Handle h_obj) {
   assert(SafepointSynchronize::is_at_safepoint(), "must only be called while at safepoint");
   oop obj = h_obj();
+  assert(obj == oopDesc::bs()->resolve_oop(obj), "expect to-space copy");
   HeuristicsResult heuristics = update_heuristics(obj, false);
   if (heuristics == HR_SINGLE_REVOKE) {
     revoke_bias(obj, false, false, NULL);
@@ -708,6 +713,7 @@
             MonitorInfo* mon_info = monitors->at(i);
             if (mon_info->owner_is_scalar_replaced()) continue;
             oop owner = mon_info->owner();
+            assert(owner == oopDesc::bs()->resolve_oop(owner), "expect to-space copy");
             if (owner != NULL) {
               markOop mark = owner->mark();
               if (mark->has_bias_pattern()) {
diff --git a/src/share/vm/runtime/deoptimization.cpp b/src/share/vm/runtime/deoptimization.cpp
--- a/src/share/vm/runtime/deoptimization.cpp
+++ b/src/share/vm/runtime/deoptimization.cpp
@@ -935,7 +935,7 @@
     if (mon_info->eliminated()) {
       assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
       if (!mon_info->owner_is_scalar_replaced()) {
-        Handle obj = Handle(mon_info->owner());
+        Handle obj = Handle(oopDesc::bs()->resolve_and_maybe_copy_oop(mon_info->owner()));
         markOop mark = obj->mark();
         if (UseBiasedLocking && mark->has_bias_pattern()) {
           // New allocated objects may have the mark set to anonymously biased.
@@ -1058,7 +1058,7 @@
       for (int j = 0; j < monitors->number_of_monitors(); j++) {
         BasicObjectLock* src = monitors->at(j);
         if (src->obj() != NULL) {
-          ObjectSynchronizer::fast_exit(src->obj(), src->lock(), thread);
+          ObjectSynchronizer::fast_exit(oopDesc::bs()->resolve_and_maybe_copy_oop(src->obj()), src->lock(), thread);
         }
       }
       array->element(i)->free_monitors(thread);
diff --git a/src/share/vm/runtime/frame.cpp b/src/share/vm/runtime/frame.cpp
--- a/src/share/vm/runtime/frame.cpp
+++ b/src/share/vm/runtime/frame.cpp
@@ -1021,7 +1021,7 @@
     guarantee(oop_adr != NULL, "bad register save location");
     return NULL;
   }
-  oop r = *oop_adr;
+  oop r = oopDesc::load_heap_oop(oop_adr);
   assert(Universe::heap()->is_in_or_null(r), err_msg("bad receiver: " INTPTR_FORMAT " (" INTX_FORMAT ")", (void *) r, (void *) r));
   return r;
 }
diff --git a/src/share/vm/runtime/globals.hpp b/src/share/vm/runtime/globals.hpp
--- a/src/share/vm/runtime/globals.hpp
+++ b/src/share/vm/runtime/globals.hpp
@@ -1518,7 +1518,126 @@
   product(bool, UseG1GC, false,                                             \
           "Use the Garbage-First garbage collector")                        \
                                                                             \
-  product(bool, UseParallelGC, false,                                       \
+  product(bool, UseShenandoahGC, false,                                     \
+          "Use the Shenandoah garbage collector")                           \
+                                                                            \
+  product(uintx, ShenandoahHeapRegionSize, 0,                               \
+          "Size of the Shenandoah regions.")                                \
+                                                                            \
+  develop(bool, ShenandoahDumpHeapBeforeConcurrentMark, false,              \
+          "Dump the ShenanodahHeap Before Each ConcurrentMark")             \
+                                                                            \
+  develop(bool, ShenandoahDumpHeapAfterConcurrentMark, false,               \
+          "Dump the ShenanodahHeap After Each Concurrent Mark")             \
+                                                                            \
+  product(bool, ShenandoahTraceFullGC, false,                               \
+          "Trace Shenandoah full GC")                                       \
+                                                                            \
+  product(bool, ShenandoahTracePhases, false,                               \
+          "Trace Shenandoah GC phases")                                     \
+                                                                            \
+  develop(bool, ShenandoahTraceJNICritical, false,                          \
+          "Trace Shenandoah stalls for JNI critical regions")               \
+                                                                            \
+  product(bool, ShenandoahTraceHumongous, false,                            \
+          "Trace Shenandoah humongous objects")                             \
+                                                                            \
+  develop(bool, ShenandoahTraceAllocations, false,                          \
+          "Trace Shenandoah Allocations")                                   \
+                                                                            \
+  develop(bool, ShenandoahTraceBrooksPointers, false,                       \
+          "Trace Brooks Pointer updates")                                   \
+                                                                            \
+  develop(bool, ShenandoahTraceEvacuations, false,                          \
+          "Trace Shenandoah Evacuations")                                   \
+                                                                            \
+  develop(bool, ShenandoahVerifyWritesToFromSpace, false,                   \
+          "Use Memory Protection to signal illegal writes to from space")   \
+                                                                            \
+  develop(bool, ShenandoahVerifyReadsToFromSpace, false,                    \
+          "Use Memory Protection to signal illegal reads to from space")    \
+                                                                            \
+  develop(bool, ShenandoahTraceConcurrentMarking, false,                    \
+          "Trace Concurrent Marking")                                       \
+                                                                            \
+  develop(bool, ShenandoahTraceUpdates, false,                              \
+          "Trace Shenandoah Updates")                                       \
+                                                                            \
+  develop(bool, ShenandoahTraceTLabs, false,                                \
+          "Trace TLabs in Shenandoah Heap")                                 \
+                                                                            \
+  product(bool, ShenandoahProcessReferences, true,                          \
+          "Enable processing of (soft/weak/..) references in Shenandoah")   \
+                                                                            \
+  develop(bool, ShenandoahTraceWeakReferences, false,                       \
+          "Trace Weak Reference Processing in Shenandoah Heap")             \
+                                                                            \
+  product(bool, ShenandoahGCVerbose, false,                                 \
+          "Verbose information about the Shenandoah garbage collector")     \
+                                                                            \
+  product(bool, ShenandoahLogConfig, false,                                 \
+          "Log information about Shenandoah's configuration settings")      \
+                                                                            \
+  develop(bool, ShenandoahVerify, false,                                    \
+          "Verify the  Shenandoah garbage collector")                       \
+                                                                            \
+  product(bool, ShenandoahParallelRootScan, true,                           \
+          "Turn on/off parallel root scanning in Shenandoah")               \
+                                                                            \
+  product(bool, ShenandoahConcurrentEvacuation, true,                       \
+          "Turn on/off concurrent evacuation in Shenandoah")                \
+                                                                            \
+  product(bool, ShenandoahConcurrentMarking, true,                          \
+          "Turn on/off concurrent marking in Shenandoah")                   \
+                                                                            \
+  product(bool, ShenandoahUpdateRefsEarly,false,                            \
+          "Turn on/off early updating of references in Shenandoah")         \
+                                                                            \
+  product(bool, ShenandoahConcurrentUpdateRefs, true,                       \
+          "Turn on/off concurrent reference updating in Shenandoah")        \
+                                                                            \
+  product(bool, ShenandoahWriteBarrier, true,                               \
+          "Turn on/off write barriers in Shenandoah")                       \
+                                                                            \
+  product(bool, ShenandoahReadBarrier, true,                                \
+          "Turn on/off read barriers in Shenandoah")                        \
+                                                                            \
+  product(ccstr, ShenandoahGCHeuristics, "dynamic",                         \
+          "The heuristics to use in Shenandoah GC; possible values: "       \
+          "statusquo, aggressive, halfway, lazy, dynamic")                  \
+                                                                            \
+  product(uintx, ShenandoahGarbageThreshold, 60,                            \
+          "Sets the percentage of garbage a region need to contain before " \
+          "it can be marked for collection. Applies to "                    \
+          "Shenandoah GC dynamic Heuristic mode only (ignored otherwise)")  \
+                                                                            \
+  product(uintx, ShenandoahFreeThreshold, 25,                               \
+          "Set the percentage of heap free in relation to the total "       \
+          "capacity before a region can enter the concurrent marking "      \
+          "phase. Applies to Shenandoah GC dynamic Heuristic mode only "    \
+          "(ignored otherwise)")                                            \
+                                                                            \
+  product(uintx, ShenandoahInitialFreeThreshold, 50,                        \
+          "Set the percentage of heap free in relation to the total "       \
+          "capacity before a region can enter the concurrent marking "      \
+          "phase. Applies to Shenandoah GC dynamic Heuristic mode only "    \
+          "(ignored otherwise)")                                            \
+                                                                            \
+  product(uintx, ShenandoahAllocationThreshold, 0,                          \
+          "Set the number of bytes allocated since last GC cycle before"    \
+          "a region can enter the concurrent marking "                      \
+          "phase. Applies to Shenandoah GC dynamic Heuristic mode only "    \
+          "(ignored otherwise)")                                            \
+                                                                            \
+  product(uintx, ShenandoahTargetHeapOccupancy, 80,                         \
+          "Sets the target maximum percentage occupance of the heap we"     \
+          "would like to maintain."                                         \
+          "Shenandoah GC newadaptive Heuristic mode only.")                 \
+                                                                            \
+  product(bool, ShenandoahPrintCollectionSet, false,                        \
+          "Print the collection set before each GC phase")                  \
+                                                                            \
+  product(bool, UseParallelGC, false,                                   \
           "Use the Parallel Scavenge garbage collector")                    \
                                                                             \
   product(bool, UseParallelOldGC, false,                                    \
diff --git a/src/share/vm/runtime/handles.cpp b/src/share/vm/runtime/handles.cpp
--- a/src/share/vm/runtime/handles.cpp
+++ b/src/share/vm/runtime/handles.cpp
@@ -36,6 +36,9 @@
 oop* HandleArea::allocate_handle(oop obj) {
   assert(_handle_mark_nesting > 1, "memory leak: allocating handle outside HandleMark");
   assert(_no_handle_mark_nesting == 0, "allocating handle inside NoHandleMark");
+  if (ShenandoahVerifyReadsToFromSpace) {
+    obj = oopDesc::bs()->resolve_oop(obj);
+  }
   assert(obj->is_oop(), err_msg("not an oop: " INTPTR_FORMAT, (intptr_t*) obj));
   return real_allocate_handle(obj);
 }
diff --git a/src/share/vm/runtime/handles.hpp b/src/share/vm/runtime/handles.hpp
--- a/src/share/vm/runtime/handles.hpp
+++ b/src/share/vm/runtime/handles.hpp
@@ -25,6 +25,7 @@
 #ifndef SHARE_VM_RUNTIME_HANDLES_HPP
 #define SHARE_VM_RUNTIME_HANDLES_HPP
 
+#include "gc/shared/barrierSet.hpp"
 #include "oops/oop.hpp"
 #include "oops/oopsHierarchy.hpp"
 
@@ -82,8 +83,8 @@
   // General access
   oop     operator () () const                   { return obj(); }
   oop     operator -> () const                   { return non_null_obj(); }
-  bool    operator == (oop o) const              { return obj() == o; }
-  bool    operator == (const Handle& h) const          { return obj() == h.obj(); }
+  bool    operator == (oop o) const              { return oopDesc::bs()->resolve_and_maybe_copy_oop(obj()) == oopDesc::bs()->resolve_and_maybe_copy_oop(o); }
+  bool    operator == (const Handle& h) const          { return oopDesc::bs()->resolve_and_maybe_copy_oop(obj()) == oopDesc::bs()->resolve_and_maybe_copy_oop(h.obj()); }
 
   // Null checks
   bool    is_null() const                        { return _handle == NULL; }
diff --git a/src/share/vm/runtime/jniHandles.hpp b/src/share/vm/runtime/jniHandles.hpp
--- a/src/share/vm/runtime/jniHandles.hpp
+++ b/src/share/vm/runtime/jniHandles.hpp
@@ -27,6 +27,7 @@
 
 #include "runtime/handles.hpp"
 #include "utilities/top.hpp"
+#include "oops/oop.hpp"
 
 class JNIHandleBlock;
 
diff --git a/src/share/vm/runtime/mutexLocker.cpp b/src/share/vm/runtime/mutexLocker.cpp
--- a/src/share/vm/runtime/mutexLocker.cpp
+++ b/src/share/vm/runtime/mutexLocker.cpp
@@ -53,6 +53,9 @@
 Mutex*   JvmtiThreadState_lock        = NULL;
 Monitor* JvmtiPendingEvent_lock       = NULL;
 Monitor* Heap_lock                    = NULL;
+Monitor* ShenandoahHeap_lock          = NULL;
+Monitor* ShenandoahFullGC_lock        = NULL;
+Monitor* ShenandoahJNICritical_lock   = NULL;
 Mutex*   ExpandHeap_lock              = NULL;
 Mutex*   AdapterHandlerLibrary_lock   = NULL;
 Mutex*   SignatureHandlerLibrary_lock = NULL;
@@ -100,6 +103,7 @@
 Mutex*   ExceptionCache_lock          = NULL;
 Monitor* ObjAllocPost_lock            = NULL;
 Mutex*   OsrList_lock                 = NULL;
+Monitor* ShenandoahMemProtect_lock    = NULL;
 
 #ifndef PRODUCT
 Mutex*   FullGCALot_lock              = NULL;
@@ -203,6 +207,14 @@
     def(StringDedupQueue_lock      , Monitor, leaf,        true,  Monitor::_safepoint_check_never);
     def(StringDedupTable_lock      , Mutex  , leaf,        true,  Monitor::_safepoint_check_never);
   }
+  if (UseShenandoahGC) {
+    def(SATB_Q_FL_lock             , Mutex  , special,     true,  Monitor::_safepoint_check_never);
+    def(SATB_Q_CBL_mon             , Monitor, nonleaf,     true,  Monitor::_safepoint_check_never);
+    def(Shared_SATB_Q_lock         , Mutex,   nonleaf,     true,  Monitor::_safepoint_check_never);
+    def(ShenandoahFullGC_lock      , Monitor, leaf,        true,  Monitor::_safepoint_check_always);
+    def(ShenandoahJNICritical_lock , Monitor, nonleaf+1,   false, Monitor::_safepoint_check_never);
+    def(ShenandoahMemProtect_lock  , Monitor, native,      false, Monitor::_safepoint_check_never);
+  }
   def(ParGCRareEvent_lock          , Mutex  , leaf     ,   true,  Monitor::_safepoint_check_sometimes);
   def(DerivedPointerTableGC_lock   , Mutex,   leaf,        true,  Monitor::_safepoint_check_never);
   def(CodeCache_lock               , Mutex  , special,     true,  Monitor::_safepoint_check_never);
@@ -257,6 +269,7 @@
   }
 
   def(Heap_lock                    , Monitor, nonleaf+1,   false, Monitor::_safepoint_check_sometimes);
+  def(ShenandoahHeap_lock          , Monitor, special,     false, Monitor::_safepoint_check_never);
   def(JfieldIdCreation_lock        , Mutex  , nonleaf+1,   true,  Monitor::_safepoint_check_always);     // jfieldID, Used in VM_Operation
   def(MemberNameTable_lock         , Mutex  , nonleaf+1,   false, Monitor::_safepoint_check_always);     // Used to protect MemberNameTable
 
diff --git a/src/share/vm/runtime/mutexLocker.hpp b/src/share/vm/runtime/mutexLocker.hpp
--- a/src/share/vm/runtime/mutexLocker.hpp
+++ b/src/share/vm/runtime/mutexLocker.hpp
@@ -45,6 +45,9 @@
 extern Mutex*   JvmtiThreadState_lock;           // a lock on modification of JVMTI thread data
 extern Monitor* JvmtiPendingEvent_lock;          // a lock on the JVMTI pending events list
 extern Monitor* Heap_lock;                       // a lock on the heap
+extern Monitor* ShenandoahHeap_lock;             // a lock on the heap, used by ShenandoahGC when evacuating at a safepoint
+extern Monitor* ShenandoahFullGC_lock;           // a monitor to wait/notify the Shenandoah background thread on full-GC requests
+extern Monitor* ShenandoahJNICritical_lock;           // a monitor to wait/notify the Shenandoah background thread on full-GC requests
 extern Mutex*   ExpandHeap_lock;                 // a lock on expanding the heap
 extern Mutex*   AdapterHandlerLibrary_lock;      // a lock on the AdapterHandlerLibrary
 extern Mutex*   SignatureHandlerLibrary_lock;    // a lock on the SignatureHandlerLibrary
@@ -102,6 +105,7 @@
 extern Mutex*   ProfilePrint_lock;               // a lock used to serialize the printing of profiles
 extern Mutex*   ExceptionCache_lock;             // a lock used to synchronize exception cache updates
 extern Mutex*   OsrList_lock;                    // a lock used to serialize access to OSR queues
+extern Monitor* ShenandoahMemProtect_lock;       // ShenandoahGC uses this for  memory protection to verify operations on the heap.
 
 #ifndef PRODUCT
 extern Mutex*   FullGCALot_lock;                 // a lock to make FullGCALot MT safe
diff --git a/src/share/vm/runtime/objectMonitor.inline.hpp b/src/share/vm/runtime/objectMonitor.inline.hpp
--- a/src/share/vm/runtime/objectMonitor.inline.hpp
+++ b/src/share/vm/runtime/objectMonitor.inline.hpp
@@ -66,6 +66,11 @@
 
 
 inline void* ObjectMonitor::object() const {
+  /*
+  if (_object != (cast_to_oop<intptr_t>(-1))) {
+    assert(oop(_object) == oopDesc::bs()->resolve_and_maybe_copy_oop(oop(_object)), "expect to-space copy");
+  }
+  */
   return _object;
 }
 
@@ -75,6 +80,9 @@
 
 inline void ObjectMonitor::set_object(void* obj) {
   _object = obj;
+  if (_object != (cast_to_oop<intptr_t>(-1))) {
+    assert(oop(_object) == oopDesc::bs()->resolve_and_maybe_copy_oop(oop(_object)), "expect to-space copy");
+  }
 }
 
 inline bool ObjectMonitor::check(TRAPS) {
diff --git a/src/share/vm/runtime/reflection.cpp b/src/share/vm/runtime/reflection.cpp
--- a/src/share/vm/runtime/reflection.cpp
+++ b/src/share/vm/runtime/reflection.cpp
@@ -405,7 +405,7 @@
     assert(lower_dim->oop_is_array(), "just checking");
     result2 = lower_dim->java_mirror();
   }
-  assert(result == result2, "results must be consistent");
+  assert(oopDesc::bs()->resolve_oop(result) == oopDesc::bs()->resolve_oop(result2), "results must be consistent");
 #endif //ASSERT
   return result;
 }
diff --git a/src/share/vm/runtime/safepoint.cpp b/src/share/vm/runtime/safepoint.cpp
--- a/src/share/vm/runtime/safepoint.cpp
+++ b/src/share/vm/runtime/safepoint.cpp
@@ -56,6 +56,7 @@
 #include "utilities/events.hpp"
 #include "utilities/macros.hpp"
 #if INCLUDE_ALL_GCS
+#include "gc/shenandoah/shenandoahConcurrentThread.hpp"
 #include "gc/cms/concurrentMarkSweepThread.hpp"
 #include "gc/g1/suspendibleThreadSet.hpp"
 #endif // INCLUDE_ALL_GCS
@@ -95,7 +96,10 @@
     ConcurrentMarkSweepThread::synchronize(false);
   } else if (UseG1GC) {
     SuspendibleThreadSet::synchronize();
+  } else if (UseShenandoahGC) {
+    ShenandoahConcurrentThread::safepoint_synchronize();
   }
+
 #endif // INCLUDE_ALL_GCS
 
   // By getting the Threads_lock, we assure that no threads are about to start or
@@ -469,6 +473,8 @@
     ConcurrentMarkSweepThread::desynchronize(false);
   } else if (UseG1GC) {
     SuspendibleThreadSet::desynchronize();
+  } else if (UseShenandoahGC) {
+    ShenandoahConcurrentThread::safepoint_desynchronize();
   }
 #endif // INCLUDE_ALL_GCS
   // record this time so VMThread can keep track how much time has elapsed
@@ -968,6 +974,9 @@
       // the other registers. In order to preserve it over GCs we need
       // to keep it in a handle.
       oop result = caller_fr.saved_oop_result(&map);
+      if (ShenandoahVerifyReadsToFromSpace) {
+        result = oopDesc::bs()->resolve_oop(result);
+      }
       assert(result == NULL || result->is_oop(), "must be oop");
       return_value = Handle(thread(), result);
       assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
diff --git a/src/share/vm/runtime/sharedRuntime.cpp b/src/share/vm/runtime/sharedRuntime.cpp
--- a/src/share/vm/runtime/sharedRuntime.cpp
+++ b/src/share/vm/runtime/sharedRuntime.cpp
@@ -211,6 +211,11 @@
 
 #endif // INCLUDE_ALL_GCS
 
+// G1 write-barrier pre: executed before a pointer store.
+JRT_LEAF(void, SharedRuntime::shenandoah_clone_barrier(oopDesc* obj))
+  oopDesc::bs()->write_region(MemRegion((HeapWord*) obj, obj->size()));
+JRT_END
+
 
 JRT_LEAF(jlong, SharedRuntime::lmul(jlong y, jlong x))
   return x * y;
@@ -1795,6 +1800,7 @@
 
 // Handles the uncommon case in locking, i.e., contention or an inflated lock.
 JRT_BLOCK_ENTRY(void, SharedRuntime::complete_monitor_locking_C(oopDesc* _obj, BasicLock* lock, JavaThread* thread))
+  _obj = oopDesc::bs()->resolve_and_maybe_copy_oop(_obj);
   // Disable ObjectSynchronizer::quick_enter() in default config
   // until JDK-8077392 is resolved.
   if ((SyncFlags & 256) != 0 && !SafepointSynchronize::is_synchronizing()) {
@@ -1824,6 +1830,7 @@
 
 // Handles the uncommon cases of monitor unlocking in compiled code
 JRT_LEAF(void, SharedRuntime::complete_monitor_unlocking_C(oopDesc* _obj, BasicLock* lock, JavaThread * THREAD))
+  _obj = oopDesc::bs()->resolve_and_maybe_copy_oop(_obj);
    oop obj(_obj);
   assert(JavaThread::current() == THREAD, "invariant");
   // I'm not convinced we need the code contained by MIGHT_HAVE_PENDING anymore
diff --git a/src/share/vm/runtime/sharedRuntime.hpp b/src/share/vm/runtime/sharedRuntime.hpp
--- a/src/share/vm/runtime/sharedRuntime.hpp
+++ b/src/share/vm/runtime/sharedRuntime.hpp
@@ -180,6 +180,7 @@
   // G1 write barriers
   static void g1_wb_pre(oopDesc* orig, JavaThread *thread);
   static void g1_wb_post(void* card_addr, JavaThread* thread);
+  static void shenandoah_clone_barrier(oopDesc* obj);
 #endif // INCLUDE_ALL_GCS
 
   // exception handling and implicit exceptions
diff --git a/src/share/vm/runtime/synchronizer.cpp b/src/share/vm/runtime/synchronizer.cpp
--- a/src/share/vm/runtime/synchronizer.cpp
+++ b/src/share/vm/runtime/synchronizer.cpp
@@ -160,6 +160,7 @@
   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
   assert(self->is_Java_thread(), "invariant");
   assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
+  assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
   No_Safepoint_Verifier nsv;
   if (obj == NULL) return false;  // slow-path for invalid obj
   const markOop mark = obj->mark();
@@ -210,6 +211,7 @@
   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
   assert(Self->is_Java_thread(), "invariant");
   assert(((JavaThread *) Self)->thread_state() == _thread_in_Java, "invariant");
+  assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
   No_Safepoint_Verifier nsv;
   if (obj == NULL) return false;       // Need to throw NPE
   const markOop mark = obj->mark();
@@ -256,6 +258,7 @@
 
 void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock,
                                     bool attempt_rebias, TRAPS) {
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     if (!SafepointSynchronize::is_at_safepoint()) {
       BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD);
@@ -275,6 +278,8 @@
 void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) {
   assert(!object->mark()->has_bias_pattern(), "should not see bias pattern here");
   // if displaced header is null, the previous enter is recursive enter, no-op
+  assert(object == oopDesc::bs()->resolve_and_maybe_copy_oop(object), "expect to-space copy");
+
   markOop dhw = lock->displaced_header();
   markOop mark;
   if (dhw == NULL) {
@@ -314,6 +319,7 @@
 // We don't need to use fast path here, because it must have been
 // failed in the interpreter/compiler code.
 void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   markOop mark = obj->mark();
   assert(!mark->has_bias_pattern(), "should not see bias pattern here");
 
@@ -364,6 +370,7 @@
 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
 intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
   TEVENT(complete_exit);
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -377,6 +384,7 @@
 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
 void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
   TEVENT(reenter);
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -392,6 +400,7 @@
 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
   // the current locking is from JNI instead of Java code
   TEVENT(jni_enter);
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -404,6 +413,7 @@
 // NOTE: must use heavy weight monitor to handle jni monitor exit
 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
   TEVENT(jni_exit);
+  assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
   if (UseBiasedLocking) {
     Handle h_obj(THREAD, obj);
     BiasedLocking::revoke_and_rebias(h_obj, false, THREAD);
@@ -446,6 +456,7 @@
 //  Wait/Notify/NotifyAll
 // NOTE: must use heavy weight monitor to handle wait()
 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -466,6 +477,7 @@
 }
 
 void ObjectSynchronizer::waitUninterruptibly(Handle obj, jlong millis, TRAPS) {
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -478,6 +490,7 @@
 }
 
 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -492,6 +505,7 @@
 
 // NOTE: see comment of notify()
 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
+  assert(obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(obj, false, THREAD);
     assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -669,6 +683,7 @@
 }
 
 intptr_t ObjectSynchronizer::FastHashCode(Thread * Self, oop obj) {
+  assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
   if (UseBiasedLocking) {
     // NOTE: many places throughout the JVM do not expect a safepoint
     // to be taken here, in particular most operations on perm gen
@@ -782,6 +797,7 @@
 
 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
                                                    Handle h_obj) {
+  assert(h_obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(h_obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     BiasedLocking::revoke_and_rebias(h_obj, false, thread);
     assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now");
@@ -854,6 +870,7 @@
 
 // FIXME: jvmti should call this
 JavaThread* ObjectSynchronizer::get_lock_owner(Handle h_obj, bool doLock) {
+  assert(h_obj() == oopDesc::bs()->resolve_and_maybe_copy_oop(h_obj()), "expect to-space copy");
   if (UseBiasedLocking) {
     if (SafepointSynchronize::is_at_safepoint()) {
       BiasedLocking::revoke_at_safepoint(h_obj);
@@ -1275,6 +1292,7 @@
 
 // Fast path code shared by multiple functions
 ObjectMonitor* ObjectSynchronizer::inflate_helper(oop obj) {
+  assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
   markOop mark = obj->mark();
   if (mark->has_monitor()) {
     assert(ObjectSynchronizer::verify_objmon_isinpool(mark->monitor()), "monitor is invalid");
@@ -1289,6 +1307,7 @@
                                                      oop object) {
   // Inflate mutates the heap ...
   // Relaxing assertion for bug 6320749.
+  assert(object == oopDesc::bs()->resolve_and_maybe_copy_oop(object), "expect to-space copy");
   assert(Universe::verify_in_progress() ||
          !SafepointSynchronize::is_at_safepoint(), "invariant");
 
@@ -1307,7 +1326,7 @@
     if (mark->has_monitor()) {
       ObjectMonitor * inf = mark->monitor();
       assert(inf->header()->is_neutral(), "invariant");
-      assert(inf->object() == object, "invariant");
+      assert((oop) inf->object() == object, "invariant");
       assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
       return inf;
     }
@@ -1513,6 +1532,7 @@
                                          ObjectMonitor** freeTailp) {
   bool deflated;
   // Normal case ... The monitor is associated with obj.
+  assert(obj == oopDesc::bs()->resolve_and_maybe_copy_oop(obj), "expect to-space copy");
   guarantee(obj->mark() == markOopDesc::encode(mid), "invariant");
   guarantee(mid == obj->mark()->monitor(), "invariant");
   guarantee(mid->header()->is_neutral(), "invariant");
@@ -1798,6 +1818,7 @@
     for (int i = 1; i < _BLOCKSIZE; i++) {
       mid = (ObjectMonitor *)(block + i);
       oop object = (oop) mid->object();
+
       if (object != NULL) {
         mid->verify();
       }
diff --git a/src/share/vm/runtime/thread.cpp b/src/share/vm/runtime/thread.cpp
--- a/src/share/vm/runtime/thread.cpp
+++ b/src/share/vm/runtime/thread.cpp
@@ -95,6 +95,7 @@
 #include "utilities/macros.hpp"
 #include "utilities/preserveException.hpp"
 #if INCLUDE_ALL_GCS
+#include "gc/shenandoah/shenandoahConcurrentThread.hpp"
 #include "gc/cms/concurrentMarkSweepThread.hpp"
 #include "gc/g1/concurrentMarkThread.inline.hpp"
 #include "gc/parallel/pcTasks.hpp"
@@ -262,6 +263,8 @@
   _MutexEvent  = ParkEvent::Allocate(this);
   _MuxEvent    = ParkEvent::Allocate(this);
 
+  _evacuating = false;
+
 #ifdef CHECK_UNHANDLED_OOPS
   if (CheckUnhandledOops) {
     _unhandled_oops = new UnhandledOops(this);
@@ -1265,6 +1268,7 @@
 }
 
 void WatcherThread::run() {
+
   assert(this == watcher_thread(), "just checking");
 
   this->record_stack_base_and_size();
@@ -1471,13 +1475,15 @@
 #if INCLUDE_ALL_GCS
 SATBMarkQueueSet JavaThread::_satb_mark_queue_set;
 DirtyCardQueueSet JavaThread::_dirty_card_queue_set;
+bool JavaThread::_evacuation_in_progress_global = false;
 #endif // INCLUDE_ALL_GCS
 
 JavaThread::JavaThread(bool is_attaching_via_jni) :
                        Thread()
 #if INCLUDE_ALL_GCS
                        , _satb_mark_queue(&_satb_mark_queue_set),
-                       _dirty_card_queue(&_dirty_card_queue_set)
+                       _dirty_card_queue(&_dirty_card_queue_set),
+                       _evacuation_in_progress(_evacuation_in_progress_global)
 #endif // INCLUDE_ALL_GCS
 {
   initialize();
@@ -1535,7 +1541,8 @@
                        Thread()
 #if INCLUDE_ALL_GCS
                        , _satb_mark_queue(&_satb_mark_queue_set),
-                       _dirty_card_queue(&_dirty_card_queue_set)
+                       _dirty_card_queue(&_dirty_card_queue_set),
+                       _evacuation_in_progress(_evacuation_in_progress_global)
 #endif // INCLUDE_ALL_GCS
 {
   initialize();
@@ -1670,7 +1677,7 @@
 
 static void ensure_join(JavaThread* thread) {
   // We do not need to grap the Threads_lock, since we are operating on ourself.
-  Handle threadObj(thread, thread->threadObj());
+  Handle threadObj(thread, oopDesc::bs()->resolve_and_maybe_copy_oop(thread->threadObj()));
   assert(threadObj.not_null(), "java thread object must exist");
   ObjectLocker lock(threadObj, thread);
   // Ignore pending exception (ThreadDeath), since we are exiting anyway
@@ -1864,9 +1871,12 @@
   // from the list of active threads. We must do this after any deferred
   // card marks have been flushed (above) so that any entries that are
   // added to the thread's dirty card queue as a result are not lost.
-  if (UseG1GC) {
+  if (UseG1GC || UseShenandoahGC) {
     flush_barrier_queues();
   }
+  if (UseShenandoahGC && UseTLAB) {
+    gclab().make_parsable(true);
+  }
 #endif // INCLUDE_ALL_GCS
 
   // Remove from list of active threads list, and notify VM thread if we are the last non-daemon thread
@@ -1901,6 +1911,21 @@
   // active field set to true.
   assert(dirty_queue.is_active(), "dirty card queue should be active");
 }
+
+bool JavaThread::evacuation_in_progress() const {
+  return _evacuation_in_progress;
+}
+
+void JavaThread::set_evacuation_in_progress(bool in_prog) {
+  _evacuation_in_progress = in_prog;
+}
+
+void JavaThread::set_evacuation_in_progress_all_threads(bool in_prog) {
+  _evacuation_in_progress_global = in_prog;
+  for (JavaThread* t = Threads::first(); t; t = t->next()) {
+    t->set_evacuation_in_progress(in_prog);
+  }
+}
 #endif // INCLUDE_ALL_GCS
 
 void JavaThread::cleanup_failed_attach_current_thread() {
@@ -1930,9 +1955,12 @@
   }
 
 #if INCLUDE_ALL_GCS
-  if (UseG1GC) {
+  if (UseG1GC || UseShenandoahGC) {
     flush_barrier_queues();
   }
+  if (UseShenandoahGC && UseTLAB) {
+    gclab().make_parsable(true);
+  }
 #endif // INCLUDE_ALL_GCS
 
   Threads::remove(this);
@@ -2881,6 +2909,7 @@
     oop thread_group = java_lang_Thread::threadGroup(thread_obj);
     if (thread_group != NULL) {
       typeArrayOop name = java_lang_ThreadGroup::name(thread_group);
+      name = typeArrayOop(oopDesc::bs()->resolve_oop(name));
       // ThreadGroup.name can be null
       if (name != NULL) {
         const char* str = UNICODE::as_utf8((jchar*) name->base(T_CHAR), name->length());
@@ -2900,6 +2929,7 @@
       oop parent = java_lang_ThreadGroup::parent(thread_group);
       if (parent != NULL) {
         typeArrayOop name = java_lang_ThreadGroup::name(parent);
+        name = typeArrayOop(oopDesc::bs()->resolve_oop(name));
         // ThreadGroup.name can be null
         if (name != NULL) {
           const char* str = UNICODE::as_utf8((jchar*) name->base(T_CHAR), name->length());
@@ -3512,9 +3542,11 @@
   // Support for ConcurrentMarkSweep. This should be cleaned up
   // and better encapsulated. The ugly nested if test would go away
   // once things are properly refactored. XXX YSR
-  if (UseConcMarkSweepGC || UseG1GC) {
+  if (UseConcMarkSweepGC || UseG1GC || UseShenandoahGC) {
     if (UseConcMarkSweepGC) {
       ConcurrentMarkSweepThread::makeSurrogateLockerThread(CHECK_JNI_ERR);
+    } else if (UseShenandoahGC) {
+      ShenandoahConcurrentThread::makeSurrogateLockerThread(CHECK_JNI_ERR);
     } else {
       ConcurrentMarkThread::makeSurrogateLockerThread(CHECK_JNI_ERR);
     }
@@ -3923,6 +3955,9 @@
 
   thread->exit(true);
 
+  // Stop GC threads.
+  Universe::heap()->shutdown();
+
   // Stop VM thread.
   {
     // 4945125 The vm thread comes to a safepoint during exit.
diff --git a/src/share/vm/runtime/thread.hpp b/src/share/vm/runtime/thread.hpp
--- a/src/share/vm/runtime/thread.hpp
+++ b/src/share/vm/runtime/thread.hpp
@@ -264,8 +264,11 @@
   friend class GC_locker;
 
   ThreadLocalAllocBuffer _tlab;                 // Thread-local eden
+  ThreadLocalAllocBuffer _gclab;                // Thread-local allocation buffer for GC (e.g. evacuation)
   jlong _allocated_bytes;                       // Cumulative number of bytes allocated on
                                                 // the Java heap
+  jlong _allocated_bytes_gclab;                 // Cumulative number of bytes allocated on
+                                                // the Java heap, in GCLABs
 
   TRACE_DATA _trace_data;                       // Thread-local data for tracing
 
@@ -281,6 +284,8 @@
   // ObjectMonitor on which this thread called Object.wait()
   ObjectMonitor* _current_waiting_monitor;
 
+  bool _evacuating;
+
   // Private thread-local objectmonitor list - a simple cache organized as a SLL.
  public:
   ObjectMonitor* omFreeList;
@@ -424,15 +429,23 @@
   ThreadLocalAllocBuffer& tlab()                 { return _tlab; }
   void initialize_tlab() {
     if (UseTLAB) {
-      tlab().initialize();
+      tlab().initialize(false);
+      gclab().initialize(true);
     }
   }
 
+  // Thread-Local GC Allocation Buffer (GCLAB) support
+  ThreadLocalAllocBuffer& gclab()                { return _gclab; }
+
   jlong allocated_bytes()               { return _allocated_bytes; }
   void set_allocated_bytes(jlong value) { _allocated_bytes = value; }
   void incr_allocated_bytes(jlong size) { _allocated_bytes += size; }
   inline jlong cooked_allocated_bytes();
 
+  jlong allocated_bytes_gclab()                { return _allocated_bytes_gclab; }
+  void set_allocated_bytes_gclab(jlong value)  { _allocated_bytes_gclab = value; }
+  void incr_allocated_bytes_gclab(jlong size)  { _allocated_bytes_gclab += size; }
+
   TRACE_DATA* trace_data()              { return &_trace_data; }
 
   const ThreadExt& ext() const          { return _ext; }
@@ -465,6 +478,14 @@
     _current_waiting_monitor = monitor;
   }
 
+  bool is_evacuating() {
+    return _evacuating;
+  }
+
+  void set_evacuating(bool evacuating) {
+    _evacuating = evacuating;
+  }
+
   // GC support
   // Apply "f->do_oop" to all root oops in "this".
   // Apply "cld_f->do_cld" to CLDs that are otherwise not kept alive.
@@ -614,6 +635,8 @@
 
 #undef TLAB_FIELD_OFFSET
 
+  static ByteSize gclab_start_offset()         { return byte_offset_of(Thread, _gclab) + ThreadLocalAllocBuffer::start_offset(); }
+
   static ByteSize allocated_bytes_offset()       { return byte_offset_of(Thread, _allocated_bytes); }
 
  public:
@@ -956,6 +979,10 @@
   static DirtyCardQueueSet _dirty_card_queue_set;
 
   void flush_barrier_queues();
+
+  bool _evacuation_in_progress;
+  static bool _evacuation_in_progress_global;
+
 #endif // INCLUDE_ALL_GCS
 
   friend class VMThread;
@@ -1375,6 +1402,9 @@
 #if INCLUDE_ALL_GCS
   static ByteSize satb_mark_queue_offset()       { return byte_offset_of(JavaThread, _satb_mark_queue); }
   static ByteSize dirty_card_queue_offset()      { return byte_offset_of(JavaThread, _dirty_card_queue); }
+
+  static ByteSize evacuation_in_progress_offset() { return byte_offset_of(JavaThread, _evacuation_in_progress); }
+
 #endif // INCLUDE_ALL_GCS
 
   // Returns the jni environment for this thread
@@ -1671,6 +1701,12 @@
   static DirtyCardQueueSet& dirty_card_queue_set() {
     return _dirty_card_queue_set;
   }
+
+  bool evacuation_in_progress() const;
+
+  void set_evacuation_in_progress(bool in_prog);
+
+  static void set_evacuation_in_progress_all_threads(bool in_prog);
 #endif // INCLUDE_ALL_GCS
 
   // This method initializes the SATB and dirty card queues before a
diff --git a/src/share/vm/runtime/vframe.hpp b/src/share/vm/runtime/vframe.hpp
--- a/src/share/vm/runtime/vframe.hpp
+++ b/src/share/vm/runtime/vframe.hpp
@@ -250,6 +250,7 @@
  public:
   // Constructor
   MonitorInfo(oop owner, BasicLock* lock, bool eliminated, bool owner_is_scalar_replaced) {
+    assert(owner == oopDesc::bs()->resolve_and_maybe_copy_oop(owner), "expect to-space copy");
     if (!owner_is_scalar_replaced) {
       _owner = owner;
       _owner_klass = NULL;
@@ -265,6 +266,7 @@
   // Accessors
   oop        owner() const {
     assert(!_owner_is_scalar_replaced, "should not be called for scalar replaced object");
+    assert(_owner == oopDesc::bs()->resolve_and_maybe_copy_oop(_owner), "expect to-space copy");
     return _owner;
   }
   oop   owner_klass() const {
diff --git a/src/share/vm/runtime/vmStructs.cpp b/src/share/vm/runtime/vmStructs.cpp
--- a/src/share/vm/runtime/vmStructs.cpp
+++ b/src/share/vm/runtime/vmStructs.cpp
@@ -208,6 +208,7 @@
 #include "opto/phaseX.hpp"
 #include "opto/regalloc.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/subnode.hpp"
 #include "opto/vectornode.hpp"
 #endif // COMPILER2
@@ -2076,6 +2077,9 @@
   declare_c2_type(OverflowAddLNode, OverflowLNode)                        \
   declare_c2_type(OverflowSubLNode, OverflowLNode)                        \
   declare_c2_type(OverflowMulLNode, OverflowLNode)                        \
+  declare_c2_type(ShenandoahBarrierNode, TypeNode)                        \
+  declare_c2_type(ShenandoahReadBarrierNode, ShenandoahBarrierNode)       \
+  declare_c2_type(ShenandoahWriteBarrierNode, ShenandoahBarrierNode)      \
                                                                           \
   /*********************/                                                 \
   /* Adapter Blob Entries */                                              \
@@ -2257,6 +2261,7 @@
   declare_constant(BarrierSet::CardTableExtension)                        \
   declare_constant(BarrierSet::G1SATBCT)                                  \
   declare_constant(BarrierSet::G1SATBCTLogging)                           \
+  declare_constant(BarrierSet::ShenandoahBarrierSet)                      \
                                                                           \
   declare_constant(BlockOffsetSharedArray::LogN)                          \
   declare_constant(BlockOffsetSharedArray::LogN_words)                    \
diff --git a/src/share/vm/runtime/vm_operations.hpp b/src/share/vm/runtime/vm_operations.hpp
--- a/src/share/vm/runtime/vm_operations.hpp
+++ b/src/share/vm/runtime/vm_operations.hpp
@@ -95,6 +95,13 @@
   template(HeapIterateOperation)                  \
   template(ReportJavaOutOfMemory)                 \
   template(JFRCheckpoint)                         \
+  template(ShenandoahFullGC)                      \
+  template(ShenandoahInitMark)                    \
+  template(ShenandoahStartEvacuation)             \
+  template(ShenandoahVerifyHeapAfterEvacuation)   \
+  template(ShenandoahEvacuation)                  \
+  template(ShenandoahUpdateRootRefs)              \
+  template(ShenandoahUpdateRefs)                  \
   template(Exit)                                  \
   template(LinuxDllLoad)                          \
   template(RotateGCLog)                           \
diff --git a/src/share/vm/services/memoryManager.cpp b/src/share/vm/services/memoryManager.cpp
--- a/src/share/vm/services/memoryManager.cpp
+++ b/src/share/vm/services/memoryManager.cpp
@@ -91,6 +91,10 @@
   return (GCMemoryManager*) new G1OldGenMemoryManager();
 }
 
+GCMemoryManager* MemoryManager::get_shenandoah_memory_manager() {
+  return (GCMemoryManager*) new ShenandoahMemoryManager();
+}
+
 instanceOop MemoryManager::get_memory_manager_instance(TRAPS) {
   // Must do an acquire so as to force ordering of subsequent
   // loads from anything _memory_mgr_obj points to or implies.
diff --git a/src/share/vm/services/memoryManager.hpp b/src/share/vm/services/memoryManager.hpp
--- a/src/share/vm/services/memoryManager.hpp
+++ b/src/share/vm/services/memoryManager.hpp
@@ -63,7 +63,7 @@
 
   void add_pool(MemoryPool* pool);
 
-  bool is_manager(instanceHandle mh)     { return mh() == _memory_mgr_obj; }
+  bool is_manager(instanceHandle mh)     { return oopDesc::bs()->resolve_and_maybe_copy_oop(mh()) == oopDesc::bs()->resolve_and_maybe_copy_oop(_memory_mgr_obj); }
 
   virtual instanceOop get_memory_manager_instance(TRAPS);
   virtual bool is_gc_memory_manager()    { return false; }
@@ -83,6 +83,7 @@
   static GCMemoryManager* get_psMarkSweep_memory_manager();
   static GCMemoryManager* get_g1YoungGen_memory_manager();
   static GCMemoryManager* get_g1OldGen_memory_manager();
+  static GCMemoryManager* get_shenandoah_memory_manager();
 };
 
 class CodeCacheMemoryManager : public MemoryManager {
@@ -253,4 +254,13 @@
   const char* name() { return "G1 Old Generation"; }
 };
 
+class ShenandoahMemoryManager : public GCMemoryManager {
+private:
+public:
+  ShenandoahMemoryManager() : GCMemoryManager() {}
+
+  const char* name()         { return "Shenandoah";}
+
+};
+
 #endif // SHARE_VM_SERVICES_MEMORYMANAGER_HPP
diff --git a/src/share/vm/services/memoryService.cpp b/src/share/vm/services/memoryService.cpp
--- a/src/share/vm/services/memoryService.cpp
+++ b/src/share/vm/services/memoryService.cpp
@@ -46,6 +46,7 @@
 #include "utilities/growableArray.hpp"
 #include "utilities/macros.hpp"
 #if INCLUDE_ALL_GCS
+#include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/cms/concurrentMarkSweepGeneration.hpp"
 #include "gc/cms/parNewGeneration.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
@@ -54,6 +55,7 @@
 #include "gc/parallel/psYoungGen.hpp"
 #include "services/g1MemoryPool.hpp"
 #include "services/psMemoryPool.hpp"
+#include "services/shenandoahMemoryPool.hpp"
 #endif // INCLUDE_ALL_GCS
 
 GrowableArray<MemoryPool*>* MemoryService::_pools_list =
@@ -98,6 +100,10 @@
       add_g1_heap_info(G1CollectedHeap::heap());
       break;
     }
+  case CollectedHeap::ShenandoahHeap : {
+    add_shenandoah_heap_info(ShenandoahHeap::heap());
+    break;
+  }
 #endif // INCLUDE_ALL_GCS
     default: {
       guarantee(false, "Unrecognized kind of heap");
@@ -115,8 +121,12 @@
 
   // All memory pools and memory managers are initialized.
   //
+  if (UseShenandoahGC) {
+    _major_gc_manager->initialize_gc_stat_info();
+  } else {
   _minor_gc_manager->initialize_gc_stat_info();
   _major_gc_manager->initialize_gc_stat_info();
+  }
 }
 
 // Add memory pools for GenCollectedHeap
@@ -187,6 +197,15 @@
   add_g1YoungGen_memory_pool(g1h, _major_gc_manager, _minor_gc_manager);
   add_g1OldGen_memory_pool(g1h, _major_gc_manager);
 }
+
+void MemoryService::add_shenandoah_heap_info(ShenandoahHeap* pgch) {
+  assert(UseShenandoahGC, "sanity");
+  _major_gc_manager = MemoryManager::get_shenandoah_memory_manager();
+  _minor_gc_manager = MemoryManager::get_shenandoah_memory_manager();
+  _managers_list->append(_major_gc_manager);
+  add_shenandoah_memory_pool(pgch, _major_gc_manager);
+}
+
 #endif // INCLUDE_ALL_GCS
 
 MemoryPool* MemoryService::add_gen(Generation* gen,
@@ -385,6 +404,19 @@
   mgr->add_pool(old_gen);
   _pools_list->append(old_gen);
 }
+
+void MemoryService::add_shenandoah_memory_pool(ShenandoahHeap* pgc,
+                                               MemoryManager* mgr) {
+  ShenandoahMemoryPool* pool = new ShenandoahMemoryPool(pgc,
+                                                        "Shenandoah",
+                                                        MemoryPool::Heap,
+                                                        false /* support_usage_threshold */);
+
+  mgr->add_pool(pool);
+  _pools_list->append(pool);
+}
+
+
 #endif // INCLUDE_ALL_GCS
 
 void MemoryService::add_code_heap_memory_pool(CodeHeap* heap, const char* name) {
diff --git a/src/share/vm/services/memoryService.hpp b/src/share/vm/services/memoryService.hpp
--- a/src/share/vm/services/memoryService.hpp
+++ b/src/share/vm/services/memoryService.hpp
@@ -46,6 +46,7 @@
 class GenCollectedHeap;
 class ParallelScavengeHeap;
 class G1CollectedHeap;
+class ShenandoahHeap;
 
 // VM Monitoring and Management Support
 
@@ -92,6 +93,9 @@
   static void add_g1OldGen_memory_pool(G1CollectedHeap* g1h,
                                        MemoryManager* mgr);
 
+  static void add_shenandoah_memory_pool(ShenandoahHeap* pgc,
+                                         MemoryManager* mgr);
+
   static MemoryPool* add_space(ContiguousSpace* space,
                                const char* name,
                                bool is_heap,
@@ -115,6 +119,7 @@
   static void add_gen_collected_heap_info(GenCollectedHeap* heap);
   static void add_parallel_scavenge_heap_info(ParallelScavengeHeap* heap);
   static void add_g1_heap_info(G1CollectedHeap* g1h);
+  static void add_shenandoah_heap_info(ShenandoahHeap* heap);
 
 public:
   static void set_universe_heap(CollectedHeap* heap);
diff --git a/src/share/vm/services/shenandoahMemoryPool.cpp b/src/share/vm/services/shenandoahMemoryPool.cpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/services/shenandoahMemoryPool.cpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "services/shenandoahMemoryPool.hpp"
+
+ShenandoahMemoryPool::ShenandoahMemoryPool(ShenandoahHeap* gen,
+                                           const char* name,
+                                           PoolType type,
+                                           bool support_usage_threshold) :
+  CollectedMemoryPool(name, type, gen->capacity(),
+                      gen->max_capacity(),
+                      support_usage_threshold),
+                      _gen(gen) {
+}
+
+MemoryUsage ShenandoahMemoryPool::get_memory_usage() {
+  size_t maxSize   = max_size();
+  size_t used      = used_in_bytes();
+  size_t committed = _gen->capacity();
+
+  return MemoryUsage(initial_size(), used, committed, maxSize);
+}
diff --git a/src/share/vm/services/shenandoahMemoryPool.hpp b/src/share/vm/services/shenandoahMemoryPool.hpp
new file mode 100644
--- /dev/null
+++ b/src/share/vm/services/shenandoahMemoryPool.hpp
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_SERVICES_SHENANDOAHMEMORYPOOL_HPP
+#define SHARE_VM_SERVICES_SHENANDOAHMEMORYPOOL_HPP
+
+#ifndef SERIALGC
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "services/memoryPool.hpp"
+#include "services/memoryUsage.hpp"
+#endif
+
+class ShenandoahMemoryPool : public CollectedMemoryPool {
+private:
+
+   ShenandoahHeap* _gen;
+
+public:
+
+  ShenandoahMemoryPool(ShenandoahHeap* pool,
+                          const char* name,
+                          PoolType type,
+                          bool support_usage_threshold);
+  MemoryUsage get_memory_usage();
+  size_t used_in_bytes()              { return _gen->used(); }
+  size_t max_size() const             { return _gen->max_capacity(); }
+};
+
+
+#endif //SHARE_VM_SERVICES_SHENANDOAHMEMORYPOOL_HPP
diff --git a/src/share/vm/services/threadService.cpp b/src/share/vm/services/threadService.cpp
--- a/src/share/vm/services/threadService.cpp
+++ b/src/share/vm/services/threadService.cpp
@@ -166,6 +166,7 @@
     // If obj == NULL, then ObjectMonitor is raw which doesn't count.
   }
 
+  obj = oopDesc::bs()->resolve_and_maybe_copy_oop(obj);
   Handle h(obj);
   return h;
 }
@@ -589,6 +590,8 @@
 bool ThreadStackTrace::is_owned_monitor_on_stack(oop object) {
   assert(SafepointSynchronize::is_at_safepoint(), "all threads are stopped");
 
+  object = oopDesc::bs()->resolve_and_maybe_copy_oop(object);
+
   bool found = false;
   int num_frames = get_stack_depth();
   for (int depth = 0; depth < num_frames; depth++) {
@@ -597,6 +600,7 @@
     GrowableArray<oop>* locked_monitors = frame->locked_monitors();
     for (int j = 0; j < len; j++) {
       oop monitor = locked_monitors->at(j);
+      monitor = oopDesc::bs()->resolve_and_maybe_copy_oop(monitor);
       assert(monitor != NULL, "must be a Java object");
       if (monitor == object) {
         found = true;