1 /*
   2  * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  26 #include "gc/shenandoah/shenandoahHeap.hpp"
  27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
  29 #include "gc/shenandoah/shenandoahRuntime.hpp"
  30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "runtime/sharedRuntime.hpp"
  34 #include "runtime/thread.hpp"
  35 #ifdef COMPILER1
  36 #include "c1/c1_LIRAssembler.hpp"
  37 #include "c1/c1_MacroAssembler.hpp"
  38 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  39 #endif
  40 
  41 #define __ masm->
  42 
  43 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
  44 
  45 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
  46                                                        Register addr, Register count, RegSet saved_regs) {
  47   if (is_oop) {
  48     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  49     if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
  50 
  51       Label done;
  52 
  53       // Avoid calling runtime if count == 0
  54       __ cbz(count, done);
  55 
  56       // Is marking active?
  57       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
  58       __ ldrb(rscratch1, gc_state);
  59       __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done);
  60 
  61       __ push(saved_regs, sp);
  62       if (count == c_rarg0) {
  63         if (addr == c_rarg1) {
  64           // exactly backwards!!
  65           __ mov(rscratch1, c_rarg0);
  66           __ mov(c_rarg0, c_rarg1);
  67           __ mov(c_rarg1, rscratch1);
  68         } else {
  69           __ mov(c_rarg1, count);
  70           __ mov(c_rarg0, addr);
  71         }
  72       } else {
  73         __ mov(c_rarg0, addr);
  74         __ mov(c_rarg1, count);
  75       }
  76       if (UseCompressedOops) {
  77         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
  78       } else {
  79         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
  80       }
  81       __ pop(saved_regs, sp);
  82       __ bind(done);
  83     }
  84   }
  85 }
  86 
  87 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
  88                                                        Register start, Register count, Register scratch, RegSet saved_regs) {
  89   if (is_oop) {
  90     __ push(saved_regs, sp);
  91     assert_different_registers(start, count, scratch);
  92     assert_different_registers(c_rarg0, count);
  93     __ mov(c_rarg0, start);
  94     __ mov(c_rarg1, count);
  95     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
  96     __ pop(saved_regs, sp);
  97   }
  98 }
  99 
 100 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 101                                                                  Register obj,
 102                                                                  Register pre_val,
 103                                                                  Register thread,
 104                                                                  Register tmp,
 105                                                                  bool tosca_live,
 106                                                                  bool expand_call) {
 107   if (ShenandoahSATBBarrier) {
 108     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 109   }
 110 }
 111 
 112 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 113                                                            Register obj,
 114                                                            Register pre_val,
 115                                                            Register thread,
 116                                                            Register tmp,
 117                                                            bool tosca_live,
 118                                                            bool expand_call) {
 119   // If expand_call is true then we expand the call_VM_leaf macro
 120   // directly to skip generating the check by
 121   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
 122 
 123   assert(thread == rthread, "must be");
 124 
 125   Label done;
 126   Label runtime;
 127 
 128   assert_different_registers(obj, pre_val, tmp, rscratch1);
 129   assert(pre_val != noreg &&  tmp != noreg, "expecting a register");
 130 
 131   Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
 132   Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 133   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 134 
 135   // Is marking active?
 136   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 137     __ ldrw(tmp, in_progress);
 138   } else {
 139     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 140     __ ldrb(tmp, in_progress);
 141   }
 142   __ cbzw(tmp, done);
 143 
 144   // Do we need to load the previous value?
 145   if (obj != noreg) {
 146     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 147   }
 148 
 149   // Is the previous value null?
 150   __ cbz(pre_val, done);
 151 
 152   // Can we store original value in the thread's buffer?
 153   // Is index == 0?
 154   // (The index field is typed as size_t.)
 155 
 156   __ ldr(tmp, index);                      // tmp := *index_adr
 157   __ cbz(tmp, runtime);                    // tmp == 0?
 158                                         // If yes, goto runtime
 159 
 160   __ sub(tmp, tmp, wordSize);              // tmp := tmp - wordSize
 161   __ str(tmp, index);                      // *index_adr := tmp
 162   __ ldr(rscratch1, buffer);
 163   __ add(tmp, tmp, rscratch1);             // tmp := tmp + *buffer_adr
 164 
 165   // Record the previous value
 166   __ str(pre_val, Address(tmp, 0));
 167   __ b(done);
 168 
 169   __ bind(runtime);
 170   // save the live input values
 171   RegSet saved = RegSet::of(pre_val);
 172   if (tosca_live) saved += RegSet::of(r0);
 173   if (obj != noreg) saved += RegSet::of(obj);
 174 
 175   __ push(saved, sp);
 176 
 177   // Calling the runtime using the regular call_VM_leaf mechanism generates
 178   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
 179   // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
 180   //
 181   // If we care generating the pre-barrier without a frame (e.g. in the
 182   // intrinsified Reference.get() routine) then ebp might be pointing to
 183   // the caller frame and so this check will most likely fail at runtime.
 184   //
 185   // Expanding the call directly bypasses the generation of the check.
 186   // So when we do not have have a full interpreter frame on the stack
 187   // expand_call should be passed true.
 188 
 189   if (expand_call) {
 190     assert(pre_val != c_rarg1, "smashed arg");
 191     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
 192   } else {
 193     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
 194   }
 195 
 196   __ pop(saved, sp);
 197 
 198   __ bind(done);
 199 }
 200 
 201 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
 202   if (ShenandoahReadBarrier) {
 203     read_barrier_impl(masm, dst);
 204   }
 205 }
 206 
 207 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
 208   assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
 209   Label is_null;
 210   __ cbz(dst, is_null);
 211   read_barrier_not_null_impl(masm, dst);
 212   __ bind(is_null);
 213 }
 214 
 215 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
 216   if (ShenandoahReadBarrier) {
 217     read_barrier_not_null_impl(masm, dst);
 218   }
 219 }
 220 
 221 
 222 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
 223   assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
 224   __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
 225 }
 226 
 227 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
 228   if (ShenandoahWriteBarrier) {
 229     write_barrier_impl(masm, dst);
 230   }
 231 }
 232 
 233 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
 234   assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
 235   assert(dst != rscratch1, "need rscratch1");
 236   assert(dst != rscratch2, "need rscratch2");
 237 
 238   Label done;
 239 
 240   Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 241   __ ldrb(rscratch1, gc_state);
 242 
 243   // Check for heap stability
 244   __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
 245   __ tst(rscratch1, rscratch2);
 246   __ br(Assembler::EQ, done);
 247 
 248   // Heap is unstable, need to perform the read-barrier even if WB is inactive
 249   __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
 250 
 251   // Check for evacuation-in-progress and jump to WB slow-path if needed
 252   __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
 253   __ tst(rscratch1, rscratch2);
 254   __ br(Assembler::EQ, done);
 255 
 256   RegSet to_save = RegSet::of(r0);
 257   if (dst != r0) {
 258     __ push(to_save, sp);
 259     __ mov(r0, dst);
 260   }
 261 
 262   __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
 263 
 264   if (dst != r0) {
 265     __ mov(dst, r0);
 266     __ pop(to_save, sp);
 267   }
 268 
 269   __ bind(done);
 270 }
 271 
 272 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
 273   if (ShenandoahStoreValEnqueueBarrier) {
 274     Label is_null;
 275     __ cbz(dst, is_null);
 276     write_barrier_impl(masm, dst);
 277     __ bind(is_null);
 278     // Save possibly live regs.
 279     RegSet live_regs = RegSet::range(r0, r4) - dst;
 280     __ push(live_regs, sp);
 281     __ strd(v0, __ pre(sp, 2 * -wordSize));
 282 
 283     satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
 284 
 285     // Restore possibly live regs.
 286     __ ldrd(v0, __ post(sp, 2 * wordSize));
 287     __ pop(live_regs, sp);
 288   }
 289   if (ShenandoahStoreValReadBarrier) {
 290     read_barrier_impl(masm, dst);
 291   }
 292 }
 293 
 294 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 295                                             Register dst, Address src, Register tmp1, Register tmp_thread) {
 296   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 297   bool in_heap = (decorators & IN_HEAP) != 0;
 298   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
 299   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
 300   bool on_reference = on_weak || on_phantom;
 301 
 302   if (in_heap) {
 303     read_barrier_not_null(masm, src.base());
 304   }
 305 
 306   BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 307   if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
 308     __ enter();
 309     satb_write_barrier_pre(masm /* masm */,
 310                            noreg /* obj */,
 311                            dst /* pre_val */,
 312                            rthread /* thread */,
 313                            tmp1 /* tmp */,
 314                            true /* tosca_live */,
 315                            true /* expand_call */);
 316     __ leave();
 317   }
 318 }
 319 
 320 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 321                                              Address dst, Register val, Register tmp1, Register tmp2) {
 322   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 323   bool in_heap = (decorators & IN_HEAP) != 0;
 324   if (in_heap) {
 325     write_barrier(masm, dst.base());
 326   }
 327   if (!on_oop) {
 328     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
 329     return;
 330   }
 331 
 332   // flatten object address if needed
 333   if (dst.index() == noreg && dst.offset() == 0) {
 334     if (dst.base() != r3) {
 335       __ mov(r3, dst.base());
 336     }
 337   } else {
 338     __ lea(r3, dst);
 339   }
 340 
 341   shenandoah_write_barrier_pre(masm,
 342                                r3 /* obj */,
 343                                tmp2 /* pre_val */,
 344                                rthread /* thread */,
 345                                tmp1  /* tmp */,
 346                                val != noreg /* tosca_live */,
 347                                false /* expand_call */);
 348 
 349   if (val == noreg) {
 350     BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
 351   } else {
 352     storeval_barrier(masm, val, tmp1);
 353     // G1 barrier needs uncompressed oop for region cross check.
 354     Register new_val = val;
 355     if (UseCompressedOops) {
 356       new_val = rscratch2;
 357       __ mov(new_val, val);
 358     }
 359     BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
 360   }
 361 
 362 }
 363 
 364 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
 365   __ cmp(op1, op2);
 366   if (ShenandoahAcmpBarrier) {
 367     Label done;
 368     __ br(Assembler::EQ, done);
 369     // The object may have been evacuated, but we won't see it without a
 370     // membar here.
 371     __ membar(Assembler::LoadStore| Assembler::LoadLoad);
 372     read_barrier(masm, op1);
 373     read_barrier(masm, op2);
 374     __ cmp(op1, op2);
 375     __ bind(done);
 376   }
 377 }
 378 
 379 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
 380                                                   Register var_size_in_bytes,
 381                                                   int con_size_in_bytes,
 382                                                   Register t1,
 383                                                   Register t2,
 384                                                   Label& slow_case) {
 385 
 386   assert_different_registers(obj, t2);
 387   assert_different_registers(obj, var_size_in_bytes);
 388   Register end = t2;
 389 
 390   __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
 391   if (var_size_in_bytes == noreg) {
 392     __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
 393   } else {
 394     __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
 395     __ lea(end, Address(obj, var_size_in_bytes));
 396   }
 397   __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
 398   __ cmp(end, rscratch1);
 399   __ br(Assembler::HI, slow_case);
 400 
 401   // update the tlab top pointer
 402   __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
 403 
 404   __ add(obj, obj, ShenandoahBrooksPointer::byte_size());
 405   __ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
 406 
 407   // recover var_size_in_bytes if necessary
 408   if (var_size_in_bytes == end) {
 409     __ sub(var_size_in_bytes, var_size_in_bytes, obj);
 410   }
 411 }
 412 
 413 void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
 414   bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
 415   bool is_write = (decorators & ACCESS_WRITE) != 0;
 416   if (is_write) {
 417     if (oop_not_null) {
 418       write_barrier(masm, obj);
 419     } else {
 420       Label done;
 421       __ cbz(obj, done);
 422       write_barrier(masm, obj);
 423       __ bind(done);
 424     }
 425   } else {
 426     if (oop_not_null) {
 427       read_barrier_not_null(masm, obj);
 428     } else {
 429       read_barrier(masm, obj);
 430     }
 431   }
 432 }
 433 
 434 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
 435                                                 bool acquire, bool release, bool weak, bool is_cae,
 436                                                 Register result) {
 437   Register tmp1 = rscratch1;
 438   Register tmp2 = rscratch2;
 439   bool is_narrow = UseCompressedOops;
 440   Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
 441 
 442   assert_different_registers(addr, expected, new_val, tmp1, tmp2);
 443 
 444   Label retry, done, fail;
 445 
 446   // CAS, using LL/SC pair.
 447   __ bind(retry);
 448   __ load_exclusive(tmp1, addr, size, acquire);
 449   if (is_narrow) {
 450     __ cmpw(tmp1, expected);
 451   } else {
 452     __ cmp(tmp1, expected);
 453   }
 454   __ br(Assembler::NE, fail);
 455   __ store_exclusive(tmp2, new_val, addr, size, release);
 456   if (weak) {
 457     __ cmpw(tmp2, 0u); // If the store fails, return NE to our caller
 458   } else {
 459     __ cbnzw(tmp2, retry);
 460   }
 461   __ b(done);
 462 
 463  __  bind(fail);
 464   // Check if rb(expected)==rb(tmp1)
 465   // Shuffle registers so that we have memory value ready for next expected.
 466   __ mov(tmp2, expected);
 467   __ mov(expected, tmp1);
 468   if (is_narrow) {
 469     __ decode_heap_oop(tmp1, tmp1);
 470     __ decode_heap_oop(tmp2, tmp2);
 471   }
 472   read_barrier_impl(masm, tmp1);
 473   read_barrier_impl(masm, tmp2);
 474   __ cmp(tmp1, tmp2);
 475   // Retry with expected now being the value we just loaded from addr.
 476   __ br(Assembler::EQ, retry);
 477   if (is_cae && is_narrow) {
 478     // For cmp-and-exchange and narrow oops, we need to restore
 479     // the compressed old-value. We moved it to 'expected' a few lines up.
 480     __ mov(tmp1, expected);
 481   }
 482   __ bind(done);
 483 
 484   if (is_cae) {
 485     __ mov(result, tmp1);
 486   } else {
 487     __ cset(result, Assembler::EQ);
 488   }
 489 }
 490 
 491 #ifdef COMPILER1
 492 
 493 #undef __
 494 #define __ ce->masm()->
 495 
 496 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
 497   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 498   // At this point we know that marking is in progress.
 499   // If do_load() is true then we have to emit the
 500   // load of the previous value; otherwise it has already
 501   // been loaded into _pre_val.
 502 
 503   __ bind(*stub->entry());
 504 
 505   assert(stub->pre_val()->is_register(), "Precondition.");
 506 
 507   Register pre_val_reg = stub->pre_val()->as_register();
 508 
 509   if (stub->do_load()) {
 510     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
 511   }
 512   __ cbz(pre_val_reg, *stub->continuation());
 513   ce->store_parameter(stub->pre_val()->as_register(), 0);
 514   __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
 515   __ b(*stub->continuation());
 516 }
 517 
 518 void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
 519 
 520   Register obj = stub->obj()->as_register();
 521   Register res = stub->result()->as_register();
 522 
 523   Label done;
 524 
 525   __ bind(*stub->entry());
 526 
 527   if (res != obj) {
 528     __ mov(res, obj);
 529   }
 530   // Check for null.
 531   if (stub->needs_null_check()) {
 532     __ cbz(res, done);
 533   }
 534 
 535   write_barrier(ce->masm(), res);
 536 
 537   __ bind(done);
 538   __ b(*stub->continuation());
 539 }
 540 
 541 #undef __
 542 
 543 #define __ sasm->
 544 
 545 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
 546   __ prologue("shenandoah_pre_barrier", false);
 547 
 548   // arg0 : previous value of memory
 549 
 550   BarrierSet* bs = BarrierSet::barrier_set();
 551 
 552   const Register pre_val = r0;
 553   const Register thread = rthread;
 554   const Register tmp = rscratch1;
 555 
 556   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 557   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 558 
 559   Label done;
 560   Label runtime;
 561 
 562   // Is marking still active?
 563   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 564   __ ldrb(tmp, gc_state);
 565   __ mov(rscratch2, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
 566   __ tst(tmp, rscratch2);
 567   __ br(Assembler::EQ, done);
 568 
 569   // Can we store original value in the thread's buffer?
 570   __ ldr(tmp, queue_index);
 571   __ cbz(tmp, runtime);
 572 
 573   __ sub(tmp, tmp, wordSize);
 574   __ str(tmp, queue_index);
 575   __ ldr(rscratch2, buffer);
 576   __ add(tmp, tmp, rscratch2);
 577   __ load_parameter(0, rscratch2);
 578   __ str(rscratch2, Address(tmp, 0));
 579   __ b(done);
 580 
 581   __ bind(runtime);
 582   __ push_call_clobbered_registers();
 583   __ load_parameter(0, pre_val);
 584   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
 585   __ pop_call_clobbered_registers();
 586   __ bind(done);
 587 
 588   __ epilogue();
 589 }
 590 
 591 #undef __
 592 
 593 #endif // COMPILER1
 594 
 595 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
 596   assert(_shenandoah_wb != NULL, "need write barrier stub");
 597   return _shenandoah_wb;
 598 }
 599 
 600 #define __ cgen->assembler()->
 601 
 602 // Shenandoah write barrier.
 603 //
 604 // Input:
 605 //   r0: OOP to evacuate.  Not null.
 606 //
 607 // Output:
 608 //   r0: Pointer to evacuated OOP.
 609 //
 610 // Trash rscratch1, rscratch2.  Preserve everything else.
 611 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
 612 
 613   __ align(6);
 614   StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
 615   address start = __ pc();
 616 
 617   Label work;
 618   __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
 619   __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 620   __ ldrb(rscratch2, Address(rscratch2, rscratch1));
 621   __ tbnz(rscratch2, 0, work);
 622   __ ret(lr);
 623   __ bind(work);
 624 
 625   Register obj = r0;
 626 
 627   __ enter(); // required for proper stackwalking of RuntimeStub frame
 628 
 629   __ push_call_clobbered_registers();
 630 
 631   __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
 632   __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
 633   __ mov(rscratch1, obj);
 634   __ pop_call_clobbered_registers();
 635   __ mov(obj, rscratch1);
 636 
 637   __ leave(); // required for proper stackwalking of RuntimeStub frame
 638   __ ret(lr);
 639 
 640   return start;
 641 }
 642 
 643 #undef __
 644 
 645 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
 646   if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
 647     int stub_code_size = 2048;
 648     ResourceMark rm;
 649     BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
 650     CodeBuffer buf(bb);
 651     StubCodeGenerator cgen(&buf);
 652     _shenandoah_wb = generate_shenandoah_wb(&cgen);
 653   }
 654 }