1 /*
   2  * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "gc/shenandoah/brooksPointer.hpp"
  26 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
  29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "runtime/sharedRuntime.hpp"
  34 #include "runtime/thread.hpp"
  35 
  36 #define __ masm->
  37 
  38 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
  39                                                        Register src, Register dst, Register count) {
  40 
  41   bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
  42   bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
  43   bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
  44   bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
  45 
  46   if (type == T_OBJECT || type == T_ARRAY) {
  47 #ifdef _LP64
  48     if (!checkcast && !obj_int) {
  49       // Save count for barrier
  50       __ movptr(r11, count);
  51     } else if (disjoint && obj_int) {
  52       // Save dst in r11 in the disjoint case
  53       __ movq(r11, dst);
  54     }
  55 #else
  56     if (disjoint) {
  57       __ mov(rdx, dst);          // save 'to'
  58     }
  59 #endif
  60 
  61     if (!dest_uninitialized) {
  62       Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
  63 #ifndef _LP64
  64       __ push(thread);
  65       __ get_thread(thread);
  66 #endif
  67 
  68       Label filtered;
  69       Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
  70       // Is marking active?
  71       if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
  72         __ cmpl(in_progress, 0);
  73       } else {
  74         assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
  75         __ cmpb(in_progress, 0);
  76       }
  77 
  78       NOT_LP64(__ pop(thread);)
  79 
  80         __ jcc(Assembler::equal, filtered);
  81 
  82       __ pusha();                      // push registers
  83 #ifdef _LP64
  84       if (count == c_rarg0) {
  85         if (dst == c_rarg1) {
  86           // exactly backwards!!
  87           __ xchgptr(c_rarg1, c_rarg0);
  88         } else {
  89           __ movptr(c_rarg1, count);
  90           __ movptr(c_rarg0, dst);
  91         }
  92       } else {
  93         __ movptr(c_rarg0, dst);
  94         __ movptr(c_rarg1, count);
  95       }
  96       if (UseCompressedOops) {
  97         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_narrow_oop_entry), 2);
  98       } else {
  99         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_oop_entry), 2);
 100       }
 101 #else
 102       __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_oop_entry),
 103                       dst, count);
 104 #endif
 105       __ popa();
 106       __ bind(filtered);
 107     }
 108   }
 109 
 110 }
 111 
 112 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 113                                                        Register src, Register dst, Register count) {
 114   bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
 115   bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
 116   bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
 117   Register tmp = rax;
 118 
 119   if (type == T_OBJECT || type == T_ARRAY) {
 120 #ifdef _LP64
 121     if (!checkcast && !obj_int) {
 122       // Save count for barrier
 123       count = r11;
 124     } else if (disjoint && obj_int) {
 125       // Use the saved dst in the disjoint case
 126       dst = r11;
 127     } else if (checkcast) {
 128       tmp = rscratch1;
 129     }
 130 #else
 131     if (disjoint) {
 132       __ mov(dst, rdx); // restore 'to'
 133     }
 134 #endif
 135 
 136     __ pusha();             // push registers (overkill)
 137 #ifdef _LP64
 138     if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
 139       assert_different_registers(c_rarg1, dst);
 140       __ mov(c_rarg1, count);
 141       __ mov(c_rarg0, dst);
 142     } else {
 143       assert_different_registers(c_rarg0, count);
 144       __ mov(c_rarg0, dst);
 145       __ mov(c_rarg1, count);
 146     }
 147     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
 148 #else
 149     __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry),
 150                     dst, count);
 151 #endif
 152     __ popa();
 153   }
 154 }
 155 
 156 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 157                                                                  Register obj,
 158                                                                  Register pre_val,
 159                                                                  Register thread,
 160                                                                  Register tmp,
 161                                                                  bool tosca_live,
 162                                                                  bool expand_call) {
 163 
 164   if (ShenandoahSATBBarrier) {
 165     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 166   }
 167 }
 168 
 169 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 170                                                            Register obj,
 171                                                            Register pre_val,
 172                                                            Register thread,
 173                                                            Register tmp,
 174                                                            bool tosca_live,
 175                                                            bool expand_call) {
 176   // If expand_call is true then we expand the call_VM_leaf macro
 177   // directly to skip generating the check by
 178   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
 179 
 180 #ifdef _LP64
 181   assert(thread == r15_thread, "must be");
 182 #endif // _LP64
 183 
 184   Label done;
 185   Label runtime;
 186 
 187   assert(pre_val != noreg, "check this code");
 188 
 189   if (obj != noreg) {
 190     assert_different_registers(obj, pre_val, tmp);
 191     assert(pre_val != rax, "check this code");
 192   }
 193 
 194   Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
 195   Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 196   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 197 
 198   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 199   __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
 200   __ jcc(Assembler::zero, done);
 201 
 202   // Do we need to load the previous value?
 203   if (obj != noreg) {
 204     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 205   }
 206 
 207   // Is the previous value null?
 208   __ cmpptr(pre_val, (int32_t) NULL_WORD);
 209   __ jcc(Assembler::equal, done);
 210 
 211   // Can we store original value in the thread's buffer?
 212   // Is index == 0?
 213   // (The index field is typed as size_t.)
 214 
 215   __ movptr(tmp, index);                   // tmp := *index_adr
 216   __ cmpptr(tmp, 0);                       // tmp == 0?
 217   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
 218 
 219   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
 220   __ movptr(index, tmp);                   // *index_adr := tmp
 221   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
 222 
 223   // Record the previous value
 224   __ movptr(Address(tmp, 0), pre_val);
 225   __ jmp(done);
 226 
 227   __ bind(runtime);
 228   // save the live input values
 229   if(tosca_live) __ push(rax);
 230 
 231   if (obj != noreg && obj != rax)
 232     __ push(obj);
 233 
 234   if (pre_val != rax)
 235     __ push(pre_val);
 236 
 237   // Calling the runtime using the regular call_VM_leaf mechanism generates
 238   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
 239   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
 240   //
 241   // If we care generating the pre-barrier without a frame (e.g. in the
 242   // intrinsified Reference.get() routine) then ebp might be pointing to
 243   // the caller frame and so this check will most likely fail at runtime.
 244   //
 245   // Expanding the call directly bypasses the generation of the check.
 246   // So when we do not have have a full interpreter frame on the stack
 247   // expand_call should be passed true.
 248 
 249   NOT_LP64( __ push(thread); )
 250 
 251   if (expand_call) {
 252     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
 253 #ifdef _LP64
 254     if (c_rarg1 != thread) {
 255       __ mov(c_rarg1, thread);
 256     }
 257     if (c_rarg0 != pre_val) {
 258       __ mov(c_rarg0, pre_val);
 259     }
 260 #else
 261     __ push(thread);
 262     __ push(pre_val);
 263 #endif
 264     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
 265   } else {
 266     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
 267   }
 268 
 269   NOT_LP64( __ pop(thread); )
 270 
 271   // save the live input values
 272   if (pre_val != rax)
 273     __ pop(pre_val);
 274 
 275   if (obj != noreg && obj != rax)
 276     __ pop(obj);
 277 
 278   if(tosca_live) __ pop(rax);
 279 
 280   __ bind(done);
 281 }
 282 
 283 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_post(MacroAssembler* masm,
 284                                                                   Register store_addr,
 285                                                                   Register new_val,
 286                                                                   Register thread,
 287                                                                   Register tmp,
 288                                                                   Register tmp2) {
 289   assert(UseShenandoahGC, "why else should we be here?");
 290 
 291   if (! UseShenandoahMatrix) {
 292     // No need for that barrier if not using matrix.
 293     return;
 294   }
 295 
 296   Label done;
 297   __ testptr(new_val, new_val);
 298   __ jcc(Assembler::zero, done);
 299   ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
 300   address matrix_addr = matrix->matrix_addr();
 301   __ movptr(rscratch1, (intptr_t) ShenandoahHeap::heap()->base());
 302   // Compute to-region index
 303   __ movptr(tmp, new_val);
 304   __ subptr(tmp, rscratch1);
 305   __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 306   // Compute from-region index
 307   __ movptr(tmp2, store_addr);
 308   __ subptr(tmp2, rscratch1);
 309   __ shrptr(tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 310   // Compute matrix index
 311   __ imulptr(tmp, tmp, matrix->stride_jint());
 312   __ addptr(tmp, tmp2);
 313   // Address is _matrix[to * stride + from]
 314   __ movptr(rscratch1, (intptr_t) matrix_addr);
 315   // Test if the element is already set.
 316   __ cmpb(Address(rscratch1, tmp, Address::times_1), 0);
 317   __ jcc(Assembler::notEqual, done);
 318   // Store true, if not yet set.
 319   __ movb(Address(rscratch1, tmp, Address::times_1), 1);
 320   __ bind(done);
 321 }
 322 
 323 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
 324   if (ShenandoahReadBarrier) {
 325     read_barrier_impl(masm, dst);
 326   }
 327 }
 328 
 329 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
 330   assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
 331   Label is_null;
 332   __ testptr(dst, dst);
 333   __ jcc(Assembler::zero, is_null);
 334   read_barrier_not_null_impl(masm, dst);
 335   __ bind(is_null);
 336 }
 337 
 338 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
 339   if (ShenandoahReadBarrier) {
 340     read_barrier_not_null_impl(masm, dst);
 341   }
 342 }
 343 
 344 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
 345   assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
 346   __ movptr(dst, Address(dst, BrooksPointer::byte_offset()));
 347 }
 348 
 349 
 350 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
 351   if (ShenandoahWriteBarrier) {
 352     write_barrier_impl(masm, dst);
 353   }
 354 }
 355 
 356 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
 357   assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
 358 #ifdef _LP64
 359   assert(dst != rscratch1, "different regs");
 360 
 361   Label done;
 362 
 363   Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 364   __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
 365 
 366   // Now check if evacuation is in progress.
 367   read_barrier_not_null(masm, dst);
 368 
 369   __ jcc(Assembler::zero, done);
 370   __ push(rscratch1);
 371   __ push(rscratch2);
 372 
 373   __ movptr(rscratch1, dst);
 374   __ shrptr(rscratch1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 375   __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
 376   __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1));
 377   __ testb(rscratch2, 0x1);
 378 
 379   __ pop(rscratch2);
 380   __ pop(rscratch1);
 381 
 382   __ jcc(Assembler::zero, done);
 383 
 384   __ push(rscratch1);
 385 
 386   // Save possibly live regs.
 387   if (dst != rax) {
 388     __ push(rax);
 389   }
 390   if (dst != rbx) {
 391     __ push(rbx);
 392   }
 393   if (dst != rcx) {
 394     __ push(rcx);
 395   }
 396   if (dst != rdx) {
 397     __ push(rdx);
 398   }
 399   if (dst != c_rarg1) {
 400     __ push(c_rarg1);
 401   }
 402 
 403   __ subptr(rsp, 2 * Interpreter::stackElementSize);
 404   __ movdbl(Address(rsp, 0), xmm0);
 405 
 406   // Call into runtime
 407   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_IRT), dst);
 408   __ mov(rscratch1, rax);
 409 
 410   // Restore possibly live regs.
 411   __ movdbl(xmm0, Address(rsp, 0));
 412   __ addptr(rsp, 2 * Interpreter::stackElementSize);
 413 
 414   if (dst != c_rarg1) {
 415     __ pop(c_rarg1);
 416   }
 417   if (dst != rdx) {
 418     __ pop(rdx);
 419   }
 420   if (dst != rcx) {
 421     __ pop(rcx);
 422   }
 423   if (dst != rbx) {
 424     __ pop(rbx);
 425   }
 426   if (dst != rax) {
 427     __ pop(rax);
 428   }
 429 
 430   // Move result into dst reg.
 431   __ mov(dst, rscratch1);
 432 
 433   __ pop(rscratch1);
 434 
 435   __ bind(done);
 436 #else
 437   Unimplemented();
 438 #endif
 439 }
 440 
 441 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
 442   if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
 443     storeval_barrier_impl(masm, dst, tmp);
 444   }
 445 }
 446 
 447 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
 448   assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
 449 
 450   if (dst == noreg) return;
 451 
 452 #ifdef _LP64
 453   if (ShenandoahStoreValEnqueueBarrier) {
 454     Label is_null;
 455     __ testptr(dst, dst);
 456     __ jcc(Assembler::zero, is_null);
 457     write_barrier_impl(masm, dst);
 458     __ bind(is_null);
 459 
 460     // The set of registers to be saved+restored is the same as in the write-barrier above.
 461     // Those are the commonly used registers in the interpreter.
 462     __ push(rbx);
 463     __ push(rcx);
 464     __ push(rdx);
 465     __ push(c_rarg1);
 466     __ subptr(rsp, 2 * Interpreter::stackElementSize);
 467     __ movdbl(Address(rsp, 0), xmm0);
 468 
 469     satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
 470     __ movdbl(xmm0, Address(rsp, 0));
 471     __ addptr(rsp, 2 * Interpreter::stackElementSize);
 472     __ pop(c_rarg1);
 473     __ pop(rdx);
 474     __ pop(rcx);
 475     __ pop(rbx);
 476   }
 477   if (ShenandoahStoreValReadBarrier) {
 478     read_barrier_impl(masm, dst);
 479   }
 480 #else
 481   Unimplemented();
 482 #endif
 483 }
 484 
 485 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 486              Register dst, Address src, Register tmp1, Register tmp_thread) {
 487   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 488   bool in_heap = (decorators & IN_HEAP) != 0;
 489   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
 490   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
 491   bool on_reference = on_weak || on_phantom;
 492   // tty->print_cr("RB src.base: %s", src.base()->name());
 493   // __ verify_oop(src.base(), "broken oop before RB");
 494   /*
 495   if (in_heap) {
 496     read_barrier_not_null(masm, src.base());
 497   }
 498   */
 499   // __ verify_oop(src.base(), "broken oop before RB");
 500   BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 501   if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
 502     const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 503     NOT_LP64(__ get_thread(thread));
 504 
 505     // Generate the SATB pre-barrier code to log the value of
 506     // the referent field in an SATB buffer.
 507     shenandoah_write_barrier_pre(masm /* masm */,
 508                                  noreg /* obj */,
 509                                  dst /* pre_val */,
 510                                  thread /* thread */,
 511                                  tmp1 /* tmp */,
 512                                  true /* tosca_live */,
 513                                  true /* expand_call */);
 514   }
 515 }
 516 
 517 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 518               Address dst, Register val, Register tmp1, Register tmp2) {
 519 
 520   bool in_heap = (decorators & IN_HEAP) != 0;
 521   bool in_concurrent_root = (decorators & IN_CONCURRENT_ROOT) != 0;
 522   /*
 523   if (in_heap) {
 524     write_barrier(masm, dst.base());
 525   }
 526   */
 527   if (type == T_OBJECT || type == T_ARRAY) {
 528     bool needs_pre_barrier = in_heap || in_concurrent_root;
 529     bool needs_post_barrier = val != noreg && in_heap && UseShenandoahMatrix;
 530 
 531     Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
 532     Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 533     // flatten object address if needed
 534     // We do it regardless of precise because we need the registers
 535     if (dst.index() == noreg && dst.disp() == 0) {
 536       if (dst.base() != tmp1) {
 537         __ movptr(tmp1, dst.base());
 538       }
 539     } else {
 540       __ lea(tmp1, dst);
 541     }
 542 
 543 #ifndef _LP64
 544     InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 545 #endif
 546 
 547     NOT_LP64(__ get_thread(rcx));
 548     NOT_LP64(imasm->save_bcp());
 549 
 550     if (needs_pre_barrier) {
 551       shenandoah_write_barrier_pre(masm /*masm*/,
 552                                    tmp1 /* obj */,
 553                                    tmp2 /* pre_val */,
 554                                    rthread /* thread */,
 555                                    tmp3  /* tmp */,
 556                                    val != noreg /* tosca_live */,
 557                                    false /* expand_call */);
 558     }
 559     if (val == noreg) {
 560       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 561     } else {
 562       storeval_barrier(masm, val, tmp3);
 563       Register new_val = val;
 564       if (needs_post_barrier) {
 565         if (UseCompressedOops) {
 566           new_val = tmp2;
 567           __ movptr(new_val, val);
 568         }
 569       }
 570       BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 571 
 572       if (needs_post_barrier) {
 573         shenandoah_write_barrier_post(masm /*masm*/,
 574                                       tmp1 /* store_adr */,
 575                                       new_val /* new_val */,
 576                                       rthread /* thread */,
 577                                       tmp3 /* tmp */,
 578                                       tmp2 /* tmp2 */);
 579       }
 580     }
 581     NOT_LP64(imasm->restore_bcp());
 582 
 583   } else {
 584     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
 585   }
 586 }
 587 
 588 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, DecoratorSet decorators, Register op1, Register op2) {
 589   __ cmpptr(op1, op2);
 590   if (ShenandoahAcmpBarrier) {
 591     Label done;
 592     __ jccb(Assembler::equal, done);
 593     read_barrier(masm, op1);
 594     read_barrier(masm, op2);
 595     __ cmpptr(op1, op2);
 596     __ bind(done);
 597   }
 598 }
 599 
 600 void ShenandoahBarrierSetAssembler::obj_equals_addr(MacroAssembler* masm, DecoratorSet decorators, Register src1, Address src2) {
 601   __ cmpptr(src1, src2);
 602   if (ShenandoahAcmpBarrier) {
 603     Label done;
 604     __ jccb(Assembler::equal, done);
 605     __ movptr(rscratch2, src2);
 606     read_barrier(masm, src1);
 607     read_barrier(masm, rscratch2);
 608     __ cmpptr(src1, rscratch2);
 609     __ bind(done);
 610   }
 611 }
 612 
 613 void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
 614   bool oop_not_null = (decorators & OOP_NOT_NULL) != 0;
 615   if (oop_not_null) {
 616     read_barrier_not_null(masm, obj);
 617   } else {
 618     read_barrier(masm, obj);
 619   }
 620 }
 621 
 622 void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
 623   write_barrier(masm, obj);
 624 }