1 /* 2 * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 28 #include "gc/shenandoah/shenandoahForwarding.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 31 #include "gc/shenandoah/shenandoahHeuristics.hpp" 32 #include "gc/shenandoah/shenandoahRuntime.hpp" 33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 34 #include "interpreter/interpreter.hpp" 35 #include "interpreter/interp_masm.hpp" 36 #include "runtime/sharedRuntime.hpp" 37 #include "runtime/thread.hpp" 38 #include "utilities/macros.hpp" 39 #ifdef COMPILER1 40 #include "c1/c1_LIRAssembler.hpp" 41 #include "c1/c1_MacroAssembler.hpp" 42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" 43 #endif 44 45 #define __ masm-> 46 47 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; 48 49 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 50 Register src, Register dst, Register count) { 51 52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; 53 54 if (is_reference_type(type)) { 55 56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { 57 #ifdef _LP64 58 Register thread = r15_thread; 59 #else 60 Register thread = rax; 61 if (thread == src || thread == dst || thread == count) { 62 thread = rbx; 63 } 64 if (thread == src || thread == dst || thread == count) { 65 thread = rcx; 66 } 67 if (thread == src || thread == dst || thread == count) { 68 thread = rdx; 69 } 70 __ push(thread); 71 __ get_thread(thread); 72 #endif 73 assert_different_registers(src, dst, count, thread); 74 75 Label done; 76 // Short-circuit if count == 0. 77 __ testptr(count, count); 78 __ jcc(Assembler::zero, done); 79 80 // Avoid runtime call when not marking. 81 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 82 int flags = ShenandoahHeap::HAS_FORWARDED; 83 if (!dest_uninitialized) { 84 flags |= ShenandoahHeap::MARKING; 85 } 86 __ testb(gc_state, flags); 87 __ jcc(Assembler::zero, done); 88 89 __ pusha(); // push registers 90 #ifdef _LP64 91 assert(src == rdi, "expected"); 92 assert(dst == rsi, "expected"); 93 assert(count == rdx, "expected"); 94 if (UseCompressedOops) { 95 if (dest_uninitialized) { 96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count); 97 } else { 98 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count); 99 } 100 } else 101 #endif 102 { 103 if (dest_uninitialized) { 104 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count); 105 } else { 106 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count); 107 } 108 } 109 __ popa(); 110 __ bind(done); 111 NOT_LP64(__ pop(thread);) 112 } 113 } 114 115 } 116 117 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, 118 Register obj, 119 Register pre_val, 120 Register thread, 121 Register tmp, 122 bool tosca_live, 123 bool expand_call) { 124 125 if (ShenandoahSATBBarrier) { 126 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call); 127 } 128 } 129 130 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, 131 Register obj, 132 Register pre_val, 133 Register thread, 134 Register tmp, 135 bool tosca_live, 136 bool expand_call) { 137 // If expand_call is true then we expand the call_VM_leaf macro 138 // directly to skip generating the check by 139 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 140 141 #ifdef _LP64 142 assert(thread == r15_thread, "must be"); 143 #endif // _LP64 144 145 Label done; 146 Label runtime; 147 148 assert(pre_val != noreg, "check this code"); 149 150 if (obj != noreg) { 151 assert_different_registers(obj, pre_val, tmp); 152 assert(pre_val != rax, "check this code"); 153 } 154 155 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); 156 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 157 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 158 159 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 160 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); 161 __ jcc(Assembler::zero, done); 162 163 // Do we need to load the previous value? 164 if (obj != noreg) { 165 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 166 } 167 168 // Is the previous value null? 169 __ cmpptr(pre_val, (int32_t) NULL_WORD); 170 __ jcc(Assembler::equal, done); 171 172 // Can we store original value in the thread's buffer? 173 // Is index == 0? 174 // (The index field is typed as size_t.) 175 176 __ movptr(tmp, index); // tmp := *index_adr 177 __ cmpptr(tmp, 0); // tmp == 0? 178 __ jcc(Assembler::equal, runtime); // If yes, goto runtime 179 180 __ subptr(tmp, wordSize); // tmp := tmp - wordSize 181 __ movptr(index, tmp); // *index_adr := tmp 182 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr 183 184 // Record the previous value 185 __ movptr(Address(tmp, 0), pre_val); 186 __ jmp(done); 187 188 __ bind(runtime); 189 // save the live input values 190 if(tosca_live) __ push(rax); 191 192 if (obj != noreg && obj != rax) 193 __ push(obj); 194 195 if (pre_val != rax) 196 __ push(pre_val); 197 198 // Calling the runtime using the regular call_VM_leaf mechanism generates 199 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 200 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL. 201 // 202 // If we care generating the pre-barrier without a frame (e.g. in the 203 // intrinsified Reference.get() routine) then ebp might be pointing to 204 // the caller frame and so this check will most likely fail at runtime. 205 // 206 // Expanding the call directly bypasses the generation of the check. 207 // So when we do not have have a full interpreter frame on the stack 208 // expand_call should be passed true. 209 210 NOT_LP64( __ push(thread); ) 211 212 #ifdef _LP64 213 // We move pre_val into c_rarg0 early, in order to avoid smashing it, should 214 // pre_val be c_rarg1 (where the call prologue would copy thread argument). 215 // Note: this should not accidentally smash thread, because thread is always r15. 216 assert(thread != c_rarg0, "smashed arg"); 217 if (c_rarg0 != pre_val) { 218 __ mov(c_rarg0, pre_val); 219 } 220 #endif 221 222 if (expand_call) { 223 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) 224 #ifdef _LP64 225 if (c_rarg1 != thread) { 226 __ mov(c_rarg1, thread); 227 } 228 // Already moved pre_val into c_rarg0 above 229 #else 230 __ push(thread); 231 __ push(pre_val); 232 #endif 233 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2); 234 } else { 235 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread); 236 } 237 238 NOT_LP64( __ pop(thread); ) 239 240 // save the live input values 241 if (pre_val != rax) 242 __ pop(pre_val); 243 244 if (obj != noreg && obj != rax) 245 __ pop(obj); 246 247 if(tosca_live) __ pop(rax); 248 249 __ bind(done); 250 } 251 252 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address src) { 253 assert(ShenandoahLoadRefBarrier, "Should be enabled"); 254 255 Label done; 256 257 #ifdef _LP64 258 Register thread = r15_thread; 259 #else 260 Register thread = rcx; 261 if (thread == dst) { 262 thread = rbx; 263 } 264 __ push(thread); 265 __ get_thread(thread); 266 #endif 267 268 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 269 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED); 270 __ jccb(Assembler::zero, done); 271 272 // Use rsi for src address 273 const Register src_addr = rsi; 274 // Setup address parameter first, if it does not clobber oop in dst 275 bool need_addr_setup = (src_addr != dst); 276 277 if (need_addr_setup) { 278 __ push(src_addr); 279 __ lea(src_addr, src); 280 281 if (dst != rax) { 282 // Move obj into rax and save rax 283 __ push(rax); 284 __ movptr(rax, dst); 285 } 286 } else { 287 // dst == rsi 288 __ push(rax); 289 __ movptr(rax, dst); 290 291 // we can clobber it, since it is outgoing register 292 __ lea(src_addr, src); 293 } 294 295 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); 296 297 if (need_addr_setup) { 298 if (dst != rax) { 299 __ movptr(dst, rax); 300 __ pop(rax); 301 } 302 __ pop(src_addr); 303 } else { 304 __ movptr(dst, rax); 305 __ pop(rax); 306 } 307 308 __ bind(done); 309 310 #ifndef _LP64 311 __ pop(thread); 312 #endif 313 } 314 315 void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src) { 316 if (!ShenandoahLoadRefBarrier) { 317 return; 318 } 319 320 Label done; 321 Label not_null; 322 Label slow_path; 323 __ block_comment("load_reference_barrier_native { "); 324 325 // null check 326 __ testptr(dst, dst); 327 __ jcc(Assembler::notZero, not_null); 328 __ jmp(done); 329 __ bind(not_null); 330 331 332 #ifdef _LP64 333 Register thread = r15_thread; 334 #else 335 Register thread = rcx; 336 if (thread == dst) { 337 thread = rbx; 338 } 339 __ push(thread); 340 __ get_thread(thread); 341 #endif 342 assert_different_registers(dst, thread); 343 344 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 345 __ testb(gc_state, ShenandoahHeap::EVACUATION); 346 #ifndef _LP64 347 __ pop(thread); 348 #endif 349 __ jccb(Assembler::notZero, slow_path); 350 __ jmp(done); 351 __ bind(slow_path); 352 353 if (dst != rax) { 354 __ push(rax); 355 } 356 __ push(rcx); 357 __ push(rdx); 358 __ push(rdi); 359 __ push(rsi); 360 #ifdef _LP64 361 __ push(r8); 362 __ push(r9); 363 __ push(r10); 364 __ push(r11); 365 __ push(r12); 366 __ push(r13); 367 __ push(r14); 368 __ push(r15); 369 #endif 370 371 assert_different_registers(dst, rsi); 372 __ lea(rsi, src); 373 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), dst, rsi); 374 375 #ifdef _LP64 376 __ pop(r15); 377 __ pop(r14); 378 __ pop(r13); 379 __ pop(r12); 380 __ pop(r11); 381 __ pop(r10); 382 __ pop(r9); 383 __ pop(r8); 384 #endif 385 __ pop(rsi); 386 __ pop(rdi); 387 __ pop(rdx); 388 __ pop(rcx); 389 390 if (dst != rax) { 391 __ movptr(dst, rax); 392 __ pop(rax); 393 } 394 395 __ bind(done); 396 __ block_comment("load_reference_barrier_native { "); 397 } 398 399 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { 400 if (ShenandoahStoreValEnqueueBarrier) { 401 storeval_barrier_impl(masm, dst, tmp); 402 } 403 } 404 405 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { 406 assert(ShenandoahStoreValEnqueueBarrier, "should be enabled"); 407 408 if (dst == noreg) return; 409 410 if (ShenandoahStoreValEnqueueBarrier) { 411 // The set of registers to be saved+restored is the same as in the write-barrier above. 412 // Those are the commonly used registers in the interpreter. 413 __ pusha(); 414 // __ push_callee_saved_registers(); 415 __ subptr(rsp, 2 * Interpreter::stackElementSize); 416 __ movdbl(Address(rsp, 0), xmm0); 417 418 #ifdef _LP64 419 Register thread = r15_thread; 420 #else 421 Register thread = rcx; 422 if (thread == dst || thread == tmp) { 423 thread = rdi; 424 } 425 if (thread == dst || thread == tmp) { 426 thread = rbx; 427 } 428 __ get_thread(thread); 429 #endif 430 assert_different_registers(dst, tmp, thread); 431 432 satb_write_barrier_pre(masm, noreg, dst, thread, tmp, true, false); 433 __ movdbl(xmm0, Address(rsp, 0)); 434 __ addptr(rsp, 2 * Interpreter::stackElementSize); 435 //__ pop_callee_saved_registers(); 436 __ popa(); 437 } 438 } 439 440 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src) { 441 if (ShenandoahLoadRefBarrier) { 442 Label done; 443 __ testptr(dst, dst); 444 __ jcc(Assembler::zero, done); 445 load_reference_barrier_not_null(masm, dst, src); 446 __ bind(done); 447 } 448 } 449 450 // 451 // Arguments: 452 // 453 // Inputs: 454 // src: oop location, might be clobbered 455 // tmp1: scratch register, might not be valid. 456 // 457 // Output: 458 // dst: oop loaded from src location 459 // 460 // Kill: 461 // tmp1 (if it is valid) 462 // 463 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 464 Register dst, Address src, Register tmp1, Register tmp_thread) { 465 // 1: non-reference load, no additional barrier is needed 466 if (!is_reference_type(type)) { 467 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 468 return; 469 } 470 471 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected"); 472 473 // 2: load a reference from src location and apply LRB if needed 474 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { 475 Register result_dst = dst; 476 bool use_tmp1_for_dst = false; 477 478 // Preserve src location for LRB 479 if (dst == src.base() || dst == src.index()) { 480 // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at() 481 if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) { 482 dst = tmp1; 483 use_tmp1_for_dst = true; 484 } else { 485 dst = rdi; 486 __ push(dst); 487 } 488 assert_different_registers(dst, src.base(), src.index()); 489 } 490 491 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 492 493 if (ShenandoahBarrierSet::use_load_reference_barrier_native(decorators, type)) { 494 load_reference_barrier_native(masm, dst, src); 495 } else { 496 load_reference_barrier(masm, dst, src); 497 } 498 499 // Move loaded oop to final destination 500 if (dst != result_dst) { 501 __ movptr(result_dst, dst); 502 503 if (!use_tmp1_for_dst) { 504 __ pop(dst); 505 } 506 507 dst = result_dst; 508 } 509 } else { 510 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 511 } 512 513 // 3: apply keep-alive barrier if needed 514 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { 515 __ push_IU_state(); 516 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); 517 assert_different_registers(dst, tmp1, tmp_thread); 518 if (!thread->is_valid()) { 519 thread = rdx; 520 } 521 NOT_LP64(__ get_thread(thread)); 522 // Generate the SATB pre-barrier code to log the value of 523 // the referent field in an SATB buffer. 524 shenandoah_write_barrier_pre(masm /* masm */, 525 noreg /* obj */, 526 dst /* pre_val */, 527 thread /* thread */, 528 tmp1 /* tmp */, 529 true /* tosca_live */, 530 true /* expand_call */); 531 __ pop_IU_state(); 532 } 533 } 534 535 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 536 Address dst, Register val, Register tmp1, Register tmp2) { 537 538 bool on_oop = is_reference_type(type); 539 bool in_heap = (decorators & IN_HEAP) != 0; 540 bool as_normal = (decorators & AS_NORMAL) != 0; 541 if (on_oop && in_heap) { 542 bool needs_pre_barrier = as_normal; 543 544 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); 545 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx); 546 // flatten object address if needed 547 // We do it regardless of precise because we need the registers 548 if (dst.index() == noreg && dst.disp() == 0) { 549 if (dst.base() != tmp1) { 550 __ movptr(tmp1, dst.base()); 551 } 552 } else { 553 __ lea(tmp1, dst); 554 } 555 556 assert_different_registers(val, tmp1, tmp2, tmp3, rthread); 557 558 #ifndef _LP64 559 __ get_thread(rthread); 560 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm); 561 imasm->save_bcp(); 562 #endif 563 564 if (needs_pre_barrier) { 565 shenandoah_write_barrier_pre(masm /*masm*/, 566 tmp1 /* obj */, 567 tmp2 /* pre_val */, 568 rthread /* thread */, 569 tmp3 /* tmp */, 570 val != noreg /* tosca_live */, 571 false /* expand_call */); 572 } 573 if (val == noreg) { 574 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); 575 } else { 576 storeval_barrier(masm, val, tmp3); 577 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); 578 } 579 NOT_LP64(imasm->restore_bcp()); 580 } else { 581 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); 582 } 583 } 584 585 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 586 Register obj, Register tmp, Label& slowpath) { 587 Label done; 588 // Resolve jobject 589 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); 590 591 // Check for null. 592 __ testptr(obj, obj); 593 __ jcc(Assembler::zero, done); 594 595 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()); 596 __ testb(gc_state, ShenandoahHeap::EVACUATION); 597 __ jccb(Assembler::notZero, slowpath); 598 __ bind(done); 599 } 600 601 // Special Shenandoah CAS implementation that handles false negatives 602 // due to concurrent evacuation. 603 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, 604 Register res, Address addr, Register oldval, Register newval, 605 bool exchange, Register tmp1, Register tmp2) { 606 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); 607 assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); 608 assert_different_registers(oldval, newval, tmp1, tmp2); 609 610 Label L_success, L_failure; 611 612 // Remember oldval for retry logic below 613 #ifdef _LP64 614 if (UseCompressedOops) { 615 __ movl(tmp1, oldval); 616 } else 617 #endif 618 { 619 __ movptr(tmp1, oldval); 620 } 621 622 // Step 1. Fast-path. 623 // 624 // Try to CAS with given arguments. If successful, then we are done. 625 626 if (os::is_MP()) __ lock(); 627 #ifdef _LP64 628 if (UseCompressedOops) { 629 __ cmpxchgl(newval, addr); 630 } else 631 #endif 632 { 633 __ cmpxchgptr(newval, addr); 634 } 635 __ jcc(Assembler::equal, L_success); 636 637 // Step 2. CAS had failed. This may be a false negative. 638 // 639 // The trouble comes when we compare the to-space pointer with the from-space 640 // pointer to the same object. To resolve this, it will suffice to resolve 641 // the value from memory -- this will give both to-space pointers. 642 // If they mismatch, then it was a legitimate failure. 643 // 644 // Before reaching to resolve sequence, see if we can avoid the whole shebang 645 // with filters. 646 647 // Filter: when offending in-memory value is NULL, the failure is definitely legitimate 648 __ testptr(oldval, oldval); 649 __ jcc(Assembler::zero, L_failure); 650 651 // Filter: when heap is stable, the failure is definitely legitimate 652 #ifdef _LP64 653 const Register thread = r15_thread; 654 #else 655 const Register thread = tmp2; 656 __ get_thread(thread); 657 #endif 658 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 659 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED); 660 __ jcc(Assembler::zero, L_failure); 661 662 #ifdef _LP64 663 if (UseCompressedOops) { 664 __ movl(tmp2, oldval); 665 __ decode_heap_oop(tmp2); 666 } else 667 #endif 668 { 669 __ movptr(tmp2, oldval); 670 } 671 672 // Decode offending in-memory value. 673 // Test if-forwarded 674 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value); 675 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded 676 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded 677 678 // Load and mask forwarding pointer 679 __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes())); 680 __ shrptr(tmp2, 2); 681 __ shlptr(tmp2, 2); 682 683 #ifdef _LP64 684 if (UseCompressedOops) { 685 __ decode_heap_oop(tmp1); // decode for comparison 686 } 687 #endif 688 689 // Now we have the forwarded offender in tmp2. 690 // Compare and if they don't match, we have legitimate failure 691 __ cmpptr(tmp1, tmp2); 692 __ jcc(Assembler::notEqual, L_failure); 693 694 // Step 3. Need to fix the memory ptr before continuing. 695 // 696 // At this point, we have from-space oldval in the register, and its to-space 697 // address is in tmp2. Let's try to update it into memory. We don't care if it 698 // succeeds or not. If it does, then the retrying CAS would see it and succeed. 699 // If this fixup fails, this means somebody else beat us to it, and necessarily 700 // with to-space ptr store. We still have to do the retry, because the GC might 701 // have updated the reference for us. 702 703 #ifdef _LP64 704 if (UseCompressedOops) { 705 __ encode_heap_oop(tmp2); // previously decoded at step 2. 706 } 707 #endif 708 709 if (os::is_MP()) __ lock(); 710 #ifdef _LP64 711 if (UseCompressedOops) { 712 __ cmpxchgl(tmp2, addr); 713 } else 714 #endif 715 { 716 __ cmpxchgptr(tmp2, addr); 717 } 718 719 // Step 4. Try to CAS again. 720 // 721 // This is guaranteed not to have false negatives, because oldval is definitely 722 // to-space, and memory pointer is to-space as well. Nothing is able to store 723 // from-space ptr into memory anymore. Make sure oldval is restored, after being 724 // garbled during retries. 725 // 726 #ifdef _LP64 727 if (UseCompressedOops) { 728 __ movl(oldval, tmp2); 729 } else 730 #endif 731 { 732 __ movptr(oldval, tmp2); 733 } 734 735 if (os::is_MP()) __ lock(); 736 #ifdef _LP64 737 if (UseCompressedOops) { 738 __ cmpxchgl(newval, addr); 739 } else 740 #endif 741 { 742 __ cmpxchgptr(newval, addr); 743 } 744 if (!exchange) { 745 __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump 746 } 747 748 // Step 5. If we need a boolean result out of CAS, set the flag appropriately. 749 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS. 750 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return. 751 752 if (exchange) { 753 __ bind(L_failure); 754 __ bind(L_success); 755 } else { 756 assert(res != NULL, "need result register"); 757 758 Label exit; 759 __ bind(L_failure); 760 __ xorptr(res, res); 761 __ jmpb(exit); 762 763 __ bind(L_success); 764 __ movptr(res, 1); 765 __ bind(exit); 766 } 767 } 768 769 #undef __ 770 771 #ifdef COMPILER1 772 773 #define __ ce->masm()-> 774 775 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { 776 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 777 // At this point we know that marking is in progress. 778 // If do_load() is true then we have to emit the 779 // load of the previous value; otherwise it has already 780 // been loaded into _pre_val. 781 782 __ bind(*stub->entry()); 783 assert(stub->pre_val()->is_register(), "Precondition."); 784 785 Register pre_val_reg = stub->pre_val()->as_register(); 786 787 if (stub->do_load()) { 788 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); 789 } 790 791 __ cmpptr(pre_val_reg, (int32_t)NULL_WORD); 792 __ jcc(Assembler::equal, *stub->continuation()); 793 ce->store_parameter(stub->pre_val()->as_register(), 0); 794 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 795 __ jmp(*stub->continuation()); 796 797 } 798 799 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { 800 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 801 __ bind(*stub->entry()); 802 803 Register obj = stub->obj()->as_register(); 804 Register res = stub->result()->as_register(); 805 Register addr = stub->addr()->as_register(); 806 Register tmp1 = stub->tmp1()->as_register(); 807 Register tmp2 = stub->tmp2()->as_register(); 808 assert_different_registers(obj, res, addr, tmp1, tmp2); 809 810 Label slow_path; 811 812 assert(res == rax, "result must arrive in rax"); 813 814 if (res != obj) { 815 __ mov(res, obj); 816 } 817 818 // Check for null. 819 __ testptr(res, res); 820 __ jcc(Assembler::zero, *stub->continuation()); 821 822 // Check for object being in the collection set. 823 __ mov(tmp1, res); 824 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 825 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); 826 #ifdef _LP64 827 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); 828 __ testbool(tmp2); 829 #else 830 // On x86_32, C1 register allocator can give us the register without 8-bit support. 831 // Do the full-register access and test to avoid compilation failures. 832 __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1)); 833 __ testptr(tmp2, 0xFF); 834 #endif 835 __ jcc(Assembler::zero, *stub->continuation()); 836 837 __ bind(slow_path); 838 ce->store_parameter(res, 0); 839 ce->store_parameter(addr, 1); 840 __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin())); 841 842 __ jmp(*stub->continuation()); 843 } 844 845 #undef __ 846 847 #define __ sasm-> 848 849 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 850 __ prologue("shenandoah_pre_barrier", false); 851 // arg0 : previous value of memory 852 853 __ push(rax); 854 __ push(rdx); 855 856 const Register pre_val = rax; 857 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); 858 const Register tmp = rdx; 859 860 NOT_LP64(__ get_thread(thread);) 861 862 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); 863 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); 864 865 Label done; 866 Label runtime; 867 868 // Is SATB still active? 869 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 870 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); 871 __ jcc(Assembler::zero, done); 872 873 // Can we store original value in the thread's buffer? 874 875 __ movptr(tmp, queue_index); 876 __ testptr(tmp, tmp); 877 __ jcc(Assembler::zero, runtime); 878 __ subptr(tmp, wordSize); 879 __ movptr(queue_index, tmp); 880 __ addptr(tmp, buffer); 881 882 // prev_val (rax) 883 __ load_parameter(0, pre_val); 884 __ movptr(Address(tmp, 0), pre_val); 885 __ jmp(done); 886 887 __ bind(runtime); 888 889 __ save_live_registers_no_oop_map(true); 890 891 // load the pre-value 892 __ load_parameter(0, rcx); 893 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread); 894 895 __ restore_live_registers(true); 896 897 __ bind(done); 898 899 __ pop(rdx); 900 __ pop(rax); 901 902 __ epilogue(); 903 } 904 905 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) { 906 __ prologue("shenandoah_load_reference_barrier", false); 907 // arg0 : object to be resolved 908 909 __ save_live_registers_no_oop_map(true); 910 911 #ifdef _LP64 912 __ load_parameter(0, c_rarg0); 913 __ load_parameter(1, c_rarg1); 914 if (UseCompressedOops) { 915 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1); 916 } else { 917 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1); 918 } 919 #else 920 __ load_parameter(0, rax); 921 __ load_parameter(1, rbx); 922 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx); 923 #endif 924 925 __ restore_live_registers_except_rax(true); 926 927 __ epilogue(); 928 } 929 930 #undef __ 931 932 #endif // COMPILER1 933 934 address ShenandoahBarrierSetAssembler::shenandoah_lrb() { 935 assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); 936 return _shenandoah_lrb; 937 } 938 939 #define __ cgen->assembler()-> 940 941 /* 942 * Incoming parameters: 943 * rax: oop 944 * rsi: load address 945 */ 946 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { 947 __ align(CodeEntryAlignment); 948 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); 949 address start = __ pc(); 950 951 Label resolve_oop, slow_path; 952 953 // We use RDI, which also serves as argument register for slow call. 954 // RAX always holds the src object ptr, except after the slow call, 955 // then it holds the result. R8/RBX is used as temporary register. 956 957 Register tmp1 = rdi; 958 Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx); 959 960 __ push(tmp1); 961 __ push(tmp2); 962 963 // Check for object being in the collection set. 964 __ mov(tmp1, rax); 965 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); 966 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); 967 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); 968 __ testbool(tmp2); 969 __ jccb(Assembler::notZero, resolve_oop); 970 __ pop(tmp2); 971 __ pop(tmp1); 972 __ ret(0); 973 974 // Test if object is already resolved. 975 __ bind(resolve_oop); 976 __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes())); 977 // Test if both lowest bits are set. We trick it by negating the bits 978 // then test for both bits clear. 979 __ notptr(tmp2); 980 __ testb(tmp2, markWord::marked_value); 981 __ jccb(Assembler::notZero, slow_path); 982 // Clear both lower bits. It's still inverted, so set them, and then invert back. 983 __ orptr(tmp2, markWord::marked_value); 984 __ notptr(tmp2); 985 // At this point, tmp2 contains the decoded forwarding pointer. 986 __ mov(rax, tmp2); 987 988 __ pop(tmp2); 989 __ pop(tmp1); 990 __ ret(0); 991 992 __ bind(slow_path); 993 994 __ push(rcx); 995 __ push(rdx); 996 __ push(rdi); 997 #ifdef _LP64 998 __ push(r8); 999 __ push(r9); 1000 __ push(r10); 1001 __ push(r11); 1002 __ push(r12); 1003 __ push(r13); 1004 __ push(r14); 1005 __ push(r15); 1006 #endif 1007 __ push(rbp); 1008 __ movptr(rbp, rsp); 1009 __ andptr(rsp, -StackAlignmentInBytes); 1010 __ push_FPU_state(); 1011 if (UseCompressedOops) { 1012 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), rax, rsi); 1013 } else { 1014 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rsi); 1015 } 1016 __ pop_FPU_state(); 1017 __ movptr(rsp, rbp); 1018 __ pop(rbp); 1019 #ifdef _LP64 1020 __ pop(r15); 1021 __ pop(r14); 1022 __ pop(r13); 1023 __ pop(r12); 1024 __ pop(r11); 1025 __ pop(r10); 1026 __ pop(r9); 1027 __ pop(r8); 1028 #endif 1029 __ pop(rdi); 1030 __ pop(rdx); 1031 __ pop(rcx); 1032 1033 __ pop(tmp2); 1034 __ pop(tmp1); 1035 __ ret(0); 1036 1037 return start; 1038 } 1039 1040 #undef __ 1041 1042 void ShenandoahBarrierSetAssembler::barrier_stubs_init() { 1043 if (ShenandoahLoadRefBarrier) { 1044 int stub_code_size = 4096; 1045 ResourceMark rm; 1046 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); 1047 CodeBuffer buf(bb); 1048 StubCodeGenerator cgen(&buf); 1049 _shenandoah_lrb = generate_shenandoah_lrb(&cgen); 1050 } 1051 }