2582 emit_opcode(cbuf, 0x0F);
2583 emit_opcode(cbuf, 0xB6);
2584 emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2585 %}
2586
2587 enc_class Push_ResultXD(regD dst) %{
2588 MacroAssembler _masm(&cbuf);
2589 __ fstp_d(Address(rsp, 0));
2590 __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2591 __ addptr(rsp, 8);
2592 %}
2593
2594 enc_class Push_SrcXD(regD src) %{
2595 MacroAssembler _masm(&cbuf);
2596 __ subptr(rsp, 8);
2597 __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2598 __ fld_d(Address(rsp, 0));
2599 %}
2600
2601
2602 // obj: object to lock
2603 // box: box address (header location) -- killed
2604 // tmp: rax -- killed
2605 // scr: rbx -- killed
2606 //
2607 // What follows is a direct transliteration of fast_lock() and fast_unlock()
2608 // from i486.ad. See that file for comments.
2609 // TODO: where possible switch from movq (r, 0) to movl(r,0) and
2610 // use the shorter encoding. (Movl clears the high-order 32-bits).
2611
2612
2613 enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
2614 %{
2615 Register objReg = as_Register((int)$obj$$reg);
2616 Register boxReg = as_Register((int)$box$$reg);
2617 Register tmpReg = as_Register($tmp$$reg);
2618 Register scrReg = as_Register($scr$$reg);
2619 MacroAssembler masm(&cbuf);
2620
2621 // Verify uniqueness of register assignments -- necessary but not sufficient
2622 assert (objReg != boxReg && objReg != tmpReg &&
2623 objReg != scrReg && tmpReg != scrReg, "invariant") ;
2624
2625 if (_counters != NULL) {
2626 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
2627 }
2628 if (EmitSync & 1) {
2629 // Without cast to int32_t a movptr will destroy r10 which is typically obj
2630 masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
2631 masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
2632 } else
2633 if (EmitSync & 2) {
2634 Label DONE_LABEL;
2635 if (UseBiasedLocking) {
2636 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
2637 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
2638 }
2639 // QQQ was movl...
2640 masm.movptr(tmpReg, 0x1);
2641 masm.orptr(tmpReg, Address(objReg, 0));
2642 masm.movptr(Address(boxReg, 0), tmpReg);
2643 if (os::is_MP()) {
2644 masm.lock();
2645 }
2646 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
2647 masm.jcc(Assembler::equal, DONE_LABEL);
2648
2649 // Recursive locking
2650 masm.subptr(tmpReg, rsp);
2651 masm.andptr(tmpReg, 7 - os::vm_page_size());
2652 masm.movptr(Address(boxReg, 0), tmpReg);
2653
2654 masm.bind(DONE_LABEL);
2655 masm.nop(); // avoid branch to branch
2656 } else {
2657 Label DONE_LABEL, IsInflated, Egress;
2658
2659 masm.movptr(tmpReg, Address(objReg, 0)) ;
2660 masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased
2661 masm.jcc (Assembler::notZero, IsInflated) ;
2662
2663 // it's stack-locked, biased or neutral
2664 // TODO: optimize markword triage order to reduce the number of
2665 // conditional branches in the most common cases.
2666 // Beware -- there's a subtle invariant that fetch of the markword
2667 // at [FETCH], below, will never observe a biased encoding (*101b).
2668 // If this invariant is not held we'll suffer exclusion (safety) failure.
2669
2670 if (UseBiasedLocking && !UseOptoBiasInlining) {
2671 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
2672 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH]
2673 }
2674
2675 // was q will it destroy high?
2676 masm.orl (tmpReg, 1) ;
2677 masm.movptr(Address(boxReg, 0), tmpReg) ;
2678 if (os::is_MP()) { masm.lock(); }
2679 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
2680 if (_counters != NULL) {
2681 masm.cond_inc32(Assembler::equal,
2682 ExternalAddress((address) _counters->fast_path_entry_count_addr()));
2683 }
2684 masm.jcc (Assembler::equal, DONE_LABEL);
2685
2686 // Recursive locking
2687 masm.subptr(tmpReg, rsp);
2688 masm.andptr(tmpReg, 7 - os::vm_page_size());
2689 masm.movptr(Address(boxReg, 0), tmpReg);
2690 if (_counters != NULL) {
2691 masm.cond_inc32(Assembler::equal,
2692 ExternalAddress((address) _counters->fast_path_entry_count_addr()));
2693 }
2694 masm.jmp (DONE_LABEL) ;
2695
2696 masm.bind (IsInflated) ;
2697 // It's inflated
2698
2699 // TODO: someday avoid the ST-before-CAS penalty by
2700 // relocating (deferring) the following ST.
2701 // We should also think about trying a CAS without having
2702 // fetched _owner. If the CAS is successful we may
2703 // avoid an RTO->RTS upgrade on the $line.
2704 // Without cast to int32_t a movptr will destroy r10 which is typically obj
2705 masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
2706
2707 masm.mov (boxReg, tmpReg) ;
2708 masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2709 masm.testptr(tmpReg, tmpReg) ;
2710 masm.jcc (Assembler::notZero, DONE_LABEL) ;
2711
2712 // It's inflated and appears unlocked
2713 if (os::is_MP()) { masm.lock(); }
2714 masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2715 // Intentional fall-through into DONE_LABEL ...
2716
2717 masm.bind (DONE_LABEL) ;
2718 masm.nop () ; // avoid jmp to jmp
2719 }
2720 %}
2721
2722 // obj: object to unlock
2723 // box: box address (displaced header location), killed
2724 // RBX: killed tmp; cannot be obj nor box
2725 enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
2726 %{
2727
2728 Register objReg = as_Register($obj$$reg);
2729 Register boxReg = as_Register($box$$reg);
2730 Register tmpReg = as_Register($tmp$$reg);
2731 MacroAssembler masm(&cbuf);
2732
2733 if (EmitSync & 4) {
2734 masm.cmpptr(rsp, 0) ;
2735 } else
2736 if (EmitSync & 8) {
2737 Label DONE_LABEL;
2738 if (UseBiasedLocking) {
2739 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
2740 }
2741
2742 // Check whether the displaced header is 0
2743 //(=> recursive unlock)
2744 masm.movptr(tmpReg, Address(boxReg, 0));
2745 masm.testptr(tmpReg, tmpReg);
2746 masm.jcc(Assembler::zero, DONE_LABEL);
2747
2748 // If not recursive lock, reset the header to displaced header
2749 if (os::is_MP()) {
2750 masm.lock();
2751 }
2752 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
2753 masm.bind(DONE_LABEL);
2754 masm.nop(); // avoid branch to branch
2755 } else {
2756 Label DONE_LABEL, Stacked, CheckSucc ;
2757
2758 if (UseBiasedLocking && !UseOptoBiasInlining) {
2759 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
2760 }
2761
2762 masm.movptr(tmpReg, Address(objReg, 0)) ;
2763 masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
2764 masm.jcc (Assembler::zero, DONE_LABEL) ;
2765 masm.testl (tmpReg, 0x02) ;
2766 masm.jcc (Assembler::zero, Stacked) ;
2767
2768 // It's inflated
2769 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2770 masm.xorptr(boxReg, r15_thread) ;
2771 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
2772 masm.jcc (Assembler::notZero, DONE_LABEL) ;
2773 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
2774 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
2775 masm.jcc (Assembler::notZero, CheckSucc) ;
2776 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2777 masm.jmp (DONE_LABEL) ;
2778
2779 if ((EmitSync & 65536) == 0) {
2780 Label LSuccess, LGoSlowPath ;
2781 masm.bind (CheckSucc) ;
2782 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2783 masm.jcc (Assembler::zero, LGoSlowPath) ;
2784
2785 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
2786 // the explicit ST;MEMBAR combination, but masm doesn't currently support
2787 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc
2788 // are all faster when the write buffer is populated.
2789 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2790 if (os::is_MP()) {
2791 masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
2792 }
2793 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2794 masm.jcc (Assembler::notZero, LSuccess) ;
2795
2796 masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX
2797 if (os::is_MP()) { masm.lock(); }
2798 masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
2799 masm.jcc (Assembler::notEqual, LSuccess) ;
2800 // Intentional fall-through into slow-path
2801
2802 masm.bind (LGoSlowPath) ;
2803 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure
2804 masm.jmp (DONE_LABEL) ;
2805
2806 masm.bind (LSuccess) ;
2807 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success
2808 masm.jmp (DONE_LABEL) ;
2809 }
2810
2811 masm.bind (Stacked) ;
2812 masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch
2813 if (os::is_MP()) { masm.lock(); }
2814 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
2815
2816 if (EmitSync & 65536) {
2817 masm.bind (CheckSucc) ;
2818 }
2819 masm.bind(DONE_LABEL);
2820 if (EmitSync & 32768) {
2821 masm.nop(); // avoid branch to branch
2822 }
2823 }
2824 %}
2825
2826
2827 enc_class enc_rethrow()
2828 %{
2829 cbuf.set_insts_mark();
2830 emit_opcode(cbuf, 0xE9); // jmp entry
2831 emit_d32_reloc(cbuf,
2832 (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2833 runtime_call_Relocation::spec(),
2834 RELOC_DISP32);
2835 %}
2836
2837 %}
2838
2839
2840
2841 //----------FRAME--------------------------------------------------------------
2842 // Definition of frame structure and management information.
2843 //
2844 // S T A C K L A Y O U T Allocators stack-slot number
2845 // | (to get allocators register number
2846 // G Owned by | | v add OptoReg::stack0())
11436 Label* l = $labl$$label;
11437 if ($cop$$cmpcode == Assembler::notEqual) {
11438 __ jccb(Assembler::parity, *l);
11439 __ jccb(Assembler::notEqual, *l);
11440 } else if ($cop$$cmpcode == Assembler::equal) {
11441 Label done;
11442 __ jccb(Assembler::parity, done);
11443 __ jccb(Assembler::equal, *l);
11444 __ bind(done);
11445 } else {
11446 ShouldNotReachHere();
11447 }
11448 %}
11449 ins_pipe(pipe_jcc);
11450 ins_short_branch(1);
11451 %}
11452
11453 // ============================================================================
11454 // inlined locking and unlocking
11455
11456 instruct cmpFastLock(rFlagsReg cr,
11457 rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
11458 %{
11459 match(Set cr (FastLock object box));
11460 effect(TEMP tmp, TEMP scr, USE_KILL box);
11461
11462 ins_cost(300);
11463 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11464 ins_encode(Fast_Lock(object, box, tmp, scr));
11465 ins_pipe(pipe_slow);
11466 %}
11467
11468 instruct cmpFastUnlock(rFlagsReg cr,
11469 rRegP object, rax_RegP box, rRegP tmp)
11470 %{
11471 match(Set cr (FastUnlock object box));
11472 effect(TEMP tmp, USE_KILL box);
11473
11474 ins_cost(300);
11475 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11476 ins_encode(Fast_Unlock(object, box, tmp));
11477 ins_pipe(pipe_slow);
11478 %}
11479
11480
11481 // ============================================================================
11482 // Safepoint Instructions
11483 instruct safePoint_poll(rFlagsReg cr)
11484 %{
11485 predicate(!Assembler::is_polling_page_far());
11486 match(SafePoint);
11487 effect(KILL cr);
11488
11489 format %{ "testl rax, [rip + #offset_to_poll_page]\t"
11490 "# Safepoint: poll for GC" %}
11491 ins_cost(125);
11492 ins_encode %{
11493 AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11494 __ testl(rax, addr);
11495 %}
11496 ins_pipe(ialu_reg_mem);
|
2582 emit_opcode(cbuf, 0x0F);
2583 emit_opcode(cbuf, 0xB6);
2584 emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2585 %}
2586
2587 enc_class Push_ResultXD(regD dst) %{
2588 MacroAssembler _masm(&cbuf);
2589 __ fstp_d(Address(rsp, 0));
2590 __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2591 __ addptr(rsp, 8);
2592 %}
2593
2594 enc_class Push_SrcXD(regD src) %{
2595 MacroAssembler _masm(&cbuf);
2596 __ subptr(rsp, 8);
2597 __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2598 __ fld_d(Address(rsp, 0));
2599 %}
2600
2601
2602 enc_class enc_rethrow()
2603 %{
2604 cbuf.set_insts_mark();
2605 emit_opcode(cbuf, 0xE9); // jmp entry
2606 emit_d32_reloc(cbuf,
2607 (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2608 runtime_call_Relocation::spec(),
2609 RELOC_DISP32);
2610 %}
2611
2612 %}
2613
2614
2615
2616 //----------FRAME--------------------------------------------------------------
2617 // Definition of frame structure and management information.
2618 //
2619 // S T A C K L A Y O U T Allocators stack-slot number
2620 // | (to get allocators register number
2621 // G Owned by | | v add OptoReg::stack0())
11211 Label* l = $labl$$label;
11212 if ($cop$$cmpcode == Assembler::notEqual) {
11213 __ jccb(Assembler::parity, *l);
11214 __ jccb(Assembler::notEqual, *l);
11215 } else if ($cop$$cmpcode == Assembler::equal) {
11216 Label done;
11217 __ jccb(Assembler::parity, done);
11218 __ jccb(Assembler::equal, *l);
11219 __ bind(done);
11220 } else {
11221 ShouldNotReachHere();
11222 }
11223 %}
11224 ins_pipe(pipe_jcc);
11225 ins_short_branch(1);
11226 %}
11227
11228 // ============================================================================
11229 // inlined locking and unlocking
11230
11231 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11232 match(Set cr (FastLock object box));
11233 effect(TEMP tmp, TEMP scr, USE_KILL box);
11234 ins_cost(300);
11235 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11236 ins_encode %{
11237 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters);
11238 %}
11239 ins_pipe(pipe_slow);
11240 %}
11241
11242 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11243 match(Set cr (FastUnlock object box));
11244 effect(TEMP tmp, USE_KILL box);
11245 ins_cost(300);
11246 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11247 ins_encode %{
11248 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
11249 %}
11250 ins_pipe(pipe_slow);
11251 %}
11252
11253
11254 // ============================================================================
11255 // Safepoint Instructions
11256 instruct safePoint_poll(rFlagsReg cr)
11257 %{
11258 predicate(!Assembler::is_polling_page_far());
11259 match(SafePoint);
11260 effect(KILL cr);
11261
11262 format %{ "testl rax, [rip + #offset_to_poll_page]\t"
11263 "# Safepoint: poll for GC" %}
11264 ins_cost(125);
11265 ins_encode %{
11266 AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11267 __ testl(rax, addr);
11268 %}
11269 ins_pipe(ialu_reg_mem);
|