2447
2448 __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2449
2450 // Load the oop from the handle
2451 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2452
2453 __ resolve(IS_NOT_NULL, obj_reg);
2454 if (UseBiasedLocking) {
2455 __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, lock_done, &slow_path_lock);
2456 }
2457
2458 // Load immediate 1 into swap_reg %rax
2459 __ movl(swap_reg, 1);
2460
2461 // Load (object->mark() | 1) into swap_reg %rax
2462 __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2463
2464 // Save (object->mark() | 1) into BasicLock's displaced header
2465 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2466
2467 if (os::is_MP()) {
2468 __ lock();
2469 }
2470
2471 // src -> dest iff dest == rax else rax <- dest
2472 __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2473 __ jcc(Assembler::equal, lock_done);
2474
2475 // Hmm should this move to the slow path code area???
2476
2477 // Test if the oopMark is an obvious stack pointer, i.e.,
2478 // 1) (mark & 3) == 0, and
2479 // 2) rsp <= mark < mark + os::pagesize()
2480 // These 3 tests can be done by evaluating the following
2481 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2482 // assuming both stack pointer and pagesize have their
2483 // least significant 2 bits clear.
2484 // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2485
2486 __ subptr(swap_reg, rsp);
2487 __ andptr(swap_reg, 3 - os::vm_page_size());
2488
2489 // Save the test result, for recursive case, the result is zero
2490 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2491 __ jcc(Assembler::notEqual, slow_path_lock);
2541 assert(pinned_slot <= stack_slots, "overflow");
2542 if (!in_regs[i].first()->is_stack()) {
2543 int offset = pinned_slot * VMRegImpl::stack_slot_size;
2544 __ movq(in_regs[i].first()->as_Register(), Address(rsp, offset));
2545 pinned_slot += VMRegImpl::slots_per_word;
2546 }
2547 gen_unpin_object(masm, in_regs[i]);
2548 }
2549 restore_native_result(masm, ret_type, stack_slots);
2550 }
2551
2552 // Switch thread to "native transition" state before reading the synchronization state.
2553 // This additional state is necessary because reading and testing the synchronization
2554 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2555 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2556 // VM thread changes sync state to synchronizing and suspends threads for GC.
2557 // Thread A is resumed to finish this native method, but doesn't block here since it
2558 // didn't see any synchronization is progress, and escapes.
2559 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2560
2561 if(os::is_MP()) {
2562 if (UseMembar) {
2563 // Force this write out before the read below
2564 __ membar(Assembler::Membar_mask_bits(
2565 Assembler::LoadLoad | Assembler::LoadStore |
2566 Assembler::StoreLoad | Assembler::StoreStore));
2567 } else {
2568 // Write serialization page so VM thread can do a pseudo remote membar.
2569 // We use the current thread pointer to calculate a thread specific
2570 // offset to write to within the page. This minimizes bus traffic
2571 // due to cache line collision.
2572 __ serialize_memory(r15_thread, rcx);
2573 }
2574 }
2575
2576 Label after_transition;
2577
2578 // check for safepoint operation in progress and/or pending suspend requests
2579 {
2580 Label Continue;
2581 Label slow_path;
2582
2583 __ safepoint_poll(slow_path, r15_thread, rscratch1);
2584
2585 __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
2586 __ jcc(Assembler::equal, Continue);
2587 __ bind(slow_path);
2588
2589 // Don't use call_VM as it will see a possible pending exception and forward it
2590 // and never return here preventing us from clearing _last_native_pc down below.
2591 // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2592 // preserved and correspond to the bcp/locals pointers. So we do a runtime call
2593 // by hand.
2594 //
2644 __ biased_locking_exit(obj_reg, old_hdr, done);
2645 }
2646
2647 // Simple recursive lock?
2648
2649 __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), (int32_t)NULL_WORD);
2650 __ jcc(Assembler::equal, done);
2651
2652 // Must save rax if if it is live now because cmpxchg must use it
2653 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2654 save_native_result(masm, ret_type, stack_slots);
2655 }
2656
2657
2658 // get address of the stack lock
2659 __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2660 // get old displaced header
2661 __ movptr(old_hdr, Address(rax, 0));
2662
2663 // Atomic swap old header if oop still contains the stack lock
2664 if (os::is_MP()) {
2665 __ lock();
2666 }
2667 __ cmpxchgptr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2668 __ jcc(Assembler::notEqual, slow_path_unlock);
2669
2670 // slow path re-enters here
2671 __ bind(unlock_done);
2672 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2673 restore_native_result(masm, ret_type, stack_slots);
2674 }
2675
2676 __ bind(done);
2677
2678 }
2679 {
2680 SkipIfEqual skip(masm, &DTraceMethodProbes, false);
2681 save_native_result(masm, ret_type, stack_slots);
2682 __ mov_metadata(c_rarg1, method());
2683 __ call_VM_leaf(
2684 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2685 r15_thread, c_rarg1);
2686 restore_native_result(masm, ret_type, stack_slots);
|
2447
2448 __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2449
2450 // Load the oop from the handle
2451 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2452
2453 __ resolve(IS_NOT_NULL, obj_reg);
2454 if (UseBiasedLocking) {
2455 __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, lock_done, &slow_path_lock);
2456 }
2457
2458 // Load immediate 1 into swap_reg %rax
2459 __ movl(swap_reg, 1);
2460
2461 // Load (object->mark() | 1) into swap_reg %rax
2462 __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2463
2464 // Save (object->mark() | 1) into BasicLock's displaced header
2465 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2466
2467 // src -> dest iff dest == rax else rax <- dest
2468 __ lock();
2469 __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2470 __ jcc(Assembler::equal, lock_done);
2471
2472 // Hmm should this move to the slow path code area???
2473
2474 // Test if the oopMark is an obvious stack pointer, i.e.,
2475 // 1) (mark & 3) == 0, and
2476 // 2) rsp <= mark < mark + os::pagesize()
2477 // These 3 tests can be done by evaluating the following
2478 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2479 // assuming both stack pointer and pagesize have their
2480 // least significant 2 bits clear.
2481 // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2482
2483 __ subptr(swap_reg, rsp);
2484 __ andptr(swap_reg, 3 - os::vm_page_size());
2485
2486 // Save the test result, for recursive case, the result is zero
2487 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2488 __ jcc(Assembler::notEqual, slow_path_lock);
2538 assert(pinned_slot <= stack_slots, "overflow");
2539 if (!in_regs[i].first()->is_stack()) {
2540 int offset = pinned_slot * VMRegImpl::stack_slot_size;
2541 __ movq(in_regs[i].first()->as_Register(), Address(rsp, offset));
2542 pinned_slot += VMRegImpl::slots_per_word;
2543 }
2544 gen_unpin_object(masm, in_regs[i]);
2545 }
2546 restore_native_result(masm, ret_type, stack_slots);
2547 }
2548
2549 // Switch thread to "native transition" state before reading the synchronization state.
2550 // This additional state is necessary because reading and testing the synchronization
2551 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2552 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2553 // VM thread changes sync state to synchronizing and suspends threads for GC.
2554 // Thread A is resumed to finish this native method, but doesn't block here since it
2555 // didn't see any synchronization is progress, and escapes.
2556 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2557
2558 if (UseMembar) {
2559 // Force this write out before the read below
2560 __ membar(Assembler::Membar_mask_bits(
2561 Assembler::LoadLoad | Assembler::LoadStore |
2562 Assembler::StoreLoad | Assembler::StoreStore));
2563 } else {
2564 // Write serialization page so VM thread can do a pseudo remote membar.
2565 // We use the current thread pointer to calculate a thread specific
2566 // offset to write to within the page. This minimizes bus traffic
2567 // due to cache line collision.
2568 __ serialize_memory(r15_thread, rcx);
2569 }
2570
2571 Label after_transition;
2572
2573 // check for safepoint operation in progress and/or pending suspend requests
2574 {
2575 Label Continue;
2576 Label slow_path;
2577
2578 __ safepoint_poll(slow_path, r15_thread, rscratch1);
2579
2580 __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
2581 __ jcc(Assembler::equal, Continue);
2582 __ bind(slow_path);
2583
2584 // Don't use call_VM as it will see a possible pending exception and forward it
2585 // and never return here preventing us from clearing _last_native_pc down below.
2586 // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2587 // preserved and correspond to the bcp/locals pointers. So we do a runtime call
2588 // by hand.
2589 //
2639 __ biased_locking_exit(obj_reg, old_hdr, done);
2640 }
2641
2642 // Simple recursive lock?
2643
2644 __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), (int32_t)NULL_WORD);
2645 __ jcc(Assembler::equal, done);
2646
2647 // Must save rax if if it is live now because cmpxchg must use it
2648 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2649 save_native_result(masm, ret_type, stack_slots);
2650 }
2651
2652
2653 // get address of the stack lock
2654 __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2655 // get old displaced header
2656 __ movptr(old_hdr, Address(rax, 0));
2657
2658 // Atomic swap old header if oop still contains the stack lock
2659 __ lock();
2660 __ cmpxchgptr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2661 __ jcc(Assembler::notEqual, slow_path_unlock);
2662
2663 // slow path re-enters here
2664 __ bind(unlock_done);
2665 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2666 restore_native_result(masm, ret_type, stack_slots);
2667 }
2668
2669 __ bind(done);
2670
2671 }
2672 {
2673 SkipIfEqual skip(masm, &DTraceMethodProbes, false);
2674 save_native_result(masm, ret_type, stack_slots);
2675 __ mov_metadata(c_rarg1, method());
2676 __ call_VM_leaf(
2677 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2678 r15_thread, c_rarg1);
2679 restore_native_result(masm, ret_type, stack_slots);
|