1990
1991 __ lea(lock_reg, Address(rbp, lock_slot_rbp_offset));
1992
1993 // Load the oop from the handle
1994 __ movptr(obj_reg, Address(oop_handle_reg, 0));
1995
1996 if (UseBiasedLocking) {
1997 // Note that oop_handle_reg is trashed during this call
1998 __ biased_locking_enter(lock_reg, obj_reg, swap_reg, oop_handle_reg, false, lock_done, &slow_path_lock);
1999 }
2000
2001 // Load immediate 1 into swap_reg %rax,
2002 __ movptr(swap_reg, 1);
2003
2004 // Load (object->mark() | 1) into swap_reg %rax,
2005 __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2006
2007 // Save (object->mark() | 1) into BasicLock's displaced header
2008 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2009
2010 if (os::is_MP()) {
2011 __ lock();
2012 }
2013
2014 // src -> dest iff dest == rax, else rax, <- dest
2015 // *obj_reg = lock_reg iff *obj_reg == rax, else rax, = *(obj_reg)
2016 __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2017 __ jcc(Assembler::equal, lock_done);
2018
2019 // Test if the oopMark is an obvious stack pointer, i.e.,
2020 // 1) (mark & 3) == 0, and
2021 // 2) rsp <= mark < mark + os::pagesize()
2022 // These 3 tests can be done by evaluating the following
2023 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2024 // assuming both stack pointer and pagesize have their
2025 // least significant 2 bits clear.
2026 // NOTE: the oopMark is in swap_reg %rax, as the result of cmpxchg
2027
2028 __ subptr(swap_reg, rsp);
2029 __ andptr(swap_reg, 3 - os::vm_page_size());
2030
2031 // Save the test result, for recursive case, the result is zero
2032 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2033 __ jcc(Assembler::notEqual, slow_path_lock);
2034 // Slow path will re-enter here
2035 __ bind(lock_done);
2074 case T_FLOAT :
2075 // Result is in st0 we'll save as needed
2076 break;
2077 case T_ARRAY: // Really a handle
2078 case T_OBJECT: // Really a handle
2079 break; // can't de-handlize until after safepoint check
2080 case T_VOID: break;
2081 case T_LONG: break;
2082 default : ShouldNotReachHere();
2083 }
2084
2085 // Switch thread to "native transition" state before reading the synchronization state.
2086 // This additional state is necessary because reading and testing the synchronization
2087 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2088 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2089 // VM thread changes sync state to synchronizing and suspends threads for GC.
2090 // Thread A is resumed to finish this native method, but doesn't block here since it
2091 // didn't see any synchronization is progress, and escapes.
2092 __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2093
2094 if(os::is_MP()) {
2095 if (UseMembar) {
2096 // Force this write out before the read below
2097 __ membar(Assembler::Membar_mask_bits(
2098 Assembler::LoadLoad | Assembler::LoadStore |
2099 Assembler::StoreLoad | Assembler::StoreStore));
2100 } else {
2101 // Write serialization page so VM thread can do a pseudo remote membar.
2102 // We use the current thread pointer to calculate a thread specific
2103 // offset to write to within the page. This minimizes bus traffic
2104 // due to cache line collision.
2105 __ serialize_memory(thread, rcx);
2106 }
2107 }
2108
2109 if (AlwaysRestoreFPU) {
2110 // Make sure the control word is correct.
2111 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
2112 }
2113
2114 Label after_transition;
2115
2116 // check for safepoint operation in progress and/or pending suspend requests
2117 { Label Continue, slow_path;
2118
2119 __ safepoint_poll(slow_path, thread, noreg);
2120
2121 __ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
2122 __ jcc(Assembler::equal, Continue);
2123 __ bind(slow_path);
2124
2125 // Don't use call_VM as it will see a possible pending exception and forward it
2126 // and never return here preventing us from clearing _last_native_pc down below.
2127 // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2182 __ biased_locking_exit(obj_reg, rbx, done);
2183 }
2184
2185 // Simple recursive lock?
2186
2187 __ cmpptr(Address(rbp, lock_slot_rbp_offset), (int32_t)NULL_WORD);
2188 __ jcc(Assembler::equal, done);
2189
2190 // Must save rax, if if it is live now because cmpxchg must use it
2191 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2192 save_native_result(masm, ret_type, stack_slots);
2193 }
2194
2195 // get old displaced header
2196 __ movptr(rbx, Address(rbp, lock_slot_rbp_offset));
2197
2198 // get address of the stack lock
2199 __ lea(rax, Address(rbp, lock_slot_rbp_offset));
2200
2201 // Atomic swap old header if oop still contains the stack lock
2202 if (os::is_MP()) {
2203 __ lock();
2204 }
2205
2206 // src -> dest iff dest == rax, else rax, <- dest
2207 // *obj_reg = rbx, iff *obj_reg == rax, else rax, = *(obj_reg)
2208 __ cmpxchgptr(rbx, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2209 __ jcc(Assembler::notEqual, slow_path_unlock);
2210
2211 // slow path re-enters here
2212 __ bind(unlock_done);
2213 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2214 restore_native_result(masm, ret_type, stack_slots);
2215 }
2216
2217 __ bind(done);
2218
2219 }
2220
2221 {
2222 SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0);
2223 // Tell dtrace about this method exit
2224 save_native_result(masm, ret_type, stack_slots);
2225 __ mov_metadata(rax, method());
2226 __ call_VM_leaf(
2227 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
|
1990
1991 __ lea(lock_reg, Address(rbp, lock_slot_rbp_offset));
1992
1993 // Load the oop from the handle
1994 __ movptr(obj_reg, Address(oop_handle_reg, 0));
1995
1996 if (UseBiasedLocking) {
1997 // Note that oop_handle_reg is trashed during this call
1998 __ biased_locking_enter(lock_reg, obj_reg, swap_reg, oop_handle_reg, false, lock_done, &slow_path_lock);
1999 }
2000
2001 // Load immediate 1 into swap_reg %rax,
2002 __ movptr(swap_reg, 1);
2003
2004 // Load (object->mark() | 1) into swap_reg %rax,
2005 __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2006
2007 // Save (object->mark() | 1) into BasicLock's displaced header
2008 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2009
2010 // src -> dest iff dest == rax, else rax, <- dest
2011 // *obj_reg = lock_reg iff *obj_reg == rax, else rax, = *(obj_reg)
2012 __ lock();
2013 __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2014 __ jcc(Assembler::equal, lock_done);
2015
2016 // Test if the oopMark is an obvious stack pointer, i.e.,
2017 // 1) (mark & 3) == 0, and
2018 // 2) rsp <= mark < mark + os::pagesize()
2019 // These 3 tests can be done by evaluating the following
2020 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2021 // assuming both stack pointer and pagesize have their
2022 // least significant 2 bits clear.
2023 // NOTE: the oopMark is in swap_reg %rax, as the result of cmpxchg
2024
2025 __ subptr(swap_reg, rsp);
2026 __ andptr(swap_reg, 3 - os::vm_page_size());
2027
2028 // Save the test result, for recursive case, the result is zero
2029 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2030 __ jcc(Assembler::notEqual, slow_path_lock);
2031 // Slow path will re-enter here
2032 __ bind(lock_done);
2071 case T_FLOAT :
2072 // Result is in st0 we'll save as needed
2073 break;
2074 case T_ARRAY: // Really a handle
2075 case T_OBJECT: // Really a handle
2076 break; // can't de-handlize until after safepoint check
2077 case T_VOID: break;
2078 case T_LONG: break;
2079 default : ShouldNotReachHere();
2080 }
2081
2082 // Switch thread to "native transition" state before reading the synchronization state.
2083 // This additional state is necessary because reading and testing the synchronization
2084 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2085 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2086 // VM thread changes sync state to synchronizing and suspends threads for GC.
2087 // Thread A is resumed to finish this native method, but doesn't block here since it
2088 // didn't see any synchronization is progress, and escapes.
2089 __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2090
2091 if (UseMembar) {
2092 // Force this write out before the read below
2093 __ membar(Assembler::Membar_mask_bits(
2094 Assembler::LoadLoad | Assembler::LoadStore |
2095 Assembler::StoreLoad | Assembler::StoreStore));
2096 } else {
2097 // Write serialization page so VM thread can do a pseudo remote membar.
2098 // We use the current thread pointer to calculate a thread specific
2099 // offset to write to within the page. This minimizes bus traffic
2100 // due to cache line collision.
2101 __ serialize_memory(thread, rcx);
2102 }
2103
2104 if (AlwaysRestoreFPU) {
2105 // Make sure the control word is correct.
2106 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
2107 }
2108
2109 Label after_transition;
2110
2111 // check for safepoint operation in progress and/or pending suspend requests
2112 { Label Continue, slow_path;
2113
2114 __ safepoint_poll(slow_path, thread, noreg);
2115
2116 __ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
2117 __ jcc(Assembler::equal, Continue);
2118 __ bind(slow_path);
2119
2120 // Don't use call_VM as it will see a possible pending exception and forward it
2121 // and never return here preventing us from clearing _last_native_pc down below.
2122 // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2177 __ biased_locking_exit(obj_reg, rbx, done);
2178 }
2179
2180 // Simple recursive lock?
2181
2182 __ cmpptr(Address(rbp, lock_slot_rbp_offset), (int32_t)NULL_WORD);
2183 __ jcc(Assembler::equal, done);
2184
2185 // Must save rax, if if it is live now because cmpxchg must use it
2186 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2187 save_native_result(masm, ret_type, stack_slots);
2188 }
2189
2190 // get old displaced header
2191 __ movptr(rbx, Address(rbp, lock_slot_rbp_offset));
2192
2193 // get address of the stack lock
2194 __ lea(rax, Address(rbp, lock_slot_rbp_offset));
2195
2196 // Atomic swap old header if oop still contains the stack lock
2197 // src -> dest iff dest == rax, else rax, <- dest
2198 // *obj_reg = rbx, iff *obj_reg == rax, else rax, = *(obj_reg)
2199 __ lock();
2200 __ cmpxchgptr(rbx, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2201 __ jcc(Assembler::notEqual, slow_path_unlock);
2202
2203 // slow path re-enters here
2204 __ bind(unlock_done);
2205 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2206 restore_native_result(masm, ret_type, stack_slots);
2207 }
2208
2209 __ bind(done);
2210
2211 }
2212
2213 {
2214 SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0);
2215 // Tell dtrace about this method exit
2216 save_native_result(masm, ret_type, stack_slots);
2217 __ mov_metadata(rax, method());
2218 __ call_VM_leaf(
2219 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
|