2979 str(Rthread, Address(Rthread, in_bytes(JavaThread::in_top_frame_unsafe_section_offset())));
2980 raw_push(Rtemp, ZR);
2981 mrs(Rtemp, Assembler::SysReg_NZCV);
2982 str(Rtemp, Address(SP, 1 * wordSize));
2983 mov_slow(Rtemp, Universe::narrow_ptrs_base());
2984 cmp(Rheap_base, Rtemp);
2985 b(ok, eq);
2986 stop(msg);
2987 bind(ok);
2988 ldr(Rtemp, Address(SP, 1 * wordSize));
2989 msr(Assembler::SysReg_NZCV, Rtemp);
2990 raw_pop(Rtemp, ZR);
2991 str(ZR, Address(Rthread, in_bytes(JavaThread::in_top_frame_unsafe_section_offset())));
2992 }
2993 }
2994 #endif // ASSERT
2995
2996 #endif // AARCH64
2997
2998 #ifdef COMPILER2
2999 void MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2 AARCH64_ONLY_ARG(Register Rscratch3))
3000 {
3001 assert(VM_Version::supports_ldrex(), "unsupported, yet?");
3002
3003 Register Rmark = Rscratch2;
3004
3005 assert(Roop != Rscratch, "");
3006 assert(Roop != Rmark, "");
3007 assert(Rbox != Rscratch, "");
3008 assert(Rbox != Rmark, "");
3009
3010 Label fast_lock, done;
3011
3012 if (UseBiasedLocking && !UseOptoBiasInlining) {
3013 Label failed;
3014 #ifdef AARCH64
3015 biased_locking_enter(Roop, Rmark, Rscratch, false, Rscratch3, done, failed);
3016 #else
3017 biased_locking_enter(Roop, Rmark, Rscratch, false, noreg, done, failed);
3018 #endif
3019 bind(failed);
3020 }
3021
3022 ldr(Rmark, Address(Roop, oopDesc::mark_offset_in_bytes()));
3023 tst(Rmark, markOopDesc::unlocked_value);
3024 b(fast_lock, ne);
3025
3026 // Check for recursive lock
3027 // See comments in InterpreterMacroAssembler::lock_object for
3028 // explanations on the fast recursive locking check.
3029 #ifdef AARCH64
3030 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
3031 Assembler::LogicalImmediate imm(mask, false);
3032 mov(Rscratch, SP);
3033 sub(Rscratch, Rmark, Rscratch);
3034 ands(Rscratch, Rscratch, imm);
3035 // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8153107)
3036 str(Rscratch, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
3037 b(done);
3038
3039 #else
3040 // -1- test low 2 bits
3041 movs(Rscratch, AsmOperand(Rmark, lsl, 30));
3042 // -2- test (hdr - SP) if the low two bits are 0
3043 sub(Rscratch, Rmark, SP, eq);
3044 movs(Rscratch, AsmOperand(Rscratch, lsr, exact_log2(os::vm_page_size())), eq);
3045 // If still 'eq' then recursive locking OK
3046 // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8153107)
3047 str(Rscratch, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
3048 b(done);
3049 #endif
3050
3051 bind(fast_lock);
3052 str(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
3053
3054 bool allow_fallthrough_on_failure = true;
3055 bool one_shot = true;
3056 cas_for_lock_acquire(Rmark, Rbox, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot);
3057
3058 bind(done);
3059
3060 }
3061
3062 void MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2 AARCH64_ONLY_ARG(Register Rscratch3))
3063 {
3064 assert(VM_Version::supports_ldrex(), "unsupported, yet?");
3065
3066 Register Rmark = Rscratch2;
3067
3068 assert(Roop != Rscratch, "");
3069 assert(Roop != Rmark, "");
3070 assert(Rbox != Rscratch, "");
3071 assert(Rbox != Rmark, "");
3072
3073 Label done;
3074
3075 if (UseBiasedLocking && !UseOptoBiasInlining) {
3076 biased_locking_exit(Roop, Rscratch, done);
3077 }
3078
3079 ldr(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
|
2979 str(Rthread, Address(Rthread, in_bytes(JavaThread::in_top_frame_unsafe_section_offset())));
2980 raw_push(Rtemp, ZR);
2981 mrs(Rtemp, Assembler::SysReg_NZCV);
2982 str(Rtemp, Address(SP, 1 * wordSize));
2983 mov_slow(Rtemp, Universe::narrow_ptrs_base());
2984 cmp(Rheap_base, Rtemp);
2985 b(ok, eq);
2986 stop(msg);
2987 bind(ok);
2988 ldr(Rtemp, Address(SP, 1 * wordSize));
2989 msr(Assembler::SysReg_NZCV, Rtemp);
2990 raw_pop(Rtemp, ZR);
2991 str(ZR, Address(Rthread, in_bytes(JavaThread::in_top_frame_unsafe_section_offset())));
2992 }
2993 }
2994 #endif // ASSERT
2995
2996 #endif // AARCH64
2997
2998 #ifdef COMPILER2
2999 void MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2, Register scratch3)
3000 {
3001 assert(VM_Version::supports_ldrex(), "unsupported, yet?");
3002
3003 Register Rmark = Rscratch2;
3004
3005 assert(Roop != Rscratch, "");
3006 assert(Roop != Rmark, "");
3007 assert(Rbox != Rscratch, "");
3008 assert(Rbox != Rmark, "");
3009
3010 Label fast_lock, done;
3011
3012 if (UseBiasedLocking && !UseOptoBiasInlining) {
3013 assert(scratch3 != noreg, "need extra temporary for -XX:-UseOptoBiasInlining");
3014 biased_locking_enter(Roop, Rmark, Rscratch, false, scratch3, done, done);
3015 // Fall through if lock not biased otherwise branch to done
3016 }
3017
3018 // Invariant: Rmark loaded below does not contain biased lock pattern
3019
3020 ldr(Rmark, Address(Roop, oopDesc::mark_offset_in_bytes()));
3021 tst(Rmark, markOopDesc::unlocked_value);
3022 b(fast_lock, ne);
3023
3024 // Check for recursive lock
3025 // See comments in InterpreterMacroAssembler::lock_object for
3026 // explanations on the fast recursive locking check.
3027 #ifdef AARCH64
3028 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
3029 Assembler::LogicalImmediate imm(mask, false);
3030 mov(Rscratch, SP);
3031 sub(Rscratch, Rmark, Rscratch);
3032 ands(Rscratch, Rscratch, imm);
3033 // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8153107)
3034 str(Rscratch, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
3035 b(done);
3036
3037 #else
3038 // -1- test low 2 bits
3039 movs(Rscratch, AsmOperand(Rmark, lsl, 30));
3040 // -2- test (hdr - SP) if the low two bits are 0
3041 sub(Rscratch, Rmark, SP, eq);
3042 movs(Rscratch, AsmOperand(Rscratch, lsr, exact_log2(os::vm_page_size())), eq);
3043 // If still 'eq' then recursive locking OK
3044 // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8153107)
3045 str(Rscratch, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
3046 b(done);
3047 #endif
3048
3049 bind(fast_lock);
3050 str(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
3051
3052 bool allow_fallthrough_on_failure = true;
3053 bool one_shot = true;
3054 cas_for_lock_acquire(Rmark, Rbox, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot);
3055
3056 bind(done);
3057
3058 // At this point flags are set as follows:
3059 // EQ -> Success
3060 // NE -> Failure, branch to slow path
3061 }
3062
3063 void MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2 AARCH64_ONLY_ARG(Register Rscratch3))
3064 {
3065 assert(VM_Version::supports_ldrex(), "unsupported, yet?");
3066
3067 Register Rmark = Rscratch2;
3068
3069 assert(Roop != Rscratch, "");
3070 assert(Roop != Rmark, "");
3071 assert(Rbox != Rscratch, "");
3072 assert(Rbox != Rmark, "");
3073
3074 Label done;
3075
3076 if (UseBiasedLocking && !UseOptoBiasInlining) {
3077 biased_locking_exit(Roop, Rscratch, done);
3078 }
3079
3080 ldr(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
|