1 /* 2 * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "c1/c1_MacroAssembler.hpp" 27 #include "c1/c1_Runtime1.hpp" 28 #include "classfile/systemDictionary.hpp" 29 #include "gc/shared/collectedHeap.hpp" 30 #include "interpreter/interpreter.hpp" 31 #include "oops/arrayOop.hpp" 32 #include "oops/markOop.hpp" 33 #include "runtime/basicLock.hpp" 34 #include "runtime/biasedLocking.hpp" 35 #include "runtime/os.hpp" 36 #include "runtime/sharedRuntime.hpp" 37 #include "runtime/stubRoutines.hpp" 38 39 // Note: Rtemp usage is this file should not impact C2 and should be 40 // correct as long as it is not implicitly used in lower layers (the 41 // arm [macro]assembler) and used with care in the other C1 specific 42 // files. 43 44 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) { 45 Label verified; 46 load_klass(Rtemp, receiver); 47 cmp(Rtemp, iCache); 48 b(verified, eq); // jump over alignment no-ops 49 #ifdef AARCH64 50 jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp); 51 #else 52 jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type); 53 #endif 54 align(CodeEntryAlignment); 55 bind(verified); 56 } 57 58 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) { 59 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect"); 60 assert((frame_size_in_bytes % StackAlignmentInBytes) == 0, "frame size should be aligned"); 61 62 #ifdef AARCH64 63 // Extra nop for MT-safe patching in NativeJump::patch_verified_entry 64 nop(); 65 #endif // AARCH64 66 67 arm_stack_overflow_check(bang_size_in_bytes, Rtemp); 68 69 // FP can no longer be used to memorize SP. It may be modified 70 // if this method contains a methodHandle call site 71 raw_push(FP, LR); 72 sub_slow(SP, SP, frame_size_in_bytes); 73 } 74 75 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) { 76 add_slow(SP, SP, frame_size_in_bytes); 77 raw_pop(FP, LR); 78 } 79 80 void C1_MacroAssembler::verified_entry() { 81 if (C1Breakpoint) { 82 breakpoint(); 83 } 84 } 85 86 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`. 87 void C1_MacroAssembler::try_allocate(Register obj, Register obj_end, Register tmp1, Register tmp2, 88 RegisterOrConstant size_expression, Label& slow_case) { 89 if (UseTLAB) { 90 tlab_allocate(obj, obj_end, tmp1, size_expression, slow_case); 91 } else { 92 eden_allocate(obj, obj_end, tmp1, tmp2, size_expression, slow_case); 93 incr_allocated_bytes(size_expression, tmp1); 94 } 95 } 96 97 98 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp) { 99 assert_different_registers(obj, klass, len, tmp); 100 101 if(UseBiasedLocking && !len->is_valid()) { 102 ldr(tmp, Address(klass, Klass::prototype_header_offset())); 103 } else { 104 mov(tmp, (intptr_t)markOopDesc::prototype()); 105 } 106 107 #ifdef AARCH64 108 if (UseCompressedClassPointers) { 109 str(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 110 encode_klass_not_null(tmp, klass); // Take care not to kill klass 111 str_w(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); 112 } else { 113 assert(oopDesc::mark_offset_in_bytes() + wordSize == oopDesc::klass_offset_in_bytes(), "adjust this code"); 114 stp(tmp, klass, Address(obj, oopDesc::mark_offset_in_bytes())); 115 } 116 #else 117 str(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 118 str(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 119 #endif // AARCH64 120 121 if (len->is_valid()) { 122 str_32(len, Address(obj, arrayOopDesc::length_offset_in_bytes())); 123 } 124 #ifdef AARCH64 125 else if (UseCompressedClassPointers) { 126 store_klass_gap(obj); 127 } 128 #endif // AARCH64 129 } 130 131 132 // Cleans object body [base..obj_end]. Clobbers `base` and `tmp` registers. 133 void C1_MacroAssembler::initialize_body(Register base, Register obj_end, Register tmp) { 134 zero_memory(base, obj_end, tmp); 135 } 136 137 138 void C1_MacroAssembler::initialize_object(Register obj, Register obj_end, Register klass, 139 Register len, Register tmp1, Register tmp2, 140 RegisterOrConstant header_size, int obj_size_in_bytes, 141 bool is_tlab_allocated) 142 { 143 assert_different_registers(obj, obj_end, klass, len, tmp1, tmp2); 144 initialize_header(obj, klass, len, tmp1); 145 146 const Register ptr = tmp2; 147 148 if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) { 149 #ifdef AARCH64 150 if (obj_size_in_bytes < 0) { 151 add_rc(ptr, obj, header_size); 152 initialize_body(ptr, obj_end, tmp1); 153 154 } else { 155 int base = instanceOopDesc::header_size() * HeapWordSize; 156 assert(obj_size_in_bytes >= base, "should be"); 157 158 const int zero_bytes = obj_size_in_bytes - base; 159 assert((zero_bytes % wordSize) == 0, "should be"); 160 161 if ((zero_bytes % (2*wordSize)) != 0) { 162 str(ZR, Address(obj, base)); 163 base += wordSize; 164 } 165 166 const int stp_count = zero_bytes / (2*wordSize); 167 168 if (zero_bytes > 8 * wordSize) { 169 Label loop; 170 add(ptr, obj, base); 171 mov(tmp1, stp_count); 172 bind(loop); 173 subs(tmp1, tmp1, 1); 174 stp(ZR, ZR, Address(ptr, 2*wordSize, post_indexed)); 175 b(loop, gt); 176 } else { 177 for (int i = 0; i < stp_count; i++) { 178 stp(ZR, ZR, Address(obj, base + i * 2 * wordSize)); 179 } 180 } 181 } 182 #else 183 if (obj_size_in_bytes >= 0 && obj_size_in_bytes <= 8 * BytesPerWord) { 184 mov(tmp1, 0); 185 const int base = instanceOopDesc::header_size() * HeapWordSize; 186 for (int i = base; i < obj_size_in_bytes; i += wordSize) { 187 str(tmp1, Address(obj, i)); 188 } 189 } else { 190 assert(header_size.is_constant() || header_size.as_register() == ptr, "code assumption"); 191 add(ptr, obj, header_size); 192 initialize_body(ptr, obj_end, tmp1); 193 } 194 #endif // AARCH64 195 } 196 197 // StoreStore barrier required after complete initialization 198 // (headers + content zeroing), before the object may escape. 199 membar(MacroAssembler::StoreStore, tmp1); 200 } 201 202 void C1_MacroAssembler::allocate_object(Register obj, Register tmp1, Register tmp2, Register tmp3, 203 int header_size, int object_size, 204 Register klass, Label& slow_case) { 205 assert_different_registers(obj, tmp1, tmp2, tmp3, klass, Rtemp); 206 assert(header_size >= 0 && object_size >= header_size, "illegal sizes"); 207 const int object_size_in_bytes = object_size * BytesPerWord; 208 209 const Register obj_end = tmp1; 210 const Register len = noreg; 211 212 if (Assembler::is_arith_imm_in_range(object_size_in_bytes)) { 213 try_allocate(obj, obj_end, tmp2, tmp3, object_size_in_bytes, slow_case); 214 } else { 215 // Rtemp should be free at c1 LIR level 216 mov_slow(Rtemp, object_size_in_bytes); 217 try_allocate(obj, obj_end, tmp2, tmp3, Rtemp, slow_case); 218 } 219 initialize_object(obj, obj_end, klass, len, tmp2, tmp3, instanceOopDesc::header_size() * HeapWordSize, object_size_in_bytes, /* is_tlab_allocated */ UseTLAB); 220 } 221 222 void C1_MacroAssembler::allocate_array(Register obj, Register len, 223 Register tmp1, Register tmp2, Register tmp3, 224 int header_size, int element_size, 225 Register klass, Label& slow_case) { 226 assert_different_registers(obj, len, tmp1, tmp2, tmp3, klass, Rtemp); 227 const int header_size_in_bytes = header_size * BytesPerWord; 228 const int scale_shift = exact_log2(element_size); 229 const Register obj_size = Rtemp; // Rtemp should be free at c1 LIR level 230 231 #ifdef AARCH64 232 mov_slow(Rtemp, max_array_allocation_length); 233 cmp_32(len, Rtemp); 234 #else 235 cmp_32(len, max_array_allocation_length); 236 #endif // AARCH64 237 b(slow_case, hs); 238 239 bool align_header = ((header_size_in_bytes | element_size) & MinObjAlignmentInBytesMask) != 0; 240 assert(align_header || ((header_size_in_bytes & MinObjAlignmentInBytesMask) == 0), "must be"); 241 assert(align_header || ((element_size & MinObjAlignmentInBytesMask) == 0), "must be"); 242 243 mov(obj_size, header_size_in_bytes + (align_header ? (MinObjAlignmentInBytes - 1) : 0)); 244 add_ptr_scaled_int32(obj_size, obj_size, len, scale_shift); 245 246 if (align_header) { 247 align_reg(obj_size, obj_size, MinObjAlignmentInBytes); 248 } 249 250 try_allocate(obj, tmp1, tmp2, tmp3, obj_size, slow_case); 251 initialize_object(obj, tmp1, klass, len, tmp2, tmp3, header_size_in_bytes, -1, /* is_tlab_allocated */ UseTLAB); 252 } 253 254 int C1_MacroAssembler::lock_object(Register hdr, Register obj, 255 Register disp_hdr, Register tmp1, 256 Label& slow_case) { 257 Label done, fast_lock, fast_lock_done; 258 int null_check_offset = 0; 259 260 const Register tmp2 = Rtemp; // Rtemp should be free at c1 LIR level 261 assert_different_registers(hdr, obj, disp_hdr, tmp1, tmp2); 262 263 assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code"); 264 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 265 const int mark_offset = BasicLock::displaced_header_offset_in_bytes(); 266 267 if (UseBiasedLocking) { 268 // load object 269 str(obj, Address(disp_hdr, obj_offset)); 270 null_check_offset = biased_locking_enter(obj, hdr/*scratched*/, tmp1, false, tmp2, done, slow_case); 271 } 272 273 assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions"); 274 275 #ifdef AARCH64 276 277 str(obj, Address(disp_hdr, obj_offset)); 278 279 if (!UseBiasedLocking) { 280 null_check_offset = offset(); 281 } 282 ldr(hdr, obj); 283 284 // Test if object is already locked 285 assert(markOopDesc::unlocked_value == 1, "adjust this code"); 286 tbnz(hdr, exact_log2(markOopDesc::unlocked_value), fast_lock); 287 288 // Check for recursive locking 289 // See comments in InterpreterMacroAssembler::lock_object for 290 // explanations on the fast recursive locking check. 291 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size()); 292 Assembler::LogicalImmediate imm(mask, false); 293 mov(tmp2, SP); 294 sub(tmp2, hdr, tmp2); 295 ands(tmp2, tmp2, imm); 296 b(slow_case, ne); 297 298 // Recursive locking: store 0 into a lock record 299 str(ZR, Address(disp_hdr, mark_offset)); 300 b(fast_lock_done); 301 302 #else // AARCH64 303 304 if (!UseBiasedLocking) { 305 null_check_offset = offset(); 306 } 307 308 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread. 309 // That would be acceptable as ether CAS or slow case path is taken in that case. 310 311 // Must be the first instruction here, because implicit null check relies on it 312 ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes())); 313 314 str(obj, Address(disp_hdr, obj_offset)); 315 tst(hdr, markOopDesc::unlocked_value); 316 b(fast_lock, ne); 317 318 // Check for recursive locking 319 // See comments in InterpreterMacroAssembler::lock_object for 320 // explanations on the fast recursive locking check. 321 // -1- test low 2 bits 322 movs(tmp2, AsmOperand(hdr, lsl, 30)); 323 // -2- test (hdr - SP) if the low two bits are 0 324 sub(tmp2, hdr, SP, eq); 325 movs(tmp2, AsmOperand(tmp2, lsr, exact_log2(os::vm_page_size())), eq); 326 // If 'eq' then OK for recursive fast locking: store 0 into a lock record. 327 str(tmp2, Address(disp_hdr, mark_offset), eq); 328 b(fast_lock_done, eq); 329 // else need slow case 330 b(slow_case); 331 332 #endif // AARCH64 333 334 bind(fast_lock); 335 // Save previous object header in BasicLock structure and update the header 336 str(hdr, Address(disp_hdr, mark_offset)); 337 338 cas_for_lock_acquire(hdr, disp_hdr, obj, tmp2, slow_case); 339 340 bind(fast_lock_done); 341 342 #ifndef PRODUCT 343 if (PrintBiasedLockingStatistics) { 344 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr()); 345 } 346 #endif // !PRODUCT 347 348 bind(done); 349 350 return null_check_offset; 351 } 352 353 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, 354 Register disp_hdr, Register tmp, 355 Label& slow_case) { 356 // Note: this method is not using its 'tmp' argument 357 358 assert_different_registers(hdr, obj, disp_hdr, Rtemp); 359 Register tmp2 = Rtemp; 360 361 assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code"); 362 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 363 const int mark_offset = BasicLock::displaced_header_offset_in_bytes(); 364 365 Label done; 366 if (UseBiasedLocking) { 367 // load object 368 ldr(obj, Address(disp_hdr, obj_offset)); 369 biased_locking_exit(obj, hdr, done); 370 } 371 372 assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions"); 373 Label retry; 374 375 // Load displaced header and object from the lock 376 ldr(hdr, Address(disp_hdr, mark_offset)); 377 // If hdr is NULL, we've got recursive locking and there's nothing more to do 378 cbz(hdr, done); 379 380 if(!UseBiasedLocking) { 381 // load object 382 ldr(obj, Address(disp_hdr, obj_offset)); 383 } 384 385 // Restore the object header 386 cas_for_lock_release(disp_hdr, hdr, obj, tmp2, slow_case); 387 388 bind(done); 389 } 390 391 392 #ifndef PRODUCT 393 394 void C1_MacroAssembler::verify_stack_oop(int stack_offset) { 395 if (!VerifyOops) return; 396 verify_oop_addr(Address(SP, stack_offset)); 397 } 398 399 void C1_MacroAssembler::verify_not_null_oop(Register r) { 400 Label not_null; 401 cbnz(r, not_null); 402 stop("non-null oop required"); 403 bind(not_null); 404 if (!VerifyOops) return; 405 verify_oop(r); 406 } 407 408 #endif // !PRODUCT