1 /*
   2  * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "c1/c1_MacroAssembler.hpp"
  27 #include "c1/c1_Runtime1.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "oops/arrayOop.hpp"
  32 #include "oops/markOop.hpp"
  33 #include "runtime/basicLock.hpp"
  34 #include "runtime/biasedLocking.hpp"
  35 #include "runtime/os.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "runtime/stubRoutines.hpp"
  38 
  39 // Note: Rtemp usage is this file should not impact C2 and should be
  40 // correct as long as it is not implicitly used in lower layers (the
  41 // arm [macro]assembler) and used with care in the other C1 specific
  42 // files.
  43 
  44 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
  45   Label verified;
  46   load_klass(Rtemp, receiver);
  47   cmp(Rtemp, iCache);
  48   b(verified, eq); // jump over alignment no-ops
  49 #ifdef AARCH64
  50   jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
  51 #else
  52   jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
  53 #endif
  54   align(CodeEntryAlignment);
  55   bind(verified);
  56 }
  57 
  58 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
  59   assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
  60   assert((frame_size_in_bytes % StackAlignmentInBytes) == 0, "frame size should be aligned");
  61 
  62 #ifdef AARCH64
  63   // Extra nop for MT-safe patching in NativeJump::patch_verified_entry
  64   nop();
  65 #endif // AARCH64
  66 
  67   arm_stack_overflow_check(bang_size_in_bytes, Rtemp);
  68 
  69   // FP can no longer be used to memorize SP. It may be modified
  70   // if this method contains a methodHandle call site
  71   raw_push(FP, LR);
  72   sub_slow(SP, SP, frame_size_in_bytes);
  73 }
  74 
  75 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) {
  76   add_slow(SP, SP, frame_size_in_bytes);
  77   raw_pop(FP, LR);
  78 }
  79 
  80 void C1_MacroAssembler::verified_entry() {
  81   if (C1Breakpoint) {
  82     breakpoint();
  83   }
  84 }
  85 
  86 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
  87 void C1_MacroAssembler::try_allocate(Register obj, Register obj_end, Register tmp1, Register tmp2,
  88                                      RegisterOrConstant size_expression, Label& slow_case) {
  89   if (UseTLAB) {
  90     tlab_allocate(obj, obj_end, tmp1, size_expression, slow_case);
  91   } else {
  92     eden_allocate(obj, obj_end, tmp1, tmp2, size_expression, slow_case);
  93   }
  94 }
  95 
  96 
  97 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp) {
  98   assert_different_registers(obj, klass, len, tmp);
  99 
 100   if(UseBiasedLocking && !len->is_valid()) {
 101     ldr(tmp, Address(klass, Klass::prototype_header_offset()));
 102   } else {
 103     mov(tmp, (intptr_t)markOopDesc::prototype());
 104   }
 105 
 106 #ifdef AARCH64
 107   if (UseCompressedClassPointers) {
 108     str(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
 109     encode_klass_not_null(tmp, klass);          // Take care not to kill klass
 110     str_w(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
 111   } else {
 112     assert(oopDesc::mark_offset_in_bytes() + wordSize == oopDesc::klass_offset_in_bytes(), "adjust this code");
 113     stp(tmp, klass, Address(obj, oopDesc::mark_offset_in_bytes()));
 114   }
 115 #else
 116   str(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
 117   str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 118 #endif // AARCH64
 119 
 120   if (len->is_valid()) {
 121     str_32(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
 122   }
 123 #ifdef AARCH64
 124   else if (UseCompressedClassPointers) {
 125     store_klass_gap(obj);
 126   }
 127 #endif // AARCH64
 128 }
 129 
 130 
 131 // Cleans object body [base..obj_end]. Clobbers `base` and `tmp` registers.
 132 void C1_MacroAssembler::initialize_body(Register base, Register obj_end, Register tmp) {
 133   zero_memory(base, obj_end, tmp);
 134 }
 135 
 136 
 137 void C1_MacroAssembler::initialize_object(Register obj, Register obj_end, Register klass,
 138                                           Register len, Register tmp1, Register tmp2,
 139                                           RegisterOrConstant header_size, int obj_size_in_bytes,
 140                                           bool is_tlab_allocated)
 141 {
 142   assert_different_registers(obj, obj_end, klass, len, tmp1, tmp2);
 143   initialize_header(obj, klass, len, tmp1);
 144 
 145   const Register ptr = tmp2;
 146 
 147   if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
 148 #ifdef AARCH64
 149     if (obj_size_in_bytes < 0) {
 150       add_rc(ptr, obj, header_size);
 151       initialize_body(ptr, obj_end, tmp1);
 152 
 153     } else {
 154       int base = instanceOopDesc::header_size() * HeapWordSize;
 155       assert(obj_size_in_bytes >= base, "should be");
 156 
 157       const int zero_bytes = obj_size_in_bytes - base;
 158       assert((zero_bytes % wordSize) == 0, "should be");
 159 
 160       if ((zero_bytes % (2*wordSize)) != 0) {
 161         str(ZR, Address(obj, base));
 162         base += wordSize;
 163       }
 164 
 165       const int stp_count = zero_bytes / (2*wordSize);
 166 
 167       if (zero_bytes > 8 * wordSize) {
 168         Label loop;
 169         add(ptr, obj, base);
 170         mov(tmp1, stp_count);
 171         bind(loop);
 172         subs(tmp1, tmp1, 1);
 173         stp(ZR, ZR, Address(ptr, 2*wordSize, post_indexed));
 174         b(loop, gt);
 175       } else {
 176         for (int i = 0; i < stp_count; i++) {
 177           stp(ZR, ZR, Address(obj, base + i * 2 * wordSize));
 178         }
 179       }
 180     }
 181 #else
 182     if (obj_size_in_bytes >= 0 && obj_size_in_bytes <= 8 * BytesPerWord) {
 183       mov(tmp1, 0);
 184       const int base = instanceOopDesc::header_size() * HeapWordSize;
 185       for (int i = base; i < obj_size_in_bytes; i += wordSize) {
 186         str(tmp1, Address(obj, i));
 187       }
 188     } else {
 189       assert(header_size.is_constant() || header_size.as_register() == ptr, "code assumption");
 190       add(ptr, obj, header_size);
 191       initialize_body(ptr, obj_end, tmp1);
 192     }
 193 #endif // AARCH64
 194   }
 195 
 196   // StoreStore barrier required after complete initialization
 197   // (headers + content zeroing), before the object may escape.
 198   membar(MacroAssembler::StoreStore, tmp1);
 199 }
 200 
 201 void C1_MacroAssembler::allocate_object(Register obj, Register tmp1, Register tmp2, Register tmp3,
 202                                         int header_size, int object_size,
 203                                         Register klass, Label& slow_case) {
 204   assert_different_registers(obj, tmp1, tmp2, tmp3, klass, Rtemp);
 205   assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
 206   const int object_size_in_bytes = object_size * BytesPerWord;
 207 
 208   const Register obj_end = tmp1;
 209   const Register len = noreg;
 210 
 211   if (Assembler::is_arith_imm_in_range(object_size_in_bytes)) {
 212     try_allocate(obj, obj_end, tmp2, tmp3, object_size_in_bytes, slow_case);
 213   } else {
 214     // Rtemp should be free at c1 LIR level
 215     mov_slow(Rtemp, object_size_in_bytes);
 216     try_allocate(obj, obj_end, tmp2, tmp3, Rtemp, slow_case);
 217   }
 218   initialize_object(obj, obj_end, klass, len, tmp2, tmp3, instanceOopDesc::header_size() * HeapWordSize, object_size_in_bytes, /* is_tlab_allocated */ UseTLAB);
 219 }
 220 
 221 void C1_MacroAssembler::allocate_array(Register obj, Register len,
 222                                        Register tmp1, Register tmp2, Register tmp3,
 223                                        int header_size, int element_size,
 224                                        Register klass, Label& slow_case) {
 225   assert_different_registers(obj, len, tmp1, tmp2, tmp3, klass, Rtemp);
 226   const int header_size_in_bytes = header_size * BytesPerWord;
 227   const int scale_shift = exact_log2(element_size);
 228   const Register obj_size = Rtemp; // Rtemp should be free at c1 LIR level
 229 
 230 #ifdef AARCH64
 231   mov_slow(Rtemp, max_array_allocation_length);
 232   cmp_32(len, Rtemp);
 233 #else
 234   cmp_32(len, max_array_allocation_length);
 235 #endif // AARCH64
 236   b(slow_case, hs);
 237 
 238   bool align_header = ((header_size_in_bytes | element_size) & MinObjAlignmentInBytesMask) != 0;
 239   assert(align_header || ((header_size_in_bytes & MinObjAlignmentInBytesMask) == 0), "must be");
 240   assert(align_header || ((element_size & MinObjAlignmentInBytesMask) == 0), "must be");
 241 
 242   mov(obj_size, header_size_in_bytes + (align_header ? (MinObjAlignmentInBytes - 1) : 0));
 243   add_ptr_scaled_int32(obj_size, obj_size, len, scale_shift);
 244 
 245   if (align_header) {
 246     align_reg(obj_size, obj_size, MinObjAlignmentInBytes);
 247   }
 248 
 249   try_allocate(obj, tmp1, tmp2, tmp3, obj_size, slow_case);
 250   initialize_object(obj, tmp1, klass, len, tmp2, tmp3, header_size_in_bytes, -1, /* is_tlab_allocated */ UseTLAB);
 251 }
 252 
 253 int C1_MacroAssembler::lock_object(Register hdr, Register obj,
 254                                    Register disp_hdr, Register tmp1,
 255                                    Label& slow_case) {
 256   Label done, fast_lock, fast_lock_done;
 257   int null_check_offset = 0;
 258 
 259   const Register tmp2 = Rtemp; // Rtemp should be free at c1 LIR level
 260   assert_different_registers(hdr, obj, disp_hdr, tmp1, tmp2);
 261 
 262   assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code");
 263   const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
 264   const int mark_offset = BasicLock::displaced_header_offset_in_bytes();
 265 
 266   if (UseBiasedLocking) {
 267     // load object
 268     str(obj, Address(disp_hdr, obj_offset));
 269     null_check_offset = biased_locking_enter(obj, hdr/*scratched*/, tmp1, false, tmp2, done, slow_case);
 270   }
 271 
 272   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 273 
 274 #ifdef AARCH64
 275 
 276   str(obj, Address(disp_hdr, obj_offset));
 277 
 278   if (!UseBiasedLocking) {
 279     null_check_offset = offset();
 280   }
 281   ldr(hdr, obj);
 282 
 283   // Test if object is already locked
 284   assert(markOopDesc::unlocked_value == 1, "adjust this code");
 285   tbnz(hdr, exact_log2(markOopDesc::unlocked_value), fast_lock);
 286 
 287   // Check for recursive locking
 288   // See comments in InterpreterMacroAssembler::lock_object for
 289   // explanations on the fast recursive locking check.
 290   intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
 291   Assembler::LogicalImmediate imm(mask, false);
 292   mov(tmp2, SP);
 293   sub(tmp2, hdr, tmp2);
 294   ands(tmp2, tmp2, imm);
 295   b(slow_case, ne);
 296 
 297   // Recursive locking: store 0 into a lock record
 298   str(ZR, Address(disp_hdr, mark_offset));
 299   b(fast_lock_done);
 300 
 301 #else // AARCH64
 302 
 303   if (!UseBiasedLocking) {
 304     null_check_offset = offset();
 305   }
 306 
 307   // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
 308   // That would be acceptable as ether CAS or slow case path is taken in that case.
 309 
 310   // Must be the first instruction here, because implicit null check relies on it
 311   ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
 312 
 313   str(obj, Address(disp_hdr, obj_offset));
 314   tst(hdr, markOopDesc::unlocked_value);
 315   b(fast_lock, ne);
 316 
 317   // Check for recursive locking
 318   // See comments in InterpreterMacroAssembler::lock_object for
 319   // explanations on the fast recursive locking check.
 320   // -1- test low 2 bits
 321   movs(tmp2, AsmOperand(hdr, lsl, 30));
 322   // -2- test (hdr - SP) if the low two bits are 0
 323   sub(tmp2, hdr, SP, eq);
 324   movs(tmp2, AsmOperand(tmp2, lsr, exact_log2(os::vm_page_size())), eq);
 325   // If 'eq' then OK for recursive fast locking: store 0 into a lock record.
 326   str(tmp2, Address(disp_hdr, mark_offset), eq);
 327   b(fast_lock_done, eq);
 328   // else need slow case
 329   b(slow_case);
 330 
 331 #endif // AARCH64
 332 
 333   bind(fast_lock);
 334   // Save previous object header in BasicLock structure and update the header
 335   str(hdr, Address(disp_hdr, mark_offset));
 336 
 337   cas_for_lock_acquire(hdr, disp_hdr, obj, tmp2, slow_case);
 338 
 339   bind(fast_lock_done);
 340 
 341 #ifndef PRODUCT
 342   if (PrintBiasedLockingStatistics) {
 343     cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
 344   }
 345 #endif // !PRODUCT
 346 
 347   bind(done);
 348 
 349   return null_check_offset;
 350 }
 351 
 352 void C1_MacroAssembler::unlock_object(Register hdr, Register obj,
 353                                       Register disp_hdr, Register tmp,
 354                                       Label& slow_case) {
 355   // Note: this method is not using its 'tmp' argument
 356 
 357   assert_different_registers(hdr, obj, disp_hdr, Rtemp);
 358   Register tmp2 = Rtemp;
 359 
 360   assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code");
 361   const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
 362   const int mark_offset = BasicLock::displaced_header_offset_in_bytes();
 363 
 364   Label done;
 365   if (UseBiasedLocking) {
 366     // load object
 367     ldr(obj, Address(disp_hdr, obj_offset));
 368     biased_locking_exit(obj, hdr, done);
 369   }
 370 
 371   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 372   Label retry;
 373 
 374   // Load displaced header and object from the lock
 375   ldr(hdr, Address(disp_hdr, mark_offset));
 376   // If hdr is NULL, we've got recursive locking and there's nothing more to do
 377   cbz(hdr, done);
 378 
 379   if(!UseBiasedLocking) {
 380     // load object
 381     ldr(obj, Address(disp_hdr, obj_offset));
 382   }
 383 
 384   // Restore the object header
 385   cas_for_lock_release(disp_hdr, hdr, obj, tmp2, slow_case);
 386 
 387   bind(done);
 388 }
 389 
 390 
 391 #ifndef PRODUCT
 392 
 393 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
 394   if (!VerifyOops) return;
 395   verify_oop_addr(Address(SP, stack_offset));
 396 }
 397 
 398 void C1_MacroAssembler::verify_not_null_oop(Register r) {
 399   Label not_null;
 400   cbnz(r, not_null);
 401   stop("non-null oop required");
 402   bind(not_null);
 403   if (!VerifyOops) return;
 404   verify_oop(r);
 405 }
 406 
 407 #endif // !PRODUCT