1 /*
   2  * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 // This file is a derivative work resulting from (and including) modifications
  26 // made by Azul Systems, Inc.  The dates of such changes are 2013-2016.
  27 // Copyright 2013-2016 Azul Systems, Inc.  All Rights Reserved.
  28 //
  29 // Please contact Azul Systems, 385 Moffett Park Drive, Suite 115, Sunnyvale,
  30 // CA 94089 USA or visit www.azul.com if you need additional information or
  31 // have any questions.
  32 
  33 #include "precompiled.hpp"
  34 #include "c1/c1_MacroAssembler.hpp"
  35 #include "c1/c1_Runtime1.hpp"
  36 #include "classfile/systemDictionary.hpp"
  37 #include "gc_interface/collectedHeap.hpp"
  38 #include "interpreter/interpreter.hpp"
  39 #include "oops/arrayOop.hpp"
  40 #include "oops/markOop.hpp"
  41 #include "runtime/basicLock.hpp"
  42 #include "runtime/biasedLocking.hpp"
  43 #include "runtime/os.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 
  47 void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
  48                                   FloatRegister f0, FloatRegister f1,
  49                                   Register result)
  50 {
  51   Label done;
  52   if (is_float) {
  53     vcmp_f32(f0, f1);
  54   } else {
  55     vcmp_f64(f0, f1);
  56   }
  57 
  58   get_fpsr();
  59 
  60   mov(result, 0);
  61   if (unordered_result < 0) {
  62     // we want -1 for unordered or less than, 0 for equal and 1 for
  63     // greater than.
  64     mov(result, 1, NE); // Not equal or unordered
  65     neg(result, result, LT);  // Less than or unordered
  66   } else {
  67     // we want -1 for less than, 0 for equal and 1 for unordered or
  68     // greater than.
  69     mov(result, 1, NE); // Not equal or unordered
  70     neg(result, result, LO);  // Less than
  71   }
  72 }
  73 
  74 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
  75   const int aligned_mask = BytesPerWord -1;
  76   const int hdr_offset = oopDesc::mark_offset_in_bytes();
  77   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
  78   Label done, fail;
  79   int null_check_offset = -1;
  80 
  81   verify_oop(obj);
  82 
  83   // save object being locked into the BasicObjectLock
  84   str(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
  85 
  86   if (UseBiasedLocking) {
  87     assert(scratch != noreg, "should have scratch register at this point");
  88     null_check_offset = biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
  89   } else {
  90     null_check_offset = offset();
  91   }
  92 
  93   // Load object header
  94   ldr(hdr, Address(obj, hdr_offset));
  95   // and mark it as unlocked
  96   orr(hdr, hdr, markOopDesc::unlocked_value);
  97   // save unlocked object header into the displaced header location on the stack
  98   str(hdr, Address(disp_hdr, 0));
  99   // test if object header is still the same (i.e. unlocked), and if so, store the
 100   // displaced header address in the object header - if it is not the same, get the
 101   // object header instead
 102   lea(rscratch2, Address(obj, hdr_offset));
 103   cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
 104   // if the object header was the same, we're done
 105   // if the object header was not the same, it is now in the hdr register
 106   // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
 107   //
 108   // 1) (hdr & aligned_mask) == 0
 109   // 2) sp <= hdr
 110   // 3) hdr <= sp + page_size
 111   //
 112   // these 3 tests can be done by evaluating the following expression:
 113   //
 114   // (hdr - sp) & (aligned_mask - page_size)
 115   //
 116   // assuming both the stack pointer and page_size have their least
 117   // significant 2 bits cleared and page_size is a power of 2
 118   mov(rscratch1, sp);
 119   sub(hdr, hdr, rscratch1);
 120   mov(rscratch2, aligned_mask - os::vm_page_size());
 121   ands(hdr, hdr, rscratch2);
 122   // for recursive locking, the result is zero => save it in the displaced header
 123   // location (NULL in the displaced hdr location indicates recursive locking)
 124   str(hdr, Address(disp_hdr, 0));
 125   // otherwise we don't care about the result and handle locking via runtime call
 126   cbnz(hdr, slow_case);
 127   // done
 128   bind(done);
 129   if (PrintBiasedLockingStatistics) {
 130     lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
 131     addmw(Address(rscratch2, 0), 1, rscratch1);
 132   }
 133   return null_check_offset;
 134 }
 135 
 136 
 137 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
 138   const int aligned_mask = BytesPerWord -1;
 139   const int hdr_offset = oopDesc::mark_offset_in_bytes();
 140   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
 141   Label done;
 142 
 143   if (UseBiasedLocking) {
 144     // load object
 145     ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
 146     biased_locking_exit(obj, hdr, done);
 147   }
 148 
 149   // load displaced header
 150   ldr(hdr, Address(disp_hdr, 0));
 151   // if the loaded hdr is NULL we had recursive locking
 152   // if we had recursive locking, we are done
 153   cbz(hdr, done);
 154   if (!UseBiasedLocking) {
 155     // load object
 156     ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
 157   }
 158   verify_oop(obj);
 159   // test if object header is pointing to the displaced header, and if so, restore
 160   // the displaced header in the object - if the object header is not pointing to
 161   // the displaced header, get the object header instead
 162   // if the object header was not pointing to the displaced header,
 163   // we do unlocking via runtime call
 164   if (hdr_offset) {
 165     lea(rscratch1, Address(obj, hdr_offset));
 166     cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
 167   } else {
 168     cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
 169   }
 170   // done
 171   bind(done);
 172 }
 173 
 174 
 175 // Defines obj, preserves var_size_in_bytes
 176 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
 177   if (UseTLAB) {
 178     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 179   } else {
 180     eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
 181     incr_allocated_bytes(noreg, var_size_in_bytes, con_size_in_bytes, t1);
 182   }
 183 }
 184 
 185 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
 186   assert_different_registers(obj, klass, len);
 187   if (UseBiasedLocking && !len->is_valid()) {
 188     assert_different_registers(obj, klass, len, t1, t2);
 189     ldr(t1, Address(klass, Klass::prototype_header_offset()));
 190   } else {
 191     // This assumes that all prototype bits fit in an int32_t
 192     mov(t1, (int32_t)(intptr_t)markOopDesc::prototype());
 193   }
 194   str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
 195   str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 196 
 197   if (len->is_valid()) {
 198     str(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
 199   }
 200 }
 201 
 202 // Zero words; len is in bytes
 203 // Destroys all registers except addr
 204 // len must be a nonzero multiple of wordSize
 205 void C1_MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
 206   assert_different_registers(addr, len, t1, rscratch1, rscratch2);
 207 
 208 #ifdef ASSERT
 209   { Label L;
 210     tst(len, BytesPerWord - 1);
 211     b(L, Assembler::EQ);
 212     stop("len is not a multiple of BytesPerWord");
 213     bind(L);
 214   }
 215 #endif
 216 
 217 #ifndef PRODUCT
 218   block_comment("zero memory");
 219 #endif
 220 
 221   Label loop;
 222   Label entry;
 223 
 224 //  Algorithm:
 225 //
 226 //    scratch1 = cnt & 7;
 227 //    cnt -= scratch1;
 228 //    p += scratch1;
 229 //    switch (scratch1) {
 230 //      do {
 231 //        cnt -= 8;
 232 //          p[-8] = 0;
 233 //        case 7:
 234 //          p[-7] = 0;
 235 //        case 6:
 236 //          p[-6] = 0;
 237 //          // ...
 238 //        case 1:
 239 //          p[-1] = 0;
 240 //        case 0:
 241 //          p += 8;
 242 //      } while (cnt);
 243 //    }
 244 
 245   const int unroll = 8; // Number of str instructions we'll unroll
 246 
 247   lsr(len, len, LogBytesPerWord);
 248   andr(rscratch1, len, unroll - 1);  // tmp1 = cnt % unroll
 249   sub(len, len, rscratch1);      // cnt -= unroll
 250   // t1 always points to the end of the region we're about to zero
 251   add(t1, addr, rscratch1, lsl(LogBytesPerWord));
 252   adr(rscratch2, entry);
 253   sub(rscratch2, rscratch2, rscratch1, lsl(2));
 254   mov(rscratch1, 0);
 255   b(rscratch2);
 256   bind(loop);
 257   sub(len, len, unroll);
 258   for (int i = -unroll; i < 0; i++)
 259     str(rscratch1, Address(t1, i * wordSize));
 260   bind(entry);
 261   add(t1, t1, unroll * wordSize);
 262   cbnz(len, loop);
 263 }
 264 
 265 // preserves obj, destroys len_in_bytes
 266 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1) {
 267   Label done;
 268   assert(obj != len_in_bytes && obj != t1 && t1 != len_in_bytes, "registers must be different");
 269   assert((hdr_size_in_bytes & (BytesPerWord - 1)) == 0, "header size is not a multiple of BytesPerWord");
 270   Register index = len_in_bytes;
 271   // index is positive and ptr sized
 272   subs(index, index, hdr_size_in_bytes);
 273   b(done, Assembler::EQ);
 274   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 275 #ifdef ASSERT
 276   { Label L;
 277     tst(index, BytesPerWord - 1);
 278     b(L, Assembler::EQ);
 279     stop("index is not a multiple of BytesPerWord");
 280     bind(L);
 281   }
 282 #endif
 283 
 284   // Preserve obj
 285   if (hdr_size_in_bytes)
 286     add(obj, obj, hdr_size_in_bytes);
 287   zero_memory(obj, index, t1);
 288   if (hdr_size_in_bytes)
 289     sub(obj, obj, hdr_size_in_bytes);
 290 
 291   // done
 292   bind(done);
 293 }
 294 
 295 
 296 void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) {
 297   assert_different_registers(obj, t1, t2); // XXX really?
 298   assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
 299 
 300   try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case);
 301 
 302   initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2);
 303 }
 304 
 305 // This method clobbers t1, t2, and rscratch1 registers.
 306 void C1_MacroAssembler::initialize_object(Register obj, Register klass,
 307                                           Register var_size_in_bytes,
 308                                           int con_size_in_bytes,
 309                                           Register t1, Register t2) {
 310   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
 311          "con_size_in_bytes is not multiple of alignment");
 312 
 313   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
 314 
 315   initialize_header(obj, klass, noreg, t1, t2);
 316 
 317   // Null out rest of allocated space
 318   const Register index = t2;
 319   const int threshold = 8 * BytesPerWord;
 320   if (var_size_in_bytes != noreg) {
 321     mov(index, var_size_in_bytes);
 322     initialize_body(obj, index, hdr_size_in_bytes, t1);
 323   } else if (con_size_in_bytes <= threshold) {
 324     // Emit required number of str instructions (unroll loop completely)
 325     mov(t1, 0);
 326     for (int i = hdr_size_in_bytes; i < con_size_in_bytes; i += BytesPerWord) {
 327       str(t1, Address(obj, i));
 328     }
 329   } else if (con_size_in_bytes > hdr_size_in_bytes) {
 330     // Use loop to null out fields
 331     int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord;
 332     mov(t1, 0);
 333 
 334     const int unroll = 4; // Number of str instructions we'll unroll
 335     mov(index, words / unroll);
 336     int remainder = words % unroll;
 337     lea(rscratch1, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord));
 338 
 339     Label entry_point, loop;
 340     b(entry_point);
 341     bind(loop);
 342     sub(index, index, 1);
 343     for (int i = -unroll; i < 0; i++) {
 344       if (-i == remainder) {
 345         bind(entry_point);
 346       }
 347       str(t1, Address(rscratch1, i * BytesPerWord));
 348     }
 349     if (remainder == 0) {
 350       bind(entry_point);
 351     }
 352     add(rscratch1, rscratch1, unroll * BytesPerWord);
 353     cbnz(index, loop);
 354   }
 355 
 356   membar(StoreStore);
 357 
 358   if (CURRENT_ENV->dtrace_alloc_probes()) {
 359     assert(obj == r0, "must be");
 360     far_call(RuntimeAddress(Runtime1::entry_for(
 361              Runtime1::dtrace_object_alloc_id)));
 362   }
 363 
 364   verify_oop(obj);
 365 }
 366 
 367 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
 368   assert_different_registers(obj, len, t1, t2, klass);
 369 
 370   // determine alignment mask
 371   assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
 372 
 373   // check for negative or excessive length
 374   mov(rscratch1, (int32_t)max_array_allocation_length);
 375   cmp(len, rscratch1);
 376   b(slow_case, Assembler::HS);
 377 
 378   const Register arr_size = t2; // okay to be the same
 379   // align object end
 380   mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
 381   add(arr_size, arr_size, len, Assembler::lsl(f));
 382   mov(t1, ~MinObjAlignmentInBytesMask);
 383   andr(arr_size, arr_size, t1);
 384 
 385   try_allocate(obj, arr_size, 0, t1, t2, slow_case);
 386 
 387   initialize_header(obj, klass, len, t1, t2);
 388 
 389   // clear rest of allocated space
 390   const Register len_zero = len;
 391   initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
 392 
 393   membar(StoreStore);
 394 
 395   if (CURRENT_ENV->dtrace_alloc_probes()) {
 396     assert(obj == r0, "must be");
 397     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
 398   }
 399 
 400   verify_oop(obj);
 401 }
 402 
 403 
 404 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
 405   verify_oop(receiver);
 406   // explicit NULL check not needed since load from [klass_offset] causes a trap
 407   // check against inline cache
 408   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
 409 
 410   cmp_klass(receiver, iCache, rscratch1);
 411 }
 412 
 413 void C1_MacroAssembler::build_frame(int frame_size_in_bytes,
 414                                     int bang_size_in_bytes) {
 415   assert(bang_size_in_bytes >= frame_size_in_bytes,
 416          "stack bang size incorrect");
 417 
 418   // If we have to make this method not-entrant, we'll overwrite its first
 419   // instruction with a jump. For this action to be legal we must ensure that
 420   // this first instruction is a B, BL, NOP, BKPT, or SVC. Make it a NOP
 421   nop();
 422 
 423   // Make sure there is enough stack space for this method's activation
 424   generate_stack_overflow_check(bang_size_in_bytes);
 425 
 426   // Push lr, rfp, and optionally update rfp. rfp points to the first stack
 427   // word used by the new frame.
 428   stmdb(sp, RegSet::of(rfp, lr).bits());
 429   if (PreserveFramePointer) {
 430     add(rfp, sp, BytesPerWord);
 431   }
 432 
 433   // Create frame. frame_size_in_bytes always comes from
 434   // LIR_Assembler::initial_frame_size_in_bytes() method, and it already
 435   // takes into account two stack words spent on saving lr and rfp.
 436   decrement(sp, frame_size_in_bytes);
 437 }
 438 
 439 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) {
 440   // Remove frame. frame_size_in_bytes always comes from
 441   // LIR_Assembler::initial_frame_size_in_bytes() method, and it already
 442   // takes into account two stack words spent on saving lr and rfp.
 443   increment(sp, frame_size_in_bytes);
 444 
 445   // Pop rfp and lr
 446   ldmia(sp, RegSet::of(rfp, lr).bits());
 447 }
 448 
 449 void C1_MacroAssembler::verified_entry() {
 450 }
 451 
 452 #ifndef PRODUCT
 453 
 454 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
 455   if (!VerifyOops) return;
 456   verify_oop_addr(Address(sp, stack_offset), "oop");
 457 }
 458 
 459 void C1_MacroAssembler::verify_not_null_oop(Register r) {
 460   if (!VerifyOops) return;
 461   Label not_null;
 462   cbnz(r, not_null);
 463   stop("non-null oop required");
 464   bind(not_null);
 465   verify_oop(r);
 466 }
 467 
 468 void C1_MacroAssembler::invalidate_registers(bool inv_r0, bool inv_r2, bool inv_r3) {
 469 #ifdef ASSERT
 470   static int nn;
 471   if (inv_r0) mov(r0, 0xDEAD);
 472   if (inv_r2) mov(r2, nn++);
 473   if (inv_r3) mov(r3, 0xDEAD);
 474 #endif
 475 }
 476 #endif // ifndef PRODUCT