1 /*
   2  * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "ci/ciEnv.hpp"
  30 #include "code/nativeInst.hpp"
  31 #include "compiler/disassembler.hpp"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/cardTable.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/collectedHeap.inline.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "memory/resourceArea.hpp"
  39 #include "oops/accessDecorators.hpp"
  40 #include "oops/klass.inline.hpp"
  41 #include "prims/methodHandles.hpp"
  42 #include "runtime/biasedLocking.hpp"
  43 #include "runtime/interfaceSupport.inline.hpp"
  44 #include "runtime/objectMonitor.hpp"
  45 #include "runtime/os.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "utilities/macros.hpp"
  49 
  50 // Implementation of AddressLiteral
  51 
  52 void AddressLiteral::set_rspec(relocInfo::relocType rtype) {
  53   switch (rtype) {
  54   case relocInfo::oop_type:
  55     // Oops are a special case. Normally they would be their own section
  56     // but in cases like icBuffer they are literals in the code stream that
  57     // we don't have a section for. We use none so that we get a literal address
  58     // which is always patchable.
  59     break;
  60   case relocInfo::external_word_type:
  61     _rspec = external_word_Relocation::spec(_target);
  62     break;
  63   case relocInfo::internal_word_type:
  64     _rspec = internal_word_Relocation::spec(_target);
  65     break;
  66   case relocInfo::opt_virtual_call_type:
  67     _rspec = opt_virtual_call_Relocation::spec();
  68     break;
  69   case relocInfo::static_call_type:
  70     _rspec = static_call_Relocation::spec();
  71     break;
  72   case relocInfo::runtime_call_type:
  73     _rspec = runtime_call_Relocation::spec();
  74     break;
  75   case relocInfo::poll_type:
  76   case relocInfo::poll_return_type:
  77     _rspec = Relocation::spec_simple(rtype);
  78     break;
  79   case relocInfo::none:
  80     break;
  81   default:
  82     ShouldNotReachHere();
  83     break;
  84   }
  85 }
  86 
  87 // Initially added to the Assembler interface as a pure virtual:
  88 //   RegisterConstant delayed_value(..)
  89 // for:
  90 //   6812678 macro assembler needs delayed binding of a few constants (for 6655638)
  91 // this was subsequently modified to its present name and return type
  92 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
  93                                                       Register tmp,
  94                                                       int offset) {
  95   ShouldNotReachHere();
  96   return RegisterOrConstant(-1);
  97 }
  98 
  99 
 100 
 101 
 102 // virtual method calling
 103 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 104                                            Register vtable_index,
 105                                            Register method_result) {
 106   const int base_offset = in_bytes(Klass::vtable_start_offset()) + vtableEntry::method_offset_in_bytes();
 107   assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 108   add(recv_klass, recv_klass, AsmOperand(vtable_index, lsl, LogBytesPerWord));
 109   ldr(method_result, Address(recv_klass, base_offset));
 110 }
 111 
 112 
 113 // Simplified, combined version, good for typical uses.
 114 // Falls through on failure.
 115 void MacroAssembler::check_klass_subtype(Register sub_klass,
 116                                          Register super_klass,
 117                                          Register temp_reg,
 118                                          Register temp_reg2,
 119                                          Register temp_reg3,
 120                                          Label& L_success) {
 121   Label L_failure;
 122   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, temp_reg2, &L_success, &L_failure, NULL);
 123   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, temp_reg2, temp_reg3, &L_success, NULL);
 124   bind(L_failure);
 125 };
 126 
 127 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 128                                                    Register super_klass,
 129                                                    Register temp_reg,
 130                                                    Register temp_reg2,
 131                                                    Label* L_success,
 132                                                    Label* L_failure,
 133                                                    Label* L_slow_path) {
 134 
 135   assert_different_registers(sub_klass, super_klass, temp_reg, temp_reg2, noreg);
 136   const Register super_check_offset = temp_reg2;
 137 
 138   Label L_fallthrough;
 139   int label_nulls = 0;
 140   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
 141   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
 142   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
 143   assert(label_nulls <= 1, "at most one NULL in the batch");
 144 
 145   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 146   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 147   Address super_check_offset_addr(super_klass, sco_offset);
 148 
 149   // If the pointers are equal, we are done (e.g., String[] elements).
 150   // This self-check enables sharing of secondary supertype arrays among
 151   // non-primary types such as array-of-interface.  Otherwise, each such
 152   // type would need its own customized SSA.
 153   // We move this check to the front of the fast path because many
 154   // type checks are in fact trivially successful in this manner,
 155   // so we get a nicely predicted branch right at the start of the check.
 156   cmp(sub_klass, super_klass);
 157   b(*L_success, eq);
 158 
 159   // Check the supertype display:
 160   ldr_u32(super_check_offset, super_check_offset_addr);
 161 
 162   Address super_check_addr(sub_klass, super_check_offset);
 163   ldr(temp_reg, super_check_addr);
 164   cmp(super_klass, temp_reg); // load displayed supertype
 165 
 166   // This check has worked decisively for primary supers.
 167   // Secondary supers are sought in the super_cache ('super_cache_addr').
 168   // (Secondary supers are interfaces and very deeply nested subtypes.)
 169   // This works in the same check above because of a tricky aliasing
 170   // between the super_cache and the primary super display elements.
 171   // (The 'super_check_addr' can address either, as the case requires.)
 172   // Note that the cache is updated below if it does not help us find
 173   // what we need immediately.
 174   // So if it was a primary super, we can just fail immediately.
 175   // Otherwise, it's the slow path for us (no success at this point).
 176 
 177   b(*L_success, eq);
 178   cmp_32(super_check_offset, sc_offset);
 179   if (L_failure == &L_fallthrough) {
 180     b(*L_slow_path, eq);
 181   } else {
 182     b(*L_failure, ne);
 183     if (L_slow_path != &L_fallthrough) {
 184       b(*L_slow_path);
 185     }
 186   }
 187 
 188   bind(L_fallthrough);
 189 }
 190 
 191 
 192 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 193                                                    Register super_klass,
 194                                                    Register temp_reg,
 195                                                    Register temp2_reg,
 196                                                    Register temp3_reg,
 197                                                    Label* L_success,
 198                                                    Label* L_failure,
 199                                                    bool set_cond_codes) {
 200   // Note: if used by code that expects a register to be 0 on success,
 201   // this register must be temp_reg and set_cond_codes must be true
 202 
 203   Register saved_reg = noreg;
 204 
 205   // get additional tmp registers
 206   if (temp3_reg == noreg) {
 207     saved_reg = temp3_reg = LR;
 208     push(saved_reg);
 209   }
 210 
 211   assert(temp2_reg != noreg, "need all the temporary registers");
 212   assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg);
 213 
 214   Register cmp_temp = temp_reg;
 215   Register scan_temp = temp3_reg;
 216   Register count_temp = temp2_reg;
 217 
 218   Label L_fallthrough;
 219   int label_nulls = 0;
 220   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
 221   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
 222   assert(label_nulls <= 1, "at most one NULL in the batch");
 223 
 224   // a couple of useful fields in sub_klass:
 225   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 226   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 227   Address secondary_supers_addr(sub_klass, ss_offset);
 228   Address super_cache_addr(     sub_klass, sc_offset);
 229 
 230 #ifndef PRODUCT
 231   inc_counter((address)&SharedRuntime::_partial_subtype_ctr, scan_temp, count_temp);
 232 #endif
 233 
 234   // We will consult the secondary-super array.
 235   ldr(scan_temp, Address(sub_klass, ss_offset));
 236 
 237   assert(! UseCompressedOops, "search_key must be the compressed super_klass");
 238   // else search_key is the
 239   Register search_key = super_klass;
 240 
 241   // Load the array length.
 242   ldr(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes()));
 243   add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes());
 244 
 245   add(count_temp, count_temp, 1);
 246 
 247   Label L_loop, L_fail;
 248 
 249   // Top of search loop
 250   bind(L_loop);
 251   // Notes:
 252   //  scan_temp starts at the array elements
 253   //  count_temp is 1+size
 254   subs(count_temp, count_temp, 1);
 255   if ((L_failure != &L_fallthrough) && (! set_cond_codes) && (saved_reg == noreg)) {
 256     // direct jump to L_failure if failed and no cleanup needed
 257     b(*L_failure, eq); // not found and
 258   } else {
 259     b(L_fail, eq); // not found in the array
 260   }
 261 
 262   // Load next super to check
 263   // In the array of super classes elements are pointer sized.
 264   int element_size = wordSize;
 265   ldr(cmp_temp, Address(scan_temp, element_size, post_indexed));
 266 
 267   // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list
 268   subs(cmp_temp, cmp_temp, search_key);
 269 
 270   // A miss means we are NOT a subtype and need to keep looping
 271   b(L_loop, ne);
 272 
 273   // Falling out the bottom means we found a hit; we ARE a subtype
 274 
 275   // Note: temp_reg/cmp_temp is already 0 and flag Z is set
 276 
 277   // Success.  Cache the super we found and proceed in triumph.
 278   str(super_klass, Address(sub_klass, sc_offset));
 279 
 280   if (saved_reg != noreg) {
 281     // Return success
 282     pop(saved_reg);
 283   }
 284 
 285   b(*L_success);
 286 
 287   bind(L_fail);
 288   // Note1: check "b(*L_failure, eq)" above if adding extra instructions here
 289   if (set_cond_codes) {
 290     movs(temp_reg, sub_klass); // clears Z and sets temp_reg to non-0 if needed
 291   }
 292   if (saved_reg != noreg) {
 293     pop(saved_reg);
 294   }
 295   if (L_failure != &L_fallthrough) {
 296     b(*L_failure);
 297   }
 298 
 299   bind(L_fallthrough);
 300 }
 301 
 302 // Returns address of receiver parameter, using tmp as base register. tmp and params_count can be the same.
 303 Address MacroAssembler::receiver_argument_address(Register params_base, Register params_count, Register tmp) {
 304   assert_different_registers(params_base, params_count);
 305   add(tmp, params_base, AsmOperand(params_count, lsl, Interpreter::logStackElementSize));
 306   return Address(tmp, -Interpreter::stackElementSize);
 307 }
 308 
 309 
 310 void MacroAssembler::align(int modulus) {
 311   while (offset() % modulus != 0) {
 312     nop();
 313   }
 314 }
 315 
 316 int MacroAssembler::set_last_Java_frame(Register last_java_sp,
 317                                         Register last_java_fp,
 318                                         bool save_last_java_pc,
 319                                         Register tmp) {
 320   int pc_offset;
 321   if (last_java_fp != noreg) {
 322     // optional
 323     str(last_java_fp, Address(Rthread, JavaThread::last_Java_fp_offset()));
 324     _fp_saved = true;
 325   } else {
 326     _fp_saved = false;
 327   }
 328   if (save_last_java_pc) {
 329     str(PC, Address(Rthread, JavaThread::last_Java_pc_offset()));
 330     pc_offset = offset() + VM_Version::stored_pc_adjustment();
 331     _pc_saved = true;
 332   } else {
 333     _pc_saved = false;
 334     pc_offset = -1;
 335   }
 336   // According to comment in javaFrameAnchorm SP must be saved last, so that other
 337   // entries are valid when SP is set.
 338 
 339   // However, this is probably not a strong constrainst since for instance PC is
 340   // sometimes read from the stack at SP... but is pushed later (by the call). Hence,
 341   // we now write the fields in the expected order but we have not added a StoreStore
 342   // barrier.
 343 
 344   // XXX: if the ordering is really important, PC should always be saved (without forgetting
 345   // to update oop_map offsets) and a StoreStore barrier might be needed.
 346 
 347   if (last_java_sp == noreg) {
 348     last_java_sp = SP; // always saved
 349   }
 350   str(last_java_sp, Address(Rthread, JavaThread::last_Java_sp_offset()));
 351 
 352   return pc_offset; // for oopmaps
 353 }
 354 
 355 void MacroAssembler::reset_last_Java_frame(Register tmp) {
 356   const Register Rzero = zero_register(tmp);
 357   str(Rzero, Address(Rthread, JavaThread::last_Java_sp_offset()));
 358   if (_fp_saved) {
 359     str(Rzero, Address(Rthread, JavaThread::last_Java_fp_offset()));
 360   }
 361   if (_pc_saved) {
 362     str(Rzero, Address(Rthread, JavaThread::last_Java_pc_offset()));
 363   }
 364 }
 365 
 366 
 367 // Implementation of call_VM versions
 368 
 369 void MacroAssembler::call_VM_leaf_helper(address entry_point, int number_of_arguments) {
 370   assert(number_of_arguments >= 0, "cannot have negative number of arguments");
 371   assert(number_of_arguments <= 4, "cannot have more than 4 arguments");
 372 
 373   // Safer to save R9 here since callers may have been written
 374   // assuming R9 survives. This is suboptimal but is not worth
 375   // optimizing for the few platforms where R9 is scratched.
 376   push(RegisterSet(R4) | R9ifScratched);
 377   mov(R4, SP);
 378   bic(SP, SP, StackAlignmentInBytes - 1);
 379   call(entry_point, relocInfo::runtime_call_type);
 380   mov(SP, R4);
 381   pop(RegisterSet(R4) | R9ifScratched);
 382 }
 383 
 384 
 385 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 386   assert(number_of_arguments >= 0, "cannot have negative number of arguments");
 387   assert(number_of_arguments <= 3, "cannot have more than 3 arguments");
 388 
 389   const Register tmp = Rtemp;
 390   assert_different_registers(oop_result, tmp);
 391 
 392   set_last_Java_frame(SP, FP, true, tmp);
 393 
 394 #if R9_IS_SCRATCHED
 395   // Safer to save R9 here since callers may have been written
 396   // assuming R9 survives. This is suboptimal but is not worth
 397   // optimizing for the few platforms where R9 is scratched.
 398 
 399   // Note: cannot save R9 above the saved SP (some calls expect for
 400   // instance the Java stack top at the saved SP)
 401   // => once saved (with set_last_Java_frame), decrease SP before rounding to
 402   // ensure the slot at SP will be free for R9).
 403   sub(SP, SP, 4);
 404   bic(SP, SP, StackAlignmentInBytes - 1);
 405   str(R9, Address(SP, 0));
 406 #else
 407   bic(SP, SP, StackAlignmentInBytes - 1);
 408 #endif // R9_IS_SCRATCHED
 409 
 410   mov(R0, Rthread);
 411   call(entry_point, relocInfo::runtime_call_type);
 412 
 413 #if R9_IS_SCRATCHED
 414   ldr(R9, Address(SP, 0));
 415 #endif
 416   ldr(SP, Address(Rthread, JavaThread::last_Java_sp_offset()));
 417 
 418   reset_last_Java_frame(tmp);
 419 
 420   // C++ interp handles this in the interpreter
 421   check_and_handle_popframe();
 422   check_and_handle_earlyret();
 423 
 424   if (check_exceptions) {
 425     // check for pending exceptions
 426     ldr(tmp, Address(Rthread, Thread::pending_exception_offset()));
 427     cmp(tmp, 0);
 428     mov(Rexception_pc, PC, ne);
 429     b(StubRoutines::forward_exception_entry(), ne);
 430   }
 431 
 432   // get oop result if there is one and reset the value in the thread
 433   if (oop_result->is_valid()) {
 434     get_vm_result(oop_result, tmp);
 435   }
 436 }
 437 
 438 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
 439   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 440 }
 441 
 442 
 443 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions) {
 444   assert (arg_1 == R1, "fixed register for arg_1");
 445   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 446 }
 447 
 448 
 449 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
 450   assert (arg_1 == R1, "fixed register for arg_1");
 451   assert (arg_2 == R2, "fixed register for arg_2");
 452   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 453 }
 454 
 455 
 456 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
 457   assert (arg_1 == R1, "fixed register for arg_1");
 458   assert (arg_2 == R2, "fixed register for arg_2");
 459   assert (arg_3 == R3, "fixed register for arg_3");
 460   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 461 }
 462 
 463 
 464 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments, bool check_exceptions) {
 465   // Not used on ARM
 466   Unimplemented();
 467 }
 468 
 469 
 470 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions) {
 471   // Not used on ARM
 472   Unimplemented();
 473 }
 474 
 475 
 476 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
 477 // Not used on ARM
 478   Unimplemented();
 479 }
 480 
 481 
 482 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
 483   // Not used on ARM
 484   Unimplemented();
 485 }
 486 
 487 // Raw call, without saving/restoring registers, exception handling, etc.
 488 // Mainly used from various stubs.
 489 void MacroAssembler::call_VM(address entry_point, bool save_R9_if_scratched) {
 490   const Register tmp = Rtemp; // Rtemp free since scratched by call
 491   set_last_Java_frame(SP, FP, true, tmp);
 492 #if R9_IS_SCRATCHED
 493   if (save_R9_if_scratched) {
 494     // Note: Saving also R10 for alignment.
 495     push(RegisterSet(R9, R10));
 496   }
 497 #endif
 498   mov(R0, Rthread);
 499   call(entry_point, relocInfo::runtime_call_type);
 500 #if R9_IS_SCRATCHED
 501   if (save_R9_if_scratched) {
 502     pop(RegisterSet(R9, R10));
 503   }
 504 #endif
 505   reset_last_Java_frame(tmp);
 506 }
 507 
 508 void MacroAssembler::call_VM_leaf(address entry_point) {
 509   call_VM_leaf_helper(entry_point, 0);
 510 }
 511 
 512 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
 513   assert (arg_1 == R0, "fixed register for arg_1");
 514   call_VM_leaf_helper(entry_point, 1);
 515 }
 516 
 517 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
 518   assert (arg_1 == R0, "fixed register for arg_1");
 519   assert (arg_2 == R1, "fixed register for arg_2");
 520   call_VM_leaf_helper(entry_point, 2);
 521 }
 522 
 523 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
 524   assert (arg_1 == R0, "fixed register for arg_1");
 525   assert (arg_2 == R1, "fixed register for arg_2");
 526   assert (arg_3 == R2, "fixed register for arg_3");
 527   call_VM_leaf_helper(entry_point, 3);
 528 }
 529 
 530 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4) {
 531   assert (arg_1 == R0, "fixed register for arg_1");
 532   assert (arg_2 == R1, "fixed register for arg_2");
 533   assert (arg_3 == R2, "fixed register for arg_3");
 534   assert (arg_4 == R3, "fixed register for arg_4");
 535   call_VM_leaf_helper(entry_point, 4);
 536 }
 537 
 538 void MacroAssembler::get_vm_result(Register oop_result, Register tmp) {
 539   assert_different_registers(oop_result, tmp);
 540   ldr(oop_result, Address(Rthread, JavaThread::vm_result_offset()));
 541   str(zero_register(tmp), Address(Rthread, JavaThread::vm_result_offset()));
 542   verify_oop(oop_result);
 543 }
 544 
 545 void MacroAssembler::get_vm_result_2(Register metadata_result, Register tmp) {
 546   assert_different_registers(metadata_result, tmp);
 547   ldr(metadata_result, Address(Rthread, JavaThread::vm_result_2_offset()));
 548   str(zero_register(tmp), Address(Rthread, JavaThread::vm_result_2_offset()));
 549 }
 550 
 551 void MacroAssembler::add_rc(Register dst, Register arg1, RegisterOrConstant arg2) {
 552   if (arg2.is_register()) {
 553     add(dst, arg1, arg2.as_register());
 554   } else {
 555     add(dst, arg1, arg2.as_constant());
 556   }
 557 }
 558 
 559 void MacroAssembler::add_slow(Register rd, Register rn, int c) {
 560   // This function is used in compiler for handling large frame offsets
 561   if ((c < 0) && (((-c) & ~0x3fc) == 0)) {
 562     return sub(rd, rn, (-c));
 563   }
 564   int low = c & 0x3fc;
 565   if (low != 0) {
 566     add(rd, rn, low);
 567     rn = rd;
 568   }
 569   if (c & ~0x3fc) {
 570     assert(AsmOperand::is_rotated_imm(c & ~0x3fc), "unsupported add_slow offset %d", c);
 571     add(rd, rn, c & ~0x3fc);
 572   } else if (rd != rn) {
 573     assert(c == 0, "");
 574     mov(rd, rn); // need to generate at least one move!
 575   }
 576 }
 577 
 578 void MacroAssembler::sub_slow(Register rd, Register rn, int c) {
 579   // This function is used in compiler for handling large frame offsets
 580   if ((c < 0) && (((-c) & ~0x3fc) == 0)) {
 581     return add(rd, rn, (-c));
 582   }
 583   int low = c & 0x3fc;
 584   if (low != 0) {
 585     sub(rd, rn, low);
 586     rn = rd;
 587   }
 588   if (c & ~0x3fc) {
 589     assert(AsmOperand::is_rotated_imm(c & ~0x3fc), "unsupported sub_slow offset %d", c);
 590     sub(rd, rn, c & ~0x3fc);
 591   } else if (rd != rn) {
 592     assert(c == 0, "");
 593     mov(rd, rn); // need to generate at least one move!
 594   }
 595 }
 596 
 597 void MacroAssembler::mov_slow(Register rd, address addr) {
 598   // do *not* call the non relocated mov_related_address
 599   mov_slow(rd, (intptr_t)addr);
 600 }
 601 
 602 void MacroAssembler::mov_slow(Register rd, const char *str) {
 603   mov_slow(rd, (intptr_t)str);
 604 }
 605 
 606 
 607 void MacroAssembler::mov_slow(Register rd, intptr_t c, AsmCondition cond) {
 608   if (AsmOperand::is_rotated_imm(c)) {
 609     mov(rd, c, cond);
 610   } else if (AsmOperand::is_rotated_imm(~c)) {
 611     mvn(rd, ~c, cond);
 612   } else if (VM_Version::supports_movw()) {
 613     movw(rd, c & 0xffff, cond);
 614     if ((unsigned int)c >> 16) {
 615       movt(rd, (unsigned int)c >> 16, cond);
 616     }
 617   } else {
 618     // Find first non-zero bit
 619     int shift = 0;
 620     while ((c & (3 << shift)) == 0) {
 621       shift += 2;
 622     }
 623     // Put the least significant part of the constant
 624     int mask = 0xff << shift;
 625     mov(rd, c & mask, cond);
 626     // Add up to 3 other parts of the constant;
 627     // each of them can be represented as rotated_imm
 628     if (c & (mask << 8)) {
 629       orr(rd, rd, c & (mask << 8), cond);
 630     }
 631     if (c & (mask << 16)) {
 632       orr(rd, rd, c & (mask << 16), cond);
 633     }
 634     if (c & (mask << 24)) {
 635       orr(rd, rd, c & (mask << 24), cond);
 636     }
 637   }
 638 }
 639 
 640 
 641 void MacroAssembler::mov_oop(Register rd, jobject o, int oop_index,
 642                              AsmCondition cond
 643                              ) {
 644 
 645   if (o == NULL) {
 646     mov(rd, 0, cond);
 647     return;
 648   }
 649 
 650   if (oop_index == 0) {
 651     oop_index = oop_recorder()->allocate_oop_index(o);
 652   }
 653   relocate(oop_Relocation::spec(oop_index));
 654 
 655   if (VM_Version::supports_movw()) {
 656     movw(rd, 0, cond);
 657     movt(rd, 0, cond);
 658   } else {
 659     ldr(rd, Address(PC), cond);
 660     // Extra nop to handle case of large offset of oop placeholder (see NativeMovConstReg::set_data).
 661     nop();
 662   }
 663 }
 664 
 665 void MacroAssembler::mov_metadata(Register rd, Metadata* o, int metadata_index) {
 666   if (o == NULL) {
 667     mov(rd, 0);
 668     return;
 669   }
 670 
 671   if (metadata_index == 0) {
 672     metadata_index = oop_recorder()->allocate_metadata_index(o);
 673   }
 674   relocate(metadata_Relocation::spec(metadata_index));
 675 
 676   if (VM_Version::supports_movw()) {
 677     movw(rd, ((int)o) & 0xffff);
 678     movt(rd, (unsigned int)o >> 16);
 679   } else {
 680     ldr(rd, Address(PC));
 681     // Extra nop to handle case of large offset of metadata placeholder (see NativeMovConstReg::set_data).
 682     nop();
 683   }
 684 }
 685 
 686 void MacroAssembler::mov_float(FloatRegister fd, jfloat c, AsmCondition cond) {
 687   Label skip_constant;
 688   union {
 689     jfloat f;
 690     jint i;
 691   } accessor;
 692   accessor.f = c;
 693 
 694   flds(fd, Address(PC), cond);
 695   b(skip_constant);
 696   emit_int32(accessor.i);
 697   bind(skip_constant);
 698 }
 699 
 700 void MacroAssembler::mov_double(FloatRegister fd, jdouble c, AsmCondition cond) {
 701   Label skip_constant;
 702   union {
 703     jdouble d;
 704     jint i[2];
 705   } accessor;
 706   accessor.d = c;
 707 
 708   fldd(fd, Address(PC), cond);
 709   b(skip_constant);
 710   emit_int32(accessor.i[0]);
 711   emit_int32(accessor.i[1]);
 712   bind(skip_constant);
 713 }
 714 
 715 void MacroAssembler::ldr_global_s32(Register reg, address address_of_global) {
 716   intptr_t addr = (intptr_t) address_of_global;
 717   mov_slow(reg, addr & ~0xfff);
 718   ldr(reg, Address(reg, addr & 0xfff));
 719 }
 720 
 721 void MacroAssembler::ldr_global_ptr(Register reg, address address_of_global) {
 722   ldr_global_s32(reg, address_of_global);
 723 }
 724 
 725 void MacroAssembler::ldrb_global(Register reg, address address_of_global) {
 726   intptr_t addr = (intptr_t) address_of_global;
 727   mov_slow(reg, addr & ~0xfff);
 728   ldrb(reg, Address(reg, addr & 0xfff));
 729 }
 730 
 731 void MacroAssembler::zero_extend(Register rd, Register rn, int bits) {
 732   if (bits <= 8) {
 733     andr(rd, rn, (1 << bits) - 1);
 734   } else if (bits >= 24) {
 735     bic(rd, rn, -1 << bits);
 736   } else {
 737     mov(rd, AsmOperand(rn, lsl, 32 - bits));
 738     mov(rd, AsmOperand(rd, lsr, 32 - bits));
 739   }
 740 }
 741 
 742 void MacroAssembler::sign_extend(Register rd, Register rn, int bits) {
 743   mov(rd, AsmOperand(rn, lsl, 32 - bits));
 744   mov(rd, AsmOperand(rd, asr, 32 - bits));
 745 }
 746 
 747 
 748 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
 749   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 750   bs->obj_equals(this, obj1, obj2);
 751 }
 752 
 753 void MacroAssembler::long_move(Register rd_lo, Register rd_hi,
 754                                Register rn_lo, Register rn_hi,
 755                                AsmCondition cond) {
 756   if (rd_lo != rn_hi) {
 757     if (rd_lo != rn_lo) { mov(rd_lo, rn_lo, cond); }
 758     if (rd_hi != rn_hi) { mov(rd_hi, rn_hi, cond); }
 759   } else if (rd_hi != rn_lo) {
 760     if (rd_hi != rn_hi) { mov(rd_hi, rn_hi, cond); }
 761     if (rd_lo != rn_lo) { mov(rd_lo, rn_lo, cond); }
 762   } else {
 763     eor(rd_lo, rd_hi, rd_lo, cond);
 764     eor(rd_hi, rd_lo, rd_hi, cond);
 765     eor(rd_lo, rd_hi, rd_lo, cond);
 766   }
 767 }
 768 
 769 void MacroAssembler::long_shift(Register rd_lo, Register rd_hi,
 770                                 Register rn_lo, Register rn_hi,
 771                                 AsmShift shift, Register count) {
 772   Register tmp;
 773   if (rd_lo != rn_lo && rd_lo != rn_hi && rd_lo != count) {
 774     tmp = rd_lo;
 775   } else {
 776     tmp = rd_hi;
 777   }
 778   assert_different_registers(tmp, count, rn_lo, rn_hi);
 779 
 780   subs(tmp, count, 32);
 781   if (shift == lsl) {
 782     assert_different_registers(rd_hi, rn_lo);
 783     assert_different_registers(count, rd_hi);
 784     mov(rd_hi, AsmOperand(rn_lo, shift, tmp), pl);
 785     rsb(tmp, count, 32, mi);
 786     if (rd_hi == rn_hi) {
 787       mov(rd_hi, AsmOperand(rn_hi, lsl, count), mi);
 788       orr(rd_hi, rd_hi, AsmOperand(rn_lo, lsr, tmp), mi);
 789     } else {
 790       mov(rd_hi, AsmOperand(rn_lo, lsr, tmp), mi);
 791       orr(rd_hi, rd_hi, AsmOperand(rn_hi, lsl, count), mi);
 792     }
 793     mov(rd_lo, AsmOperand(rn_lo, shift, count));
 794   } else {
 795     assert_different_registers(rd_lo, rn_hi);
 796     assert_different_registers(rd_lo, count);
 797     mov(rd_lo, AsmOperand(rn_hi, shift, tmp), pl);
 798     rsb(tmp, count, 32, mi);
 799     if (rd_lo == rn_lo) {
 800       mov(rd_lo, AsmOperand(rn_lo, lsr, count), mi);
 801       orr(rd_lo, rd_lo, AsmOperand(rn_hi, lsl, tmp), mi);
 802     } else {
 803       mov(rd_lo, AsmOperand(rn_hi, lsl, tmp), mi);
 804       orr(rd_lo, rd_lo, AsmOperand(rn_lo, lsr, count), mi);
 805     }
 806     mov(rd_hi, AsmOperand(rn_hi, shift, count));
 807   }
 808 }
 809 
 810 void MacroAssembler::long_shift(Register rd_lo, Register rd_hi,
 811                                 Register rn_lo, Register rn_hi,
 812                                 AsmShift shift, int count) {
 813   assert(count != 0 && (count & ~63) == 0, "must be");
 814 
 815   if (shift == lsl) {
 816     assert_different_registers(rd_hi, rn_lo);
 817     if (count >= 32) {
 818       mov(rd_hi, AsmOperand(rn_lo, lsl, count - 32));
 819       mov(rd_lo, 0);
 820     } else {
 821       mov(rd_hi, AsmOperand(rn_hi, lsl, count));
 822       orr(rd_hi, rd_hi, AsmOperand(rn_lo, lsr, 32 - count));
 823       mov(rd_lo, AsmOperand(rn_lo, lsl, count));
 824     }
 825   } else {
 826     assert_different_registers(rd_lo, rn_hi);
 827     if (count >= 32) {
 828       if (count == 32) {
 829         mov(rd_lo, rn_hi);
 830       } else {
 831         mov(rd_lo, AsmOperand(rn_hi, shift, count - 32));
 832       }
 833       if (shift == asr) {
 834         mov(rd_hi, AsmOperand(rn_hi, asr, 0));
 835       } else {
 836         mov(rd_hi, 0);
 837       }
 838     } else {
 839       mov(rd_lo, AsmOperand(rn_lo, lsr, count));
 840       orr(rd_lo, rd_lo, AsmOperand(rn_hi, lsl, 32 - count));
 841       mov(rd_hi, AsmOperand(rn_hi, shift, count));
 842     }
 843   }
 844 }
 845 
 846 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 847   // This code pattern is matched in NativeIntruction::skip_verify_oop.
 848   // Update it at modifications.
 849   if (!VerifyOops) return;
 850 
 851   char buffer[64];
 852 #ifdef COMPILER1
 853   if (CommentedAssembly) {
 854     snprintf(buffer, sizeof(buffer), "verify_oop at %d", offset());
 855     block_comment(buffer);
 856   }
 857 #endif
 858   const char* msg_buffer = NULL;
 859   {
 860     ResourceMark rm;
 861     stringStream ss;
 862     ss.print("%s at offset %d (%s:%d)", s, offset(), file, line);
 863     msg_buffer = code_string(ss.as_string());
 864   }
 865 
 866   save_all_registers();
 867 
 868   if (reg != R2) {
 869       mov(R2, reg);                              // oop to verify
 870   }
 871   mov(R1, SP);                                   // register save area
 872 
 873   Label done;
 874   InlinedString Lmsg(msg_buffer);
 875   ldr_literal(R0, Lmsg);                         // message
 876 
 877   // call indirectly to solve generation ordering problem
 878   ldr_global_ptr(Rtemp, StubRoutines::verify_oop_subroutine_entry_address());
 879   call(Rtemp);
 880 
 881   restore_all_registers();
 882 
 883   b(done);
 884 #ifdef COMPILER2
 885   int off = offset();
 886 #endif
 887   bind_literal(Lmsg);
 888 #ifdef COMPILER2
 889   if (offset() - off == 1 * wordSize) {
 890     // no padding, so insert nop for worst-case sizing
 891     nop();
 892   }
 893 #endif
 894   bind(done);
 895 }
 896 
 897 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 898   if (!VerifyOops) return;
 899 
 900   const char* msg_buffer = NULL;
 901   {
 902     ResourceMark rm;
 903     stringStream ss;
 904     if ((addr.base() == SP) && (addr.index()==noreg)) {
 905       ss.print("verify_oop_addr SP[%d]: %s", (int)addr.disp(), s);
 906     } else {
 907       ss.print("verify_oop_addr: %s", s);
 908     }
 909     ss.print(" (%s:%d)", file, line);
 910     msg_buffer = code_string(ss.as_string());
 911   }
 912 
 913   int push_size = save_all_registers();
 914 
 915   if (addr.base() == SP) {
 916     // computes an addr that takes into account the push
 917     if (addr.index() != noreg) {
 918       Register new_base = addr.index() == R2 ? R1 : R2; // avoid corrupting the index
 919       add(new_base, SP, push_size);
 920       addr = addr.rebase(new_base);
 921     } else {
 922       addr = addr.plus_disp(push_size);
 923     }
 924   }
 925 
 926   ldr(R2, addr);                                 // oop to verify
 927   mov(R1, SP);                                   // register save area
 928 
 929   Label done;
 930   InlinedString Lmsg(msg_buffer);
 931   ldr_literal(R0, Lmsg);                         // message
 932 
 933   // call indirectly to solve generation ordering problem
 934   ldr_global_ptr(Rtemp, StubRoutines::verify_oop_subroutine_entry_address());
 935   call(Rtemp);
 936 
 937   restore_all_registers();
 938 
 939   b(done);
 940   bind_literal(Lmsg);
 941   bind(done);
 942 }
 943 
 944 void MacroAssembler::c2bool(Register x)
 945 {
 946   tst(x, 0xff);   // Only look at the lowest byte
 947   mov(x, 1, ne);
 948 }
 949 
 950 void MacroAssembler::null_check(Register reg, Register tmp, int offset) {
 951   if (needs_explicit_null_check(offset)) {
 952     assert_different_registers(reg, tmp);
 953     if (tmp == noreg) {
 954       tmp = Rtemp;
 955       assert((! Thread::current()->is_Compiler_thread()) ||
 956              (! (ciEnv::current()->task() == NULL)) ||
 957              (! (ciEnv::current()->comp_level() == CompLevel_full_optimization)),
 958              "Rtemp not available in C2"); // explicit tmp register required
 959       // XXX: could we mark the code buffer as not compatible with C2 ?
 960     }
 961     ldr(tmp, Address(reg));
 962   }
 963 }
 964 
 965 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
 966 void MacroAssembler::eden_allocate(Register obj, Register obj_end, Register tmp1, Register tmp2,
 967                                  RegisterOrConstant size_expression, Label& slow_case) {
 968   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 969   bs->eden_allocate(this, obj, obj_end, tmp1, tmp2, size_expression, slow_case);
 970 }
 971 
 972 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
 973 void MacroAssembler::tlab_allocate(Register obj, Register obj_end, Register tmp1,
 974                                  RegisterOrConstant size_expression, Label& slow_case) {
 975   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 976   bs->tlab_allocate(this, obj, obj_end, tmp1, size_expression, slow_case);
 977 }
 978 
 979 // Fills memory regions [start..end] with zeroes. Clobbers `start` and `tmp` registers.
 980 void MacroAssembler::zero_memory(Register start, Register end, Register tmp) {
 981   Label loop;
 982   const Register ptr = start;
 983 
 984   mov(tmp, 0);
 985   bind(loop);
 986   cmp(ptr, end);
 987   str(tmp, Address(ptr, wordSize, post_indexed), lo);
 988   b(loop, lo);
 989 }
 990 
 991 void MacroAssembler::arm_stack_overflow_check(int frame_size_in_bytes, Register tmp) {
 992   // Version of AbstractAssembler::generate_stack_overflow_check optimized for ARM
 993   if (UseStackBanging) {
 994     const int page_size = os::vm_page_size();
 995 
 996     sub_slow(tmp, SP, JavaThread::stack_shadow_zone_size());
 997     strb(R0, Address(tmp));
 998     for (; frame_size_in_bytes >= page_size; frame_size_in_bytes -= 0xff0) {
 999       strb(R0, Address(tmp, -0xff0, pre_indexed));
1000     }
1001   }
1002 }
1003 
1004 void MacroAssembler::arm_stack_overflow_check(Register Rsize, Register tmp) {
1005   if (UseStackBanging) {
1006     Label loop;
1007 
1008     mov(tmp, SP);
1009     add_slow(Rsize, Rsize, JavaThread::stack_shadow_zone_size() - os::vm_page_size());
1010     bind(loop);
1011     subs(Rsize, Rsize, 0xff0);
1012     strb(R0, Address(tmp, -0xff0, pre_indexed));
1013     b(loop, hi);
1014   }
1015 }
1016 
1017 void MacroAssembler::stop(const char* msg) {
1018   // This code pattern is matched in NativeIntruction::is_stop.
1019   // Update it at modifications.
1020 #ifdef COMPILER1
1021   if (CommentedAssembly) {
1022     block_comment("stop");
1023   }
1024 #endif
1025 
1026   InlinedAddress Ldebug(CAST_FROM_FN_PTR(address, MacroAssembler::debug));
1027   InlinedString Lmsg(msg);
1028 
1029   // save all registers for further inspection
1030   save_all_registers();
1031 
1032   ldr_literal(R0, Lmsg);                     // message
1033   mov(R1, SP);                               // register save area
1034 
1035   ldr_literal(PC, Ldebug);                   // call MacroAssembler::debug
1036 
1037   bind_literal(Lmsg);
1038   bind_literal(Ldebug);
1039 }
1040 
1041 void MacroAssembler::warn(const char* msg) {
1042 #ifdef COMPILER1
1043   if (CommentedAssembly) {
1044     block_comment("warn");
1045   }
1046 #endif
1047 
1048   InlinedAddress Lwarn(CAST_FROM_FN_PTR(address, warning));
1049   InlinedString Lmsg(msg);
1050   Label done;
1051 
1052   int push_size = save_caller_save_registers();
1053 
1054 
1055   ldr_literal(R0, Lmsg);                    // message
1056   ldr_literal(LR, Lwarn);                   // call warning
1057 
1058   call(LR);
1059 
1060   restore_caller_save_registers();
1061 
1062   b(done);
1063   bind_literal(Lmsg);
1064   bind_literal(Lwarn);
1065   bind(done);
1066 }
1067 
1068 
1069 int MacroAssembler::save_all_registers() {
1070   // This code pattern is matched in NativeIntruction::is_save_all_registers.
1071   // Update it at modifications.
1072   push(RegisterSet(R0, R12) | RegisterSet(LR) | RegisterSet(PC));
1073   return 15*wordSize;
1074 }
1075 
1076 void MacroAssembler::restore_all_registers() {
1077   pop(RegisterSet(R0, R12) | RegisterSet(LR));   // restore registers
1078   add(SP, SP, wordSize);                         // discard saved PC
1079 }
1080 
1081 int MacroAssembler::save_caller_save_registers() {
1082 #if R9_IS_SCRATCHED
1083   // Save also R10 to preserve alignment
1084   push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR) | RegisterSet(R9,R10));
1085   return 8*wordSize;
1086 #else
1087   push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR));
1088   return 6*wordSize;
1089 #endif
1090 }
1091 
1092 void MacroAssembler::restore_caller_save_registers() {
1093 #if R9_IS_SCRATCHED
1094   pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR) | RegisterSet(R9,R10));
1095 #else
1096   pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR));
1097 #endif
1098 }
1099 
1100 void MacroAssembler::debug(const char* msg, const intx* registers) {
1101   // In order to get locks to work, we need to fake a in_VM state
1102   JavaThread* thread = JavaThread::current();
1103   thread->set_thread_state(_thread_in_vm);
1104 
1105   if (ShowMessageBoxOnError) {
1106     ttyLocker ttyl;
1107     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
1108       BytecodeCounter::print();
1109     }
1110     if (os::message_box(msg, "Execution stopped, print registers?")) {
1111       // saved registers: R0-R12, LR, PC
1112       const int nregs = 15;
1113       const Register regs[nregs] = {R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, LR, PC};
1114 
1115       for (int i = 0; i < nregs; i++) {
1116         tty->print_cr("%s = " INTPTR_FORMAT, regs[i]->name(), registers[i]);
1117       }
1118 
1119       // derive original SP value from the address of register save area
1120       tty->print_cr("%s = " INTPTR_FORMAT, SP->name(), p2i(&registers[nregs]));
1121     }
1122     BREAKPOINT;
1123   } else {
1124     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
1125   }
1126   assert(false, "DEBUG MESSAGE: %s", msg);
1127   fatal("%s", msg); // returning from MacroAssembler::debug is not supported
1128 }
1129 
1130 void MacroAssembler::unimplemented(const char* what) {
1131   const char* buf = NULL;
1132   {
1133     ResourceMark rm;
1134     stringStream ss;
1135     ss.print("unimplemented: %s", what);
1136     buf = code_string(ss.as_string());
1137   }
1138   stop(buf);
1139 }
1140 
1141 
1142 // Implementation of FixedSizeCodeBlock
1143 
1144 FixedSizeCodeBlock::FixedSizeCodeBlock(MacroAssembler* masm, int size_in_instrs, bool enabled) :
1145 _masm(masm), _start(masm->pc()), _size_in_instrs(size_in_instrs), _enabled(enabled) {
1146 }
1147 
1148 FixedSizeCodeBlock::~FixedSizeCodeBlock() {
1149   if (_enabled) {
1150     address curr_pc = _masm->pc();
1151 
1152     assert(_start < curr_pc, "invalid current pc");
1153     guarantee(curr_pc <= _start + _size_in_instrs * Assembler::InstructionSize, "code block is too long");
1154 
1155     int nops_count = (_start - curr_pc) / Assembler::InstructionSize + _size_in_instrs;
1156     for (int i = 0; i < nops_count; i++) {
1157       _masm->nop();
1158     }
1159   }
1160 }
1161 
1162 
1163 // Serializes memory. Potentially blows flags and reg.
1164 // tmp is a scratch for v6 co-processor write op (could be noreg for other architecure versions)
1165 // preserve_flags takes a longer path in LoadStore case (dmb rather then control dependency) to preserve status flags. Optional.
1166 // load_tgt is an ordered load target in a LoadStore case only, to create dependency between the load operation and conditional branch. Optional.
1167 void MacroAssembler::membar(Membar_mask_bits order_constraint,
1168                             Register tmp,
1169                             bool preserve_flags,
1170                             Register load_tgt) {
1171 
1172   if (order_constraint == StoreStore) {
1173     dmb(DMB_st, tmp);
1174   } else if ((order_constraint & StoreLoad)  ||
1175              (order_constraint & LoadLoad)   ||
1176              (order_constraint & StoreStore) ||
1177              (load_tgt == noreg)             ||
1178              preserve_flags) {
1179     dmb(DMB_all, tmp);
1180   } else {
1181     // LoadStore: speculative stores reordeing is prohibited
1182 
1183     // By providing an ordered load target register, we avoid an extra memory load reference
1184     Label not_taken;
1185     bind(not_taken);
1186     cmp(load_tgt, load_tgt);
1187     b(not_taken, ne);
1188   }
1189 }
1190 
1191 
1192 // If "allow_fallthrough_on_failure" is false, we always branch to "slow_case"
1193 // on failure, so fall-through can only mean success.
1194 // "one_shot" controls whether we loop and retry to mitigate spurious failures.
1195 // This is only needed for C2, which for some reason does not rety,
1196 // while C1/interpreter does.
1197 // TODO: measure if it makes a difference
1198 
1199 void MacroAssembler::cas_for_lock_acquire(Register oldval, Register newval,
1200   Register base, Register tmp, Label &slow_case,
1201   bool allow_fallthrough_on_failure, bool one_shot)
1202 {
1203 
1204   bool fallthrough_is_success = false;
1205 
1206   // ARM Litmus Test example does prefetching here.
1207   // TODO: investigate if it helps performance
1208 
1209   // The last store was to the displaced header, so to prevent
1210   // reordering we must issue a StoreStore or Release barrier before
1211   // the CAS store.
1212 
1213   membar(MacroAssembler::StoreStore, noreg);
1214 
1215   if (one_shot) {
1216     ldrex(tmp, Address(base, oopDesc::mark_offset_in_bytes()));
1217     cmp(tmp, oldval);
1218     strex(tmp, newval, Address(base, oopDesc::mark_offset_in_bytes()), eq);
1219     cmp(tmp, 0, eq);
1220   } else {
1221     atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
1222   }
1223 
1224   // MemBarAcquireLock barrier
1225   // According to JSR-133 Cookbook, this should be LoadLoad | LoadStore,
1226   // but that doesn't prevent a load or store from floating up between
1227   // the load and store in the CAS sequence, so play it safe and
1228   // do a full fence.
1229   membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), noreg);
1230   if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
1231     b(slow_case, ne);
1232   }
1233 }
1234 
1235 void MacroAssembler::cas_for_lock_release(Register oldval, Register newval,
1236   Register base, Register tmp, Label &slow_case,
1237   bool allow_fallthrough_on_failure, bool one_shot)
1238 {
1239 
1240   bool fallthrough_is_success = false;
1241 
1242   assert_different_registers(oldval,newval,base,tmp);
1243 
1244   // MemBarReleaseLock barrier
1245   // According to JSR-133 Cookbook, this should be StoreStore | LoadStore,
1246   // but that doesn't prevent a load or store from floating down between
1247   // the load and store in the CAS sequence, so play it safe and
1248   // do a full fence.
1249   membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), tmp);
1250 
1251   if (one_shot) {
1252     ldrex(tmp, Address(base, oopDesc::mark_offset_in_bytes()));
1253     cmp(tmp, oldval);
1254     strex(tmp, newval, Address(base, oopDesc::mark_offset_in_bytes()), eq);
1255     cmp(tmp, 0, eq);
1256   } else {
1257     atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
1258   }
1259   if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
1260     b(slow_case, ne);
1261   }
1262 
1263   // ExitEnter
1264   // According to JSR-133 Cookbook, this should be StoreLoad, the same
1265   // barrier that follows volatile store.
1266   // TODO: Should be able to remove on armv8 if volatile loads
1267   // use the load-acquire instruction.
1268   membar(StoreLoad, noreg);
1269 }
1270 
1271 #ifndef PRODUCT
1272 
1273 // Preserves flags and all registers.
1274 // On SMP the updated value might not be visible to external observers without a sychronization barrier
1275 void MacroAssembler::cond_atomic_inc32(AsmCondition cond, int* counter_addr) {
1276   if (counter_addr != NULL) {
1277     InlinedAddress counter_addr_literal((address)counter_addr);
1278     Label done, retry;
1279     if (cond != al) {
1280       b(done, inverse(cond));
1281     }
1282 
1283     push(RegisterSet(R0, R3) | RegisterSet(Rtemp));
1284     ldr_literal(R0, counter_addr_literal);
1285 
1286     mrs(CPSR, Rtemp);
1287 
1288     bind(retry);
1289     ldr_s32(R1, Address(R0));
1290     add(R2, R1, 1);
1291     atomic_cas_bool(R1, R2, R0, 0, R3);
1292     b(retry, ne);
1293 
1294     msr(CPSR_fsxc, Rtemp);
1295 
1296     pop(RegisterSet(R0, R3) | RegisterSet(Rtemp));
1297 
1298     b(done);
1299     bind_literal(counter_addr_literal);
1300 
1301     bind(done);
1302   }
1303 }
1304 
1305 #endif // !PRODUCT
1306 
1307 
1308 // Building block for CAS cases of biased locking: makes CAS and records statistics.
1309 // The slow_case label is used to transfer control if CAS fails. Otherwise leaves condition codes set.
1310 void MacroAssembler::biased_locking_enter_with_cas(Register obj_reg, Register old_mark_reg, Register new_mark_reg,
1311                                                  Register tmp, Label& slow_case, int* counter_addr) {
1312 
1313   cas_for_lock_acquire(old_mark_reg, new_mark_reg, obj_reg, tmp, slow_case);
1314 #ifdef ASSERT
1315   breakpoint(ne); // Fallthrough only on success
1316 #endif
1317 #ifndef PRODUCT
1318   if (counter_addr != NULL) {
1319     cond_atomic_inc32(al, counter_addr);
1320   }
1321 #endif // !PRODUCT
1322 }
1323 
1324 int MacroAssembler::biased_locking_enter(Register obj_reg, Register swap_reg, Register tmp_reg,
1325                                          bool swap_reg_contains_mark,
1326                                          Register tmp2,
1327                                          Label& done, Label& slow_case,
1328                                          BiasedLockingCounters* counters) {
1329   // obj_reg must be preserved (at least) if the bias locking fails
1330   // tmp_reg is a temporary register
1331   // swap_reg was used as a temporary but contained a value
1332   //   that was used afterwards in some call pathes. Callers
1333   //   have been fixed so that swap_reg no longer needs to be
1334   //   saved.
1335   // Rtemp in no longer scratched
1336 
1337   assert(UseBiasedLocking, "why call this otherwise?");
1338   assert_different_registers(obj_reg, swap_reg, tmp_reg, tmp2);
1339   guarantee(swap_reg!=tmp_reg, "invariant");
1340   assert(tmp_reg != noreg, "must supply tmp_reg");
1341 
1342 #ifndef PRODUCT
1343   if (PrintBiasedLockingStatistics && (counters == NULL)) {
1344     counters = BiasedLocking::counters();
1345   }
1346 #endif
1347 
1348   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1349   Address mark_addr(obj_reg, oopDesc::mark_offset_in_bytes());
1350 
1351   // Biased locking
1352   // See whether the lock is currently biased toward our thread and
1353   // whether the epoch is still valid
1354   // Note that the runtime guarantees sufficient alignment of JavaThread
1355   // pointers to allow age to be placed into low bits
1356   // First check to see whether biasing is even enabled for this object
1357   Label cas_label;
1358 
1359   // The null check applies to the mark loading, if we need to load it.
1360   // If the mark has already been loaded in swap_reg then it has already
1361   // been performed and the offset is irrelevant.
1362   int null_check_offset = offset();
1363   if (!swap_reg_contains_mark) {
1364     ldr(swap_reg, mark_addr);
1365   }
1366 
1367   // On MP platform loads could return 'stale' values in some cases.
1368   // That is acceptable since either CAS or slow case path is taken in the worst case.
1369 
1370   andr(tmp_reg, swap_reg, (uintx)markOopDesc::biased_lock_mask_in_place);
1371   cmp(tmp_reg, markOopDesc::biased_lock_pattern);
1372 
1373   b(cas_label, ne);
1374 
1375   // The bias pattern is present in the object's header. Need to check
1376   // whether the bias owner and the epoch are both still current.
1377   load_klass(tmp_reg, obj_reg);
1378   ldr(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset()));
1379   orr(tmp_reg, tmp_reg, Rthread);
1380   eor(tmp_reg, tmp_reg, swap_reg);
1381 
1382   bics(tmp_reg, tmp_reg, ((int) markOopDesc::age_mask_in_place));
1383 
1384 #ifndef PRODUCT
1385   if (counters != NULL) {
1386     cond_atomic_inc32(eq, counters->biased_lock_entry_count_addr());
1387   }
1388 #endif // !PRODUCT
1389 
1390   b(done, eq);
1391 
1392   Label try_revoke_bias;
1393   Label try_rebias;
1394 
1395   // At this point we know that the header has the bias pattern and
1396   // that we are not the bias owner in the current epoch. We need to
1397   // figure out more details about the state of the header in order to
1398   // know what operations can be legally performed on the object's
1399   // header.
1400 
1401   // If the low three bits in the xor result aren't clear, that means
1402   // the prototype header is no longer biased and we have to revoke
1403   // the bias on this object.
1404   tst(tmp_reg, (uintx)markOopDesc::biased_lock_mask_in_place);
1405   b(try_revoke_bias, ne);
1406 
1407   // Biasing is still enabled for this data type. See whether the
1408   // epoch of the current bias is still valid, meaning that the epoch
1409   // bits of the mark word are equal to the epoch bits of the
1410   // prototype header. (Note that the prototype header's epoch bits
1411   // only change at a safepoint.) If not, attempt to rebias the object
1412   // toward the current thread. Note that we must be absolutely sure
1413   // that the current epoch is invalid in order to do this because
1414   // otherwise the manipulations it performs on the mark word are
1415   // illegal.
1416   tst(tmp_reg, (uintx)markOopDesc::epoch_mask_in_place);
1417   b(try_rebias, ne);
1418 
1419   // tmp_reg has the age, epoch and pattern bits cleared
1420   // The remaining (owner) bits are (Thread ^ current_owner)
1421 
1422   // The epoch of the current bias is still valid but we know nothing
1423   // about the owner; it might be set or it might be clear. Try to
1424   // acquire the bias of the object using an atomic operation. If this
1425   // fails we will go in to the runtime to revoke the object's bias.
1426   // Note that we first construct the presumed unbiased header so we
1427   // don't accidentally blow away another thread's valid bias.
1428 
1429   // Note that we know the owner is not ourself. Hence, success can
1430   // only happen when the owner bits is 0
1431 
1432   // until the assembler can be made smarter, we need to make some assumptions about the values
1433   // so we can optimize this:
1434   assert((markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place) == 0x1ff, "biased bitmasks changed");
1435 
1436   mov(swap_reg, AsmOperand(swap_reg, lsl, 23));
1437   mov(swap_reg, AsmOperand(swap_reg, lsr, 23)); // markOop with thread bits cleared (for CAS)
1438 
1439   orr(tmp_reg, swap_reg, Rthread); // new mark
1440 
1441   biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
1442         (counters != NULL) ? counters->anonymously_biased_lock_entry_count_addr() : NULL);
1443 
1444   // If the biasing toward our thread failed, this means that
1445   // another thread succeeded in biasing it toward itself and we
1446   // need to revoke that bias. The revocation will occur in the
1447   // interpreter runtime in the slow case.
1448 
1449   b(done);
1450 
1451   bind(try_rebias);
1452 
1453   // At this point we know the epoch has expired, meaning that the
1454   // current "bias owner", if any, is actually invalid. Under these
1455   // circumstances _only_, we are allowed to use the current header's
1456   // value as the comparison value when doing the cas to acquire the
1457   // bias in the current epoch. In other words, we allow transfer of
1458   // the bias from one thread to another directly in this situation.
1459 
1460   // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
1461 
1462   eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
1463 
1464   // owner bits 'random'. Set them to Rthread.
1465   mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
1466   mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
1467 
1468   orr(tmp_reg, tmp_reg, Rthread); // new mark
1469 
1470   biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
1471         (counters != NULL) ? counters->rebiased_lock_entry_count_addr() : NULL);
1472 
1473   // If the biasing toward our thread failed, then another thread
1474   // succeeded in biasing it toward itself and we need to revoke that
1475   // bias. The revocation will occur in the runtime in the slow case.
1476 
1477   b(done);
1478 
1479   bind(try_revoke_bias);
1480 
1481   // The prototype mark in the klass doesn't have the bias bit set any
1482   // more, indicating that objects of this data type are not supposed
1483   // to be biased any more. We are going to try to reset the mark of
1484   // this object to the prototype value and fall through to the
1485   // CAS-based locking scheme. Note that if our CAS fails, it means
1486   // that another thread raced us for the privilege of revoking the
1487   // bias of this particular object, so it's okay to continue in the
1488   // normal locking code.
1489 
1490   // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
1491 
1492   eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
1493 
1494   // owner bits 'random'. Clear them
1495   mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
1496   mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
1497 
1498   biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, cas_label,
1499         (counters != NULL) ? counters->revoked_lock_entry_count_addr() : NULL);
1500 
1501   // Fall through to the normal CAS-based lock, because no matter what
1502   // the result of the above CAS, some thread must have succeeded in
1503   // removing the bias bit from the object's header.
1504 
1505   bind(cas_label);
1506 
1507   return null_check_offset;
1508 }
1509 
1510 
1511 void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done) {
1512   assert(UseBiasedLocking, "why call this otherwise?");
1513 
1514   // Check for biased locking unlock case, which is a no-op
1515   // Note: we do not have to check the thread ID for two reasons.
1516   // First, the interpreter checks for IllegalMonitorStateException at
1517   // a higher level. Second, if the bias was revoked while we held the
1518   // lock, the object could not be rebiased toward another thread, so
1519   // the bias bit would be clear.
1520   ldr(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1521 
1522   andr(tmp_reg, tmp_reg, (uintx)markOopDesc::biased_lock_mask_in_place);
1523   cmp(tmp_reg, markOopDesc::biased_lock_pattern);
1524   b(done, eq);
1525 }
1526 
1527 
1528 void MacroAssembler::resolve_jobject(Register value,
1529                                      Register tmp1,
1530                                      Register tmp2) {
1531   assert_different_registers(value, tmp1, tmp2);
1532   Label done, not_weak;
1533   cbz(value, done);             // Use NULL as-is.
1534   STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
1535   tbz(value, 0, not_weak);      // Test for jweak tag.
1536 
1537   // Resolve jweak.
1538   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
1539                  Address(value, -JNIHandles::weak_tag_value), value, tmp1, tmp2, noreg);
1540   b(done);
1541   bind(not_weak);
1542   // Resolve (untagged) jobject.
1543   access_load_at(T_OBJECT, IN_NATIVE,
1544                  Address(value, 0), value, tmp1, tmp2, noreg);
1545   verify_oop(value);
1546   bind(done);
1547 }
1548 
1549 
1550 //////////////////////////////////////////////////////////////////////////////////
1551 
1552 
1553 void MacroAssembler::load_sized_value(Register dst, Address src,
1554                                     size_t size_in_bytes, bool is_signed, AsmCondition cond) {
1555   switch (size_in_bytes) {
1556     case  4: ldr(dst, src, cond); break;
1557     case  2: is_signed ? ldrsh(dst, src, cond) : ldrh(dst, src, cond); break;
1558     case  1: is_signed ? ldrsb(dst, src, cond) : ldrb(dst, src, cond); break;
1559     default: ShouldNotReachHere();
1560   }
1561 }
1562 
1563 
1564 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes, AsmCondition cond) {
1565   switch (size_in_bytes) {
1566     case  4: str(src, dst, cond); break;
1567     case  2: strh(src, dst, cond);   break;
1568     case  1: strb(src, dst, cond);   break;
1569     default: ShouldNotReachHere();
1570   }
1571 }
1572 
1573 // Look up the method for a megamorphic invokeinterface call.
1574 // The target method is determined by <Rinterf, Rindex>.
1575 // The receiver klass is in Rklass.
1576 // On success, the result will be in method_result, and execution falls through.
1577 // On failure, execution transfers to the given label.
1578 void MacroAssembler::lookup_interface_method(Register Rklass,
1579                                              Register Rintf,
1580                                              RegisterOrConstant itable_index,
1581                                              Register method_result,
1582                                              Register Rscan,
1583                                              Register Rtmp,
1584                                              Label& L_no_such_interface) {
1585 
1586   assert_different_registers(Rklass, Rintf, Rscan, Rtmp);
1587 
1588   const int entry_size = itableOffsetEntry::size() * HeapWordSize;
1589   assert(itableOffsetEntry::interface_offset_in_bytes() == 0, "not added for convenience");
1590 
1591   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
1592   const int base = in_bytes(Klass::vtable_start_offset());
1593   const int scale = exact_log2(vtableEntry::size_in_bytes());
1594   ldr_s32(Rtmp, Address(Rklass, Klass::vtable_length_offset())); // Get length of vtable
1595   add(Rscan, Rklass, base);
1596   add(Rscan, Rscan, AsmOperand(Rtmp, lsl, scale));
1597 
1598   // Search through the itable for an interface equal to incoming Rintf
1599   // itable looks like [intface][offset][intface][offset][intface][offset]
1600 
1601   Label loop;
1602   bind(loop);
1603   ldr(Rtmp, Address(Rscan, entry_size, post_indexed));
1604   cmp(Rtmp, Rintf);  // set ZF and CF if interface is found
1605   cmn(Rtmp, 0, ne);  // check if tmp == 0 and clear CF if it is
1606   b(loop, ne);
1607 
1608   // CF == 0 means we reached the end of itable without finding icklass
1609   b(L_no_such_interface, cc);
1610 
1611   if (method_result != noreg) {
1612     // Interface found at previous position of Rscan, now load the method
1613     ldr_s32(Rtmp, Address(Rscan, itableOffsetEntry::offset_offset_in_bytes() - entry_size));
1614     if (itable_index.is_register()) {
1615       add(Rtmp, Rtmp, Rklass); // Add offset to Klass*
1616       assert(itableMethodEntry::size() * HeapWordSize == wordSize, "adjust the scaling in the code below");
1617       assert(itableMethodEntry::method_offset_in_bytes() == 0, "adjust the offset in the code below");
1618       ldr(method_result, Address::indexed_ptr(Rtmp, itable_index.as_register()));
1619     } else {
1620       int method_offset = itableMethodEntry::size() * HeapWordSize * itable_index.as_constant() +
1621                           itableMethodEntry::method_offset_in_bytes();
1622       add_slow(method_result, Rklass, method_offset);
1623       ldr(method_result, Address(method_result, Rtmp));
1624     }
1625   }
1626 }
1627 
1628 #ifdef COMPILER2
1629 // TODO: 8 bytes at a time? pre-fetch?
1630 // Compare char[] arrays aligned to 4 bytes.
1631 void MacroAssembler::char_arrays_equals(Register ary1, Register ary2,
1632                                         Register limit, Register result,
1633                                       Register chr1, Register chr2, Label& Ldone) {
1634   Label Lvector, Lloop;
1635 
1636   // if (ary1 == ary2)
1637   //     return true;
1638   cmpoop(ary1, ary2);
1639   b(Ldone, eq);
1640 
1641   // Note: limit contains number of bytes (2*char_elements) != 0.
1642   tst(limit, 0x2); // trailing character ?
1643   b(Lvector, eq);
1644 
1645   // compare the trailing char
1646   sub(limit, limit, sizeof(jchar));
1647   ldrh(chr1, Address(ary1, limit));
1648   ldrh(chr2, Address(ary2, limit));
1649   cmp(chr1, chr2);
1650   mov(result, 0, ne);     // not equal
1651   b(Ldone, ne);
1652 
1653   // only one char ?
1654   tst(limit, limit);
1655   mov(result, 1, eq);
1656   b(Ldone, eq);
1657 
1658   // word by word compare, dont't need alignment check
1659   bind(Lvector);
1660 
1661   // Shift ary1 and ary2 to the end of the arrays, negate limit
1662   add(ary1, limit, ary1);
1663   add(ary2, limit, ary2);
1664   neg(limit, limit);
1665 
1666   bind(Lloop);
1667   ldr_u32(chr1, Address(ary1, limit));
1668   ldr_u32(chr2, Address(ary2, limit));
1669   cmp_32(chr1, chr2);
1670   mov(result, 0, ne);     // not equal
1671   b(Ldone, ne);
1672   adds(limit, limit, 2*sizeof(jchar));
1673   b(Lloop, ne);
1674 
1675   // Caller should set it:
1676   // mov(result_reg, 1);  //equal
1677 }
1678 #endif
1679 
1680 void MacroAssembler::inc_counter(address counter_addr, Register tmpreg1, Register tmpreg2) {
1681   mov_slow(tmpreg1, counter_addr);
1682   ldr_s32(tmpreg2, tmpreg1);
1683   add_32(tmpreg2, tmpreg2, 1);
1684   str_32(tmpreg2, tmpreg1);
1685 }
1686 
1687 void MacroAssembler::floating_cmp(Register dst) {
1688   vmrs(dst, FPSCR);
1689   orr(dst, dst, 0x08000000);
1690   eor(dst, dst, AsmOperand(dst, lsl, 3));
1691   mov(dst, AsmOperand(dst, asr, 30));
1692 }
1693 
1694 void MacroAssembler::restore_default_fp_mode() {
1695 #ifndef __SOFTFP__
1696   // Round to Near mode, IEEE compatible, masked exceptions
1697   mov(Rtemp, 0);
1698   vmsr(FPSCR, Rtemp);
1699 #endif // !__SOFTFP__
1700 }
1701 
1702 // 24-bit word range == 26-bit byte range
1703 bool check26(int offset) {
1704   // this could be simplified, but it mimics encoding and decoding
1705   // an actual branch insrtuction
1706   int off1 = offset << 6 >> 8;
1707   int encoded = off1 & ((1<<24)-1);
1708   int decoded = encoded << 8 >> 6;
1709   return offset == decoded;
1710 }
1711 
1712 // Perform some slight adjustments so the default 32MB code cache
1713 // is fully reachable.
1714 static inline address first_cache_address() {
1715   return CodeCache::low_bound() + sizeof(HeapBlock::Header);
1716 }
1717 static inline address last_cache_address() {
1718   return CodeCache::high_bound() - Assembler::InstructionSize;
1719 }
1720 
1721 
1722 // Can we reach target using unconditional branch or call from anywhere
1723 // in the code cache (because code can be relocated)?
1724 bool MacroAssembler::_reachable_from_cache(address target) {
1725 #ifdef __thumb__
1726   if ((1 & (intptr_t)target) != 0) {
1727     // Return false to avoid 'b' if we need switching to THUMB mode.
1728     return false;
1729   }
1730 #endif
1731 
1732   address cl = first_cache_address();
1733   address ch = last_cache_address();
1734 
1735   if (ForceUnreachable) {
1736     // Only addresses from CodeCache can be treated as reachable.
1737     if (target < CodeCache::low_bound() || CodeCache::high_bound() < target) {
1738       return false;
1739     }
1740   }
1741 
1742   intptr_t loffset = (intptr_t)target - (intptr_t)cl;
1743   intptr_t hoffset = (intptr_t)target - (intptr_t)ch;
1744 
1745   return check26(loffset - 8) && check26(hoffset - 8);
1746 }
1747 
1748 bool MacroAssembler::reachable_from_cache(address target) {
1749   assert(CodeCache::contains(pc()), "not supported");
1750   return _reachable_from_cache(target);
1751 }
1752 
1753 // Can we reach the entire code cache from anywhere else in the code cache?
1754 bool MacroAssembler::_cache_fully_reachable() {
1755   address cl = first_cache_address();
1756   address ch = last_cache_address();
1757   return _reachable_from_cache(cl) && _reachable_from_cache(ch);
1758 }
1759 
1760 bool MacroAssembler::cache_fully_reachable() {
1761   assert(CodeCache::contains(pc()), "not supported");
1762   return _cache_fully_reachable();
1763 }
1764 
1765 void MacroAssembler::jump(address target, relocInfo::relocType rtype, Register scratch, AsmCondition cond) {
1766   assert((rtype == relocInfo::runtime_call_type) || (rtype == relocInfo::none), "not supported");
1767   if (reachable_from_cache(target)) {
1768     relocate(rtype);
1769     b(target, cond);
1770     return;
1771   }
1772 
1773   // Note: relocate is not needed for the code below,
1774   // encoding targets in absolute format.
1775   if (ignore_non_patchable_relocations()) {
1776     rtype = relocInfo::none;
1777   }
1778 
1779   if (VM_Version::supports_movw() && (scratch != noreg) && (rtype == relocInfo::none)) {
1780     // Note: this version cannot be (atomically) patched
1781     mov_slow(scratch, (intptr_t)target, cond);
1782     bx(scratch, cond);
1783   } else {
1784     Label skip;
1785     InlinedAddress address_literal(target);
1786     if (cond != al) {
1787       b(skip, inverse(cond));
1788     }
1789     relocate(rtype);
1790     ldr_literal(PC, address_literal);
1791     bind_literal(address_literal);
1792     bind(skip);
1793   }
1794 }
1795 
1796 // Similar to jump except that:
1797 // - near calls are valid only if any destination in the cache is near
1798 // - no movt/movw (not atomically patchable)
1799 void MacroAssembler::patchable_jump(address target, relocInfo::relocType rtype, Register scratch, AsmCondition cond) {
1800   assert((rtype == relocInfo::runtime_call_type) || (rtype == relocInfo::none), "not supported");
1801   if (cache_fully_reachable()) {
1802     // Note: this assumes that all possible targets (the initial one
1803     // and the addressed patched to) are all in the code cache.
1804     assert(CodeCache::contains(target), "target might be too far");
1805     relocate(rtype);
1806     b(target, cond);
1807     return;
1808   }
1809 
1810   // Discard the relocation information if not needed for CacheCompiledCode
1811   // since the next encodings are all in absolute format.
1812   if (ignore_non_patchable_relocations()) {
1813     rtype = relocInfo::none;
1814   }
1815 
1816   {
1817     Label skip;
1818     InlinedAddress address_literal(target);
1819     if (cond != al) {
1820       b(skip, inverse(cond));
1821     }
1822     relocate(rtype);
1823     ldr_literal(PC, address_literal);
1824     bind_literal(address_literal);
1825     bind(skip);
1826   }
1827 }
1828 
1829 void MacroAssembler::call(address target, RelocationHolder rspec, AsmCondition cond) {
1830   Register scratch = LR;
1831   assert(rspec.type() == relocInfo::runtime_call_type || rspec.type() == relocInfo::none, "not supported");
1832   if (reachable_from_cache(target)) {
1833     relocate(rspec);
1834     bl(target, cond);
1835     return;
1836   }
1837 
1838   // Note: relocate is not needed for the code below,
1839   // encoding targets in absolute format.
1840   if (ignore_non_patchable_relocations()) {
1841     // This assumes the information was needed only for relocating the code.
1842     rspec = RelocationHolder::none;
1843   }
1844 
1845   if (VM_Version::supports_movw() && (rspec.type() == relocInfo::none)) {
1846     // Note: this version cannot be (atomically) patched
1847     mov_slow(scratch, (intptr_t)target, cond);
1848     blx(scratch, cond);
1849     return;
1850   }
1851 
1852   {
1853     Label ret_addr;
1854     if (cond != al) {
1855       b(ret_addr, inverse(cond));
1856     }
1857 
1858 
1859     InlinedAddress address_literal(target);
1860     relocate(rspec);
1861     adr(LR, ret_addr);
1862     ldr_literal(PC, address_literal);
1863 
1864     bind_literal(address_literal);
1865     bind(ret_addr);
1866   }
1867 }
1868 
1869 
1870 int MacroAssembler::patchable_call(address target, RelocationHolder const& rspec, bool c2) {
1871   assert(rspec.type() == relocInfo::static_call_type ||
1872          rspec.type() == relocInfo::none ||
1873          rspec.type() == relocInfo::opt_virtual_call_type, "not supported");
1874 
1875   // Always generate the relocation information, needed for patching
1876   relocate(rspec); // used by NativeCall::is_call_before()
1877   if (cache_fully_reachable()) {
1878     // Note: this assumes that all possible targets (the initial one
1879     // and the addresses patched to) are all in the code cache.
1880     assert(CodeCache::contains(target), "target might be too far");
1881     bl(target);
1882   } else {
1883     Label ret_addr;
1884     InlinedAddress address_literal(target);
1885     adr(LR, ret_addr);
1886     ldr_literal(PC, address_literal);
1887     bind_literal(address_literal);
1888     bind(ret_addr);
1889   }
1890   return offset();
1891 }
1892 
1893 // ((OopHandle)result).resolve();
1894 void MacroAssembler::resolve_oop_handle(Register result) {
1895   // OopHandle::resolve is an indirection.
1896   ldr(result, Address(result, 0));
1897 }
1898 
1899 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
1900   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
1901   ldr(tmp, Address(method, Method::const_offset()));
1902   ldr(tmp, Address(tmp,  ConstMethod::constants_offset()));
1903   ldr(tmp, Address(tmp, ConstantPool::pool_holder_offset_in_bytes()));
1904   ldr(mirror, Address(tmp, mirror_offset));
1905   resolve_oop_handle(mirror);
1906 }
1907 
1908 
1909 ///////////////////////////////////////////////////////////////////////////////
1910 
1911 // Compressed pointers
1912 
1913 
1914 void MacroAssembler::load_klass(Register dst_klass, Register src_oop, AsmCondition cond) {
1915   ldr(dst_klass, Address(src_oop, oopDesc::klass_offset_in_bytes()), cond);
1916 }
1917 
1918 
1919 // Blows src_klass.
1920 void MacroAssembler::store_klass(Register src_klass, Register dst_oop) {
1921   str(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
1922 }
1923 
1924 
1925 
1926 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1927   access_load_at(T_OBJECT, IN_HEAP | decorators, src, dst, tmp1, tmp2, tmp3);
1928 }
1929 
1930 // Blows src and flags.
1931 void MacroAssembler::store_heap_oop(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1932   access_store_at(T_OBJECT, IN_HEAP | decorators, obj, new_val, tmp1, tmp2, tmp3, false);
1933 }
1934 
1935 void MacroAssembler::store_heap_oop_null(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1936   access_store_at(T_OBJECT, IN_HEAP, obj, new_val, tmp1, tmp2, tmp3, true);
1937 }
1938 
1939 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
1940                                     Address src, Register dst, Register tmp1, Register tmp2, Register tmp3) {
1941   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1942   decorators = AccessInternal::decorator_fixup(decorators);
1943   bool as_raw = (decorators & AS_RAW) != 0;
1944   if (as_raw) {
1945     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
1946   } else {
1947     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
1948   }
1949 }
1950 
1951 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
1952                                      Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, bool is_null) {
1953   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1954   decorators = AccessInternal::decorator_fixup(decorators);
1955   bool as_raw = (decorators & AS_RAW) != 0;
1956   if (as_raw) {
1957     bs->BarrierSetAssembler::store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
1958   } else {
1959     bs->store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
1960   }
1961 }
1962 
1963 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
1964   // Use stronger ACCESS_WRITE|ACCESS_READ by default.
1965   if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
1966     decorators |= ACCESS_READ | ACCESS_WRITE;
1967   }
1968   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1969   return bs->resolve(this, decorators, obj);
1970 }
1971 
1972 
1973 #ifdef COMPILER2
1974 void MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2)
1975 {
1976   assert(VM_Version::supports_ldrex(), "unsupported, yet?");
1977 
1978   Register Rmark      = Rscratch2;
1979 
1980   assert(Roop != Rscratch, "");
1981   assert(Roop != Rmark, "");
1982   assert(Rbox != Rscratch, "");
1983   assert(Rbox != Rmark, "");
1984 
1985   Label fast_lock, done;
1986 
1987   if (UseBiasedLocking && !UseOptoBiasInlining) {
1988     Label failed;
1989     biased_locking_enter(Roop, Rmark, Rscratch, false, noreg, done, failed);
1990     bind(failed);
1991   }
1992 
1993   ldr(Rmark, Address(Roop, oopDesc::mark_offset_in_bytes()));
1994   tst(Rmark, markOopDesc::unlocked_value);
1995   b(fast_lock, ne);
1996 
1997   // Check for recursive lock
1998   // See comments in InterpreterMacroAssembler::lock_object for
1999   // explanations on the fast recursive locking check.
2000   // -1- test low 2 bits
2001   movs(Rscratch, AsmOperand(Rmark, lsl, 30));
2002   // -2- test (hdr - SP) if the low two bits are 0
2003   sub(Rscratch, Rmark, SP, eq);
2004   movs(Rscratch, AsmOperand(Rscratch, lsr, exact_log2(os::vm_page_size())), eq);
2005   // If still 'eq' then recursive locking OK
2006   // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8153107)
2007   str(Rscratch, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
2008   b(done);
2009 
2010   bind(fast_lock);
2011   str(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
2012 
2013   bool allow_fallthrough_on_failure = true;
2014   bool one_shot = true;
2015   cas_for_lock_acquire(Rmark, Rbox, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot);
2016 
2017   bind(done);
2018 
2019 }
2020 
2021 void MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2)
2022 {
2023   assert(VM_Version::supports_ldrex(), "unsupported, yet?");
2024 
2025   Register Rmark      = Rscratch2;
2026 
2027   assert(Roop != Rscratch, "");
2028   assert(Roop != Rmark, "");
2029   assert(Rbox != Rscratch, "");
2030   assert(Rbox != Rmark, "");
2031 
2032   Label done;
2033 
2034   if (UseBiasedLocking && !UseOptoBiasInlining) {
2035     biased_locking_exit(Roop, Rscratch, done);
2036   }
2037 
2038   ldr(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
2039   // If hdr is NULL, we've got recursive locking and there's nothing more to do
2040   cmp(Rmark, 0);
2041   b(done, eq);
2042 
2043   // Restore the object header
2044   bool allow_fallthrough_on_failure = true;
2045   bool one_shot = true;
2046   cas_for_lock_release(Rmark, Rbox, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot);
2047 
2048   bind(done);
2049 
2050 }
2051 #endif // COMPILER2